ethereum-etl export data every time write over file. Filebeat always reload file then ELK receive repeat records…

So

.env

  
STARTBLOCK=01205866  
ENDBLOCK=01205888

startetl.sh

IP_PORT is go-ethereum-node1 outside ip & port. here is docker-machine
source get file that path need to be careful

  
#!/bin/bash  
  
IP_PORT=192.168.99.100:18545  
ETH_METHOD=eth_blockNumber  
BLOCKNUMBER_JSON_HEX=$(curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"'$ETH_METHOD'","params":[],"id":1}' $IP_PORT | jq '.result' | tr -d '"')  
BLOCKNUMBER_DEX=$(printf "%08d\n" $BLOCKNUMBER_JSON_HEX)  
  
printf "\n===== Now Geth BlockerNumber =====\n"  
printf "HEX: %s\n" $BLOCKNUMBER_JSON_HEX  
printf "DEC: %s\n" $BLOCKNUMBER_DEX  
  
source .env  
#source ~/ethereum_etl/.env  
  
printf "\n===== .env BLOCK VARS =====\n"  
printf "STARTBLOCK: %s\n" $STARTBLOCK  
printf "ENDBLOCK: %s\n" $ENDBLOCK  
  
if [[ $BLOCKNUMBER_DEX =~ ^?[0-9]+$ ]] || [ -z $BLOCKNUMBER_DEX ]; then  
  printf "NOT Number Geth BLOCKNUMBER, check Geth\n"  
  exit  
fi  
  
if [[ $STARTBLOCK =~ ^?[0-9]+$ ]] || [ -z $STARTBLOCK ]; then  
  printf "NOT Number .env STARTBLOCK, check .env\n"  
  exit  
fi  
  
if [[ $ENDBLOCK =~ ^?[0-9]+$ ]] || [ -z $ENDBLOCK ]; then  
  printf "NOT Number .env ENDBLOCK, check .env\n"  
  exit  
fi  
  
#printf "\n%s\n" $BLOCKNUMBER_DEX  
#printf "%s" $STARTBLOCK  
#printf "\n%s\n" $((10#$BLOCKNUMBER_DEX - 10#$STARTBLOCK))  
  
if (( (( 10#$BLOCKNUMBER_DEX - 10#$STARTBLOCK -1 )) > 0 )); then  
  printf "\n===== Change .env =====\n"  
  printf "startblock: %s => %s\n" $STARTBLOCK $((10#$ENDBLOCK+1))  
  printf "endblock: %s => %s\n" $ENDBLOCK $BLOCKNUMBER_DEX    
  
  STARTBLOCK=$(printf "%08d\n" $((10#$ENDBLOCK+1)) )  
  echo STARTBLOCK=$STARTBLOCK > ~/ethereum_etl/.env  
  echo ENDBLOCK=$BLOCKNUMBER_DEX >> .env  
  #echo STARTBLOCK=$STARTBLOCK > ~/ethereum_etl/.env  
  #echo ENDBLOCK=$BLOCKNUMBER_DEX >> ~ethereum_etl/.env  
fi  
  
docker-compose up --build ethereum_etl

docker-compose.yml

  
version: '3.3'  
  
services:  
  go-ethereum-node1:  
    build:  
      context: go-ethereum-node1/  
    volumes:  
      #- ./go-ethereum/keystore:/root/.ethereum/devchain/keystore:rw  
      - ./go-ethereum-node1/genesis/poa_for_dev.json:/root/genesis/poa_for_dev.json:ro  
      - alldata:/root/.ethereum/devchain  
      - /etc/localtime:/etc/localtime:ro  
      #- ./go-ethereum/log:/root/log:rw  
    entrypoint: /root/start.sh  
    ports:  
      - "18545:8545"  
      - "30313:30303"  
      - "30313:30303/udp"  
    networks:  
      - etl  
  
  ethereum_etl:  
    build:  
      context: ethereum-etl/  
    env_file: .env  
    volumes:  
      - alldata:/ethereum-etl/output:rw  
      #- /root/go/src/github.com/ethereum/go-ethereum/build/bin/data:/ethereum-etl/ipc  
    #restart: unless-stopped  
    networks:  
      - etl  
  
volumes:    
  alldata:   
  
networks:  
  etl:  
    driver: bridge

ethereum_etl DOCKERFILE

  
FROM python:3.6-alpine  
MAINTAINER Eric Lim   
ENV PROJECT_DIR=ethereum-etl  
  
RUN apk add unzip  
RUN wget https://github.com/blockchain-etl/ethereum-etl/archive/develop.zip \  
&& unzip develop.zip && rm develop.zip  
  
RUN mv ethereum-etl-develop /$PROJECT_DIR  
  
WORKDIR /$PROJECT_DIR  
RUN apk add --no-cache gcc musl-dev  #for C libraries:    
RUN pip install --upgrade pip && pip install -e /$PROJECT_DIR/  
  
#CMD ["export_all", "-s", "01990000", "-e", "99999999", "-p", "http://xxx.xxx.xxx.xxx:8545", "-o", "output"]  
#CMD ["sh","-c", "echo startblock=$STARTBLOCK  endblock=$ENDBLOCK"]  
  
CMD ["sh","-c","python ethereumetl export_all -s $STARTBLOCK -e $ENDBLOCK  -p http://xxx.xxx.xxx.xxx:8545 -o output"]

go-ethereum-node1

see https://sueboy.blogspot.com/search?q=poa