ElastAlert elasticsearch elk

docker-compose

  
version: '3.3'  
  
services:  
  elasticsearch:  
    build:  
      context: elasticsearch/  
    volumes:  
      #- ./elasticsearch/esdata:/usr/share/elasticsearch/data:rw  
      - alldata:/usr/share/elasticsearch/data:rw  
      #- ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro  
    ports:  
      - "9200:9200"  
      - "9300:9300"  
    environment:  
      - bootstrap.memory_lock=true  
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"  
    ulimits:  
      memlock:  
        soft: -1  
        hard: -1  
      nofile:  
        soft: 65536  
        hard: 65536  
    networks:  
      - fastdev  
  
  logstash:  
    build:  
      context: logstash/  
    volumes:  
      - ./logstash/config/logstash.yml:/etc/logstash/logstash.yml:ro  
      - ./logstash/pipeline:/etc/logstash/conf.d:ro  
    ports:  
      - "5000:5000"  
      - "5044:5044"  
    environment:  
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"  
    networks:  
      - fastdev  
    depends_on:  
      - elasticsearch  
  
  kibana:  
    build:  
      context: kibana/  
    volumes:  
      - ./kibana/config/:/usr/share/kibana/config:ro  
    ports:  
      - "5601:5601"  
    networks:  
      - fastdev  
    depends_on:  
      - elasticsearch  
  
  fakelog:  
    build:  
      context: fakelog/  
    volumes:  
      - ./fakelog/logs:/ethereum-etl/output/:rw  
    networks:  
      - fastdev  
  
  elastalert:  
    build:  
      context: elastalert/  
    volumes:  
      - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json:ro  
      - ./elastalert/config/elastalert.yaml:/opt/elastalert/config.yaml:ro  
      - ./elastalert/rules:/opt/elastalert/rules:rw  
      - ./elastalert/rule_templates:/opt/elastalert/rule_templates:ro  
    ports:  
      - "3030:3030"  
    #  - "3333:3333"  
    environment:  
      - net=elasticsearch  
    networks:  
      - fastdev  
  
volumes:    
  alldata:   
  
networks:  
  fastdev:  
    driver: bridge  

Only elasticsearch/Dockerfile some error happen because debian mirror. Other YAML see before blogs

  
FROM java:8-jre  
  
# grab gosu for easy step-down from root  
ENV GOSU_VERSION 1.7  
RUN set -x \  
 && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \  
 && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \  
 && export GNUPGHOME="$(mktemp -d)" \  
 && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \  
 && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \  
 && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \  
 && chmod +x /usr/local/bin/gosu \  
 && gosu nobody true  
  
# https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-repositories.html  
# https://packages.elasticsearch.org/GPG-KEY-elasticsearch  
#RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4  
  
#ENV ELASTICSEARCH_MAJOR 1.7  
#ENV ELASTICSEARCH_VERSION 1.7.5  
#ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/1.7/debian  
  
#RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list  
  
#RUN set -x \  
# && apt-get clean && apt-get update \  
# && apt-get install -y --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \  
# && rm -rf /var/lib/apt/lists/*  
  
RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add -  
RUN echo "deb http://packages.elastic.co/elasticsearch/1.7/debian stable main" | tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list  
RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie.list  
RUN echo "deb [check-valid-until=no] http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list  
RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list  
RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list  
RUN apt-get -o Acquire::Check-Valid-Until=false update && apt-get install elasticsearch  
  
ENV PATH /usr/share/elasticsearch/bin:$PATH  
  
WORKDIR /usr/share/elasticsearch  
  
RUN set -ex \  
 && for path in \  
  ./data \  
  ./logs \  
  ./config \  
  ./config/scripts \  
 ; do \  
  mkdir -p "$path"; \  
  chown -R elasticsearch:elasticsearch "$path"; \  
 done  
  
COPY config ./config  
  
#VOLUME /usr/share/elasticsearch/data  
  
COPY docker-entrypoint.sh /  
RUN chmod +x /docker-entrypoint.sh  
  
EXPOSE 9200 9300  
ENTRYPOINT ["/docker-entrypoint.sh"]  
CMD ["elasticsearch"]  

ElastAlert/Dockerfile

  
FROM bitsensor/elastalert:latest  

ElastAlert/config/config.json

  
{  
  "appName": "elastalert-server",  
  "port": 3030,  
  "wsport": 3333,  
  "elastalertPath": "/opt/elastalert",  
  "verbose": false,  
  "es_debug": false,  
  "debug": false,  
  "rulesPath": {  
    "relative": true,  
    "path": "/rules"  
  },  
  "templatesPath": {  
    "relative": true,  
    "path": "/rule_templates"  
  },  
  "es_host": "elasticsearch",  
  "es_port": 9200,  
  "writeback_index": "elastalert_status"  
}  

ElastAlert/config/elastalert.yaml

  
es_host: elasticsearch  
es_port: 9200  
rules_folder: rules  
  
# How often ElastAlert will query elasticsearch                  
# The unit can be anything from weeks to seconds  
run_every:  
  seconds: 5  
  #minutes: 1  
# ElastAlert will buffer results from the most recent                                                        
# period of time, in case some log sources are not in real time  
buffer_time:  
  minutes: 15  
# If an alert fails for some reason, ElastAlert will retry                                                   
# sending the alert until this time period has elapsed   
alert_time_limit:  
  minutes: 2  
  #days: 2  
    
writeback_index: elastalert_status  
  
#smtp_host: ""  

ElastAlert/config/rules/test_metric.yaml

  
name: Example rule  
  
# How often ElastAlert will query elasticsearch                  
# The unit can be anything from weeks to seconds  
run_every:  
  seconds: 5  
  #minutes: 1  
# ElastAlert will buffer results from the most recent                                                        
# period of time, in case some log sources are not in real time  
buffer_time:  
  minutes: 15  
# If an alert fails for some reason, ElastAlert will retry                                                   
# sending the alert until this time period has elapsed   
alert_time_limit:  
  minutes: 2  
  #days: 2  
      
type: metric_aggregation  
  
index: filebeat-*  
  
metric_agg_key: project  
metric_agg_type: value_count  
  
#query_key: "name.keyword"  
doc_type: _type  
  
max_threshold: 32  
min_threshold: 19  
  
filter:  
- query:  
    query_string:  
      query: "srctype: geth AND project: hr AND mined AND geth_ip: \"xxx.xxx.xxx.xxx\""  
  
#alert_subject: "No data on dashboard"  
# "|" means remove useless characters  
alert_text: |  
    test attach  
alert_text_args: []  
  
alert:  
- "email"  
email:  
- "1803110@narlabes.org.tw"  

ElastAlert/config/rule_templates

copy file from github or not copy.
https://github.com/Yelp/elastalert/tree/master/example_rules

run ElastAlert or ElastAlert test

  
cd /opt/elastalert  
  
a. elastalert  
  
b. elastalert-test-rule ./rules/test_metric.yaml --start 2019-04-10T08:40:00.000  
  
#At /opt/elastalert run command, it auto load config.yaml on this path. You can find "ls config.yaml".  

And rules (test_metric.yaml) put on path /opt/elastalert/rules

elastalert.yml and config.json put on path /opt/elastalert**-server**/config

`