diff --git a/Dockerfile b/Dockerfile index 056ff5af..c7e1caae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest as py-ea -ARG ELASTALERT_VERSION=v0.2.0b2 +ARG ELASTALERT_VERSION=v0.2.4 ENV ELASTALERT_VERSION=${ELASTALERT_VERSION} # URL from which to download Elastalert. ARG ELASTALERT_URL=https://github.com/Yelp/elastalert/archive/$ELASTALERT_VERSION.zip @@ -9,7 +9,7 @@ ENV ELASTALERT_HOME /opt/elastalert WORKDIR /opt -RUN apk add --update --no-cache ca-certificates openssl-dev openssl python2-dev python2 py2-pip py2-yaml libffi-dev gcc musl-dev wget && \ +RUN apk add --update --no-cache ca-certificates openssl-dev openssl python3-dev python3 py3-pip py3-yaml libffi-dev gcc musl-dev wget cargo && \ # Download and unpack Elastalert. wget -O elastalert.zip "${ELASTALERT_URL}" && \ unzip elastalert.zip && \ @@ -20,18 +20,25 @@ WORKDIR "${ELASTALERT_HOME}" # Install Elastalert. # see: https://github.com/Yelp/elastalert/issues/1654 -RUN sed -i 's/jira>=1.0.10/jira>=1.0.10,<1.0.15/g' setup.py && \ - python setup.py install && \ - pip install -r requirements.txt +RUN pip install -r requirements.txt && \ + pip install "setuptools>=11.3" + +# see https://github.com/Yelp/elastalert/issues/2475 +RUN pip uninstall -y jira && \ + pip install jira>2.0.0 + +RUN python3 setup.py install FROM node:alpine LABEL maintainer="BitSensor " # Set timezone for this container ENV TZ Etc/UTC -RUN apk add --update --no-cache curl tzdata python2 make libmagic +RUN apk add --update --no-cache curl tzdata python3 make libmagic + +RUN ln -s /usr/bin/python3 /usr/bin/python -COPY --from=py-ea /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages +COPY --from=py-ea /usr/lib/python3.8/site-packages /usr/lib/python3.8/site-packages COPY --from=py-ea /opt/elastalert /opt/elastalert COPY --from=py-ea /usr/bin/elastalert* /usr/bin/ diff --git a/kubernetes/config-map.yaml b/kubernetes/config-map.yaml new file mode 100644 index 00000000..74db3db9 --- /dev/null +++ b/kubernetes/config-map.yaml @@ -0,0 +1,120 @@ +apiVersion: v1 +data: + config.json: | + { + "appName": "elastalert", + "port": 3030, + "wsport": 3333, + "elastalertPath": "/opt/elastalert", + "verbose": false, + "es_debug": false, + "debug": false, + "rulesPath": { + "relative": true, + "path": "/rules" + }, + "templatesPath": { + "relative": true, + "path": "/rule_templates" + }, + "es_host": "elastic-cluster-es-http", + "es_port": 9200, + "writeback_index": "elastalert_status" + } + config-test.yaml: | + # NOTE: This config is used when testing a rule + + # The elasticsearch hostname for metadata writeback + # Note that every rule can have its own elasticsearch host + es_host: elastic-cluster-es-http + + # The elasticsearch port + es_port: 9200 + + # This is the folder that contains the rule yaml files + # Any .yaml file will be loaded as a rule + rules_folder: rules + + # How often ElastAlert will query elasticsearch + # The unit can be anything from weeks to seconds + run_every: + seconds: 5 + + # ElastAlert will buffer results from the most recent + # period of time, in case some log sources are not in real time + buffer_time: + minutes: 1 + + # Optional URL prefix for elasticsearch + #es_url_prefix: elasticsearch + + # Connect with TLS to elasticsearch + #use_ssl: True + + # Verify TLS certificates + #verify_certs: True + + # GET request with body is the default option for Elasticsearch. + # If it fails for some reason, you can pass 'GET', 'POST' or 'source'. + # See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport + # for details + #es_send_get_body_as: GET + + # The index on es_host which is used for metadata storage + # This can be a unmapped index, but it is recommended that you run + # elastalert-create-index to set a mapping + writeback_index: elastalert_status + + # If an alert fails for some reason, ElastAlert will retry + # sending the alert until this time period has elapsed + alert_time_limit: + days: 2 + config.yaml: | + # The elasticsearch hostname for metadata writeback + # Note that every rule can have its own elasticsearch host + es_host: elastic-cluster-es-http + + # The elasticsearch port + es_port: 9200 + + # This is the folder that contains the rule yaml files + # Any .yaml file will be loaded as a rule + rules_folder: rules + + # How often ElastAlert will query elasticsearch + # The unit can be anything from weeks to seconds + run_every: + seconds: 5 + + # ElastAlert will buffer results from the most recent + # period of time, in case some log sources are not in real time + buffer_time: + minutes: 1 + + # Optional URL prefix for elasticsearch + #es_url_prefix: elasticsearch + + # Connect with TLS to elasticsearch + #use_ssl: True + + # Verify TLS certificates + #verify_certs: True + + # GET request with body is the default option for Elasticsearch. + # If it fails for some reason, you can pass 'GET', 'POST' or 'source'. + # See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport + # for details + #es_send_get_body_as: GET + + # The index on es_host which is used for metadata storage + # This can be a unmapped index, but it is recommended that you run + # elastalert-create-index to set a mapping + writeback_index: elastalert_status + + # If an alert fails for some reason, ElastAlert will retry + # sending the alert until this time period has elapsed + alert_time_limit: + days: 2 +kind: ConfigMap +metadata: + name: elastalert-configs diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml new file mode 100644 index 00000000..bea43c0b --- /dev/null +++ b/kubernetes/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: elastalert + name: elastalert +spec: + replicas: 1 + selector: + matchLabels: + app: elastalert + template: + metadata: + labels: + app: elastalert + spec: + volumes: + - name: config-yaml + configMap: + name: elastalert-configs + items: + - key: config.yaml + path: config.yaml + - name: config-test-yaml + configMap: + name: elastalert-configs + items: + - key: config-test.yaml + path: config-test.yaml + - name: config-json + configMap: + name: elastalert-configs + items: + - key: config.json + path: config.json + - name: rule-templates + configMap: + name: rule-templates + - name: rules-dir + persistentVolumeClaim: + claimName: rules-dir + containers: + - image: bitsensor/elastalert:3.0.0-beta.0 + name: elastalert + env: + - name: ES_USERNAME + value: elastic + - name: ES_PASSWORD + valueFrom: + secretKeyRef: + name: elastic-cluster-es-elastic-user + key: elastic + volumeMounts: + - name: rules-dir + mountPath: /opt/elastalert/rules + - name: config-yaml + mountPath: /opt/elastalert/config.yaml + subPath: config.yaml + - name: config-test-yaml + mountPath: /opt/elastalert/config-test.yaml + subPath: config-test.yaml + - name: config-json + mountPath: /opt/elastalert-server/config/config.json + subPath: config.json + - name: rule-templates + mountPath: /opt/elastalert/rule_templates diff --git a/kubernetes/persistent-volume-claim.yaml b/kubernetes/persistent-volume-claim.yaml new file mode 100644 index 00000000..db520f59 --- /dev/null +++ b/kubernetes/persistent-volume-claim.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rules-dir +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi diff --git a/kubernetes/rule-templates-config-map.yaml b/kubernetes/rule-templates-config-map.yaml new file mode 100644 index 00000000..3208a4f8 --- /dev/null +++ b/kubernetes/rule-templates-config-map.yaml @@ -0,0 +1,516 @@ +apiVersion: v1 +data: + detection_template.yaml: | + # Rule name, must be unique + name: Alert on any detection + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + + # Type of alert. + type: any + realert: + seconds: 0 + + # A list of elasticsearch filters used for find events + # These filters are joined with AND and nested in a filtered query + # For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html + filter: + - query: + query_string: + query: "_exists_:detections" + + include: + - endpoint.location + - endpoint.name + - context.http.userAgent + - context.ip + - context.php.session.sessionId + - detections + - meta.user + + + # Enhancement for converting 'detections' array into object, ex. get merged detection type by + # 'detections_parsed.type' or get first detection type by 'detection_parsed.0.type' + match_enhancements: + - "elastalert_modules.bitsensor_enhancement.AlertTextEnhancement" + run_enhancements_first: true + + + alert_subject: ":exclamation: Detection on {}" + alert_subject_args: + - endpoint.name + + alert_text_type: alert_text_only + alert_text: "Triggered at _{}_\n\n*Attacker:*\nIP: {} \nUser-Agent: {}\nDetection: `{}`\n\n:Id: {}\nUser: {}" + alert_text_args: + - endpoint.localtime + - context.ip + - context.http.userAgent + - detections_parsed.type + - _id + - meta.user + + # The alert is use when a match is found + alert: + - slack + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + error_jira_template.yaml: | + # Rule name, must be unique + name: Alert on any error + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + + # Type of alert. + type: any + realert: + seconds: 0 + + # A list of elasticsearch filters used for find events + # These filters are joined with AND and nested in a filtered query + # For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html + filter: + - query: + query_string: + query: "_exists_:errors" + + include: + - endpoint.location + - endpoint.name + - context.http.userAgent + - context.ip + - errors + - meta.user + + + # Enhancement for converting 'detections' array into object, ex. get merged detection type by + # 'detections_parsed.type' or get first detection type by 'detection_parsed.0.type' + match_enhancements: + - "elastalert_modules.bitsensor_enhancement.AlertTextEnhancement" + run_enhancements_first: true + + + alert_subject: "Error on {}" + alert_subject_args: + - endpoint.name + + alert_text_type: alert_text_only + alert_text: "Triggered at _{}_\n\n*Attacker:*\nIP: {} \nUser-Agent: {}\nError: *{}*\n\nId: {}\nUser: {}" + alert_text_args: + - endpoint.localtime + - context.ip + - context.http.userAgent + - errors_parsed.type + - _id + - meta.user + + # The alert is use when a match is found + alert: + - jira + + jira_server: https://bitsensor.atlassian.net + jira_project: SA + jira_issuetype: Story + jira_labels: error + + # Add jira_acct.txt to rules folder + # The file is yaml formatted and must contain fields: 'user', 'password' + jira_account_file: "rules/jira_acct.txt" + integration_started_template.yaml: | + # Rule name, must be unique + name: Integration Started + + # Alert on x events in y seconds + type: frequency + + # Alert when this many documents matching the query occur within a timeframe + num_events: 1 + + # num_events must occur within this amount of time to trigger an alert + timeframe: + hours: 1 + + # When the attacker continues, send a new alert after x minutes + realert: + days: 7 + + query_key: + - meta.provider + - endpoint.name + + include: + - meta.provider + - endpoint.name + + alert_subject: "Integration started on <{}> | <{}|Show Dashboard>" + alert_subject_args: + - endpoint.name + - kibana_link + + alert_text: |- + Integration on {} has started with plugin {}. + + alert_text_args: + - endpoint.name + - meta.provider + + # The alert when a match is found + alert: + - slack + + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + + # Alert body only cointains a title and text + alert_text_type: alert_text_only + + # Link to BitSensor Kibana Dashboard + use_kibana4_dashboard: "https://kibana.dashboard.io/app/kibana#/dashboard" + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + doc_type: datapoint + no_data_template.yaml: | + # Alert when no data has been received for more then 30 seconds. + + # Rule name, must be unique + name: No Data + + # Type of alert. + type: flatline + + # Alert when this many documents matching the query occur within a timeframe + threshold: 1 + use_terms: true + + # num_events must occur within this amount of time to trigger an alert + timeframe: + seconds: 30 + + realert: + minutes: 10 + + exponential_realert: + hours: 1 + + doc_type: datapoint + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + + alert_subject: "No data on dashboard" + + alert_text_type: alert_text_only + alert_text: "The stack receives no data. It might be down :(" + + # The alert is use when a match is found + alert: + - slack + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + relevant_attack_template.yaml: | + # Index to search, wildcard supported + name: Known Attacks + + # Alert on each event + type: any + + # A list of elasticsearch filters used for find events + # These filters are joined with AND and nested in a filtered query + # For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html + filter: + - query: + query_string: + query: "detections.reason:KNOWN_ATTACK" + + index: bitsensor-detections-* + timestamp_field: endpoint.localtime + + # Key per profile + query_key: + - context.ip + - context.http.userAgent + + # When the attacker continues, send a new alert after x minutes + realert: + minutes: 10 + + # Index to search, wildcard supported + include: + - endpoint.location + - endpoint.name + - context.http.userAgent + - context.ip + - context.php.session.sessionId + - detections + + alert_subject: "Attack on <{}> of type {} | <{}|Show Dashboard>" + alert_subject_args: + - endpoint.name + - detections_parsed.type + - kibana_link + + alert_text: |- + An attack on {} is detected. + Detection name: {} + Detection type: {} + + The attacker looks like: + IP: {} + User-Agent: {} + + alert_text_args: + - endpoint.name + - detections_parsed.name + - detections_parsed.type + - context.ip + - context.http.userAgent + + # Specify your services here + alert: + - slack + + # How To Generate your API: + # Click on your Workspace name (upper left corner) + # Go to "Manage Apps", then "Custom Integrations", "Incoming Webhooks" + # Press "Add Configuration", and choose your channel. Now paste it here: + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "BitSensor Alerting" + + # Alert body only cointains a title and text + alert_text_type: alert_text_only + + # Link to BitSensor Kibana Dashboard + use_kibana4_dashboard: "https://kibana.dashboard.io/app/kibana#/dashboard" + + # Enhancement for converting 'detections' array into object, ex. get merged detection type by + # 'detections_parsed.type' or get first detection type by 'detection_parsed.0.type' + match_enhancements: + - "elastalert_modules.bitsensor_enhancement.AlertTextEnhancement" + run_enhancements_first: true + spike_template.yml: | + # Rule name, must be unique + name: Spike in attacks on server + + # Type of alert. + type: spike + + # num_events must occur within this amount of time to trigger an alert + timeframe: + seconds: 60 + spike_height: 10 + spike_type: up + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + + query_key: + - endpoint.name + + alert_subject: "Surge in attacks on {}" + alert_subject_args: + - endpoint.name + + alert_text_type: alert_text_only + alert_text: "Surge in attacks on {}" + alert_text_args: + - endpoint.name + + # The alert is use when a match is found + alert: + - slack + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + successful_attack_template.yaml: | + # Rule name, must be unique + name: Alert on Successful Attack + + # Type of alert. + type: any + + realert: + seconds: 0 + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + + # A list of elasticsearch filters used for find events + # These filters are joined with AND and nested in a filtered query + # For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html + filter: + - query: + query_string: + query: "detections.successful:true" + + include: + - endpoint.location + - endpoint.name + - context.http.userAgent + - context.ip + - context.php.session.sessionId + - detections.type + - detections.name + - meta.user + - errors + + alert_subject: "Successful attack on {}" + alert_subject_args: + - endpoint.name + + alert_text_type: alert_text_only + alert_text: "Detection triggered at {}\nIP: {} \nUser-Agent: {}\n\nID: {}\nUser: {}" + alert_text_args: + - endpoint.localtime + - context.ip + - context.http.userAgent + - _id + - meta.user + + # The alert is use when a match is found + alert: + - slack + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + threshold_template.yml: | + # Alert when there are 500 discovery detection events coming from the same ip, userAgent within 30 seconds. + + # Rule name, must be unique + name: Attack threshold exceeded + + # Type of alert. + type: percentage_match + + # Alert when this many documents matching the query occur within a timeframe + max_percentage: 10 + + # num_events must occur within this amount of time to trigger an alert + timeframe: + seconds: 60 + + # Index to search, wildcard supported + index: bitsensor + timestamp_field: endpoint.localtime + + query_key: + - context.ip + + # A list of elasticsearch filters used for find events + # These filters are joined with AND and nested in a filtered query + # For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html + match_bucket_filter: + - query: + query_string: + query: "_exists_:detections" + + include: + - endpoint.location + - endpoint.name + - context.http.userAgent + - context.ip + - context.php.session.sessionId + - detections.type + - detections.name + - meta.user + - errors + + alert_subject: "Attack threshold exceeded by {}" + alert_subject_args: + - context.ip + + alert_text_type: alert_text_only + alert_text: "Time: {}\nIP: {} \nUser-Agent: {}\n\nID: {}\nUser: {}" + alert_text_args: + - endpoint.localtime + - context.ip + - context.http.userAgent + - _id + - meta.user + + # The alert is use when a match is found + alert: + - slack + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + volumetric_alert_template.yaml: | + # Rule name, must be unique + name: Bad/Bot Behaviour + + # Alert on x events in y seconds + type: frequency + + # Alert when this many documents matching the query occur within a timeframe + num_events: 20 + + # num_events must occur within this amount of time to trigger an alert + timeframe: + seconds: 30 + + # A list of elasticsearch filters used for find events + # These filters are joined with AND and nested in a filtered query + # For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html + filter: + - query: + query_string: + query: "detections.reason:BEHAVIOUR" + + # Index to search, wildcard supported + index: bitsensor-detections-* + timestamp_field: endpoint.localtime + doc_type: datapoint + + # When the attacker continues, send a new alert after x minutes + realert: + minutes: 1 + + query_key: + - context.ip + - context.http.userAgent + + include: + - endpoint.location + - endpoint.name + - context.http.userAgent + - context.ip + - context.php.session.sessionId + + alert_subject: "Bad/Bot behaviour on <{}> | <{}|Show Dashboard>" + alert_subject_args: + - endpoint.name + - kibana_link + + alert_text: |- + An attack on {} is detected. + + The attacker looks like: + IP: {} + Tool: {} + + alert_text_args: + - endpoint.name + - context.ip + - context.http.userAgent + + # The alert is use when a match is found + alert: + - slack + + slack_webhook_url: "https://hooks.slack.com/services/" + slack_username_override: "ElastAlert" + + # Alert body only cointains a title and text + alert_text_type: alert_text_only + + # Link to BitSensor Kibana Dashboard + use_kibana4_dashboard: "https://kibana.dashboard.io/app/kibana#/dashboard" +kind: ConfigMap +metadata: + creationTimestamp: null + name: rule-templates diff --git a/kubernetes/service.yaml b/kubernetes/service.yaml new file mode 100644 index 00000000..646133a4 --- /dev/null +++ b/kubernetes/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: elastalert + name: elastalert +spec: + ports: + - port: 3030 + protocol: TCP + targetPort: 3030 + name: "http" + - port: 3333 + protocol: TCP + targetPort: 3333 + name: "ws" + selector: + app: elastalert