Merge pull request #68 from 2023-Summer-Bootcamp-Team-K/(#67)Fix/ELK

(#67)fix/elk

.gitignore

@@ -1 +1,2 @@
-/alertmanager/config.yml
+/alertmanager/config.yml
+.env

backend/config/settings.py

@@ -22,6 +22,7 @@
     'django.contrib.staticfiles',
     'django_prometheus',
     'django_redis',
+    'django_elasticsearch_dsl',
     'rest_framework',
     'drf_yasg',
     'storages',
@@ -149,3 +150,9 @@
         }
     }
 }
+
+ELASTICSEARCH_DSL = {
+    'default': {
+        'hosts': 'localhost:9200',
+    },
+}

backend/pet/documents.py

@@ -0,0 +1,28 @@
+from django_elasticsearch_dsl import Document
+from django_elasticsearch_dsl.registries import registry
+from .models import Pet
+
+@registry.register_document
+class PetDocument(Document):
+    class Index:
+        name = 'pets'
+        settings = {'number_of_shards': 1,
+                    'number_of_replicas': 0}
+
+    class Django:
+        model = Pet
+        fields = [
+            'name',
+            'age',
+            'species',
+            'gender',
+            'weight',
+            'started_date',
+            'feed',
+            'sore_spot',
+            'profile_url',
+            'profile_image',
+            'created_at',
+            'updated_at',
+            'is_deleted',
+        ]

backend/requirements.txt

@@ -1,12 +1,10 @@
 APScheduler==3.10.1
 asgiref==3.7.2
+async-timeout==4.0.2
 attrs==23.1.0
-blinker==1.6.2
-boto3==1.28.0
-botocore==1.31.0
+boto3==1.28.7
+botocore==1.31.7
 certifi==2023.5.7
-click==8.1.4
-dart==1.0
 Django==4.0.3
 django-apscheduler==0.6.2
 django-dotenv==1.4.2
@@ -26,16 +24,17 @@ exceptiongroup==1.1.2
 Flask==2.3.2
 Flask-SQLAlchemy==3.0.5
 gunicorn==20.1.0
+ecs-logging==2.0.2
+filebeat==0.0.2
 h11==0.14.0
 idna==3.4
 inflection==0.5.1
-itsdangerous==2.1.2
-Jinja2==3.1.2
 jmespath==1.0.1
 jsonschema==4.18.0
 jsonschema-specifications==2023.6.1
 Markdown==3.4.3
 MarkupSafe==2.1.3
+logstash==0.1.dev0
 mysqlclient==2.2.0
 numpy==1.25.0
 outcome==1.2.0
@@ -47,8 +46,9 @@ PyMySQL==1.1.0
 PySocks==1.7.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
+python-logstash==0.4.8
 pytz==2023.3
-PyYAML==6.0
+PyYAML==6.0.1
 redis==4.6.0
 referencing==0.29.1
 rpds-py==0.8.10
@@ -57,7 +57,6 @@ selenium==4.10.0
 six==1.16.0
 sniffio==1.3.0
 sortedcontainers==2.4.0
-SQLAlchemy==2.0.18
 sqlparse==0.4.4
 trio==0.22.2
 trio-websocket==0.10.3
@@ -66,6 +65,4 @@ tzdata==2023.3
 tzlocal==5.0.1
 uritemplate==4.1.1
 urllib3==1.26.16
-users==1.0.dev0
-Werkzeug==2.3.6
 wsproto==1.2.0

docker-compose.yml

@@ -72,6 +72,114 @@ services:
       - backend
     restart: always
 
+  setup:
+    profiles:
+      - setup
+    build:
+      context: setup/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    init: true
+    volumes:
+      - ./setup/entrypoint.sh:/entrypoint.sh:ro,Z
+      - ./setup/lib.sh:/lib.sh:ro,Z
+      - ./setup/roles:/roles:ro,Z
+    environment:
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+      METRICBEAT_INTERNAL_PASSWORD: ${METRICBEAT_INTERNAL_PASSWORD:-}
+      FILEBEAT_INTERNAL_PASSWORD: ${FILEBEAT_INTERNAL_PASSWORD:-}
+      HEARTBEAT_INTERNAL_PASSWORD: ${HEARTBEAT_INTERNAL_PASSWORD:-}
+      MONITORING_INTERNAL_PASSWORD: ${MONITORING_INTERNAL_PASSWORD:-}
+      BEATS_SYSTEM_PASSWORD: ${BEATS_SYSTEM_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+
+  elasticsearch:
+    build:
+      context: docker-elk/elasticsearch/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    volumes:
+      - ./docker-elk/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,Z
+      - elasticsearch:/usr/share/elasticsearch/data:Z
+    ports:
+      - 9200:9200
+      - 9300:9300
+    environment:
+      node.name: elasticsearch
+      ES_JAVA_OPTS: -Xms512m -Xmx512m
+      # Bootstrap password.
+      # Used to initialize the keystore during the initial startup of
+      # Elasticsearch. Ignored on subsequent runs.
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      # Use single node discovery in order to disable production mode and avoid bootstrap checks.
+      # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
+      discovery.type: single-node
+    networks:
+      - elk
+    restart: unless-stopped
+
+  logstash:
+    build:
+      context: docker-elk/logstash/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    volumes:
+      - ./docker-elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z
+      - ./docker-elk/logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
+    ports:
+      - 5044:5044
+      - 50000:50000/tcp
+      - 50000:50000/udp
+      - 9600:9600
+    environment:
+      LS_JAVA_OPTS: -Xms256m -Xmx256m
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+    restart: unless-stopped
+
+  kibana:
+    build:
+      context: docker-elk/kibana/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    volumes:
+      - ./docker-elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
+    ports:
+      - 5601:5601
+    environment:
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
+    restart: unless-stopped
+
+  filebeat:
+    build:
+      context: ./docker-elk/filebeat
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    entrypoint: "filebeat -e -strict.perms=false"
+    volumes:
+      - ./docker-elk/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
+      - ./nginx/log:/var/log/nginx # nginx log path (require same option on nginx container)
+    depends_on:
+      - nginx
+      - backend
+      - logstash
+      - elasticsearch
+      - kibana
+    links:
+      - logstash
+
   node-exporter:
     image: prom/node-exporter
     container_name: node-exporter
@@ -115,5 +223,10 @@ services:
       - app-tier
 
 networks:
+  elk:
+    driver: bridge
   app-tier:
     driver: bridge
+
+volumes:
+  elasticsearch:

docker-elk/.gitattributes

@@ -0,0 +1,2 @@
+# Declare files that will always have LF line endings on checkout.
+*.sh text eol=lf

docker-elk/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+- name: Discuss the Elastic Stack
+  url: https://discuss.elastic.co
+  about: Please ask questions related to the usage of Elastic products in those forums.
+- name: Docker Community Forums
+  url: https://forums.docker.com
+  about: Please ask questions related to the usage of Docker products in those forums.
+- name: docker-elk Gitter chat room
+  url: https://app.gitter.im/#/room/#deviantony_docker-elk:gitter.im
+  about: General questions regarding this project can also be asked in the chat.

docker-elk/.github/ISSUE_TEMPLATE/issue_report.md

@@ -0,0 +1,87 @@
+---
+name: Issue report
+about: Report a problem with the docker-elk integration or its documentation.
+---
+
+<!--
+Thanks for your issue report!
+
+In order for us to be able to reproduce the problem and identify the root cause
+quickly, we kindly ask you to include *all* the information requested below in
+your issue report. It saves us a lot of effort and allows us to provide you
+with a solution with as little delay as possible.
+
+Issues submitted without the requested information will be closed.
+Thank you for your understanding.
+-->
+
+
+### Problem description
+
+<!--
+Please be as descriptive as possible regarding the encountered issue versus the
+expected outcome.
+-->
+
+### Extra information
+
+#### Stack configuration
+
+<!--
+Please mention all changes performed to the default configuration, including to Dockerfiles.
+If possible, provide the output of the `git diff` command.
+-->
+
+#### Docker setup
+
+<!--
+Please paste the full output of the `docker version` command below.
+
+Example:
+
+Client: Docker Engine - Community
+ Version:           20.10.2
+ API version:       1.41
+ ...
+-->
+
+```console
+$ docker version
+
+[OUTPUT HERE]
+```
+
+<!--
+Please paste the full output of the `docker-compose version` command below.
+
+Example:
+
+docker-compose version 1.27.4, build 40524192
+docker-py version: 4.3.1
+...
+-->
+
+```console
+$ docker-compose version
+
+[OUTPUT HERE]
+```
+
+#### Container logs
+
+<!--
+Please paste the full output of the `docker-compose logs` command below.
+
+Example:
+
+elasticsearch_1  | Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
+elasticsearch_1  | {"@timestamp":"2021-01-16T21:53:38.331Z", "log.level": "INFO", "message":"version...
+kibana_1         | {"type":"log","@timestamp":"2021-01-16T21:54:10+00:00","tags":["info","plugins-system"],...
+...
+-->
+
+```console
+$ docker-compose logs
+
+[OUTPUT HERE]
+```

docker-elk/.github/dependabot.yml

@@ -0,0 +1,14 @@
+# Dependabot configuration
+#
+# For more information, please refer to:
+#   https://docs.github.com/en/code-security/dependabot/dependabot-version-updates
+
+version: 2
+
+updates:
+
+# Maintain dependencies for GitHub Actions
+- package-ecosystem: github-actions
+  directory: /
+  schedule:
+    interval: weekly