reorganize docker-compose for production use

.github/workflows/frontend.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Build frontend
-      run: make wait_web regenerate_frontend
+      run: docker-compose up gulp
     - name: Show git status
       run: git status
     - name: Require rebuild content

.github/workflows/web.yml

@@ -9,6 +9,10 @@ on:
 jobs:
   django:
     runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ matrix.python }}
+      DJANGO_VERSION: ${{ matrix.django }}
+      METADEFENDER_API_KEY: ${{ secrets.METADEFENDER_API_KEY }}
     strategy:
       matrix:
         python: 
@@ -23,18 +27,12 @@ jobs:
         lfs: true
     - name: Build application
       run: docker-compose build web
-      env:
-        PYTHON_VERSION: ${{ matrix.python }}
-        DJANGO_VERSION: ${{ matrix.django }}
-        METADEFENDER_API_KEY: ${{ secrets.METADEFENDER_API_KEY }}
-    - name: Run side services
-      run: make wait_mysql wait_elasticsearch wait_tika
     - name: Migration check
       run: make check
     - name: Show settings
       run: make settings
-    - name: Build test
-      run: make test
+    - name: Run test
+      run: make test-es
 #     - name: Send coverage report
 #       run: make coverage_send
 #       env:
@@ -51,4 +49,4 @@ jobs:
       if: failure()
     - name: Show docker process
       run: docker ps
-      if: always()
+      if: always()

Dockerfile

@@ -1,25 +1,32 @@
-# This is Dockerfile for development purposes only.
-ARG PYTHON_VERSION='3'
-FROM python:${PYTHON_VERSION}-slim
+ARG PYTHON_VERSION='3.6'
+FROM python:${PYTHON_VERSION}-slim as build
 RUN mkdir /code
 WORKDIR /code
 
 # Install python dependencies
 ENV PYTHONUNBUFFERED 1
-ENV DJANGO_SETTINGS_MODULE="config.settings.production"
 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
    default-libmysqlclient-dev \
    gcc \
    build-essential \
    git \
    curl \
-   gettext libgettextpo-dev wait-for-it \
+   gettext libgettextpo-dev \
 && rm -rf /var/lib/apt/lists/*
 COPY requirements/*.txt ./requirements/
 ARG DJANGO_VERSION='==2.22.*'
+FROM build as development
 # TODO: Move to /requirements/base.txt after fixing following bug:
 #       https://github.com/readthedocs/readthedocs-docker-images/issues/158
+RUN pip install mysqlclient==2.0.3
+RUN bash -c "if [[ "${DJANGO_VERSION}" == 'master' ]]; then \
+pip install --no-cache-dir -r requirements/dev.txt https://github.com/django/django/archive/master.tar.gz; else \
+pip install --no-cache-dir -r requirements/dev.txt \"django${DJANGO_VERSION}\"; fi"
+COPY ./ /code/
+CMD python manage.py runserver 0.0.0.0:8000
+FROM build as production
+ENV DJANGO_SETTINGS_MODULE="config.settings.production"
 RUN pip install --no-cache-dir mysqlclient==2.0.3 -r requirements/production.txt
 COPY ./ /code/
 RUN DJANGO_SECRET_KEY=x \

Makefile

@@ -6,7 +6,6 @@ clean:
 	docker-compose down
 
 regenerate_frontend:
-	docker-compose run web python manage.py collectstatic -c --noinput
 	docker-compose up gulp
 
 build:
@@ -15,28 +14,17 @@ build:
 test:
 	docker-compose run web coverage run manage.py test --keepdb --verbosity=2 ${TEST}
 
+test-es:
+	docker-compose -f docker-compose.yml -f docker-compose.es.yml run python manage.py test --keepdb --verbosity=2 ${TEST}
+
 coverage_html:
 	docker-compose run web coverage html
 
 coverage_send:
 	docker-compose run -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_HEAD_REF -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_TOKEN -e COVERALLS_REPO_TOKEN web coveralls
 
-wait_web: wait_mysql wait_elasticsearch wait_tika
-
-wait_mysql:
-	docker-compose up -d db
-	docker-compose run web bash -c 'wait-for-it -t 30 db:3306' || (docker-compose logs db; exit -1)
-
-wait_elasticsearch:
-	docker-compose up -d elasticsearch
-	docker-compose run web bash -c 'wait-for-it -t 30 elasticsearch:9200' || (docker-compose logs elasticsearch; exit -1)
-
-wait_tika:
-	docker-compose up -d tika
-	docker-compose run web bash -c 'wait-for-it -t 60 tika:9998' || (docker-compose logs tika; exit -1)
-
 migrate:
-	docker-compose run web python manage.py migrate
+	docker-compose up migration
 
 lint: # lint currently staged files
 	pre-commit run

docker-compose.es.yml

@@ -0,0 +1,29 @@
+version: '3.1'
+# To play with this file use following command: 
+# $ docker-compose -f docker-compose.yml docker-compose.es.yml up 
+
+services:
+  web:
+    environment: &webEnvironment
+      DATABASE_URL: mysql://root:password@db/feder
+      ELASTICSEARCH_URL: http://elasticsearch:9200/
+      APACHE_TIKA_URL: 'http://tika:9998/'
+      VIRUSTOTAL_API_KEY:
+      ATTACHMENTSCANNER_API_KEY:
+      ATTACHMENTSCANNER_API_URL:
+      METADEFENDER_API_KEY:
+
+  worker:
+    command: python manage.py process_tasks
+    environment: *webEnvironment
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
+    environment:
+      discovery.type: single-node
+      cluster.routing.allocation.disk.threshold_enabled: 'false'
+    ports:
+      - "9200:9200"
+
+  tika:
+    image: quay.io/watchdogpolska/apache-tika

docker-compose.yml

@@ -1,6 +1,12 @@
-version: '2.1'
+version: '3.1'
 # This is application’s services configuration for development purposes only.
 
+x-node: &node
+  image: node:15
+  working_dir: /app
+  volumes:
+    - .:/app
+
 services:
   db:
     image: mysql:5.7
@@ -12,20 +18,32 @@ services:
       MYSQL_DATABASE: feder
     volumes:
       - mysql-data:/var/lib/mysql
-
+    healthcheck:
+        test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+        timeout: 20s
+        retries: 10
+  # Services to regenerate frontend
+  # Regenerated front-end is stored in project directory on host
+  npm:
+    <<: *node
+    command: sh -c 'id && npm install --unsafe-perm && npm rebuild --unsafe-perm node-sass'
   gulp:
-    build:
-      context: .
-      dockerfile: .contrib/docker/Dockerfile.gulp
+    <<: *node
+    command: npx gulp build
     ports:
       - "35729:35729"
-    volumes:
-      - .:/app
+    depends_on:
+      npm:
+        condition: service_completed_successfully
+      collectstatic:
+        condition: service_completed_successfully
 
+  # Services for Django application
   web:
     build: &webBuild
       context: .
       dockerfile: .contrib/docker/Dockerfile.web
+      target: development
       args:
         # Match to production environment
         PYTHON_VERSION: '${PYTHON_VERSION:-3.6.9}'
@@ -34,12 +52,17 @@ services:
       - .:/code
     environment: &webEnvironment
       DATABASE_URL: mysql://root:password@db/feder
-      ELASTICSEARCH_URL: http://elasticsearch:9200/
-      APACHE_TIKA_URL: 'http://tika:9998/'
       VIRUSTOTAL_API_KEY:
       ATTACHMENTSCANNER_API_KEY:
       ATTACHMENTSCANNER_API_URL:
       METADEFENDER_API_KEY:
+    depends_on:
+      migration:
+        condition: service_completed_successfully
+      collectstatic:
+        condition: service_completed_successfully
+      db:
+        condition: service_healthy
     ports:
       - "8000:8000"
     # Following allows to execute `docker attach feder_web_1`
@@ -52,17 +75,18 @@ services:
     build: *webBuild
     volumes: *webVolumes
     environment: *webEnvironment
-
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
-    environment:
-      discovery.type: single-node
-      cluster.routing.allocation.disk.threshold_enabled: 'false'
-    ports:
-      - "9200:9200"
-
-  tika:
-    image: quay.io/watchdogpolska/apache-tika
-
+  collectstatic:
+    command: python manage.py collectstatic --no-input
+    build: *webBuild
+    volumes: *webVolumes
+    environment: *webEnvironment
+  migration:
+    command: python manage.py migrate
+    build: *webBuild
+    volumes: *webVolumes
+    environment: *webEnvironment
+    depends_on:
+      db:
+        condition: service_healthy
 volumes:
   mysql-data: