Optimize application for DigitalOcean Apps

.do/deploy.template.yaml

@@ -0,0 +1,56 @@
+alerts:
+- rule: DEPLOYMENT_FAILED
+- rule: DOMAIN_FAILED
+name: feder
+services:
+- dockerfile_path: Dockerfile
+  envs:
+  - key: DJANGO_SECRET_KEY
+    scope: RUN_AND_BUILD_TIME
+    type: SECRET
+    value: ...
+  - key: DJANGO_SERVER_EMAIL
+    scope: RUN_AND_BUILD_TIME
+    value: x
+  - key: DATABASE_URL
+    scope: RUN_AND_BUILD_TIME
+    type: SECRET
+    value: ...
+  - key: EMAILLABS_APP_KEY
+    scope: RUN_AND_BUILD_TIME
+    value: x
+  - key: EMAILLABS_SECRET_KEY
+    scope: RUN_AND_BUILD_TIME
+    type: SECRET
+    value: ...
+  - key: LETTER_RECEIVE_SECRET
+    scope: RUN_AND_BUILD_TIME
+    type: SECRET
+    value: ...
+  - key: AWS_S3_ACCESS_KEY_ID
+    scope: RUN_AND_BUILD_TIME
+    value: access-key
+  - key: AWS_S3_SECRET_ACCESS_KEY
+    scope: RUN_AND_BUILD_TIME
+    type: SECRET
+    value: secret-key
+  - key: AWS_S3_ENDPOINT_URL
+    scope: RUN_AND_BUILD_TIME
+    value: http://minio:9000/
+  - key: AWS_STORAGE_BUCKET_NAME
+    scope: RUN_AND_BUILD_TIME
+    value: 'feder'
+  github:
+    branch: do
+    deploy_on_push: true
+    repo: watchdogpolska/feder
+  health_check:
+    http_path: /
+  http_port: 8000
+  instance_count: 1
+  instance_size_slug: basic-xxs
+  name: feder
+  routes:
+  - path: /
+  source_dir: /
+# TODO: add workers

.dockerignore

@@ -5,8 +5,10 @@ node_modules
 **/*.pyc
 docs
 **/*.rst
+**/__pycache__
 **/*.md
 media
 feder/media
-feder/staticfiles
+feder/static
+staticfiles
 htmlcov

.github/workflows/frontend.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Build frontend
-      run: make wait_web regenerate_frontend
+      run: docker-compose up gulp
     - name: Show git status
       run: git status
     - name: Require rebuild content

.github/workflows/web.yml

@@ -9,6 +9,10 @@ on:
 jobs:
   django:
     runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ matrix.python }}
+      DJANGO_VERSION: ${{ matrix.django }}
+      METADEFENDER_API_KEY: ${{ secrets.METADEFENDER_API_KEY }}
     strategy:
       matrix:
         python: 
@@ -23,18 +27,12 @@ jobs:
         lfs: true
     - name: Build application
       run: docker-compose build web
-      env:
-        PYTHON_VERSION: ${{ matrix.python }}
-        DJANGO_VERSION: ${{ matrix.django }}
-        METADEFENDER_API_KEY: ${{ secrets.METADEFENDER_API_KEY }}
-    - name: Run side services
-      run: make wait_mysql wait_elasticsearch wait_tika
     - name: Migration check
       run: make check
     - name: Show settings
       run: make settings
-    - name: Build test
-      run: make test
+    - name: Run test
+      run: make test-es
 #     - name: Send coverage report
 #       run: make coverage_send
 #       env:
@@ -51,4 +49,4 @@ jobs:
       if: failure()
     - name: Show docker process
       run: docker ps
-      if: always()
+      if: always()

Dockerfile

@@ -0,0 +1,42 @@
+ARG PYTHON_VERSION='3.6'
+FROM python:${PYTHON_VERSION}-slim as build
+RUN mkdir /code
+WORKDIR /code
+
+# Install python dependencies
+ENV PYTHONUNBUFFERED 1
+RUN apt-get update \
+&& apt-get install -y --no-install-recommends \
+   default-libmysqlclient-dev \
+   gcc \
+   build-essential \
+   git \
+   curl \
+   gettext libgettextpo-dev \
+&& rm -rf /var/lib/apt/lists/*
+COPY requirements/*.txt ./requirements/
+ARG DJANGO_VERSION='==2.22.*'
+FROM build as development
+# TODO: Move to /requirements/base.txt after fixing following bug:
+#       https://github.com/readthedocs/readthedocs-docker-images/issues/158
+RUN pip install mysqlclient==2.0.3
+RUN bash -c "if [[ \"${DJANGO_VERSION}\" == 'master' ]]; then \
+pip install --no-cache-dir -r requirements/dev.txt https://github.com/django/django/archive/master.tar.gz; else \
+pip install --no-cache-dir -r requirements/dev.txt \"django${DJANGO_VERSION}\"; fi"
+COPY ./ /code/
+CMD python manage.py runserver 0.0.0.0:8000
+FROM build as production
+ENV DJANGO_SETTINGS_MODULE="config.settings.production"
+RUN pip install --no-cache-dir mysqlclient==2.0.3 -r requirements/production.txt
+COPY ./ /code/
+RUN mkdir -p /code/feder/static && DJANGO_SECRET_KEY=x \
+   DJANGO_SERVER_EMAIL=x \
+   DATABASE_URL=sqlite:// \
+   EMAILLABS_APP_KEY=x \
+   EMAILLABS_SECRET_KEY=x \
+   LETTER_RECEIVE_SECRET=x \
+   AWS_S3_ACCESS_KEY_ID=x \
+   AWS_S3_SECRET_ACCESS_KEY=x \
+   AWS_STORAGE_BUCKET_NAME=x \
+   python manage.py collectstatic --no-input
+CMD ["gunicorn", "--worker-tmp-dir", "/dev/shm", "--capture-output", "config.wsgi"]

Makefile

@@ -6,7 +6,6 @@ clean:
 	docker-compose down
 
 regenerate_frontend:
-	docker-compose run web python manage.py collectstatic -c --noinput
 	docker-compose up gulp
 
 build:
@@ -15,39 +14,28 @@ build:
 test:
 	docker-compose run web coverage run manage.py test --keepdb --verbosity=2 ${TEST}
 
+test-es:
+	docker-compose -f docker-compose.yml -f docker-compose.es.yml run python manage.py test --keepdb --verbosity=2 ${TEST}
+
 coverage_html:
 	docker-compose run web coverage html
 
 coverage_send:
 	docker-compose run -e GITHUB_ACTIONS -e GITHUB_REF -e GITHUB_SHA -e GITHUB_HEAD_REF -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -e GITHUB_TOKEN -e COVERALLS_REPO_TOKEN web coveralls
 
-wait_web: wait_mysql wait_elasticsearch wait_tika
-
-wait_mysql:
-	docker-compose up -d db
-	docker-compose run web bash -c 'wait-for-it -t 30 db:3306' || (docker-compose logs db; exit -1)
-
-wait_elasticsearch:
-	docker-compose up -d elasticsearch
-	docker-compose run web bash -c 'wait-for-it -t 30 elasticsearch:9200' || (docker-compose logs elasticsearch; exit -1)
-
-wait_tika:
-	docker-compose up -d tika
-	docker-compose run web bash -c 'wait-for-it -t 60 tika:9998' || (docker-compose logs tika; exit -1)
-
 migrate:
-	docker-compose run web python manage.py migrate
+	docker-compose up migration
 
 lint: # lint currently staged files
 	pre-commit run
 
 lint-all: # lint all files in repository
 	pre-commit run --all-files
 
-check: wait_mysql
+check:
 	docker-compose run web python manage.py makemigrations --check
 
-migrations: wait_mysql
+migrations:
 	docker-compose run web python manage.py makemigrations
 
 settings:

README.rst

@@ -10,6 +10,10 @@
      :target: https://david-dm.org/watchdogpolska/feder#info=devDependencies
      :alt: devDependency Status
 
+ .. image:: https://www.deploytodo.com/do-btn-blue.svg
+     :target: https://cloud.digitalocean.com/apps/new?repo=https://github.com/watchdogpolska/feder/tree/do
+     :alt: Deploy to DO
+
 Feder
 =====
 

config/settings/common.py

@@ -98,6 +98,7 @@
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "reversion.middleware.RevisionMiddleware",
+    "whitenoise.middleware.WhiteNoiseMiddleware",
 )
 
 # MIGRATIONS CONFIGURATION
@@ -316,8 +317,6 @@
 
 INSTALLED_APPS += ("github_revision",)
 GITHUB_REVISION_REPO_URL = "https://github.com/watchdogpolska/feder"
-SENDFILE_BACKEND = "django_sendfile.backends.development"
-SENDFILE_ROOT = MEDIA_ROOT
 
 DATA_UPLOAD_MAX_MEMORY_SIZE = 200000000  # 200MB
 
@@ -358,3 +357,8 @@
 APACHE_TIKA_URL = env("APACHE_TIKA_URL", default="http://localhost:9998/tika")
 
 ELASTICSEARCH_SHOW_SIMILAR = env("ELASTICSEARCH_SHOW_SIMILAR", default=False)
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+AWS_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
+AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", default=None)

config/settings/production.py

@@ -40,10 +40,7 @@
 # Uploaded Media Files
 # ------------------------
 # See: http://django-storages.readthedocs.org/en/latest/index.html
-SENDFILE_BACKEND = "django_sendfile.backends.nginx"
 MEDIA_URL = "/media_internal/"
-SENDFILE_ROOT = MEDIA_ROOT
-SENDFILE_URL = MEDIA_URL
 
 # EMAIL
 # ------------------------------------------------------------------------------

docker-compose.es.yml

@@ -0,0 +1,29 @@
+version: '3.1'
+# To play with this file use following command: 
+# $ docker-compose -f docker-compose.yml docker-compose.es.yml up 
+
+services:
+  web:
+    environment: &webEnvironment
+      DATABASE_URL: mysql://root:password@db/feder
+      ELASTICSEARCH_URL: http://elasticsearch:9200/
+      APACHE_TIKA_URL: 'http://tika:9998/'
+      VIRUSTOTAL_API_KEY:
+      ATTACHMENTSCANNER_API_KEY:
+      ATTACHMENTSCANNER_API_URL:
+      METADEFENDER_API_KEY:
+
+  worker:
+    command: python manage.py process_tasks
+    environment: *webEnvironment
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
+    environment:
+      discovery.type: single-node
+      cluster.routing.allocation.disk.threshold_enabled: 'false'
+    ports:
+      - "9200:9200"
+
+  tika:
+    image: quay.io/watchdogpolska/apache-tika