store assets in s3, skip es tests (if no es)

.dockerignore

@@ -5,8 +5,10 @@ node_modules
 **/*.pyc
 docs
 **/*.rst
+**/__pycache__
 **/*.md
 media
 feder/media
-feder/staticfiles
+feder/static
+staticfiles
 htmlcov

Makefile

@@ -32,10 +32,10 @@ lint: # lint currently staged files
 lint-all: # lint all files in repository
 	pre-commit run --all-files
 
-check: wait_mysql
+check:
 	docker-compose run web python manage.py makemigrations --check
 
-migrations: wait_mysql
+migrations:
 	docker-compose run web python manage.py makemigrations
 
 settings:

config/settings/common.py

@@ -317,8 +317,6 @@
 
 INSTALLED_APPS += ("github_revision",)
 GITHUB_REVISION_REPO_URL = "https://github.com/watchdogpolska/feder"
-SENDFILE_BACKEND = "django_sendfile.backends.development"
-SENDFILE_ROOT = MEDIA_ROOT
 
 DATA_UPLOAD_MAX_MEMORY_SIZE = 200000000  # 200MB
 
@@ -359,3 +357,8 @@
 APACHE_TIKA_URL = env("APACHE_TIKA_URL", default="http://localhost:9998/tika")
 
 ELASTICSEARCH_SHOW_SIMILAR = env("ELASTICSEARCH_SHOW_SIMILAR", default=False)
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+AWS_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
+AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL", default=None)

config/settings/production.py

@@ -40,10 +40,7 @@
 # Uploaded Media Files
 # ------------------------
 # See: http://django-storages.readthedocs.org/en/latest/index.html
-SENDFILE_BACKEND = "django_sendfile.backends.nginx"
 MEDIA_URL = "/media_internal/"
-SENDFILE_ROOT = MEDIA_ROOT
-SENDFILE_URL = MEDIA_URL
 
 # EMAIL
 # ------------------------------------------------------------------------------

docker-compose.yml

@@ -41,12 +41,10 @@ services:
   # Services for Django application
   web:
     build: &webBuild
-      context: .
-      dockerfile: .contrib/docker/Dockerfile.web
       target: development
       args:
         # Match to production environment
-        PYTHON_VERSION: '${PYTHON_VERSION:-3.6.9}'
+        PYTHON_VERSION: '${PYTHON_VERSION:-3.7}'
         DJANGO_VERSION: '${DJANGO_VERSION:-==2.2.*}'
     volumes: &webVolumes
       - .:/code
@@ -56,13 +54,19 @@ services:
       ATTACHMENTSCANNER_API_KEY:
       ATTACHMENTSCANNER_API_URL:
       METADEFENDER_API_KEY:
+      AWS_S3_ACCESS_KEY_ID: &s3-access access-key
+      AWS_S3_SECRET_ACCESS_KEY: &s3-secret secret-key
+      AWS_S3_ENDPOINT_URL: http://minio:9000/
+      AWS_STORAGE_BUCKET_NAME: 'feder'
     depends_on:
       migration:
         condition: service_completed_successfully
       collectstatic:
         condition: service_completed_successfully
       db:
         condition: service_healthy
+      minio:
+        condition: service_healthy
     ports:
       - "8000:8000"
     # Following allows to execute `docker attach feder_web_1`
@@ -88,5 +92,22 @@ services:
     depends_on:
       db:
         condition: service_healthy
+  minio:
+    image: "minio/minio"
+    entrypoint: sh
+    # hack to precreate bucket, see https://github.com/minio/minio/issues/4769
+    command: -c 'mkdir -p /data/feder && minio server /data'
+    restart: always
+    environment:
+      MINIO_ACCESS_KEY: *s3-access
+      MINIO_SECRET_KEY: *s3-secret
+    volumes:
+      - "minio-data:/data"
+    healthcheck:
+        test: ["CMD", "curl" ,"localhost:9000"]
+        timeout: 20s
+        retries: 10
+
 volumes:
   mysql-data:
+  minio-data:

feder/es_search/tests.py

@@ -3,20 +3,18 @@
 from django.test import TestCase
 from ..letters.factories import IncomingLetterFactory, AttachmentFactory
 from .tasks import index_letter
-from time import sleep
-from elasticsearch_dsl import Search, Index
-from elasticsearch_dsl.query import MultiMatch, Match, Q, MoreLikeThis
-from elasticsearch_dsl.connections import get_connection, connections
-from elasticsearch.exceptions import ElasticsearchException
+from elasticsearch_dsl.connections import get_connection
 from .documents import LetterDocument
 from django.core.management import call_command
-import time
 from collections.abc import Iterable
-from elasticsearch.exceptions import ConflictError
 from .queries import more_like_this, search_keywords, find_document, delete_document
-import time
+from unittest import skipIf
+from .settings import ELASTICSEARCH_URL
 
 
+@skipIf(
+    not ELASTICSEARCH_URL, "Elasticsearch disabled. Set ELASTICSEARCH_URL to enable"
+)
 class ESMixin:
     connection_alias = "default"
     _index_suffix = "_test"

feder/letters/factories.py

@@ -43,7 +43,7 @@ class FileTextField(FileField):
     DEFAULT_FILENAME = "content.txt"
 
     def _make_data(self, params):
-        return params.get("text", get_text())
+        return params.get("text", get_text().encode("utf-8"))
 
 
 class LetterFactory(factory.django.DjangoModelFactory):

feder/letters/migrations/0027_auto_20211004_0220.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.24 on 2021-10-04 02:20
+
+from django.db import migrations, models
+import feder.letters.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('letters', '0026_auto_20210505_1327'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='letter',
+            name='eml',
+            field=models.FileField(blank=True, null=True, upload_to=feder.letters.models.messages_eml, verbose_name='File'),
+        ),
+    ]

feder/letters/models.py

@@ -25,6 +25,7 @@
 from django.utils import timezone
 from ..es_search.queries import more_like_this, find_document
 from feder.cases.models import enforce_quarantined_queryset
+from feder.main.utils import date_random_path
 
 logger = logging.getLogger(__name__)
 
@@ -90,6 +91,8 @@ def get_queryset(self):
         )
 
 
+def messages_eml():
+    return date_random_path("messages")()
 class Letter(AbstractRecord):
     SPAM = Choices(
         (0, "unknown", _("Unknown")),
@@ -158,7 +161,10 @@ class Letter(AbstractRecord):
         max_length=500,
     )
     eml = models.FileField(
-        upload_to="messages/%Y/%m/%d", verbose_name=_("File"), null=True, blank=True
+        upload_to=messages_eml,
+        verbose_name=_("File"),
+        null=True,
+        blank=True,
     )
     objects = LetterManager()
     objects_with_spam = LetterQuerySet.as_manager()

feder/letters/tests/test_views.py

@@ -1,7 +1,7 @@
 import codecs
 import json
 import os
-
+import requests
 from django.core import mail
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import SimpleUploadedFile
@@ -71,7 +71,7 @@ def test_content(self):
         self.assertContains(response, "Wniosek")
 
 
-class LetterDetailViewTestCase(ESMixin, ObjectMixin, PermissionStatusMixin, TestCase):
+class LetterDetailViewTestCase(ObjectMixin, PermissionStatusMixin, TestCase):
     status_anonymous = 200
     status_no_permission = 200
     permission = []
@@ -109,6 +109,21 @@ def test_contains_link_to_attachment(self):
             self.client.get(self.get_url()), attachment.get_absolute_url()
         )
 
+
+class IndexLetterDetailViewTestCase(ESMixin, LetterDetailViewTestCase):
+    def test_contains_link_to_attachment(self):
+        attachment = AttachmentFactory(letter=self.letter)
+        self.assertNotContains(
+            self.client.get(self.get_url()), attachment.get_absolute_url()
+        )
+        AttachmentRequestFactory(
+            content_object=attachment,
+            status=ScanRequest.STATUS.not_detected,
+        )
+        self.assertContains(
+            self.client.get(self.get_url()), attachment.get_absolute_url()
+        )
+
     def test_not_contain_link_to_similiar_on_disabled(self):
         similiar = IncomingLetterFactory(body=self.letter.body)
         self.index([similiar, self.letter])
@@ -144,9 +159,9 @@ def test_contain_link_to_similiar_on_enabled(self):
 
 class LetterMessageXSendFileView(PermissionStatusMixin, TestCase):
     permission = []
-    status_has_permission = 200
-    status_anonymous = 200
-    status_no_permission = 200
+    status_has_permission = 302
+    status_anonymous = 302
+    status_no_permission = 302
 
     def setUp(self):
         super().setUp()
@@ -200,19 +215,23 @@ class LetterDeleteViewTestCase(ObjectMixin, PermissionStatusMixin, TransactionTe
     def get_url(self):
         return reverse("letters:delete", kwargs={"pk": self.from_user.pk})
 
+    def check_url(self, url):
+        resp = requests.get(url)
+        return resp.status_code == 200
+
     def test_remove_eml_file(self):
         self.login_permitted_user()
-        self.assertTrue(os.path.isfile(self.from_user.eml.file.name))
+        self.assertTrue(self.check_url(self.from_user.eml.url))
         self.client.post(self.get_url())
-        self.assertFalse(os.path.isfile(self.from_user.eml.file.name))
+        self.assertFalse(self.check_url(self.from_user.eml.url))
 
     def test_remove_letter_with_attachment(self):
         # TransactionTestCase has to be used to test file cleanup feature.
         self.login_permitted_user()
         attachment = AttachmentFactory(letter=self.from_user)
-        self.assertTrue(os.path.isfile(attachment.attachment.file.name))
+        self.assertTrue(self.check_url(attachment.attachment.url))
         self.client.post(self.get_url())
-        self.assertFalse(os.path.isfile(attachment.attachment.file.name))
+        self.assertFalse(self.check_url(attachment.attachment.url))
 
 
 class LetterReplyViewTestCase(ObjectMixin, PermissionStatusMixin, TestCase):
@@ -309,7 +328,7 @@ def get_url(self):
         return reverse("letters:atom")
 
     def test_item_enclosure_url(self):
-        self.from_institution.eml.save("msg.eml", ContentFile("Foo"), save=True)
+        self.from_institution.eml.save("msg.eml", ContentFile(b"Foo"), save=True)
         resp = self.client.get(self.get_url())
         self.assertContains(resp, self.from_institution.eml.name)
 
@@ -470,9 +489,9 @@ def get_url(self):
 
 class StandardAttachmentXSendFileViewTestCase(PermissionStatusMixin, TestCase):
     permission = []
-    status_has_permission = 200
-    status_anonymous = 200
-    status_no_permission = 200
+    status_has_permission = 302
+    status_anonymous = 302
+    status_no_permission = 302
     spam_status = Letter.SPAM.non_spam
 
     def setUp(self):

feder/letters/views.py

@@ -1,6 +1,5 @@
 import json
 import uuid
-from os import path
 from atom.ext.django_filters.views import UserKwargFilterSetMixin
 from atom.views import (
     CreateMessageMixin,
@@ -45,7 +44,6 @@
 from feder.main.mixins import (
     AttrPermissionRequiredMixin,
     RaisePermissionRequiredMixin,
-    BaseXSendFileView,
 )
 from feder.monitorings.models import Monitoring
 from feder.records.models import Record
@@ -55,20 +53,11 @@
 from .models import Letter, Attachment
 from feder.monitorings.tasks import send_mass_draft
 from feder.virus_scan.models import Request as ScanRequest
+from feder.main.mixins import BaseDetailFileRedirect
 
 _("Letters index")
 
 
-class MixinGzipXSendFile:
-    def get_sendfile_kwargs(self, context):
-        kwargs = super().get_sendfile_kwargs(context)
-        if kwargs["filename"] and kwargs["filename"].endswith(".gz"):
-            kwargs["encoding"] = "gzip"
-            filename = path.basename(kwargs["filename"][: -len(".gz")])
-            kwargs["attachment_filename"] = filename
-        return kwargs
-
-
 class CaseRequiredMixin:
     def get_queryset(self):
         qs = super().get_queryset().exclude(record__case=None)
@@ -142,14 +131,10 @@ def get_queryset(self):
         return qs.for_user(self.request.user)
 
 
-class LetterMessageXSendFileView(MixinGzipXSendFile, BaseXSendFileView):
+class LetterMessageXSendFileView(BaseDetailFileRedirect):
+    # todo: rename class to modern
     model = Letter
     file_field = "eml"
-    send_as_attachment = True
-
-    def get_queryset(self):
-        qs = super().get_queryset()
-        return qs.for_user(self.request.user)
 
 
 class LetterCreateView(
@@ -540,32 +525,22 @@ def form_valid(self, form):
         return super().form_valid(form)
 
 
-class AttachmentXSendFileView(MixinGzipXSendFile, BaseXSendFileView):
+class AttachmentXSendFileView(BaseDetailFileRedirect):
     model = Attachment
     file_field = "attachment"
-    send_as_attachment = True
 
-    def get_queryset(self):
-        return super().get_queryset().for_user(self.request.user)
-
-    def get_sendfile_kwargs(self, context):
-        kwargs = super().get_sendfile_kwargs(context)
-        if kwargs["filename"].endswith(".gz"):
-            kwargs["encoding"] = "gzip"
-        return kwargs
-
-    def render_to_response(self, context):
-        if context["object"].is_infected():
+    def get(self, *args, **kwargs):
+        response = super().get(*args, **kwargs)
+        if self.object.is_infected():
             raise PermissionDenied(
                 "You do not have permission to view that file. "
                 "The file was considered dangerous."
             )
-        return super().render_to_response(context)
+        return response
 
 
 class AttachmentRequestCreateView(ActionMessageMixin, ActionView):
     template_name_suffix = "_request_scan"
-    model = Attachment
 
     def get_object(self, *args, **kwargs):
         if not hasattr(self, "object"):