From 6df1a83a4befbb8687a951d45fe6910deba83272 Mon Sep 17 00:00:00 2001 From: yelinz Date: Fri, 30 Jun 2023 09:15:31 +0200 Subject: [PATCH] feat: add django storages and settings for s3 storage --- CONFIGURATION.md | 23 ++++++++++++++++++++++- document_merge_service/settings.py | 16 +++++++++++++--- poetry.lock | 29 ++++++++++++++++++++++++++--- pyproject.toml | 1 + 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index e580ec8e..ed083467 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -87,7 +87,28 @@ If either `EMAIL_HOST_USER` or `EMAIL_HOST_PASSWORD` is empty, Django won't atte * `ADMINS`: list of people who will get code error notifications. Items in the list should follow this example: `Test Example ,Test2 ` ## Sentry -* `SENTRY_DSN`: identifier (data source name) for where to send events to. If no value is provided, sentry won't be activated (default: ") +* `SENTRY_DSN`: identifier (data source name) for where to send events to. If no value is provided, sentry won't be activated (default: "") * `SENTRY_ENVIRONMENT`: which app environment sent an event to sentry (default: `development`) * `SENTRY_TRACES_SAMPLE_RATE`: percentage chance a given transaction will be sent to Sentry (default: `1.0`) * `SENTRY_SEND_DEFAULT_PII`: enable send PII data that associates users to errors (default: `True`) + +## Template storage +* `FILE_STORAGE`: Django file storage backend (default: `django.core.files.storage.FileSystemStorage`) +* `MEDIA_ROOT`: Absolute filesystem path to the directory that will hold user-uploaded files. (default: "") +* `MEDIA_URL`: URL that handles the media served from MEDIA_ROOT, used for managing stored files. When using buckets this needs to be changed. (default: `api/v1/template/`) + + +### [django-storages](https://django-storages.readthedocs.io/en/1.13.2/backends/amazon-S3.html) S3 settings +Refer to for example [Digital Ocean](https://django-storages.readthedocs.io/en/1.13.2/backends/digital-ocean-spaces.html) configuration if using a S3 compatible storage which isn't AWS. + +Required to use S3 storage: +* `AWS_ACCESS_KEY_ID`: AWS access key id +* `AWS_S3_SECRET_ACCESS_KEY`: AWS secret access key +* `AWS_STORAGE_BUCKET_NAME`: Storage bucket name + +Optional: +* `AWS_S3_ENDPOINT_URL`: Custom S3 URL to use when connecting to S3, including scheme. (default: "") +* `AWS_S3_REGION_NAME`: Region of the storage (default: "") +* `AWS_LOCATION`: A path prefix that will be prepended to all uploads (default: "") +* `AWS_S3_FILE_OVERWRITE`: If `True` Files with the same name will overwrite each other. Otherwise extra characters are appended. (default: `False`) +* `AWS_S3_SIGNATURE_VERSION`: S3 signature version to use (default: `s2`) \ No newline at end of file diff --git a/document_merge_service/settings.py b/document_merge_service/settings.py index cd6072d2..b095d002 100644 --- a/document_merge_service/settings.py +++ b/document_merge_service/settings.py @@ -152,11 +152,21 @@ def parse_admins(admins): "FILE_STORAGE", default="django.core.files.storage.FileSystemStorage" ) MEDIA_ROOT = env.str("MEDIA_ROOT", "") - # TODO: This should be removed in favor of storing the files in a bucket # https://code.djangoproject.com/ticket/32991 -MEDIA_URL = "api/v1/template/" - +MEDIA_URL = env.str("MEDIA_URL", "api/v1/template/") + +# django-storages S3 settings +AWS_S3_ACCESS_KEY_ID = env.str("AWS_S3_ACCESS_KEY_ID", "") +AWS_S3_SECRET_ACCESS_KEY = env.str("AWS_S3_SECRET_ACCESS_KEY", "") +AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME", "") +AWS_S3_ENDPOINT_URL = env.str("AWS_S3_ENDPOINT_URL", "") +AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", "") +AWS_LOCATION = env.str("AWS_LOCATION", "") +AWS_S3_FILE_OVERWRITE = env.bool("AWS_S3_FILE_OVERWRITE", False) +AWS_S3_SIGNATURE_VERSION = env.str("AWS_S3_SIGNATURE_VERSION", "v2") + +# unoconv UNOCONV_ALLOWED_TYPES = env.list("UNOCOV_ALLOWED_TYPES", default=["pdf"]) UNOCONV_PYTHON = env.str("UNOCONV_PYTHON", default="/usr/bin/python3") UNOCONV_PATH = env.str("UNOCONV_PATH", default="/usr/bin/unoconv") diff --git a/poetry.lock b/poetry.lock index 405a55da..378ef062 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "arrow" @@ -536,6 +536,29 @@ files = [ [package.dependencies] django = ">=2.2" +[[package]] +name = "django-storages" +version = "1.13.2" +description = "Support for many storage backends in Django" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "django-storages-1.13.2.tar.gz", hash = "sha256:cbadd15c909ceb7247d4ffc503f12a9bec36999df8d0bef7c31e57177d512688"}, + {file = "django_storages-1.13.2-py3-none-any.whl", hash = "sha256:31dc5a992520be571908c4c40d55d292660ece3a55b8141462b4e719aa38eab3"}, +] + +[package.dependencies] +Django = ">=3.2" + +[package.extras] +azure = ["azure-storage-blob (>=12.0.0)"] +boto3 = ["boto3 (>=1.4.4)"] +dropbox = ["dropbox (>=7.2.1)"] +google = ["google-cloud-storage (>=1.27.0)"] +libcloud = ["apache-libcloud"] +sftp = ["paramiko (>=1.10.0)"] + [[package]] name = "django-stubs" version = "1.13.0" @@ -920,7 +943,7 @@ Click = [ ] sh = [ {version = ">=1.13.0", markers = "sys_platform != \"win32\""}, - {version = "1.14.2", optional = true, markers = "sys_platform != \"win32\" and extra == \"trusted-deps\""}, + {version = "1.14.2", optional = true, markers = "sys_platform != \"win32\""}, ] [package.extras] @@ -2565,4 +2588,4 @@ pgsql = ["psycopg2-binary"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "2d6d22f3d0be07b01f55b6562b0536f7ec7f1ab25652c6bd5f6722354eed7453" +content-hash = "a15d96561240b581d8254fff058f1cf8d6d8705afd29a30068f2c3cfc5520f35" diff --git a/pyproject.toml b/pyproject.toml index 1f015ddb..4b5545a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ xltpl = "^0.16" openpyxl = "^3.0.10" django-generic-api-permissions = "^0.2.0" sentry-sdk = "^1.12.1" +django-storages = "^1.13.2" [tool.poetry.group.dev.dependencies] black = "22.10.0"