Merge pull request #3745 from alephdata/release/3.17.0

Release/3.17.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.15.6
+current_version = 3.17.0
 tag_name = {new_version}
 commit = True
 tag = True

.github/workflows/docs.yml

@@ -6,6 +6,8 @@ on:
       - "docs/**"
     branches:
       - "**"
+    pr:
+      - "docs/**"
   workflow_dispatch: {}
 
 jobs:

.github/workflows/e2e.yml

@@ -30,6 +30,12 @@ jobs:
           docker --version
           docker-compose --version
 
+      - name: Build e2e image
+        run: make build-e2e
+
+      - name: Pull and start services
+        run: make services-e2e
+
       - name: Run tests
         run: make e2e
 

CONTRIBUTING.md

@@ -16,9 +16,9 @@ If you're trying to install Aleph and need help, or if you are running Aleph and
 
 1. [Read the Aleph Support Policy](https://github.com/alephdata/aleph/blob/main/SUPPORT.md) and understand under what
    rules we provide support to others. If you feel you fall within that
-   group, please [get in touch](https://docs.alephdata.org/get-in-touch).
+   group, please [get in touch](https://docs.aleph.occrp.org/get-in-touch).
 
-2. Make sure you've [read the documentation](https://docs.alephdata.org),
+2. Make sure you've [read the documentation](https://docs.aleph.occrp.org),
    especially the technical troubleshooting sections.
 
 3. Only once you're **certain that the problem you are seeing is a bug**
@@ -30,7 +30,7 @@ If you're trying to install Aleph and need help, or if you are running Aleph and
 
 A great place to start when looking for something to work on is our [project board](https://github.com/orgs/alephdata/projects/10). For experienced Aleph developers feel free to check our the [workboard](https://github.com/orgs/alephdata/projects/10/views/1). If this is your first time working on Aleph then it's probably a good idea to pick [something easy to work on](https://github.com/orgs/alephdata/projects/10/views/10). You can also find these issues on our [contribute page](https://github.com/alephdata/aleph/contribute). Regardless, once you've selected an issue please take a moment to assign it to yourself, this will stop anyone else from working on your issue.
 
-If, after reviewing the list (or selecting an issue to work on) you'd like to reach out and talk to us then [get in touch](https://docs.alephdata.org/get-in-touch).
+If, after reviewing the list (or selecting an issue to work on) you'd like to reach out and talk to us then [get in touch](https://docs.aleph.occrp.org/get-in-touch).
 
 ### Forking and Branching
 

Dockerfile

@@ -49,4 +49,4 @@ ENV ALEPH_ELASTICSEARCH_URI=http://elasticsearch:9200/ \
 RUN mkdir /run/prometheus
 
 # Run the green unicorn
-CMD gunicorn --config /aleph/gunicorn.conf.py --workers 5 --log-level info --log-file - aleph.manage:app
+CMD gunicorn --config /aleph/gunicorn.conf.py --workers 6 --log-level debug --log-file -

Makefile

@@ -3,7 +3,7 @@ COMPOSE_E2E=docker-compose -f docker-compose.dev.yml -f docker-compose.e2e.yml
 APPDOCKER=$(COMPOSE) run --rm app
 UIDOCKER=$(COMPOSE) run --no-deps --rm ui
 ALEPH_TAG=latest
-BLACK_FLAGS=--exclude aleph/migrate
+BLACK_OPTS=--extend-exclude aleph/migrate
 
 all: build upgrade web
 
@@ -29,13 +29,13 @@ lint-ui:
 	$(UIDOCKER) npm run lint
 
 format:
-	black $(BLACK_FLAGS) aleph/
+	black $(BLACK_OPTS) aleph/
 
 format-ui:
 	$(UIDOCKER) npm run format
 
 format-check:
-	black --check $(BLACK_FLAGS) aleph/
+	black --check $(BLACK_OPTS) aleph/
 
 format-check-ui:
 	$(UIDOCKER) npm run format:check
@@ -112,7 +112,7 @@ e2e: services-e2e e2e/test-results
 	$(COMPOSE_E2E) run --rm app aleph upgrade
 	$(COMPOSE_E2E) run --rm app aleph createuser --name="E2E Admin" --admin --password="admin" [email protected]
 	$(COMPOSE_E2E) up -d api ui worker
-	BASE_URL=http://ui:8080 $(COMPOSE_E2E) run --rm --build-arg PLAYWRIGHT_VERSION=$(shell awk -F'==' '/^playwright==/ { print $$2 }' e2e/requirements.txt) e2e pytest -s -v --output=/e2e/test-results/ --screenshot=only-on-failure --video=retain-on-failure e2e/
+	BASE_URL=http://ui:8080 $(COMPOSE_E2E) run --rm e2e pytest -s -v --output=/e2e/test-results/ --screenshot=only-on-failure --video=retain-on-failure e2e/
 
 e2e-local-setup: dev
 	python3 -m pip install -q -r e2e/requirements.txt

README.rst

@@ -15,9 +15,8 @@ companies) against watchlists, e.g. from prior research or public datasets.
 For further details on the software, how to use it, install it or manage data
 imports, please check the documentation at: 
 
-* https://docs.alephdata.org
-* Installation: https://docs.alephdata.org/developers/installation
-* Changes: https://docs.alephdata.org/developers/changelog
+* https://docs.aleph.occrp.org
+* Installation: https://docs.aleph.occrp.org/developers/installation
 
 
 Support
@@ -28,7 +27,7 @@ If you're interested in participating in this process, please read the support
 policy (`SUPPORT.md`), the contribution rules (`CONTRIBUTING.md`), and the code of conduct (`CODE_OF_CONDUCT.md`) and then get
 in touch:
 
-* https://docs.alephdata.org/get-in-touch
+* https://docs.aleph.occrp.org/get-in-touch
 
 Release process
 ---------------

aleph/logic/html.py

@@ -58,7 +58,7 @@ def sanitize_html(html_text, base_url, encoding=None):
             if el.tag == "a":
                 el.set("target", "_blank")
                 el.set("rel", "nofollow noreferrer external noopener")
-        return tostring(doc)
+        return tostring(doc, method="html", encoding="unicode")
     except Exception as exc:
         log.warning("HTML sanitizer failure [%s]: %s", type(exc), exc)
         return gettext("[HTML removed: could not be sanitized]")

aleph/pages/about.en.md

@@ -6,4 +6,4 @@ menu: true
 
 Aleph is a powerful tool for people who follow the money. It helps investigators to securely access and search large amounts of data - no matter whether they are a government database or a leaked email archive.
 
-Please refer to the [documentation](https://docs.alephdata.org/) for more details.
+Please refer to the [documentation](https://docs.aleph.occrp.org/) for more details.

aleph/tests/test_view_util.py

@@ -36,6 +36,11 @@ def test_sanitize_html(self):
         assert html.find(".//a").get("target") == "_blank", html
         assert "nofollow" in html.find(".//a").get("rel"), html
 
+    def test_sanitize_html_no_self_closing_tags(self):
+        html_str = '<html><body><noscript><script src="https://example.org></script></noscript></body></html>'
+        processed = sanitize_html(html_str, "https://example.org")
+        assert processed == "<html><body><div><noscript></noscript></div></body></html>"
+
     def test_validate_returns_errors_for_paths(self):
         # given
         schema = "RoleCreate"  # name min length 4, password min length 6

aleph/translations/ar/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2023 ORGANIZATION
+# Copyright (C) 2024 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
 # 
 # Translators:
 # Eman Alqaisi <[email protected]>, 2020
@@ -13,7 +13,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-11-27 13:54+0100\n"
+"POT-Creation-Date: 2024-05-22 15:09+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Mohammed AlKawmani <[email protected]>, 2020\n"
 "Language-Team: Arabic (https://app.transifex.com/aleph/teams/76591/ar/)\n"
@@ -32,7 +32,7 @@ msgstr "ألف"
 msgid "No schema on entity"
 msgstr "لا جزئية في هذا الكل "
 
-#: aleph/logic/html.py:63
+#: aleph/logic/html.py:64
 msgid "[HTML removed: could not be sanitized]"
 msgstr "[تمت إزالة HTML: تعذر تنظيف المحتوى]"
 
@@ -228,7 +228,7 @@ msgstr ""
 msgid "{{export}} is ready for download"
 msgstr ""
 
-#: aleph/search/__init__.py:76
+#: aleph/search/__init__.py:77
 msgid "No schema is specified for the query."
 msgstr "لم يتم تحديد مخطط للاستعلام."
 
@@ -395,11 +395,11 @@ msgid ""
 "Aleph administrator"
 msgstr ""
 
-#: aleph/views/sessions_api.py:61
+#: aleph/views/sessions_api.py:69
 msgid "Invalid user or password."
 msgstr "اسم مستخدم أو كلمة مرور غير صحيحة"
 
-#: aleph/views/sessions_api.py:95
+#: aleph/views/sessions_api.py:104
 msgid "Authentication has failed."
 msgstr "فشل التوثيق "
 

aleph/translations/bs/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2023 ORGANIZATION
+# Copyright (C) 2024 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
 # 
 # Translators:
 # pudo <[email protected]>, 2018
@@ -14,7 +14,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-11-27 13:54+0100\n"
+"POT-Creation-Date: 2024-05-22 15:09+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Tajna Biscevic <[email protected]>, 2019\n"
 "Language-Team: Bosnian (https://app.transifex.com/aleph/teams/76591/bs/)\n"
@@ -33,7 +33,7 @@ msgstr "Aleph"
 msgid "No schema on entity"
 msgstr ""
 
-#: aleph/logic/html.py:63
+#: aleph/logic/html.py:64
 msgid "[HTML removed: could not be sanitized]"
 msgstr ""
 
@@ -229,7 +229,7 @@ msgstr ""
 msgid "{{export}} is ready for download"
 msgstr ""
 
-#: aleph/search/__init__.py:76
+#: aleph/search/__init__.py:77
 msgid "No schema is specified for the query."
 msgstr "Nije navedena shema za upit."
 
@@ -396,11 +396,11 @@ msgid ""
 "Aleph administrator"
 msgstr ""
 
-#: aleph/views/sessions_api.py:61
+#: aleph/views/sessions_api.py:69
 msgid "Invalid user or password."
 msgstr ""
 
-#: aleph/views/sessions_api.py:95
+#: aleph/views/sessions_api.py:104
 msgid "Authentication has failed."
 msgstr ""
 

aleph/translations/de/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2023 ORGANIZATION
+# Copyright (C) 2024 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
 # 
 # Translators:
 # pudo <[email protected]>, 2020
@@ -11,7 +11,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-11-27 13:54+0100\n"
+"POT-Creation-Date: 2024-05-22 15:09+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: pudo <[email protected]>, 2020\n"
 "Language-Team: German (https://app.transifex.com/aleph/teams/76591/de/)\n"
@@ -30,7 +30,7 @@ msgstr "Aleph"
 msgid "No schema on entity"
 msgstr "Dieses Objekt hat keinen Typ."
 
-#: aleph/logic/html.py:63
+#: aleph/logic/html.py:64
 msgid "[HTML removed: could not be sanitized]"
 msgstr "[HTML entfernt: konnte nicht gesichert werden]"
 
@@ -227,7 +227,7 @@ msgstr "Vollständige Exporte"
 msgid "{{export}} is ready for download"
 msgstr "{{export}} ist zum Download bereit"
 
-#: aleph/search/__init__.py:76
+#: aleph/search/__init__.py:77
 msgid "No schema is specified for the query."
 msgstr "Diese Anfrage muss ein 'schema' enthalten."
 
@@ -407,11 +407,11 @@ msgid ""
 "Aleph administrator"
 msgstr ""
 
-#: aleph/views/sessions_api.py:61
+#: aleph/views/sessions_api.py:69
 msgid "Invalid user or password."
 msgstr "Der Nutzername oder das Passwort ist falsch."
 
-#: aleph/views/sessions_api.py:95
+#: aleph/views/sessions_api.py:104
 msgid "Authentication has failed."
 msgstr "Die Anmeldung ist fehlgeschlagen."
 

aleph/translations/es/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2023 ORGANIZATION
+# Copyright (C) 2024 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
 # 
 # Translators:
 # John Ospina <[email protected]>, 2018
@@ -13,7 +13,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-11-27 13:54+0100\n"
+"POT-Creation-Date: 2024-05-22 15:09+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Nathan Jaccard <[email protected]>, 2020\n"
 "Language-Team: Spanish (https://app.transifex.com/aleph/teams/76591/es/)\n"
@@ -32,7 +32,7 @@ msgstr "Aleph"
 msgid "No schema on entity"
 msgstr "Sin esquema de la entidad"
 
-#: aleph/logic/html.py:63
+#: aleph/logic/html.py:64
 msgid "[HTML removed: could not be sanitized]"
 msgstr "[HTML eliminado: no se pudo depurar]"
 
@@ -228,7 +228,7 @@ msgstr ""
 msgid "{{export}} is ready for download"
 msgstr ""
 
-#: aleph/search/__init__.py:76
+#: aleph/search/__init__.py:77
 msgid "No schema is specified for the query."
 msgstr "No se especificó esquema para la consulta."
 
@@ -401,11 +401,11 @@ msgid ""
 "Aleph administrator"
 msgstr ""
 
-#: aleph/views/sessions_api.py:61
+#: aleph/views/sessions_api.py:69
 msgid "Invalid user or password."
 msgstr "Contraseña o usuario incorrecto."
 
-#: aleph/views/sessions_api.py:95
+#: aleph/views/sessions_api.py:104
 msgid "Authentication has failed."
 msgstr "Error en la autenticación."
 

aleph/translations/fr/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2023 ORGANIZATION
+# Copyright (C) 2024 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2024.
 # 
 # Translators:
 # Mathieu Morey <[email protected]>, 2020
@@ -12,7 +12,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-11-27 13:54+0100\n"
+"POT-Creation-Date: 2024-05-22 15:09+0000\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Jean-Philippe Menotti, 2022\n"
 "Language-Team: French (https://app.transifex.com/aleph/teams/76591/fr/)\n"
@@ -31,7 +31,7 @@ msgstr "Aleph"
 msgid "No schema on entity"
 msgstr "Pas de schéma sur l’entité"
 
-#: aleph/logic/html.py:63
+#: aleph/logic/html.py:64
 msgid "[HTML removed: could not be sanitized]"
 msgstr "[HTML retiré : n’a pas pu être assaini]"
 
@@ -228,7 +228,7 @@ msgstr "Exports terminés"
 msgid "{{export}} is ready for download"
 msgstr "{{export}} est prêt au téléchargement"
 
-#: aleph/search/__init__.py:76
+#: aleph/search/__init__.py:77
 msgid "No schema is specified for the query."
 msgstr "Aucun schéma n’est spécifié pour la requête."
 
@@ -408,11 +408,11 @@ msgid ""
 "Aleph administrator"
 msgstr ""
 
-#: aleph/views/sessions_api.py:61
+#: aleph/views/sessions_api.py:69
 msgid "Invalid user or password."
 msgstr "Nom d’utilisateur ou mot de passe non valable."
 
-#: aleph/views/sessions_api.py:95
+#: aleph/views/sessions_api.py:104
 msgid "Authentication has failed."
 msgstr "L’authentification a échoué."