From 7034e2735787cad0d7a5ab6b0290fc197ca85468 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Thu, 21 Mar 2024 15:19:58 +0100
Subject: [PATCH 01/43] Revert "Add more details about the lemmatizer at the
 new corpus phase. (#317)"

This reverts commit f41d822052b7231aaeb7605d811ee3485c6ae8d7.
---
 .github/workflows/test.yml                 |   2 +-
 app/__init__.py                            |   2 +-
 app/lemmatizers.py                         |  12 -
 app/templates/main/corpus_new.html         |  40 +-
 config.py                                  |  22 +-
 dummy_lemmatizer_service.py                |  22 -
 tests/test_selenium/base.py                |   1 +
 tests/test_selenium/test_bookmark.py       |   1 +
 tests/test_selenium/test_control_lists.py  |   2 +
 tests/test_selenium/test_corpus_init.py    | 103 +--
 tests/test_selenium/test_export.py         |   2 +
 translations/bo_CN/LC_MESSAGES/messages.po | 879 ++++++---------------
 12 files changed, 256 insertions(+), 832 deletions(-)
 delete mode 100644 app/lemmatizers.py
 delete mode 100644 dummy_lemmatizer_service.py

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a5a64446..6845d1a3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,7 +36,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install pytest pytest-cov coveralls flake8 pytest-sugar
+        pip install pytest pytest-cov coveralls flake8
     - name: Set up TEST_DATABASE_URL
       run: |
         echo "TEST_DATABASE_URL=postgresql://pyrrha:pyrrha@localhost:5432/data-test" >> $GITHUB_ENV
diff --git a/app/__init__.py b/app/__init__.py
index bf8a8516..ac3a4548 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1,6 +1,7 @@
 import os
 from flask import Flask, g
 
+from config import config
 from flask_compress import Compress
 from flask_login import LoginManager
 from flask_mail import Mail
@@ -28,7 +29,6 @@
 
 def create_app(config_name="dev"):
     """ Create the application """
-    from config import config
 
     app = Flask(
         __name__,
diff --git a/app/lemmatizers.py b/app/lemmatizers.py
deleted file mode 100644
index 169ef566..00000000
--- a/app/lemmatizers.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from typing import Optional
-from dataclasses import dataclass
-
-
-@dataclass
-class LemmatizerService:
-    title: str   # e.g. "Old French"
-    uri: str  # Current address
-    provider: str  # e.g. Deucalion at Ecole nationale des Chartes
-    bibtex: str  # Citation Scheme
-    apa: str  # APA equivalent
-    ui: Optional[str] = None
diff --git a/app/templates/main/corpus_new.html b/app/templates/main/corpus_new.html
index 51ea0143..c70a4543 100644
--- a/app/templates/main/corpus_new.html
+++ b/app/templates/main/corpus_new.html
@@ -72,28 +72,10 @@ <h1>{{ _('Create a new corpus') }}</h1>
             <div class="col-md-9">
                 <div class="input-group">
                   <select class="form-control" id="language-model">
-                      <option selected value="disabled" class="text-muted disabled">{{_('Select a service')}}</option>
-                      {%- for service in lemmatizers %}
-                    <option value="{{ service.uri }}">{{ service.title }}</option>
+                      {%- for lang, address in lemmatizers %}
+                    <option value="{{ address }}">{{ lang }}</option>
                       {% endfor -%}
                   </select>
-                  {%- for service in lemmatizers %}
-                    <div class="alert alert-info lemmatizer-details" data-uri="{{service.uri}}" style="display: none;">
-                        {% trans name=service.title, provider=service.provider %}
-                            <em>{{ name }}</em> is a lemmatization service provided by <i>{{ provider }}</i>.<br />
-                        {% endtrans %}
-                        {% if service.ui %}
-                            <em>{{_("If you only want to lemmatize (and not correct data), you can use their user interface here:")}}</em>
-                            <a href="{{service.ui}}">{{service.ui}}</a><br/>
-                        {% endif %}
-                        {{ _("Please cite tools accordingly. You can use the following APA or Bibtex: ")}}
-                        <ul>
-                            <li><em>APA:</em> <code>{{service.apa }}</code></li>
-                            <li><em>BIB:</em> <code style="padding-left: 1em; display: block;">{{service.bibtex}}</code></li>
-                        </ul>
-                    </div>
-                  {% endfor %}
-
                   <div class="input-group-append">
                     <button id="submit-model" class="btn btn-outline-secondary" type="button">{{ _('Lemmatize') }}</button>
                   </div>
@@ -218,14 +200,7 @@ <h1>{{ _('Create a new corpus') }}</h1>
 <script type="text/javascript">
 
 $(document).ready(function() {
-    $("#language-model").on("change", function (event) {
-        let uri = $("#language-model").val();
-        document.querySelectorAll(".lemmatizer-details").forEach(el => el.style.display = "none");
-        if (uri === "disabled") {
-            return;
-        }
-        document.querySelectorAll(`.lemmatizer-details[data-uri='${uri}']`).forEach(el => el.style.display = "block");
-    });
+
     $("#submit-model").on("click", function(event) {
         event.preventDefault();
         // Get the parameters
@@ -238,12 +213,11 @@ <h1>{{ _('Create a new corpus') }}</h1>
             tokens_fail = document.getElementById("tokens-fail"),
             approximate_count = document.getElementById("tokens-approximate");
 
-        if (uri === "disabled") { return; }
-
         // Create the form
         var formData = new FormData();
         formData.append("data", text_data.value);
 
+
         // Set up our HTTP request
         var xhr = new XMLHttpRequest();
         var update = function(xhr) {
@@ -254,16 +228,14 @@ <h1>{{ _('Create a new corpus') }}</h1>
         };
         // Setup our listener to process compeleted requests
         xhr.onreadystatechange = function () {
-            // Only run if the request is complete
-            if (typeof xhr.status !== "number") return null;
 
-            if (xhr.readyState < 3) return null;
+            // Only run if the request is complete
+            if (xhr.readyState < 3 || xhr.readyState > 4) return null;
 
             if (xhr.readyState === 3) {
                 update(xhr);
                 return null;
             }
-            if (xhr.status === 0) return null;
 
             // Process our return data
             if (xhr.status >= 200 && xhr.status < 300) {
diff --git a/config.py b/config.py
index df7795be..7183fdc2 100644
--- a/config.py
+++ b/config.py
@@ -1,6 +1,5 @@
 import os
-from app.lemmatizers import LemmatizerService
-from typing import List
+
 
 basedir = os.path.abspath(os.path.dirname(__file__))
 
@@ -39,7 +38,7 @@ class Config:
     PAGINATION_DEFAULT_TOKENS = 100
 
     # Lemmatizer (until Deucalion client)
-    LEMMATIZERS: List[LemmatizerService] = []
+    LEMMATIZERS = []
 
     # Change automatically the Postgresql instance language if not english
     FORCE_PSQL_EN_LOCALE = True
@@ -102,23 +101,6 @@ class BaseTestConfig(Config):
     EMAIL_SUBJECT_PREFIX = '[{}]'.format(Config.APP_NAME)
     EMAIL_SENDER = '{app_name} Admin <{email}>'.format(app_name=Config.APP_NAME, email=MAIL_USERNAME)
 
-    LEMMATIZERS = [
-        LemmatizerService(
-            "Dummy lemmatizer",
-            "http://localhost:4567/lemma",
-            provider="ProviderInstitution",
-            ui="someui.com",
-            apa="Clérice et al. 2019",
-            bibtex="""@article{camps2021corpus,
-	title        = {Corpus and Models for Lemmatisation and POS-tagging of Old French},
-	author       = {Camps, Jean-Baptiste and Cl{\'e}rice, Thibault and Duval, Fr{\'e}d{\'e}ric and Kanaoka, Naomi and Pinche, Ariane and others},
-	year         = 2021,
-	journal      = {arXiv preprint arXiv:2109.11442},
-	keywords     = {Old French}
-}"""
-        )
-    ]
-
 
 class SQLiteTestConfig(BaseTestConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or \
diff --git a/dummy_lemmatizer_service.py b/dummy_lemmatizer_service.py
deleted file mode 100644
index 69201b98..00000000
--- a/dummy_lemmatizer_service.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from flask import Flask, request, Response, stream_with_context
-app = Flask("fixture")
-
-
-@app.route("/lemma", methods=["POST"])
-def lemmatizing():
-    r = Response(
-        "\n".join(
-            ["token\tlemma"] +
-            ["\t".join([tok, f"{idx}"]) for idx, tok in enumerate(request.form.get("data").split())]
-        ),
-        200,
-        headers={
-            'Content-Type': 'text/plain; charset=utf-8',
-            'Access-Control-Allow-Origin': "*"
-        }
-    )
-    return r
-
-
-if __name__ == "__main__":
-    app.run(port="4567")
diff --git a/tests/test_selenium/base.py b/tests/test_selenium/base.py
index 19fbe2cc..f0b2d006 100644
--- a/tests/test_selenium/base.py
+++ b/tests/test_selenium/base.py
@@ -731,6 +731,7 @@ def get_field(row, f):
             self.driver_find_element_by_class_name("pagination"),
             "a"
         )
+        self.driver.save_screenshot("first.results.png")
         for page_index in range(0, len(pagination)):
             # self.driver_find_element_by_class_name("pagination").find_elements_by_tag_name("a")[page_index].click()
             self.element_find_elements_by_tag_name(
diff --git a/tests/test_selenium/test_bookmark.py b/tests/test_selenium/test_bookmark.py
index 99917c9a..1a57f737 100644
--- a/tests/test_selenium/test_bookmark.py
+++ b/tests/test_selenium/test_bookmark.py
@@ -9,6 +9,7 @@ class TestBookmark(TestBase):
 
     def set_bookmark(self, tok_id, page=None):
         self.driver.get(self.url_for_with_port("main.tokens_correct", corpus_id="1", page=page))
+        self.driver.save_screenshot("token.png")
         self.driver_find_element_by_id("dd_t"+str(tok_id)).click()
         self.driver.implicitly_wait(2)
         dd = self.driver_find_element_by_css_selector("*[aria-labelledby='dd_t{}']".format(tok_id))
diff --git a/tests/test_selenium/test_control_lists.py b/tests/test_selenium/test_control_lists.py
index 68f6b56f..cfa24eea 100644
--- a/tests/test_selenium/test_control_lists.py
+++ b/tests/test_selenium/test_control_lists.py
@@ -147,8 +147,10 @@ def test_action_as_owner(self):
         # Check that we can edit informations about markdown or whatever
         self.driver.get(self.url_for_with_port("control_lists_bp.information_edit", control_list_id=1))
         self.driver_find_element_by_id("cl_notes").send_keys("# This is some notes")
+        self.driver.save_screenshot("HELLOOOO1.png")
         self.driver_find_element_by_id("submit").click()
         self.driver.get(self.url_for_with_port("control_lists_bp.information_read", control_list_id=1))
+        self.driver.save_screenshot("HELLOOOO2.png")
         self.assertEqual(
             self.driver_find_element_by_tag_name("h1").text, "This is some notes",
             "Check that edition works"
diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 82575100..efa7c17e 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -3,14 +3,9 @@
 from app import db
 from tests.test_selenium.base import TestBase
 from tests.fixtures import PLAINTEXT_CORPORA
-from selenium.webdriver.support.select import Select
 import csv
 import os
 
-from flask import Flask, request, Response
-from multiprocessing import Process
-from selenium.webdriver.support.wait import WebDriverWait
-
 
 class TestCorpusRegistration(TestBase):
     """ Test creation of Corpus """
@@ -466,7 +461,6 @@ def test_registration_with_wrongly_formated_input(self):
 	dieu	NOMpro	NOMB.=s|GENRE=m|CAS=n
 vos	vos1	PROper	PERS.=2|NOMB.=p|GENRE=m|CAS=r
 soit	estre1	VERcjg	MODE=sub|TEMPS=pst|PERS.=3|NOMB.=s""")
-
         self.driver_find_element_by_id("label_checkbox_reuse").click()
         self.driver_find_element_by_id("control_list_select").click()
         self.driver_find_element_by_id("cl_opt_" + str(target_cl.id)).click()
@@ -474,11 +468,7 @@ def test_registration_with_wrongly_formated_input(self):
         self.driver_find_element_by_id("submit").click()
 
         self.assertEqual(
-            sorted([
-                e.text.strip()
-                for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")
-                if e.text.strip()
-            ]),
+            sorted([e.text.strip() for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")]),
             sorted([
                 'At least one line of your corpus is missing a token/form. Check line 1'
             ]),
@@ -503,12 +493,9 @@ def test_registration_with_no_tsv_input(self):
         self.driver_find_element_by_id("cl_opt_" + str(target_cl.id)).click()
 
         self.driver_find_element_by_id("submit").click()
+
         self.assertEqual(
-            sorted([
-                e.text.strip()
-                for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")
-                if e.text.strip()
-            ]),
+            sorted([e.text.strip() for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")]),
             sorted([
                 'You did not input any text.'
             ]),
@@ -635,87 +622,3 @@ def test_corpus_name_unique_user(self):
         self.driver_find_element_by_id("submit").click()
         self.driver.implicitly_wait(5)
         self.assertFalse(self.driver_find_elements_by_css_selector(".alert.alert-danger"))
-
-    def test_lemmatization_service(self):
-        """
-        Test that a user can create a corpus and that this corpus has its data well recorded
-        """
-        # Click register menu link
-        self.driver_find_element_by_id("new_corpus_link").click()
-        self.driver.implicitly_wait(15)
-
-        # Fill in registration form
-        self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
-        self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
-
-        # Default
-        details = self.driver_find_elements_by_css_selector(".lemmatizer-details")
-        self.assertEqual(details[0].is_displayed(), False, "Nothing should be displayed by default")
-
-        # Select a lemmatizer
-        s = Select(self.driver_find_elements_by_css_selector("#language-model")[0])
-        s.select_by_visible_text("Dummy lemmatizer")
-        self.assertEqual(details[0].is_displayed(), True, "Something should be displayed")
-        self.assertIn("Dummy lemmatizer is a lemmatization service provided by ProviderInstitution.", details[0].text)
-
-        # Unselect
-        s.select_by_visible_text("Select a service")
-        self.assertEqual(details[0].is_displayed(), False, "Nothing should be now")
-
-    def test_lemmatization_service_runs(self):
-        # Create a second fixture app
-        app = Flask("fixture")
-
-        @app.route("/lemma", methods=["POST"])
-        def lemmatizing():
-            return Response(
-                "\n".join(
-                    ["token\tlemma"] +
-                    ["\t".join([tok, f"{idx}"]) for idx, tok in enumerate(request.form.get("data").split())]
-                ),
-                200,
-                headers={
-                    'Content-Type': 'text/plain; charset=utf-8',
-                    'Access-Control-Allow-Origin': "*"
-                }
-            )
-        # Start it
-        second_app = Process(target=app.run, daemon=True, kwargs=dict(host="localhost", port="4567"))
-        second_app.start()
-
-        self.driver_find_element_by_id("new_corpus_link").click()
-        self.driver.implicitly_wait(15)
-
-        # Fill in registration form
-        self.driver_find_element_by_id("corpusName").send_keys("Test")
-        self.writeMultiline(self.driver_find_element_by_id("tokens"), "Je suis")
-
-        # Select the lemmatizer
-        s = Select(self.driver_find_elements_by_css_selector("#language-model")[0])
-        s.select_by_index(1)
-        self.driver.implicitly_wait(5)
-
-        # Lemmatize
-        self.driver_find_element_by_id("submit-model").click()
-
-        # Wait
-        wait = WebDriverWait(self.driver, timeout=5)
-        wait.until(lambda x: self.driver_find_element_by_id("tokens-success").is_displayed())
-
-        # Check results
-        self.assertEqual(
-            self.driver_find_element_by_id("tokens").get_property("value").split("\n"),
-            ["token\tlemma", "Je\t0", "suis\t1"]
-        )
-
-        self.assertEqual(
-            self.driver_find_element_by_id("tokens-success").text,
-            "Operation finished with success ! 2 tokens analyzed in total.",
-            "Lemmatization happened"
-        )
-        # Kill second app
-        try:
-            second_app.terminate()
-            second_app.join(2)
-        finally:
-            second_app.close()
diff --git a/tests/test_selenium/test_export.py b/tests/test_selenium/test_export.py
index 67282adb..399207ec 100644
--- a/tests/test_selenium/test_export.py
+++ b/tests/test_selenium/test_export.py
@@ -102,8 +102,10 @@ def test_tei_geste_export(self):
         self.driver.refresh()
 
         self.driver.get(self.url_for_with_port("main.tokens_export", corpus_id=1))
+        self.driver.get_screenshot_as_file("./beforedownload.png")
         self.driver_find_element_by_id("geste-tei").click()
 
+        self.driver.get_screenshot_as_file("./afterdownload.png")
         time.sleep(5)
 
         with open(os.path.join(self.download_path, "wauchier.xml")) as f:
diff --git a/translations/bo_CN/LC_MESSAGES/messages.po b/translations/bo_CN/LC_MESSAGES/messages.po
index 6d39a5d8..04aa2adf 100644
--- a/translations/bo_CN/LC_MESSAGES/messages.po
+++ b/translations/bo_CN/LC_MESSAGES/messages.po
@@ -6,26 +6,21 @@
 # Translators:
 # Ngawang Trinley <ngawangtrinley@gmail.com>, 2022
 # Tashi Dhondup <dhonduptiger658@gmail.com>, 2022
+#
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2024-03-19 18:26+0100\n"
+"POT-Creation-Date: 2022-05-12 23:50+0530\n"
 "PO-Revision-Date: 2022-06-30 14:23+0000\n"
 "Last-Translator: Tashi Dhondup <dhonduptiger658@gmail.com>, 2022\n"
-"Language: bo_CN\n"
-"Language-Team: Tibetan (China) "
-"(https://www.transifex.com/OpenPecha/teams/139966/bo_CN/)\n"
-"Plural-Forms: nplurals=1; plural=0;\n"
+"Language-Team: Tibetan (China) (https://www.transifex.com/OpenPecha/teams/139966/bo_CN/)\n"
 "MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=utf-8\n"
+"Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Generated-By: Babel 2.14.0\n"
-
-#: app/templates/account/join_invite.html:8
-#, fuzzy
-msgid "Create your password"
-msgstr "གསང་གྲངས་མི་དྲན་ནམ།"
+"Generated-By: Babel 2.10.3\n"
+"Language: bo_CN\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
 
 #: app/templates/account/login.html:7
 msgid "Log in"
@@ -35,188 +30,22 @@ msgstr "ཐོ་འཇུག"
 msgid "Forgot password?"
 msgstr "གསང་གྲངས་མི་དྲན་ནམ།"
 
-#: app/templates/account/manage.html:25 app/templates/admin/manage_user.html:29
-#, fuzzy
-msgid "Full name"
-msgstr "རུས་མིང་།"
-
-#: app/templates/account/manage.html:26 app/templates/admin/manage_user.html:33
-#: app/templates/macros/dashboard_macros.html:28
-msgid "Email address"
-msgstr "གློག་འཕྲིན་ཁ་བྱང་།"
-
-#: app/templates/account/manage.html:27 app/templates/admin/manage_user.html:37
-#: app/templates/macros/dashboard_macros.html:7
-#: app/templates/macros/dashboard_macros.html:30
-msgid "Account type"
-msgstr "ཁ་བྱང་གི་རིགས།"
-
-#: app/templates/account/manage.html:35
-#, fuzzy
-msgid "Account Settings"
-msgstr "ཁ་བྱང་གི་རིགས།"
-
-#: app/templates/account/manage.html:37
-msgid "Manage your account settings and change your login information."
-msgstr ""
-
 #: app/templates/account/register.html:8
 msgid "Create an account"
 msgstr "ཐོ་འགོད།"
 
-#: app/templates/account/reset_password.html:7
-#, fuzzy
-msgid "Reset your password"
-msgstr "གསང་གྲངས་མི་དྲན་ནམ།"
-
-#: app/templates/account/unconfirmed.html:8
-msgid "You need to confirm your account before continuing."
-msgstr ""
-
-#: app/templates/account/unconfirmed.html:15
-msgid "I didn't receive an email!"
-msgstr ""
-
-#: app/templates/account/unconfirmed.html:17
-msgid "No problem! Just click the button below and we'll send you another one."
-msgstr ""
-
-#: app/templates/account/unconfirmed.html:18
-msgid "Resend confirmation email"
-msgstr ""
-
-#: app/templates/account/email/change_email.html:1
-#: app/templates/account/email/confirm.html:1
-#: app/templates/account/email/invite.html:1
-#: app/templates/account/email/reset_password.html:1
-#, fuzzy
-msgid "Dear"
-msgstr "འཚོལ་ཞིབ།"
-
-#: app/templates/account/email/change_email.html:3
-msgid "To confirm your new email address"
-msgstr ""
-
-#: app/templates/account/email/change_email.html:3
-#: app/templates/account/email/confirm.html:5
-#: app/templates/account/email/invite.html:5
-msgid "click here"
-msgstr ""
-
-#: app/templates/account/email/change_email.html:5
-msgid ""
-"Alternatively, you can paste the following link in your browser's address"
-" bar:"
-msgstr ""
-
-#: app/templates/account/email/change_email.html:9
-#: app/templates/account/email/confirm.html:11
-#: app/templates/account/email/invite.html:13
-#: app/templates/account/email/reset_password.html:11
-msgid "Sincerely,"
-msgstr ""
-
-#: app/templates/account/email/change_email.html:13
-#: app/templates/account/email/confirm.html:15
-#: app/templates/account/email/invite.html:17
-#: app/templates/account/email/reset_password.html:15
-#: app/templates/control_lists/email/contact.html:10
-msgid "Note: replies to this email address are not monitored."
-msgstr ""
-
-#: app/templates/account/email/confirm.html:3
-#, fuzzy
-msgid "Welcome to"
-msgstr "Pyrrha ལ་ཕེབས་པར་དགའ་བསུ་ཞུ།"
-
-#: app/templates/account/email/confirm.html:5
-msgid "To confirm your account, please"
-msgstr ""
-
-#: app/templates/account/email/invite.html:3
-msgid "You are invited to join"
-msgstr ""
-
-#: app/templates/account/email/invite.html:5
-msgid "To set your password, please"
-msgstr ""
-
-#: app/templates/account/email/reset_password.html:9
-msgid "If you have not requested a password reset, simply ignore this message."
-msgstr ""
-
-#: app/templates/admin/manage_user.html:41
-#, fuzzy
-msgid "Account status"
-msgstr "ཁ་བྱང་གི་རིགས།"
-
-#: app/templates/admin/manage_user.html:53
-#, fuzzy
-msgid "Back to all users"
-msgstr "སྤྱོད་མཁན་ལ་ཆོག་མཆན་སྤྲོད་པ།"
-
-#: app/templates/admin/manage_user.html:67
-msgid "This action is permanent"
-msgstr ""
-
-#: app/templates/admin/manage_user.html:69
-msgid ""
-"Deleting a user account is not a reversible change. Any information "
-"associated\n"
-"                                with this account will be removed, and "
-"cannot be recovered."
-msgstr ""
-
-#: app/templates/admin/manage_user.html:77
-msgid "I understand that this action cannot be undone."
-msgstr ""
-
-#: app/templates/admin/manage_user.html:79
-msgid ""
-"Delete\n"
-"                        this user"
-msgstr ""
-
-#: app/templates/admin/new_user.html:13
-#: app/templates/admin/registered_users.html:9
-#: app/templates/macros/dashboard_macros.html:158
-msgid "Back to dashboard"
-msgstr "གཙོ་ངོས་སུ་ཕྱིར་ལོག"
-
-#: app/templates/admin/new_user.html:16 app/templates/main/dashboard.html:12
-msgid "Add New User"
-msgstr "སྤྱོད་མཁན་གསར་པ་སྣོན།"
-
-#: app/templates/admin/new_user.html:17 app/templates/main/dashboard.html:13
-msgid "Create a new user account"
-msgstr "སྤྱོད་མཁན་གྱི་ཁ་བྱང་གསར་པ་ཞིག་བཟོ།"
-
-#: app/templates/admin/registered_users.html:12
-#: app/templates/main/dashboard.html:10
-msgid "Registered Users"
-msgstr "ཐོ་འགོད་བྱས་ཟིན་པའི་སྤྱོད་མཁན།"
-
-#: app/templates/admin/registered_users.html:14
-#, fuzzy
-msgid "View and manage currently registered users."
-msgstr "སྐད་མཛོད་སྤྱོད་མཁན་རྣམས་འཆར་སྟོན་དང་དོ་དམ།"
-
 #: app/templates/control_lists/contact.html:8
 msgid "Contact control lists administrators"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་གི་འཛིན་སྐྱོང་བར་འབྲེལ་གཏུགས།"
 
 #: app/templates/control_lists/contact.html:10
 msgid ""
-"This page is built so that you can contact administrators of a control "
-"list to propose\n"
-"            new lemma, POS or morphologies accepted values. We have "
-"preferred to use email so that discussion could be rich\n"
+"This page is built so that you can contact administrators of a control list to propose\n"
+"            new lemma, POS or morphologies accepted values. We have preferred to use email so that discussion could be rich\n"
 "            when needed."
 msgstr ""
-"This page is built so that you can contact administrators of a control "
-"list to propose\n"
-"            new lemma, POS or morphologies accepted values. We have "
-"preferred to use email so that discussion could be rich\n"
+"This page is built so that you can contact administrators of a control list to propose\n"
+"            new lemma, POS or morphologies accepted values. We have preferred to use email so that discussion could be rich\n"
 "            when needed."
 
 #: app/templates/control_lists/control_list.html:10
@@ -225,7 +54,6 @@ msgid "Description"
 msgstr "འགྲེལ་བཤད།"
 
 #: app/templates/control_lists/control_list.html:13
-#: app/templates/control_lists/information_edit.html:23
 #: app/templates/control_lists/information_read.html:15
 msgid "Bibliography"
 msgstr "དཔེ་ཐོ།"
@@ -248,8 +76,8 @@ msgstr "བསྐྱར་འབྲི་བྱེད།"
 
 #: app/templates/control_lists/control_list.html:30
 msgid ""
-"The following pages are made to completely rewrite control lists. Use "
-"with caution !"
+"The following pages are made to completely rewrite control lists. Use with "
+"caution !"
 msgstr ""
 "གཤམ་བཀོད་ཀྱི་ཤོག་ངོས་དག་ནི་སྟངས་འཛིན་ཐོ་གཞུང་ཡོངས་སུ་བསྐྱར་འབྲི་བྱས་པ་ལས་གྲུབ་པ་ཡིན།"
 " བཀོལ་སྤྱོད་སྐབས་གཟབ་ནན་བྱེད་དགོས།"
@@ -275,16 +103,6 @@ msgstr "གཟབ་དགོས་སའི་གནས།"
 msgid "Make public"
 msgstr "སྤྱི་སྤྱོད་ཅན་དུ་བཟོས་པ།"
 
-#: app/templates/control_lists/edit.html:18
-#: app/templates/control_lists/information_edit.html:35
-#: app/templates/main/control_list_switch.html:29
-#: app/templates/main/corpus_custom_dictionary.html:55
-#: app/templates/main/corpus_edit_allowed_values.html:18
-#: app/templates/main/corpus_new.html:211
-#: app/templates/main/corpus_preferences.html:86
-msgid "Submit"
-msgstr "འབུལ་སྤྲོད།"
-
 #: app/templates/control_lists/information_edit.html:11
 msgid "Description of the control list"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་དེའི་འགྲེལ་བཤད།"
@@ -310,21 +128,26 @@ msgstr "སློབ་སྟོན།"
 #: app/templates/control_lists/information_edit.html:30
 msgid ""
 "All the relevant information that might be of help\n"
-"        about how to tag using your control list. Can link to external "
-"data and use"
+"        about how to tag using your control list. Can link to external data and use"
 msgstr ""
 "All the relevant information that might be of help\n"
-"        about how to tag using your control list. Can link to external "
-"data and use"
+"        about how to tag using your control list. Can link to external data and use"
+
+#: app/templates/control_lists/information_edit.html:35
+#: app/templates/main/control_list_switch.html:29
+#: app/templates/main/corpus_custom_dictionary.html:55
+#: app/templates/main/corpus_new.html:198
+#: app/templates/main/corpus_preferences.html:86
+msgid "Submit"
+msgstr "འབུལ་སྤྲོད།"
 
 #: app/templates/control_lists/macros.html:11
 msgid "Manage Lists"
 msgstr "དོ་དམ་ཐོ་གཞུང་།"
 
 #: app/templates/control_lists/macros.html:13
-#: app/templates/macros/tokens_macros.html:514
 #: app/templates/main/corpus_custom_dictionary.html:12
-#: app/templates/main/corpus_info.html:129
+#: app/templates/main/corpus_info.html:127
 #: app/templates/main/tokens_similar_to_token.html:19
 #: app/templates/main/tokens_similar_to_token.html:29
 #: app/templates/main/tokens_similar_to_token.html:52
@@ -332,10 +155,9 @@ msgid "Lemma"
 msgstr "མ་ཚིག"
 
 #: app/templates/control_lists/macros.html:14
-#: app/templates/macros/nav_macros.html:255
-#: app/templates/macros/tokens_macros.html:517
+#: app/templates/macros/nav_macros.html:245
 #: app/templates/main/corpus_custom_dictionary.html:42
-#: app/templates/main/corpus_info.html:155
+#: app/templates/main/corpus_info.html:153
 #: app/templates/main/tokens_similar_to_token.html:22
 #: app/templates/main/tokens_similar_to_token.html:32
 #: app/templates/main/tokens_similar_to_token.html:55
@@ -362,39 +184,6 @@ msgstr "མིང་གསར་བརྗེ།"
 msgid "Edit informations"
 msgstr "གནས་ཚུལ་རྩོམ་སྒྲིག་བྱེད།"
 
-#: app/templates/control_lists/propose_as_public.html:8
-msgid "Propose the list as public list"
-msgstr ""
-
-#: app/templates/control_lists/propose_as_public.html:10
-msgid ""
-"Through this form, you will contact the administrators to ask them for\n"
-"            making this control list public. A public control list can "
-"not be made private again."
-msgstr ""
-
-#: app/templates/control_lists/propose_as_public.html:11
-msgid ""
-"By making this\n"
-"                list public, other users will be able to freely access "
-"this control list. They will be able\n"
-"            to contact the administrators of this list to propose new "
-"allowed values such as lemma."
-msgstr ""
-
-#: app/templates/control_lists/read.html:7
-#, fuzzy
-msgid "Allowed values for"
-msgstr "ཆོག་མཆན་ཐོབ་པའི་ lemma"
-
-#: app/templates/control_lists/read.html:10
-msgid "Readable"
-msgstr ""
-
-#: app/templates/control_lists/read.html:28
-msgid "No allowed value in this category"
-msgstr ""
-
 #: app/templates/control_lists/read_lemma.html:8
 msgid "Control List for Lemmas"
 msgstr "Lemmas ཡི་སྟངས་འཛིན་ཐོ་གཞུང་།"
@@ -431,46 +220,14 @@ msgstr "བཙལ་བྱ།"
 msgid "Show help"
 msgstr "རམ་འདེགས་སྟོན།"
 
-#: app/templates/control_lists/read_lemma.html:45
-#, fuzzy
-msgid "Options :"
-msgstr "མཆན་འགྲེལ།"
-
-#: app/templates/control_lists/read_lemma.html:47
-#: app/templates/main/tokens_search_through_fields.html:64
-msgid "* can be used to match partial words, eg."
-msgstr "* can be used to match partial words, eg."
-
-#: app/templates/control_lists/read_lemma.html:48
-#: app/templates/main/tokens_search_through_fields.html:65
-msgid "! can be used to negate a match, eg."
-msgstr "! can be used to negate a match, eg."
-
-#: app/templates/control_lists/read_lemma.html:49
-#: app/templates/main/tokens_search_through_fields.html:66
-msgid "| can be used to perform an OR operation, eg."
-msgstr "| can be used to perform an OR operation, eg."
-
-#: app/templates/control_lists/read_lemma.html:50
-#, fuzzy
-msgid "To search forms which do not contain 'e', eg."
-msgstr "འདུས་མེད་པའི་འགེངས་ཤོག་འཚོལ་ཕྱིར།"
-
 #: app/templates/control_lists/rename.html:8
 msgid "Rename the control list"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་དེ་ལ་མིང་བསྐྱར་འདོགས་བྱེད།"
 
-#: app/templates/errors/403.html:5
-msgid "Forbidden: You do not have the right to access this page."
-msgstr ""
-
-#: app/templates/errors/404.html:5
-msgid "Page not found"
-msgstr ""
-
-#: app/templates/errors/500.html:5
-msgid "Internal Server Error"
-msgstr ""
+#: app/templates/macros/dashboard_macros.html:7
+#: app/templates/macros/dashboard_macros.html:30
+msgid "Account type"
+msgstr "ཁ་བྱང་གི་རིགས།"
 
 #: app/templates/macros/dashboard_macros.html:9
 msgid "All account types"
@@ -491,6 +248,10 @@ msgstr "དངོས་མིང་།"
 msgid "Last name"
 msgstr "རུས་མིང་།"
 
+#: app/templates/macros/dashboard_macros.html:28
+msgid "Email address"
+msgstr "གློག་འཕྲིན་ཁ་བྱང་།"
+
 #: app/templates/macros/dashboard_macros.html:32
 msgid "Status"
 msgstr "གནས་བབས།"
@@ -527,10 +288,13 @@ msgstr "སྐད་མཛོད།"
 
 #: app/templates/macros/dashboard_macros.html:142
 #: app/templates/macros/nav_macros.html:89
-#: app/templates/main/corpus_new.html:131
 msgid "Control Lists"
 msgstr "གཏན་འབེབ་ཟིན་པའི་ཐོ་གཞུང་།"
 
+#: app/templates/macros/dashboard_macros.html:158
+msgid "Back to dashboard"
+msgstr "གཙོ་ངོས་སུ་ཕྱིར་ལོག"
+
 #: app/templates/macros/dashboard_macros.html:165
 msgid "View and manage"
 msgstr "འཆར་སྟོན་དང་དོ་དམ།"
@@ -570,19 +334,16 @@ msgstr "སྤྱོད་མཁན་ཞིག་ལ་ཆོག་མཆན་
 
 #: app/templates/macros/dashboard_macros.html:226
 msgid ""
-"The user will be able to use the post-correction tools but also grant and"
-" remove\n"
+"The user will be able to use the post-correction tools but also grant and remove\n"
 "                                    accesses to this"
 msgstr ""
-"The user will be able to use the post-correction tools but also grant and"
-" remove\n"
+"The user will be able to use the post-correction tools but also grant and remove\n"
 "                                    accesses to this"
 
 #: app/templates/macros/header_macros.html:2
-#: app/templates/macros/nav_macros.html:192
+#: app/templates/macros/nav_macros.html:182
 #: app/templates/main/tokens_add_row.html:7
 #: app/templates/main/tokens_correct.html:8
-#: app/templates/main/tokens_correct_unallowed.html:8
 #: app/templates/main/tokens_del_row.html:7
 #: app/templates/main/tokens_edit_form.html:7
 #: app/templates/main/tokens_edit_history.html:6
@@ -598,11 +359,6 @@ msgstr "སྐད་མཛོད།"
 msgid "Dashboard"
 msgstr "གཙོ་ངོས།"
 
-#: app/templates/macros/nav_macros.html:37
-#, fuzzy
-msgid "Your Account"
-msgstr "ཐོ་འགོད།"
-
 #: app/templates/macros/nav_macros.html:38
 msgid "Log out"
 msgstr "ཕྱིར་ཐོན།"
@@ -619,250 +375,105 @@ msgstr "ནང་འཛུལ།"
 msgid "New Corpus"
 msgstr "སྐད་མཛོད་གསར་པ།"
 
-#: app/templates/macros/nav_macros.html:113
+#: app/templates/macros/nav_macros.html:103
 msgid "Bug or request ?"
 msgstr "གནད་དོན་ཡིན་ནམ་ཡང་ན་རེ་ཞུ་ཡིན།"
 
-#: app/templates/macros/nav_macros.html:114
+#: app/templates/macros/nav_macros.html:104
 msgid "Help"
 msgstr "རམ་འདེགས།"
 
-#: app/templates/macros/nav_macros.html:150
+#: app/templates/macros/nav_macros.html:140
 msgid "Go to"
 msgstr "དེར་འགྲོ།"
 
-#: app/templates/macros/nav_macros.html:186
+#: app/templates/macros/nav_macros.html:176
 msgid "Menu"
 msgstr "ཐོ།"
 
-#: app/templates/macros/nav_macros.html:195
+#: app/templates/macros/nav_macros.html:185
 msgid "Home"
 msgstr "མདུན་ངོས།"
 
-#: app/templates/macros/nav_macros.html:197
+#: app/templates/macros/nav_macros.html:187
 #: app/templates/main/corpus_preferences.html:5
 msgid "Preferences"
 msgstr "རྒྱུ་ཆ་ཁག"
 
-#: app/templates/macros/nav_macros.html:202
+#: app/templates/macros/nav_macros.html:192
 msgid "Control List"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་།"
 
-#: app/templates/macros/nav_macros.html:205
-#: app/templates/macros/tokens_macros.html:455
+#: app/templates/macros/nav_macros.html:195
 msgid "Edit"
 msgstr "རྩོམ་སྒྲིག"
 
-#: app/templates/macros/nav_macros.html:209
+#: app/templates/macros/nav_macros.html:199
 #: app/templates/main/tokens_similar_to_record.html:14
 msgid "Change"
 msgstr "བསྒྱུར་བཅོས།"
 
-#: app/templates/macros/nav_macros.html:213
+#: app/templates/macros/nav_macros.html:203
 msgid "Corpus Custom Dictionary"
 msgstr "སྐད་མཛོད་འདི་ཉིད་ཀྱི་ཚིག་མཛོད།"
 
-#: app/templates/macros/nav_macros.html:217
+#: app/templates/macros/nav_macros.html:207
 msgid "Text history"
 msgstr "སློབ་དེབ་ཀྱི་ལོ་རྒྱུས།"
 
-#: app/templates/macros/nav_macros.html:221
+#: app/templates/macros/nav_macros.html:211
 msgid "Delete the corpus (2 steps)"
 msgstr "སྐད་མཛོད་དེ་ (2 steps) སུབ།"
 
-#: app/templates/macros/nav_macros.html:227
+#: app/templates/macros/nav_macros.html:217
 msgid "Go to bookmark"
 msgstr "དཔེ་རྟགས་ཡོད་ས་དེར་འགྲོ།"
 
-#: app/templates/macros/nav_macros.html:231
+#: app/templates/macros/nav_macros.html:221
 msgid "Annotations"
 msgstr "མཆན་འགྲེལ།"
 
-#: app/templates/macros/nav_macros.html:234
+#: app/templates/macros/nav_macros.html:224
 #: app/templates/main/tokens_edit_form.html:34
 msgid "Correct"
 msgstr "དག་སྒྲིག"
 
-#: app/templates/macros/nav_macros.html:237
+#: app/templates/macros/nav_macros.html:227
 #: app/templates/main/tokens_search_through_fields.html:70
 msgid "Search"
 msgstr "འཚོལ་ཞིབ།"
 
-#: app/templates/macros/nav_macros.html:240
+#: app/templates/macros/nav_macros.html:230
 msgid "Export"
 msgstr "ཕྱིར་འདྲེན།"
 
-#: app/templates/macros/nav_macros.html:243
+#: app/templates/macros/nav_macros.html:233
 msgid "History"
 msgstr "ལོ་རྒྱུས།"
 
-#: app/templates/macros/nav_macros.html:247
+#: app/templates/macros/nav_macros.html:237
 msgid "Correct unallowed ..."
 msgstr "ཆོག་མཆན་མ་སྤྲད་པ་དག་སྒྲིག"
 
-#: app/templates/macros/nav_macros.html:250
+#: app/templates/macros/nav_macros.html:240
 msgid "lemma"
 msgstr "མ་ཚིག"
 
-#: app/templates/macros/nav_macros.html:260
+#: app/templates/macros/nav_macros.html:250
 msgid "morphology"
 msgstr "ཚིག་གི་གྲུབ་ཆ།"
 
-#: app/templates/macros/nav_macros.html:274
+#: app/templates/macros/nav_macros.html:264
 msgid "Your corpora"
 msgstr "ཁྱེད་ཀྱི་སྐད་མཛོད།"
 
-#: app/templates/macros/tokens_macros.html:16
-#, fuzzy
-msgid "Save checked similar lemma"
-msgstr "མ་ཚིག་གསར་པ་ཉར།"
-
-#: app/templates/macros/tokens_macros.html:18
-#: app/templates/macros/tokens_macros.html:81
-#, fuzzy
-msgid "Save"
-msgstr "ཉར་ཟིན།"
-
-#: app/templates/macros/tokens_macros.html:19
-msgid "Select All"
-msgstr ""
-
-#: app/templates/macros/tokens_macros.html:20
-msgid "Deselect All"
-msgstr ""
-
-#: app/templates/macros/tokens_macros.html:26
-msgid ""
-"An error occured. Contact the administrator\n"
-"        with all the information you can so they can replicate your error."
-msgstr ""
-
-#: app/templates/macros/tokens_macros.html:32
-#, fuzzy
-msgid "Id"
-msgstr "ID"
-
-#: app/templates/macros/tokens_macros.html:34
-#: app/templates/macros/tokens_macros.html:512
-#: app/templates/main/tokens_add_row.html:13
-#: app/templates/main/tokens_del_row.html:13
-#: app/templates/main/tokens_edit_form.html:13
-#: app/templates/main/tokens_search_through_fields.html:19
-#: app/templates/main/tokens_similar_to_token.html:49
-msgid "Form"
-msgstr "འགེངས་ཤོག"
-
-#: app/templates/macros/tokens_macros.html:89
-#, fuzzy
-msgid "Edit the form"
-msgstr "གནས་ཚུལ་རྩོམ་སྒྲིག་བྱེད།"
-
-#: app/templates/macros/tokens_macros.html:90
-#, fuzzy
-msgid "Delete the row"
-msgstr "སུབ།"
-
-#: app/templates/macros/tokens_macros.html:91
-msgid "Add a token after this one"
-msgstr ""
-
-#: app/templates/macros/tokens_macros.html:92
-#, fuzzy
-msgid "Set as bookmark"
-msgstr "དཔེ་རྟགས་ཡོད་ས་དེར་འགྲོ།"
-
-#: app/templates/macros/tokens_macros.html:380
-msgid "Add <code>$val$</code> to custom dictionary of <em>$key$</em>"
-msgstr ""
-
-#: app/templates/macros/tokens_macros.html:454
-#: app/templates/main/tokens_edit_history.html:11
-#: app/templates/main/tokens_similar_to_record.html:17
-msgid "User"
-msgstr "སྤྱོད་མཁན།"
-
-#: app/templates/macros/tokens_macros.html:456
-#: app/templates/macros/tokens_macros.html:522
-#: app/templates/main/tokens_add_row.html:14
-#: app/templates/main/tokens_del_row.html:14
-#: app/templates/main/tokens_edit_form.html:14
-#: app/templates/main/tokens_similar_to_record.html:26
-#: app/templates/main/tokens_similar_to_token.html:50
-msgid "Context"
-msgstr "སྐབས་བསྟུན།"
-
-#: app/templates/macros/tokens_macros.html:458
-#: app/templates/main/tokens_similar_to_record.html:28
-msgid "Old Lemma"
-msgstr "མ་ཚིག་རྙིང་བ།"
-
-#: app/templates/macros/tokens_macros.html:459
-#: app/templates/main/tokens_similar_to_record.html:29
-msgid "Corr Lemma"
-msgstr "Corr མ་ཚིག"
-
-#: app/templates/macros/tokens_macros.html:462
-#: app/templates/main/tokens_similar_to_record.html:32
-msgid "Previous POS"
-msgstr "སྔ་མའི་ POS"
-
-#: app/templates/macros/tokens_macros.html:463
-#: app/templates/main/tokens_similar_to_record.html:33
-msgid "Actual POS"
-msgstr "དངོས་ཡོད་ཀྱི་ POS"
-
-#: app/templates/macros/tokens_macros.html:466
-#: app/templates/main/tokens_similar_to_record.html:36
-msgid "Previous Morph"
-msgstr "སྔ་མའི་ཚིག་གི་གྲུབ་ཆ།"
-
-#: app/templates/macros/tokens_macros.html:467
-#: app/templates/main/tokens_similar_to_record.html:37
-msgid "Actual Morph"
-msgstr "དངོས་ཡོད་ཀྱི་ཚིག་གི་གྲུབ་ཆ།"
-
-#: app/templates/macros/tokens_macros.html:469
-#, fuzzy
-msgid "Similar"
-msgstr "tokens འདྲ་མཚུངས་ཁག"
-
-#: app/templates/macros/tokens_macros.html:520
-#: app/templates/main/corpus_custom_dictionary.html:26
-#: app/templates/main/tokens_similar_to_token.html:25
-#: app/templates/main/tokens_similar_to_token.html:35
-#: app/templates/main/tokens_similar_to_token.html:58
-msgid "Morph"
-msgstr "ཚིག་གི་གྲུབ་ཆ།"
-
-#: app/templates/macros/upload_text.html:4
-msgid "Text file"
-msgstr ""
-
-#: app/templates/macros/upload_text.html:5
-msgid "Upload a text file (.txt) which will be read"
-msgstr ""
-
-#: app/templates/macros/utils.html:3
-msgid "Private"
-msgstr ""
-
-#: app/templates/macros/utils.html:4
-msgid "Sent for publicization"
-msgstr ""
-
-#: app/templates/macros/utils.html:5
-#, fuzzy
-msgid "Public"
-msgstr "སྤྱི་སྤྱོད་ཅན་དུ་བཟོས་པ།"
-
 #: app/templates/main/control_list_switch.html:9
 msgid "Switch control list"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་བརྗེ་བ།"
 
 #: app/templates/main/control_list_switch.html:13
-#, fuzzy
-msgid "Current control list:"
-msgstr "སྟངས་འཛིན་ཐོ་གཞུང་དེ་ལ་མིང་བསྐྱར་འདོགས་བྱེད།"
+msgid "Current control list: {{ current_control_lists.name }}"
+msgstr "ད་ལྟའི་སྟངས་འཛིན་ཐོ་གཞུང་ནི། {{ current_control_lists.name }}"
 
 #: app/templates/main/corpus_custom_dictionary.html:6
 msgid "Custom dictionary"
@@ -876,17 +487,20 @@ msgstr "ཚིག་མཛོད་རྣམས།"
 msgid "List of supplementaries lemma for this corpus. One per line"
 msgstr "སྐད་མཛོད་འདིའི་ཁ་གསབ་མ་ཚིག་གི་ཐོ་གཞུང་། རེ་རེ་ཐིག་ཕྲེང་རེ་ཡིན་དགོས།"
 
+#: app/templates/main/corpus_custom_dictionary.html:26
+#: app/templates/main/tokens_similar_to_token.html:25
+#: app/templates/main/tokens_similar_to_token.html:35
+#: app/templates/main/tokens_similar_to_token.html:58
+msgid "Morph"
+msgstr "ཚིག་གི་གྲུབ་ཆ།"
+
 #: app/templates/main/corpus_custom_dictionary.html:34
 msgid ""
-"List of supplementaries morph for this corpus. One per line, a human "
-"readable version of the code\n"
-"                    can be given after a tabulation (4 spaces are "
-"automatically transformed into tabulation). eg."
+"List of supplementaries morph for this corpus. One per line, a human readable version of the code\n"
+"                    can be given after a tabulation (4 spaces are automatically transformed into tabulation). eg."
 msgstr ""
-"List of supplementaries morph for this corpus. One per line, a human "
-"readable version of the code\n"
-"                    can be given after a tabulation (4 spaces are "
-"automatically transformed into tabulation). eg."
+"List of supplementaries morph for this corpus. One per line, a human readable version of the code\n"
+"                    can be given after a tabulation (4 spaces are automatically transformed into tabulation). eg."
 
 #: app/templates/main/corpus_custom_dictionary.html:36
 msgid "Morph=MyMorph    My human readable morph"
@@ -900,31 +514,6 @@ msgstr "བར་ཚེགས་ཀྱིས་བར་ཕྱེས་པའི
 msgid "MyPos,MyPos2"
 msgstr "MyPos,MyPos2"
 
-#: app/templates/main/corpus_delete.html:8
-#, fuzzy
-msgid "Delete the corpus"
-msgstr "སྐད་མཛོད་དེ་ (2 steps) སུབ།"
-
-#: app/templates/main/corpus_edit_allowed_values.html:11
-#, fuzzy
-msgid "Edit allowed"
-msgstr "ཆོག་མཆན་མ་ཐོབ་པ།"
-
-#: app/templates/main/corpus_edit_allowed_values.html:14
-#, fuzzy
-msgid "Allowed"
-msgstr "ཆོག་མཆན་མ་ཐོབ་པ།"
-
-#: app/templates/main/corpus_generate_fixtures.html:10
-#, fuzzy
-msgid "Allowed Lemma"
-msgstr "ཆོག་མཆན་ཐོབ་པའི་ lemma"
-
-#: app/templates/main/corpus_generate_fixtures.html:15
-#: app/templates/main/corpus_new.html:199
-msgid "Allowed POS"
-msgstr "ཆོག་མཆན་ཐོབ་པའི་ཚིག་གཤིས།"
-
 #: app/templates/main/corpus_info.html:15
 msgid "Statistics"
 msgstr "བསྡོམས་རྩིས།"
@@ -958,61 +547,62 @@ msgid "edited"
 msgstr "རྩོམ་སྒྲིག་ཟིན་པ།"
 
 #: app/templates/main/corpus_info.html:57
-#, fuzzy, python-format
-msgid ""
-"lemma were correct from lemmatization. It means you corrected %(count)s "
-"lemma."
+msgid "lemma were correct from lemmatization. It means you corrected"
 msgstr ""
 "མ་ཚིག་རྣམས་མ་ཚིག་གི་གྲུབ་ཚུལ་ནས་དག་སྒྲིག་བྱས་འདུག "
 "འདིའི་ནང་དོན་ནི་ཁྱེད་ཀྱིས་དག་སྒྲིག་བྱས་འདུག་ཅེས་པའི་དོན་ཡིན།"
 
-#: app/templates/main/corpus_info.html:79
-#, fuzzy, python-format
-msgid ""
-"morphologies were correct from lemmatization. It means you corrected "
-"%(count)s morphologies."
+#: app/templates/main/corpus_info.html:57
+msgid "lemma."
+msgstr "མ་ཚིག"
+
+#: app/templates/main/corpus_info.html:78
+msgid "morphologies were correct from lemmatization. It means you corrected"
 msgstr ""
 "ཚིག་གི་གྲུབ་ཆ་རྣམས་མ་ཚིག་གི་གྲུབ་ཚུལ་ནས་དག་སྒྲིག་བྱས་འདུག "
 "འདིའི་ནང་དོན་ནི་ཁྱེད་ཀྱིས་དག་སྒྲིག་བྱས་འདུག་ཅེས་པའི་དོན་ཡིན།"
 
-#: app/templates/main/corpus_info.html:102
-#, fuzzy, python-format
-msgid "POS were correct from lemmatization. It means you corrected %(count)s POS."
+#: app/templates/main/corpus_info.html:78
+msgid "morphologies."
+msgstr "ཚིག་གི་གྲུབ་ཆ་རྣམས།"
+
+#: app/templates/main/corpus_info.html:101
+msgid "POS were correct from lemmatization. It means you corrected"
 msgstr ""
 "POS རྣམས་མ་ཚིག་གི་གྲུབ་ཚུལ་ནས་དག་སྒྲིག་བྱས་འདུག "
 "འདིའི་ནང་དོན་ནི་ཁྱེད་ཀྱིས་དག་སྒྲིག་བྱས་འདུག་ཅེས་པའི་དོན་ཡིན།"
 
-#: app/templates/main/corpus_info.html:109
+#: app/templates/main/corpus_info.html:101
+msgid "POS."
+msgstr "ཚིག་གཤིས།"
+
+#: app/templates/main/corpus_info.html:107
 msgid "Annotations over time"
 msgstr "ཆུ་ཚོད་ལས་བརྒལ་བའི་འགྲེལ་བརྗོད་ཁག"
 
-#: app/templates/main/corpus_info.html:115
+#: app/templates/main/corpus_info.html:113
 msgid "Most common corrections"
 msgstr "ཆེས་སྤྱིར་བཏང་གི་ཕྱོགས་བསྒྲིགས་རྣམས།"
 
-#: app/templates/main/corpus_info.html:122
-msgid "See more"
-msgstr ""
-
-#: app/templates/main/corpus_info.html:131
-#: app/templates/main/corpus_info.html:144
-#: app/templates/main/corpus_info.html:157
+#: app/templates/main/corpus_info.html:129
+#: app/templates/main/corpus_info.html:142
+#: app/templates/main/corpus_info.html:155
 msgid "Count"
 msgstr "གྲངས་བརྩི།"
 
-#: app/templates/main/corpus_info.html:131
-#: app/templates/main/corpus_info.html:144
-#: app/templates/main/corpus_info.html:157
+#: app/templates/main/corpus_info.html:129
+#: app/templates/main/corpus_info.html:142
+#: app/templates/main/corpus_info.html:155
 msgid "Corrected"
 msgstr "དག་སྒྲིག་ཟིན་པ།"
 
-#: app/templates/main/corpus_info.html:131
-#: app/templates/main/corpus_info.html:144
-#: app/templates/main/corpus_info.html:157
+#: app/templates/main/corpus_info.html:129
+#: app/templates/main/corpus_info.html:142
+#: app/templates/main/corpus_info.html:155
 msgid "Original"
 msgstr "མ་ཕྱི།"
 
-#: app/templates/main/corpus_info.html:142
+#: app/templates/main/corpus_info.html:140
 msgid "Morphology"
 msgstr "ཚིག་གི་གྲུབ་ཆ་རྣམས།"
 
@@ -1021,7 +611,6 @@ msgid "Create a new corpus"
 msgstr "སྐད་མཛོད་གསར་བཟོ།"
 
 #: app/templates/main/corpus_new.html:10
-#: app/templates/main/corpus_preferences.html:8
 msgid "Metadata"
 msgstr "གཞི་གྲངས་རྒྱས་པ།"
 
@@ -1041,7 +630,8 @@ msgstr "Left Context"
 #: app/templates/main/corpus_new.html:19
 #: app/templates/main/corpus_preferences.html:41
 msgid "Number of words to display on the left of the word to annotate"
-msgstr "འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡོན་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
+msgstr ""
+"འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡོན་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
 
 #: app/templates/main/corpus_new.html:22
 #: app/templates/main/corpus_preferences.html:45
@@ -1051,7 +641,8 @@ msgstr "Right Context"
 #: app/templates/main/corpus_new.html:24
 #: app/templates/main/corpus_preferences.html:58
 msgid "Number of words to display on the right of the word to annotate"
-msgstr "འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡས་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
+msgstr ""
+"འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡས་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
 
 #: app/templates/main/corpus_new.html:27
 #: app/templates/main/corpus_preferences.html:10
@@ -1059,10 +650,11 @@ msgid "Delimiter token"
 msgstr "Delimiter token"
 
 #: app/templates/main/corpus_new.html:29
-msgid "[Optional] A specific token that is used as a delimiter between passages."
+msgid ""
+"[Optional] A specific token that is used as a delimiter between passages."
 msgstr ""
-"[Optional] "
-"བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ token ཞིག"
+"[Optional] བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ "
+"token ཞིག"
 
 #: app/templates/main/corpus_new.html:33
 #: app/templates/main/corpus_preferences.html:63
@@ -1092,56 +684,36 @@ msgstr ""
 msgid "Example :"
 msgstr "དཔེར་བརྗོད་ནི།"
 
-#: app/templates/main/corpus_new.html:68 app/templates/main/corpus_new.html:93
+#: app/templates/main/corpus_new.html:68 app/templates/main/corpus_new.html:80
 msgid "Lemmatize"
 msgstr "མ་ཚིག་འགར།"
 
 #: app/templates/main/corpus_new.html:70
 msgid ""
-"If your text is not lemmatized, select the language and click on "
-"lemmatize."
+"If your text is not lemmatized, select the language and click on lemmatize."
 msgstr ""
 "གལ་སྲིད་ཁྱེད་ཀྱི་ཡིག་ཆ་མ་ཚིག་བཟོས་མེད་ན། "
 "སྐད་ཡིག་དེ་བདམས་ཏེ་མ་ཚིག་བཟོ་ཟེར་བ་དེའི་སྒང་ལ་སྣུན། "
 "གལ་སྲིད་ཁྱེད་ཀྱི་ཡིག་ཆ་མ་ཚིག་བཟོས་མེད་ན། "
 "སྐད་ཡིག་དེ་བདམས་ཏེ་མ་ཚིག་བཟོ་ཟེར་བ་དེའི་སྒང་ལ་སྣུན།"
 
-#: app/templates/main/corpus_new.html:75
-msgid "Select a service"
-msgstr ""
-
-#: app/templates/main/corpus_new.html:82
-#, python-format
-msgid "%(name) is a lemmatization service provided by %(provider)"
-msgstr ""
-
 #: app/templates/main/corpus_new.html:84
 msgid ""
-"If you only want to lemmatize (and not correct data), you can use their "
-"user interface here:"
-msgstr ""
-
-#: app/templates/main/corpus_new.html:87
-msgid "Please cite tools accordingly. You can use the following APA or Bibtex: "
-msgstr ""
-
-#: app/templates/main/corpus_new.html:97
-msgid ""
 "tokens analyzed... Total is approximately computed and could differ "
 "depending on the language..."
 msgstr ""
 "དབྱེ་ཞིབ་བྱས་ཟིན་པའི་tokens "
 "ཡི་སྤྱི་སྡོམ་ནི་ཧ་ལམ་རྩིས་བརྒྱབ་ཟིན་པ་དང་སྐད་ཡིག་དེར་གཞིགས་ཏེ་ཁྱད་པར་འབྱེད་ཐུབ་་་"
 
-#: app/templates/main/corpus_new.html:100
+#: app/templates/main/corpus_new.html:87
 msgid "Operation finished with success !"
 msgstr "ལས་འགུལ་བདེ་ལེགས་ངང་གྲུབ་སོང་།"
 
-#: app/templates/main/corpus_new.html:100
+#: app/templates/main/corpus_new.html:87
 msgid "tokens analyzed in total."
 msgstr "སྤྱི་བསྡོམས་ཀྱིས་དབྱེ་ཞིབ་བྱས་པའི་ tokens"
 
-#: app/templates/main/corpus_new.html:103
+#: app/templates/main/corpus_new.html:90
 msgid ""
 "Operation failed! Contact the administrator with the corpus you tried to "
 "lemmatize to investigate the issue."
@@ -1149,123 +721,117 @@ msgstr ""
 "ལས་འགུལ་ལམ་ལྷོངས་བྱུང་མ་སོང་། "
 "གནད་དོན་དེར་ཞིབ་འཇུག་བྱེད་པ་ལ་ཁྱེད་ཀྱིས་མ་ཚིག་བཟོ་ཐབས་བྱེད་བཞིན་པའི་སྐད་མཛོད་དེའི་འཛིན་སྐྱོང་བར་འབྲེལ་བ་བྱོས།"
 
-#: app/templates/main/corpus_new.html:111
+#: app/templates/main/corpus_new.html:98
 msgid "Tokenize (beta)"
 msgstr "Tokenize (beta)"
 
-#: app/templates/main/corpus_new.html:113
+#: app/templates/main/corpus_new.html:100
 msgid ""
-"If your text is not tokenized and you do not need to pre-lemmatize it, "
-"you can use this function"
+"If your text is not tokenized and you do not need to pre-lemmatize it, you "
+"can use this function"
 msgstr ""
 "གལ་སྲིད་ཁྱེད་ཀྱི་ཡིག་ཆ་ལ་མ་ཚིག་བཟོས་མེད་པ་དང་ཁྱེད་ལ་དེའི་སྔོན་འགྲོའི་མ་ཚིག་བཟོས་མི་དགོས་པ་ཡིན་ན།"
 " ཁྱེད་ཀྱིས་ཁྱད་ཆོས་འདི་བཀོལ་ཆོག"
 
-#: app/templates/main/corpus_new.html:118
+#: app/templates/main/corpus_new.html:105
 msgid "Remove hyphens (Be careful with this function)"
 msgstr "འཐེན་རྟགས་རྣམས་སུབ། (Be careful with this function)"
 
-#: app/templates/main/corpus_new.html:122
+#: app/templates/main/corpus_new.html:109
 msgid "Keep punctuation"
 msgstr "ཚེག་རྟགས་ཉོར།"
 
-#: app/templates/main/corpus_new.html:126
+#: app/templates/main/corpus_new.html:113
 msgid "Tokenize"
 msgstr "Tokenize"
 
-#: app/templates/main/corpus_new.html:136
+#: app/templates/main/corpus_new.html:123
 msgid "Use an existing control list"
 msgstr "ད་སྔ་ཡོད་བསྡད་པའི་སྟངས་འཛིན་ཐོ་གཞུང་ཞིག་བཀོལ་སྤྱོད་བྱེད།"
 
-#: app/templates/main/corpus_new.html:139
+#: app/templates/main/corpus_new.html:126
 msgid ""
-"By using shared control lists, you ensure that you stick to accepted "
-"values in\n"
-"                    the academic community. You will be able to propose "
-"new values to the administrators of control lists."
+"By using shared control lists, you ensure that you stick to accepted values in\n"
+"                    the academic community. You will be able to propose new values to the administrators of control lists."
 msgstr ""
-"By using shared control lists, you ensure that you stick to accepted "
-"values in\n"
-"                    the academic community. You will be able to propose "
-"new values to the administrators of control lists."
+"By using shared control lists, you ensure that you stick to accepted values in\n"
+"                    the academic community. You will be able to propose new values to the administrators of control lists."
 
-#: app/templates/main/corpus_new.html:161
+#: app/templates/main/corpus_new.html:148
 msgid "Write your own"
 msgstr "ཁྱེད་རང་རང་གིས་ཕྲིས།"
 
-#: app/templates/main/corpus_new.html:163
+#: app/templates/main/corpus_new.html:150
 msgid ""
-"In case the configurations provided do not fit your need, you can create "
-"your own configuration.\n"
-"                    You will be able to share it with collaborators, "
-"propose it as a canon setting for the whole base of users.\n"
-"                    You can also later add bibliographic information "
-"about your settings."
+"In case the configurations provided do not fit your need, you can create your own configuration.\n"
+"                    You will be able to share it with collaborators, propose it as a canon setting for the whole base of users.\n"
+"                    You can also later add bibliographic information about your settings."
 msgstr ""
-"In case the configurations provided do not fit your need, you can create "
-"your own configuration.\n"
-"                    You will be able to share it with collaborators, "
-"propose it as a canon setting for the whole base of users.\n"
-"                    You can also later add bibliographic information "
-"about your settings."
-
-#: app/templates/main/corpus_new.html:170
+"In case the configurations provided do not fit your need, you can create your own configuration.\n"
+"                    You will be able to share it with collaborators, propose it as a canon setting for the whole base of users.\n"
+"                    You can also later add bibliographic information about your settings."
+
+#: app/templates/main/corpus_new.html:157
 msgid "Load a configuration"
 msgstr "གྲུབ་ཆ་གོ་སྒྲིག་ཅིག་ནང་འདྲེན་བྱེད།"
 
-#: app/templates/main/corpus_new.html:171
+#: app/templates/main/corpus_new.html:158
 msgid "This helps kickstarting configuration"
 msgstr "འདིས་གཞི་རྩའི་གྲུབ་ཆའི་སྒྲིག་བཀོད་ལ་རམ་འདེགས་བྱེད།"
 
-#: app/templates/main/corpus_new.html:177
+#: app/templates/main/corpus_new.html:164
 msgid "Ancien Français"
 msgstr "Ancien Français"
 
-#: app/templates/main/corpus_new.html:178
+#: app/templates/main/corpus_new.html:165
 msgid "Latin (LASLA)"
 msgstr "ལཱ་ཏིན། (LASLA)"
 
-#: app/templates/main/corpus_new.html:181
+#: app/templates/main/corpus_new.html:168
 msgid "Load"
 msgstr "ནང་འདྲེན།"
 
-#: app/templates/main/corpus_new.html:186
+#: app/templates/main/corpus_new.html:173
 msgid "Lemma loaded"
 msgstr "ནང་འདྲེན་བྱས་པའི་ Lemma"
 
-#: app/templates/main/corpus_new.html:187
+#: app/templates/main/corpus_new.html:174
 msgid "POS loaded"
 msgstr "ནང་འདྲེན་བྱས་པའི་ཚིག་གཤིས།"
 
-#: app/templates/main/corpus_new.html:188
+#: app/templates/main/corpus_new.html:175
 msgid "Morphology loaded"
 msgstr "ནང་འདྲེན་བྱས་པའི་ Morphology"
 
-#: app/templates/main/corpus_new.html:194
+#: app/templates/main/corpus_new.html:181
 msgid "Allowed lemma"
 msgstr "ཆོག་མཆན་ཐོབ་པའི་ lemma"
 
-#: app/templates/main/corpus_new.html:195
+#: app/templates/main/corpus_new.html:182
 msgid "This should be formatted as a list of lemma separated by new line"
 msgstr ""
 "འདི་ནི་ངེས་པར་དུ་ཐིག་ཕྲེང་གསར་པ་རེས་བར་ཕྱེས་པའི་lemma "
 "ཡི་ཐོ་གཞུང་ཞིག་གི་རྣམ་པ་རུ་ཡོད་དགོས།"
 
-#: app/templates/main/corpus_new.html:200
+#: app/templates/main/corpus_new.html:186
+msgid "Allowed POS"
+msgstr "ཆོག་མཆན་ཐོབ་པའི་ཚིག་གཤིས།"
+
+#: app/templates/main/corpus_new.html:187
 msgid "This should be formatted as a list of POS separated by comma"
 msgstr ""
 "འདི་ནི་ངེས་པར་དུ་བར་ཚེག་གིས་བར་ཕྱེས་པའི་POS "
 "ཡི་ཐོ་གཞུང་ཞིག་གི་རྣམ་པ་རུ་ཡོད་དགོས།"
 
-#: app/templates/main/corpus_new.html:200
+#: app/templates/main/corpus_new.html:187
 msgid "and no space"
 msgstr "དང་བར་སྟོང་མེད་པ།"
 
-#: app/templates/main/corpus_new.html:204
+#: app/templates/main/corpus_new.html:191
 msgid "Allowed Morph (as TSV content)"
 msgstr "ཆོག་མཆན་ཐོབ་པའི་ Morph (as TSV content)"
 
-#: app/templates/main/corpus_new.html:205
+#: app/templates/main/corpus_new.html:192
 msgid ""
 "The TSV should at least have the header : label and could have a "
 "<code>readable</code> column for human"
@@ -1278,65 +844,50 @@ msgid ""
 "[Optional] A specific token that is used as a delimiter\n"
 "                    between passages."
 msgstr ""
-"[Optional]  "
-"བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ token ཞིག"
+"[Optional]  བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ "
+"token ཞིག"
 
 #: app/templates/main/dashboard.html:9
 msgid "User Dashboard"
 msgstr "སྤྱོད་མཁན་གྱི་གཙོ་ངོས།"
 
+#: app/templates/main/dashboard.html:10
+msgid "Registered Users"
+msgstr "ཐོ་འགོད་བྱས་ཟིན་པའི་སྤྱོད་མཁན།"
+
 #: app/templates/main/dashboard.html:11
 msgid "View and manage user accounts"
 msgstr "སྤྱོད་མཁན་གྱི་ཁ་བྱང་འཆར་སྟོན་དང་དོ་དམ།"
 
+#: app/templates/main/dashboard.html:12
+msgid "Add New User"
+msgstr "སྤྱོད་མཁན་གསར་པ་སྣོན།"
+
+#: app/templates/main/dashboard.html:13
+msgid "Create a new user account"
+msgstr "སྤྱོད་མཁན་གྱི་ཁ་བྱང་གསར་པ་ཞིག་བཟོ།"
+
 #: app/templates/main/dashboard.html:14
 msgid "Invite New User"
 msgstr "སྤྱོད་མཁན་གསར་པ་གདན་འདྲེན།"
 
 #: app/templates/main/dashboard.html:15
 msgid "Invites a new user to create their own account"
-msgstr "ཁོ་ཚོ་རང་ཉིད་ཀྱི་ཁ་བྱང་གསར་བཟོའི་ཕྱིར་སྤྱོད་མཁན་གསར་པ་ཞིག་གདན་འདྲེན་བྱེད།"
-
-#: app/templates/main/dashboard_corpus_table.html:5
-msgid "Management of corpora"
-msgstr ""
-
-#: app/templates/main/dashboard_corpus_table.html:10
-msgid "Links"
 msgstr ""
-
-#: app/templates/main/dashboard_corpus_table.html:12
-#, fuzzy
-msgid "Corpus name"
-msgstr "སྐད་མཛོད་མིང་།"
-
-#: app/templates/main/dashboard_corpus_table.html:13
-#, fuzzy
-msgid "Last modified"
-msgstr "རུས་མིང་།"
-
-#: app/templates/main/dashboard_corpus_table.html:14
-#, fuzzy
-msgid "Users"
-msgstr "སྤྱོད་མཁན།"
+"ཁོ་ཚོ་རང་ཉིད་ཀྱི་ཁ་བྱང་གསར་བཟོའི་ཕྱིར་སྤྱོད་མཁན་གསར་པ་ཞིག་གདན་འདྲེན་བྱེད།"
 
 #: app/templates/main/index.html:5 app/templates/main/index_loggedin.html:6
 msgid "Welcome to Pyrrha !"
 msgstr "Pyrrha ལ་ཕེབས་པར་དགའ་བསུ་ཞུ།"
 
-#: app/templates/main/index.html:6
-msgid ""
-"Pyrrha is a simple Python Flask WebApp to fasten the post-correction of "
-"lemmatized and morpho-syntactic tagged corpora."
-msgstr ""
-
 #: app/templates/main/tokens_add_row.html:7
 msgid "Add a token"
 msgstr "token ཞིག་སྣོན།"
 
 #: app/templates/main/tokens_add_row.html:8
 msgid "This page allows you to add a token after the following one"
-msgstr "ཤོག་ངོས་འདིས་ཁྱེད་ལ་ཡོང་བཞིན་པ་དེའི་རྗེས་སུ་token ཞིག་སྣོན་འཇུག་པ་ཡིན།"
+msgstr ""
+"ཤོག་ངོས་འདིས་ཁྱེད་ལ་ཡོང་བཞིན་པ་དེའི་རྗེས་སུ་token ཞིག་སྣོན་འཇུག་པ་ཡིན།"
 
 #: app/templates/main/tokens_add_row.html:12
 #: app/templates/main/tokens_del_row.html:12
@@ -1344,6 +895,22 @@ msgstr "ཤོག་ངོས་འདིས་ཁྱེད་ལ་ཡོང་
 msgid "Token ID"
 msgstr "Token ID"
 
+#: app/templates/main/tokens_add_row.html:13
+#: app/templates/main/tokens_del_row.html:13
+#: app/templates/main/tokens_edit_form.html:13
+#: app/templates/main/tokens_search_through_fields.html:19
+#: app/templates/main/tokens_similar_to_token.html:49
+msgid "Form"
+msgstr "འགེངས་ཤོག"
+
+#: app/templates/main/tokens_add_row.html:14
+#: app/templates/main/tokens_del_row.html:14
+#: app/templates/main/tokens_edit_form.html:14
+#: app/templates/main/tokens_similar_to_record.html:26
+#: app/templates/main/tokens_similar_to_token.html:50
+msgid "Context"
+msgstr "སྐབས་བསྟུན།"
+
 #: app/templates/main/tokens_add_row.html:27
 msgid "Form of the new token to insert"
 msgstr "བར་འཇུག་བྱ་རྒྱུའི་token གསར་པ་དེའི་འགེངས་ཤོག"
@@ -1359,12 +926,15 @@ msgid "Go back to correcting annotations"
 msgstr "ཕྱིར་དག་སྒྲིག་བྱེད་བཞིན་པའི་མཆན་འགྲེལ་ལ་ཕེབས།"
 
 #: app/templates/main/tokens_correct.html:8
-#: app/templates/main/tokens_correct_unallowed.html:8
 #: app/templates/main/tokens_history.html:7
 #: app/templates/main/tokens_similar_to_record.html:10
 msgid "List of tokens"
 msgstr "token ཡི་ཐོ་གཞུང་།"
 
+#: app/templates/main/tokens_correct_unallowed.html:8
+msgid "Corpus {{corpus.name}} - List of tokens"
+msgstr "{{corpus.name}} སྐད་མཛོད་ཀྱི་ཚིག་ཐོ།"
+
 #: app/templates/main/tokens_del_row.html:7
 msgid "Remove a token"
 msgstr "token ཞིག་སུབ།"
@@ -1399,6 +969,11 @@ msgstr "འགེངས་ཤོག་གསར་པ།"
 msgid "History of tokens edition"
 msgstr "tokens ཡི་ཐོན་རིམ་གྱི་ལོ་རྒྱུས།"
 
+#: app/templates/main/tokens_edit_history.html:11
+#: app/templates/main/tokens_similar_to_record.html:17
+msgid "User"
+msgstr "སྤྱོད་མཁན།"
+
 #: app/templates/main/tokens_edit_history.html:11
 #: app/templates/main/tokens_similar_to_record.html:19
 msgid "Date"
@@ -1424,10 +999,46 @@ msgstr "ཕབ་ལེན།"
 msgid "Search tokens within the corpus"
 msgstr "སྐད་མཛོད་ནང་དུ་མ་ཚིག་འཚོལ་བཤེར།"
 
+#: app/templates/main/tokens_search_through_fields.html:64
+msgid "* can be used to match partial words, eg."
+msgstr "* can be used to match partial words, eg."
+
+#: app/templates/main/tokens_search_through_fields.html:65
+msgid "! can be used to negate a match, eg."
+msgstr "! can be used to negate a match, eg."
+
+#: app/templates/main/tokens_search_through_fields.html:66
+msgid "| can be used to perform an OR operation, eg."
+msgstr "| can be used to perform an OR operation, eg."
+
 #: app/templates/main/tokens_search_through_fields.html:67
 msgid "To search forms which do not contain"
 msgstr "འདུས་མེད་པའི་འགེངས་ཤོག་འཚོལ་ཕྱིར།"
 
+#: app/templates/main/tokens_similar_to_record.html:28
+msgid "Old Lemma"
+msgstr "མ་ཚིག་རྙིང་བ།"
+
+#: app/templates/main/tokens_similar_to_record.html:29
+msgid "Corr Lemma"
+msgstr "Corr མ་ཚིག"
+
+#: app/templates/main/tokens_similar_to_record.html:32
+msgid "Previous POS"
+msgstr "སྔ་མའི་ POS"
+
+#: app/templates/main/tokens_similar_to_record.html:33
+msgid "Actual POS"
+msgstr "དངོས་ཡོད་ཀྱི་ POS"
+
+#: app/templates/main/tokens_similar_to_record.html:36
+msgid "Previous Morph"
+msgstr "སྔ་མའི་ཚིག་གི་གྲུབ་ཆ།"
+
+#: app/templates/main/tokens_similar_to_record.html:37
+msgid "Actual Morph"
+msgstr "དངོས་ཡོད་ཀྱི་ཚིག་གི་གྲུབ་ཆ།"
+
 #: app/templates/main/tokens_similar_to_record.html:60
 msgid "Similar matching:"
 msgstr "འདྲ་ཤས་ཆེ་བ་རྣམས་ནི།"
@@ -1458,20 +1069,4 @@ msgstr "མ་ཕྱིའི་ token"
 
 #: app/templates/main/tokens_similar_to_token.html:79
 msgid "Similar matching"
-msgstr "འདྲ་ཤས་ཆེ་བ་རྣམས་ནི།"
-
-#~ msgid "Current control list: {{ current_control_lists.name }}"
-#~ msgstr "ད་ལྟའི་སྟངས་འཛིན་ཐོ་གཞུང་ནི། {{ current_control_lists.name }}"
-
-#~ msgid "lemma."
-#~ msgstr "མ་ཚིག"
-
-#~ msgid "morphologies."
-#~ msgstr "ཚིག་གི་གྲུབ་ཆ་རྣམས།"
-
-#~ msgid "POS."
-#~ msgstr "ཚིག་གཤིས།"
-
-#~ msgid "Corpus {{corpus.name}} - List of tokens"
-#~ msgstr "{{corpus.name}} སྐད་མཛོད་ཀྱི་ཚིག་ཐོ།"
-
+msgstr "འདྲ་ཤས་ཆེ་བ་རྣམས་ནི།"
\ No newline at end of file

From 6e202f01cb29eadff388c65e5ecda253744840b1 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Thu, 13 Jun 2024 16:08:20 +0200
Subject: [PATCH 02/43] ajout filtres de correction token

---
 app/main/views/tokens.py             | 48 ++++++++++++++++++---
 app/models/corpus.py                 | 64 +++++++++++++++++++++-------
 app/templates/macros/nav_macros.html | 22 ++++++++++
 3 files changed, 112 insertions(+), 22 deletions(-)

diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index ff4d448d..a0c42dfc 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -51,20 +51,56 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     :param corpus_id: Id of the corpus
     :param allowed_type: Type of allowed value to check agains (lemma, POS, morph)
     """
+
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
-    tokens = corpus\
-        .get_unallowed(allowed_type)\
+    tokens = corpus \
+        .get_unallowed(allowed_type) \
         .paginate(
-            page=int_or(request.args.get("page"), 1),
-            per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
-        )
+        page=int_or(request.args.get("page"), 1),
+        per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
+    )
     return render_template_with_nav_info(
         'main/tokens_correct_unallowed.html',
         corpus=corpus,
         tokens=tokens,
         allowed_type=allowed_type,
-        changed=corpus.changed(tokens.items)
+        changed=corpus.changed(tokens.items))
+
+
+@main.route('/corpus/<corpus_id>/tokens/unallowed/<allowed_type>/correct/filter', methods=['POST'])
+@login_required
+@requires_corpus_access("corpus_id")
+def tokens_correct_unallowed_filter(corpus_id, allowed_type):
+    corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
+    list_filter=[]
+    if request.method=="POST":
+        list_filter.append(request.form.get("punct"))
+        list_filter.append(request.form.get("numeral"))
+        list_filter.append(request.form.get("ignore"))
+        list_filter.append(request.form.get("metadata"))
+        filtered_filter=[]
+        for el in list_filter:
+            if el != None:
+                filtered_filter.append(el)
+        filter = " ".join(filtered_filter)
+        print(filter)
+
+
+    tokens = corpus \
+        .get_unallowed(allowed_type, ignore=filter) \
+        .paginate(
+        page=int_or(request.args.get("page"), 1),
+        per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
     )
+    return render_template_with_nav_info(
+        'main/tokens_correct_unallowed.html',
+        corpus=corpus,
+        tokens=tokens,
+        allowed_type=allowed_type,
+        changed=corpus.changed(tokens.items))
+
+
+
 
 
 @main.route('/corpus/<int:corpus_id>/tokens/changes/similar/<int:record_id>')
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 57fd2bdb..9469d385 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -7,6 +7,7 @@
 # PIP Packages
 import unidecode
 import sqlalchemy.exc
+import re
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import backref
 from sqlalchemy import func, literal, not_, or_, and_
@@ -297,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, allowed_type="lemma"):
+    def get_unallowed(self, allowed_type="lemma", ignore=False):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -317,22 +318,53 @@ def get_unallowed(self, allowed_type="lemma"):
         else:
             raise ValueError("Get Allowed value had %s and it's not from the lemma, POS, morph set" % allowed_type)
 
-        allowed = db.session.query(cls).filter(
-            cls.control_list == self.control_lists_id,
-            cls.label == prop
-        )
-        custom_dict = db.session.query(CorpusCustomDictionary).filter(
-            CorpusCustomDictionary.corpus == self.id,
-            CorpusCustomDictionary.category == allowed_type,
-            CorpusCustomDictionary.label == prop
-        )
-        return db.session.query(WordToken).filter(
-            db.and_(
-                WordToken.corpus == self.id,
-                not_(allowed.exists()),
-                not_(custom_dict.exists())
+        if ignore:
+            regex_liste = []
+            if "metadata" in ignore:
+                regex_liste.append(r'^(?!\[[^\]]+:[^\]]*\]$).*')
+            if "ignore" in ignore:
+                regex_liste.append(r'^(?!^\[IGNORE\]$)')
+            if "punct" in ignore:
+                regex_liste.append(r"(?!^[^\w\s]$).")
+            if "numeral" in ignore:
+                regex_liste.append(r'(?!^\d+$).+')
+            regex = "".join(regex_liste)
+            print(regex)
+            allowed = db.session.query(cls).filter(
+                cls.control_list == self.control_lists_id,
+                cls.label == prop
+            )
+            custom_dict = db.session.query(CorpusCustomDictionary).filter(
+                CorpusCustomDictionary.corpus == self.id,
+                CorpusCustomDictionary.category == allowed_type,
+                CorpusCustomDictionary.label == prop
+            )
+            return db.session.query(WordToken).filter(
+                    db.and_(
+                        WordToken.corpus == self.id,
+                        not_(allowed.exists()),
+                        not_(custom_dict.exists()),
+                        WordToken.form.op('~')(regex)
+                    )
+                ).order_by(WordToken.order_id)
+
+        else:
+            allowed = db.session.query(cls).filter(
+                cls.control_list == self.control_lists_id,
+                cls.label == prop
             )
-        ).order_by(WordToken.order_id)
+            custom_dict = db.session.query(CorpusCustomDictionary).filter(
+                CorpusCustomDictionary.corpus == self.id,
+                CorpusCustomDictionary.category == allowed_type,
+                CorpusCustomDictionary.label == prop
+            )
+            return db.session.query(WordToken).filter(
+                db.and_(
+                    WordToken.corpus == self.id,
+                    not_(allowed.exists()),
+                    not_(custom_dict.exists())
+                )
+            ).order_by(WordToken.order_id)
 
     @property
     def tokens_count(self):
diff --git a/app/templates/macros/nav_macros.html b/app/templates/macros/nav_macros.html
index 83e2bcbe..8b0cfa3e 100644
--- a/app/templates/macros/nav_macros.html
+++ b/app/templates/macros/nav_macros.html
@@ -261,6 +261,27 @@
                 </a>
             {% endif %}
         </div>
+        <div>
+             <div class="btn-group btn-group-sm" role="group">
+                  <a class="btn btn-outline-dark title">{{ _('Ignore Invalide Lemmas: ') }}</a>
+                 {% set url = request.path %}
+                 {% if 'lemma' in request.path %}
+                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='lemma')}}">
+                 {% endif %}
+                 {% if 'morph' in request.path %}
+                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='morph')}}">
+                 {% endif %}
+                 {% if 'POS' in request.path %}
+                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='POS')}}">
+                 {% endif %}
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="punct" name="punct" title="Allow punctuation as lemma"> Punctuation</a>
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="numeral" name="numeral" title="Allow numerals as lemma"> Numeral</a>
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="ignore" name="ignore" title="Allow lemma [IGNORE]"> Ignore</a>
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="metadata" name="metadata" title="Allow metadata such as [METADATA:something] and [REF:1.2.3]"> Metadata</a>
+                 <button type="submit"  class="btn btn-outline-dark title">Submit</button>
+             </form></div>
+             </div>
+        </div>
     </div></div>
     </div>
     <hr />
@@ -294,6 +315,7 @@ <h3 class="list-group-item bg-dark text-white">Corpora</h3>
                     </div>
                  </div>
             {% endfor %}
+
         </div>
         <hr />
         {{ render_pagination(corpora, endpoint, get_params=[]) }}

From 49016d06431cb99c715f2c3ea7ae2bd75cb5bb94 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Thu, 13 Jun 2024 16:23:38 +0200
Subject: [PATCH 03/43] Revert "ajout filtres de correction token"

This reverts commit 6e202f01cb29eadff388c65e5ecda253744840b1.
---
 app/main/views/tokens.py             | 48 +++------------------
 app/models/corpus.py                 | 64 +++++++---------------------
 app/templates/macros/nav_macros.html | 22 ----------
 3 files changed, 22 insertions(+), 112 deletions(-)

diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index a0c42dfc..ff4d448d 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -51,56 +51,20 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     :param corpus_id: Id of the corpus
     :param allowed_type: Type of allowed value to check agains (lemma, POS, morph)
     """
-
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
-    tokens = corpus \
-        .get_unallowed(allowed_type) \
+    tokens = corpus\
+        .get_unallowed(allowed_type)\
         .paginate(
-        page=int_or(request.args.get("page"), 1),
-        per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
-    )
+            page=int_or(request.args.get("page"), 1),
+            per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
+        )
     return render_template_with_nav_info(
         'main/tokens_correct_unallowed.html',
         corpus=corpus,
         tokens=tokens,
         allowed_type=allowed_type,
-        changed=corpus.changed(tokens.items))
-
-
-@main.route('/corpus/<corpus_id>/tokens/unallowed/<allowed_type>/correct/filter', methods=['POST'])
-@login_required
-@requires_corpus_access("corpus_id")
-def tokens_correct_unallowed_filter(corpus_id, allowed_type):
-    corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
-    list_filter=[]
-    if request.method=="POST":
-        list_filter.append(request.form.get("punct"))
-        list_filter.append(request.form.get("numeral"))
-        list_filter.append(request.form.get("ignore"))
-        list_filter.append(request.form.get("metadata"))
-        filtered_filter=[]
-        for el in list_filter:
-            if el != None:
-                filtered_filter.append(el)
-        filter = " ".join(filtered_filter)
-        print(filter)
-
-
-    tokens = corpus \
-        .get_unallowed(allowed_type, ignore=filter) \
-        .paginate(
-        page=int_or(request.args.get("page"), 1),
-        per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
+        changed=corpus.changed(tokens.items)
     )
-    return render_template_with_nav_info(
-        'main/tokens_correct_unallowed.html',
-        corpus=corpus,
-        tokens=tokens,
-        allowed_type=allowed_type,
-        changed=corpus.changed(tokens.items))
-
-
-
 
 
 @main.route('/corpus/<int:corpus_id>/tokens/changes/similar/<int:record_id>')
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 9469d385..57fd2bdb 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -7,7 +7,6 @@
 # PIP Packages
 import unidecode
 import sqlalchemy.exc
-import re
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import backref
 from sqlalchemy import func, literal, not_, or_, and_
@@ -298,7 +297,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, allowed_type="lemma", ignore=False):
+    def get_unallowed(self, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -318,53 +317,22 @@ def get_unallowed(self, allowed_type="lemma", ignore=False):
         else:
             raise ValueError("Get Allowed value had %s and it's not from the lemma, POS, morph set" % allowed_type)
 
-        if ignore:
-            regex_liste = []
-            if "metadata" in ignore:
-                regex_liste.append(r'^(?!\[[^\]]+:[^\]]*\]$).*')
-            if "ignore" in ignore:
-                regex_liste.append(r'^(?!^\[IGNORE\]$)')
-            if "punct" in ignore:
-                regex_liste.append(r"(?!^[^\w\s]$).")
-            if "numeral" in ignore:
-                regex_liste.append(r'(?!^\d+$).+')
-            regex = "".join(regex_liste)
-            print(regex)
-            allowed = db.session.query(cls).filter(
-                cls.control_list == self.control_lists_id,
-                cls.label == prop
-            )
-            custom_dict = db.session.query(CorpusCustomDictionary).filter(
-                CorpusCustomDictionary.corpus == self.id,
-                CorpusCustomDictionary.category == allowed_type,
-                CorpusCustomDictionary.label == prop
-            )
-            return db.session.query(WordToken).filter(
-                    db.and_(
-                        WordToken.corpus == self.id,
-                        not_(allowed.exists()),
-                        not_(custom_dict.exists()),
-                        WordToken.form.op('~')(regex)
-                    )
-                ).order_by(WordToken.order_id)
-
-        else:
-            allowed = db.session.query(cls).filter(
-                cls.control_list == self.control_lists_id,
-                cls.label == prop
-            )
-            custom_dict = db.session.query(CorpusCustomDictionary).filter(
-                CorpusCustomDictionary.corpus == self.id,
-                CorpusCustomDictionary.category == allowed_type,
-                CorpusCustomDictionary.label == prop
+        allowed = db.session.query(cls).filter(
+            cls.control_list == self.control_lists_id,
+            cls.label == prop
+        )
+        custom_dict = db.session.query(CorpusCustomDictionary).filter(
+            CorpusCustomDictionary.corpus == self.id,
+            CorpusCustomDictionary.category == allowed_type,
+            CorpusCustomDictionary.label == prop
+        )
+        return db.session.query(WordToken).filter(
+            db.and_(
+                WordToken.corpus == self.id,
+                not_(allowed.exists()),
+                not_(custom_dict.exists())
             )
-            return db.session.query(WordToken).filter(
-                db.and_(
-                    WordToken.corpus == self.id,
-                    not_(allowed.exists()),
-                    not_(custom_dict.exists())
-                )
-            ).order_by(WordToken.order_id)
+        ).order_by(WordToken.order_id)
 
     @property
     def tokens_count(self):
diff --git a/app/templates/macros/nav_macros.html b/app/templates/macros/nav_macros.html
index 8b0cfa3e..83e2bcbe 100644
--- a/app/templates/macros/nav_macros.html
+++ b/app/templates/macros/nav_macros.html
@@ -261,27 +261,6 @@
                 </a>
             {% endif %}
         </div>
-        <div>
-             <div class="btn-group btn-group-sm" role="group">
-                  <a class="btn btn-outline-dark title">{{ _('Ignore Invalide Lemmas: ') }}</a>
-                 {% set url = request.path %}
-                 {% if 'lemma' in request.path %}
-                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='lemma')}}">
-                 {% endif %}
-                 {% if 'morph' in request.path %}
-                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='morph')}}">
-                 {% endif %}
-                 {% if 'POS' in request.path %}
-                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='POS')}}">
-                 {% endif %}
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="punct" name="punct" title="Allow punctuation as lemma"> Punctuation</a>
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="numeral" name="numeral" title="Allow numerals as lemma"> Numeral</a>
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="ignore" name="ignore" title="Allow lemma [IGNORE]"> Ignore</a>
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="metadata" name="metadata" title="Allow metadata such as [METADATA:something] and [REF:1.2.3]"> Metadata</a>
-                 <button type="submit"  class="btn btn-outline-dark title">Submit</button>
-             </form></div>
-             </div>
-        </div>
     </div></div>
     </div>
     <hr />
@@ -315,7 +294,6 @@ <h3 class="list-group-item bg-dark text-white">Corpora</h3>
                     </div>
                  </div>
             {% endfor %}
-
         </div>
         <hr />
         {{ render_pagination(corpora, endpoint, get_params=[]) }}

From 4308ecc768a295bf6adceb73b1b28375509888cd Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Thu, 13 Jun 2024 16:28:42 +0200
Subject: [PATCH 04/43] =?UTF-8?q?ajout=20fonction=20filtres=20tokens=20?=
 =?UTF-8?q?=C3=A0=20corriger?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/main/views/tokens.py             | 33 +++++++++++++++
 app/models/corpus.py                 | 62 +++++++++++++++++++++-------
 app/templates/macros/nav_macros.html | 19 +++++++++
 3 files changed, 98 insertions(+), 16 deletions(-)

diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index ff4d448d..d9be6f5f 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -67,6 +67,39 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     )
 
 
+@main.route('/corpus/<corpus_id>/tokens/unallowed/<allowed_type>/correct/filter', methods=['POST'])
+@login_required
+@requires_corpus_access("corpus_id")
+def tokens_correct_unallowed_filter(corpus_id, allowed_type):
+    corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
+    list_filter=[]
+    if request.method=="POST":
+        list_filter.append(request.form.get("punct"))
+        list_filter.append(request.form.get("numeral"))
+        list_filter.append(request.form.get("ignore"))
+        list_filter.append(request.form.get("metadata"))
+        filtered_filter=[]
+        for el in list_filter:
+            if el != None:
+                filtered_filter.append(el)
+        filter = " ".join(filtered_filter)
+        print(filter)
+
+
+    tokens = corpus \
+        .get_unallowed(allowed_type, ignore=filter) \
+        .paginate(
+        page=int_or(request.args.get("page"), 1),
+        per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
+    )
+    return render_template_with_nav_info(
+        'main/tokens_correct_unallowed.html',
+        corpus=corpus,
+        tokens=tokens,
+        allowed_type=allowed_type,
+        changed=corpus.changed(tokens.items))
+
+
 @main.route('/corpus/<int:corpus_id>/tokens/changes/similar/<int:record_id>')
 @login_required
 @requires_corpus_access("corpus_id")
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 57fd2bdb..1c9a5443 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -297,7 +297,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, allowed_type="lemma"):
+    def get_unallowed(self, allowed_type="lemma", ignore=False):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -317,22 +317,52 @@ def get_unallowed(self, allowed_type="lemma"):
         else:
             raise ValueError("Get Allowed value had %s and it's not from the lemma, POS, morph set" % allowed_type)
 
-        allowed = db.session.query(cls).filter(
-            cls.control_list == self.control_lists_id,
-            cls.label == prop
-        )
-        custom_dict = db.session.query(CorpusCustomDictionary).filter(
-            CorpusCustomDictionary.corpus == self.id,
-            CorpusCustomDictionary.category == allowed_type,
-            CorpusCustomDictionary.label == prop
-        )
-        return db.session.query(WordToken).filter(
-            db.and_(
-                WordToken.corpus == self.id,
-                not_(allowed.exists()),
-                not_(custom_dict.exists())
+        if ignore:
+            regex_liste = []
+            if "metadata" in ignore:
+                regex_liste.append(r'^(?!\[[^\]]+:[^\]]*\]$).*')
+            if "ignore" in ignore:
+                regex_liste.append(r'^(?!^\[IGNORE\]$)')
+            if "punct" in ignore:
+                regex_liste.append(r"(?!^[^\w\s]$).")
+            if "numeral" in ignore:
+                regex_liste.append(r'(?!^\d+$).+')
+            regex = "".join(regex_liste)
+            allowed = db.session.query(cls).filter(
+                cls.control_list == self.control_lists_id,
+                cls.label == prop
+            )
+            custom_dict = db.session.query(CorpusCustomDictionary).filter(
+                CorpusCustomDictionary.corpus == self.id,
+                CorpusCustomDictionary.category == allowed_type,
+                CorpusCustomDictionary.label == prop
+            )
+            return db.session.query(WordToken).filter(
+                db.and_(
+                    WordToken.corpus == self.id,
+                    not_(allowed.exists()),
+                    not_(custom_dict.exists()),
+                    WordToken.form.op('~')(regex)
+                )
+            ).order_by(WordToken.order_id)
+
+        else:
+            allowed = db.session.query(cls).filter(
+                cls.control_list == self.control_lists_id,
+                cls.label == prop
             )
-        ).order_by(WordToken.order_id)
+            custom_dict = db.session.query(CorpusCustomDictionary).filter(
+                CorpusCustomDictionary.corpus == self.id,
+                CorpusCustomDictionary.category == allowed_type,
+                CorpusCustomDictionary.label == prop
+            )
+            return db.session.query(WordToken).filter(
+                db.and_(
+                    WordToken.corpus == self.id,
+                    not_(allowed.exists()),
+                    not_(custom_dict.exists())
+                )
+            ).order_by(WordToken.order_id)
 
     @property
     def tokens_count(self):
diff --git a/app/templates/macros/nav_macros.html b/app/templates/macros/nav_macros.html
index 83e2bcbe..ebf4c3ed 100644
--- a/app/templates/macros/nav_macros.html
+++ b/app/templates/macros/nav_macros.html
@@ -261,6 +261,25 @@
                 </a>
             {% endif %}
         </div>
+        <div>
+             <div class="btn-group btn-group-sm" role="group">
+                  <a class="btn btn-outline-dark title">{{ _('Correction: Ignore Invalide Lemmas: ') }}</a>
+                 {% set url = request.path %}
+                 {% if 'lemma' in request.path %}
+                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='lemma')}}">
+                 {% endif %}
+                 {% if 'morph' in request.path %}
+                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='morph')}}">
+                 {% endif %}
+                 {% if 'POS' in request.path %}
+                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='POS')}}">
+                 {% endif %}
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="punct" name="punct" title="Allow punctuation as lemma"> Punctuation</a>
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="numeral" name="numeral" title="Allow numerals as lemma"> Numeral</a>
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="ignore" name="ignore" title="Allow lemma [IGNORE]"> Ignore</a>
+                 <a class="btn btn-outline-dark title"><input type="checkbox" value="metadata" name="metadata" title="Allow metadata such as [METADATA:something] and [REF:1.2.3]"> Metadata</a>
+                 <button type="submit"  class="btn btn-outline-dark title">Submit</button>
+                 </form></div></div>
     </div></div>
     </div>
     <hr />

From 09768ec80452b3197edb1ba49c597ea0088055d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?=
 <1929830+PonteIneptique@users.noreply.github.com>
Date: Wed, 20 Mar 2024 09:14:42 +0100
Subject: [PATCH 05/43] Add more details about the lemmatizer at the new corpus
 phase. (#317)

* Add more details about the lemmatizer at the new corpus phase.
* Add test for lemmatization services
* Add more verbosity because tests hang
* Remove various screenshots
---
 .github/workflows/test.yml                 |   2 +-
 app/__init__.py                            |   2 +-
 app/lemmatizers.py                         |  12 +
 app/templates/main/corpus_new.html         |  40 +-
 config.py                                  |  22 +-
 dummy_lemmatizer_service.py                |  22 +
 tests/test_selenium/base.py                |   1 -
 tests/test_selenium/test_bookmark.py       |   1 -
 tests/test_selenium/test_control_lists.py  |   2 -
 tests/test_selenium/test_corpus_init.py    | 103 ++-
 tests/test_selenium/test_export.py         |   2 -
 translations/bo_CN/LC_MESSAGES/messages.po | 879 +++++++++++++++------
 12 files changed, 832 insertions(+), 256 deletions(-)
 create mode 100644 app/lemmatizers.py
 create mode 100644 dummy_lemmatizer_service.py

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6845d1a3..a5a64446 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -36,7 +36,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements.txt
-        pip install pytest pytest-cov coveralls flake8
+        pip install pytest pytest-cov coveralls flake8 pytest-sugar
     - name: Set up TEST_DATABASE_URL
       run: |
         echo "TEST_DATABASE_URL=postgresql://pyrrha:pyrrha@localhost:5432/data-test" >> $GITHUB_ENV
diff --git a/app/__init__.py b/app/__init__.py
index ac3a4548..bf8a8516 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1,7 +1,6 @@
 import os
 from flask import Flask, g
 
-from config import config
 from flask_compress import Compress
 from flask_login import LoginManager
 from flask_mail import Mail
@@ -29,6 +28,7 @@
 
 def create_app(config_name="dev"):
     """ Create the application """
+    from config import config
 
     app = Flask(
         __name__,
diff --git a/app/lemmatizers.py b/app/lemmatizers.py
new file mode 100644
index 00000000..169ef566
--- /dev/null
+++ b/app/lemmatizers.py
@@ -0,0 +1,12 @@
+from typing import Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class LemmatizerService:
+    title: str   # e.g. "Old French"
+    uri: str  # Current address
+    provider: str  # e.g. Deucalion at Ecole nationale des Chartes
+    bibtex: str  # Citation Scheme
+    apa: str  # APA equivalent
+    ui: Optional[str] = None
diff --git a/app/templates/main/corpus_new.html b/app/templates/main/corpus_new.html
index c70a4543..51ea0143 100644
--- a/app/templates/main/corpus_new.html
+++ b/app/templates/main/corpus_new.html
@@ -72,10 +72,28 @@ <h1>{{ _('Create a new corpus') }}</h1>
             <div class="col-md-9">
                 <div class="input-group">
                   <select class="form-control" id="language-model">
-                      {%- for lang, address in lemmatizers %}
-                    <option value="{{ address }}">{{ lang }}</option>
+                      <option selected value="disabled" class="text-muted disabled">{{_('Select a service')}}</option>
+                      {%- for service in lemmatizers %}
+                    <option value="{{ service.uri }}">{{ service.title }}</option>
                       {% endfor -%}
                   </select>
+                  {%- for service in lemmatizers %}
+                    <div class="alert alert-info lemmatizer-details" data-uri="{{service.uri}}" style="display: none;">
+                        {% trans name=service.title, provider=service.provider %}
+                            <em>{{ name }}</em> is a lemmatization service provided by <i>{{ provider }}</i>.<br />
+                        {% endtrans %}
+                        {% if service.ui %}
+                            <em>{{_("If you only want to lemmatize (and not correct data), you can use their user interface here:")}}</em>
+                            <a href="{{service.ui}}">{{service.ui}}</a><br/>
+                        {% endif %}
+                        {{ _("Please cite tools accordingly. You can use the following APA or Bibtex: ")}}
+                        <ul>
+                            <li><em>APA:</em> <code>{{service.apa }}</code></li>
+                            <li><em>BIB:</em> <code style="padding-left: 1em; display: block;">{{service.bibtex}}</code></li>
+                        </ul>
+                    </div>
+                  {% endfor %}
+
                   <div class="input-group-append">
                     <button id="submit-model" class="btn btn-outline-secondary" type="button">{{ _('Lemmatize') }}</button>
                   </div>
@@ -200,7 +218,14 @@ <h1>{{ _('Create a new corpus') }}</h1>
 <script type="text/javascript">
 
 $(document).ready(function() {
-
+    $("#language-model").on("change", function (event) {
+        let uri = $("#language-model").val();
+        document.querySelectorAll(".lemmatizer-details").forEach(el => el.style.display = "none");
+        if (uri === "disabled") {
+            return;
+        }
+        document.querySelectorAll(`.lemmatizer-details[data-uri='${uri}']`).forEach(el => el.style.display = "block");
+    });
     $("#submit-model").on("click", function(event) {
         event.preventDefault();
         // Get the parameters
@@ -213,11 +238,12 @@ <h1>{{ _('Create a new corpus') }}</h1>
             tokens_fail = document.getElementById("tokens-fail"),
             approximate_count = document.getElementById("tokens-approximate");
 
+        if (uri === "disabled") { return; }
+
         // Create the form
         var formData = new FormData();
         formData.append("data", text_data.value);
 
-
         // Set up our HTTP request
         var xhr = new XMLHttpRequest();
         var update = function(xhr) {
@@ -228,14 +254,16 @@ <h1>{{ _('Create a new corpus') }}</h1>
         };
         // Setup our listener to process compeleted requests
         xhr.onreadystatechange = function () {
-
             // Only run if the request is complete
-            if (xhr.readyState < 3 || xhr.readyState > 4) return null;
+            if (typeof xhr.status !== "number") return null;
+
+            if (xhr.readyState < 3) return null;
 
             if (xhr.readyState === 3) {
                 update(xhr);
                 return null;
             }
+            if (xhr.status === 0) return null;
 
             // Process our return data
             if (xhr.status >= 200 && xhr.status < 300) {
diff --git a/config.py b/config.py
index 7183fdc2..df7795be 100644
--- a/config.py
+++ b/config.py
@@ -1,5 +1,6 @@
 import os
-
+from app.lemmatizers import LemmatizerService
+from typing import List
 
 basedir = os.path.abspath(os.path.dirname(__file__))
 
@@ -38,7 +39,7 @@ class Config:
     PAGINATION_DEFAULT_TOKENS = 100
 
     # Lemmatizer (until Deucalion client)
-    LEMMATIZERS = []
+    LEMMATIZERS: List[LemmatizerService] = []
 
     # Change automatically the Postgresql instance language if not english
     FORCE_PSQL_EN_LOCALE = True
@@ -101,6 +102,23 @@ class BaseTestConfig(Config):
     EMAIL_SUBJECT_PREFIX = '[{}]'.format(Config.APP_NAME)
     EMAIL_SENDER = '{app_name} Admin <{email}>'.format(app_name=Config.APP_NAME, email=MAIL_USERNAME)
 
+    LEMMATIZERS = [
+        LemmatizerService(
+            "Dummy lemmatizer",
+            "http://localhost:4567/lemma",
+            provider="ProviderInstitution",
+            ui="someui.com",
+            apa="Clérice et al. 2019",
+            bibtex="""@article{camps2021corpus,
+	title        = {Corpus and Models for Lemmatisation and POS-tagging of Old French},
+	author       = {Camps, Jean-Baptiste and Cl{\'e}rice, Thibault and Duval, Fr{\'e}d{\'e}ric and Kanaoka, Naomi and Pinche, Ariane and others},
+	year         = 2021,
+	journal      = {arXiv preprint arXiv:2109.11442},
+	keywords     = {Old French}
+}"""
+        )
+    ]
+
 
 class SQLiteTestConfig(BaseTestConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or \
diff --git a/dummy_lemmatizer_service.py b/dummy_lemmatizer_service.py
new file mode 100644
index 00000000..69201b98
--- /dev/null
+++ b/dummy_lemmatizer_service.py
@@ -0,0 +1,22 @@
+from flask import Flask, request, Response, stream_with_context
+app = Flask("fixture")
+
+
+@app.route("/lemma", methods=["POST"])
+def lemmatizing():
+    r = Response(
+        "\n".join(
+            ["token\tlemma"] +
+            ["\t".join([tok, f"{idx}"]) for idx, tok in enumerate(request.form.get("data").split())]
+        ),
+        200,
+        headers={
+            'Content-Type': 'text/plain; charset=utf-8',
+            'Access-Control-Allow-Origin': "*"
+        }
+    )
+    return r
+
+
+if __name__ == "__main__":
+    app.run(port="4567")
diff --git a/tests/test_selenium/base.py b/tests/test_selenium/base.py
index f0b2d006..19fbe2cc 100644
--- a/tests/test_selenium/base.py
+++ b/tests/test_selenium/base.py
@@ -731,7 +731,6 @@ def get_field(row, f):
             self.driver_find_element_by_class_name("pagination"),
             "a"
         )
-        self.driver.save_screenshot("first.results.png")
         for page_index in range(0, len(pagination)):
             # self.driver_find_element_by_class_name("pagination").find_elements_by_tag_name("a")[page_index].click()
             self.element_find_elements_by_tag_name(
diff --git a/tests/test_selenium/test_bookmark.py b/tests/test_selenium/test_bookmark.py
index 1a57f737..99917c9a 100644
--- a/tests/test_selenium/test_bookmark.py
+++ b/tests/test_selenium/test_bookmark.py
@@ -9,7 +9,6 @@ class TestBookmark(TestBase):
 
     def set_bookmark(self, tok_id, page=None):
         self.driver.get(self.url_for_with_port("main.tokens_correct", corpus_id="1", page=page))
-        self.driver.save_screenshot("token.png")
         self.driver_find_element_by_id("dd_t"+str(tok_id)).click()
         self.driver.implicitly_wait(2)
         dd = self.driver_find_element_by_css_selector("*[aria-labelledby='dd_t{}']".format(tok_id))
diff --git a/tests/test_selenium/test_control_lists.py b/tests/test_selenium/test_control_lists.py
index cfa24eea..68f6b56f 100644
--- a/tests/test_selenium/test_control_lists.py
+++ b/tests/test_selenium/test_control_lists.py
@@ -147,10 +147,8 @@ def test_action_as_owner(self):
         # Check that we can edit informations about markdown or whatever
         self.driver.get(self.url_for_with_port("control_lists_bp.information_edit", control_list_id=1))
         self.driver_find_element_by_id("cl_notes").send_keys("# This is some notes")
-        self.driver.save_screenshot("HELLOOOO1.png")
         self.driver_find_element_by_id("submit").click()
         self.driver.get(self.url_for_with_port("control_lists_bp.information_read", control_list_id=1))
-        self.driver.save_screenshot("HELLOOOO2.png")
         self.assertEqual(
             self.driver_find_element_by_tag_name("h1").text, "This is some notes",
             "Check that edition works"
diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index efa7c17e..82575100 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -3,9 +3,14 @@
 from app import db
 from tests.test_selenium.base import TestBase
 from tests.fixtures import PLAINTEXT_CORPORA
+from selenium.webdriver.support.select import Select
 import csv
 import os
 
+from flask import Flask, request, Response
+from multiprocessing import Process
+from selenium.webdriver.support.wait import WebDriverWait
+
 
 class TestCorpusRegistration(TestBase):
     """ Test creation of Corpus """
@@ -461,6 +466,7 @@ def test_registration_with_wrongly_formated_input(self):
 	dieu	NOMpro	NOMB.=s|GENRE=m|CAS=n
 vos	vos1	PROper	PERS.=2|NOMB.=p|GENRE=m|CAS=r
 soit	estre1	VERcjg	MODE=sub|TEMPS=pst|PERS.=3|NOMB.=s""")
+
         self.driver_find_element_by_id("label_checkbox_reuse").click()
         self.driver_find_element_by_id("control_list_select").click()
         self.driver_find_element_by_id("cl_opt_" + str(target_cl.id)).click()
@@ -468,7 +474,11 @@ def test_registration_with_wrongly_formated_input(self):
         self.driver_find_element_by_id("submit").click()
 
         self.assertEqual(
-            sorted([e.text.strip() for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")]),
+            sorted([
+                e.text.strip()
+                for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")
+                if e.text.strip()
+            ]),
             sorted([
                 'At least one line of your corpus is missing a token/form. Check line 1'
             ]),
@@ -493,9 +503,12 @@ def test_registration_with_no_tsv_input(self):
         self.driver_find_element_by_id("cl_opt_" + str(target_cl.id)).click()
 
         self.driver_find_element_by_id("submit").click()
-
         self.assertEqual(
-            sorted([e.text.strip() for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")]),
+            sorted([
+                e.text.strip()
+                for e in self.driver_find_elements_by_css_selector(".alert.alert-danger")
+                if e.text.strip()
+            ]),
             sorted([
                 'You did not input any text.'
             ]),
@@ -622,3 +635,87 @@ def test_corpus_name_unique_user(self):
         self.driver_find_element_by_id("submit").click()
         self.driver.implicitly_wait(5)
         self.assertFalse(self.driver_find_elements_by_css_selector(".alert.alert-danger"))
+
+    def test_lemmatization_service(self):
+        """
+        Test that a user can create a corpus and that this corpus has its data well recorded
+        """
+        # Click register menu link
+        self.driver_find_element_by_id("new_corpus_link").click()
+        self.driver.implicitly_wait(15)
+
+        # Fill in registration form
+        self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
+        self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
+
+        # Default
+        details = self.driver_find_elements_by_css_selector(".lemmatizer-details")
+        self.assertEqual(details[0].is_displayed(), False, "Nothing should be displayed by default")
+
+        # Select a lemmatizer
+        s = Select(self.driver_find_elements_by_css_selector("#language-model")[0])
+        s.select_by_visible_text("Dummy lemmatizer")
+        self.assertEqual(details[0].is_displayed(), True, "Something should be displayed")
+        self.assertIn("Dummy lemmatizer is a lemmatization service provided by ProviderInstitution.", details[0].text)
+
+        # Unselect
+        s.select_by_visible_text("Select a service")
+        self.assertEqual(details[0].is_displayed(), False, "Nothing should be now")
+
+    def test_lemmatization_service_runs(self):
+        # Create a second fixture app
+        app = Flask("fixture")
+
+        @app.route("/lemma", methods=["POST"])
+        def lemmatizing():
+            return Response(
+                "\n".join(
+                    ["token\tlemma"] +
+                    ["\t".join([tok, f"{idx}"]) for idx, tok in enumerate(request.form.get("data").split())]
+                ),
+                200,
+                headers={
+                    'Content-Type': 'text/plain; charset=utf-8',
+                    'Access-Control-Allow-Origin': "*"
+                }
+            )
+        # Start it
+        second_app = Process(target=app.run, daemon=True, kwargs=dict(host="localhost", port="4567"))
+        second_app.start()
+
+        self.driver_find_element_by_id("new_corpus_link").click()
+        self.driver.implicitly_wait(15)
+
+        # Fill in registration form
+        self.driver_find_element_by_id("corpusName").send_keys("Test")
+        self.writeMultiline(self.driver_find_element_by_id("tokens"), "Je suis")
+
+        # Select the lemmatizer
+        s = Select(self.driver_find_elements_by_css_selector("#language-model")[0])
+        s.select_by_index(1)
+        self.driver.implicitly_wait(5)
+
+        # Lemmatize
+        self.driver_find_element_by_id("submit-model").click()
+
+        # Wait
+        wait = WebDriverWait(self.driver, timeout=5)
+        wait.until(lambda x: self.driver_find_element_by_id("tokens-success").is_displayed())
+
+        # Check results
+        self.assertEqual(
+            self.driver_find_element_by_id("tokens").get_property("value").split("\n"),
+            ["token\tlemma", "Je\t0", "suis\t1"]
+        )
+
+        self.assertEqual(
+            self.driver_find_element_by_id("tokens-success").text,
+            "Operation finished with success ! 2 tokens analyzed in total.",
+            "Lemmatization happened"
+        )
+        # Kill second app
+        try:
+            second_app.terminate()
+            second_app.join(2)
+        finally:
+            second_app.close()
diff --git a/tests/test_selenium/test_export.py b/tests/test_selenium/test_export.py
index 399207ec..67282adb 100644
--- a/tests/test_selenium/test_export.py
+++ b/tests/test_selenium/test_export.py
@@ -102,10 +102,8 @@ def test_tei_geste_export(self):
         self.driver.refresh()
 
         self.driver.get(self.url_for_with_port("main.tokens_export", corpus_id=1))
-        self.driver.get_screenshot_as_file("./beforedownload.png")
         self.driver_find_element_by_id("geste-tei").click()
 
-        self.driver.get_screenshot_as_file("./afterdownload.png")
         time.sleep(5)
 
         with open(os.path.join(self.download_path, "wauchier.xml")) as f:
diff --git a/translations/bo_CN/LC_MESSAGES/messages.po b/translations/bo_CN/LC_MESSAGES/messages.po
index 04aa2adf..6d39a5d8 100644
--- a/translations/bo_CN/LC_MESSAGES/messages.po
+++ b/translations/bo_CN/LC_MESSAGES/messages.po
@@ -6,21 +6,26 @@
 # Translators:
 # Ngawang Trinley <ngawangtrinley@gmail.com>, 2022
 # Tashi Dhondup <dhonduptiger658@gmail.com>, 2022
-#
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-05-12 23:50+0530\n"
+"POT-Creation-Date: 2024-03-19 18:26+0100\n"
 "PO-Revision-Date: 2022-06-30 14:23+0000\n"
 "Last-Translator: Tashi Dhondup <dhonduptiger658@gmail.com>, 2022\n"
-"Language-Team: Tibetan (China) (https://www.transifex.com/OpenPecha/teams/139966/bo_CN/)\n"
-"MIME-Version: 1.0\n"
-"Content-Type: text/plain; charset=UTF-8\n"
-"Content-Transfer-Encoding: 8bit\n"
-"Generated-By: Babel 2.10.3\n"
 "Language: bo_CN\n"
+"Language-Team: Tibetan (China) "
+"(https://www.transifex.com/OpenPecha/teams/139966/bo_CN/)\n"
 "Plural-Forms: nplurals=1; plural=0;\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.14.0\n"
+
+#: app/templates/account/join_invite.html:8
+#, fuzzy
+msgid "Create your password"
+msgstr "གསང་གྲངས་མི་དྲན་ནམ།"
 
 #: app/templates/account/login.html:7
 msgid "Log in"
@@ -30,22 +35,188 @@ msgstr "ཐོ་འཇུག"
 msgid "Forgot password?"
 msgstr "གསང་གྲངས་མི་དྲན་ནམ།"
 
+#: app/templates/account/manage.html:25 app/templates/admin/manage_user.html:29
+#, fuzzy
+msgid "Full name"
+msgstr "རུས་མིང་།"
+
+#: app/templates/account/manage.html:26 app/templates/admin/manage_user.html:33
+#: app/templates/macros/dashboard_macros.html:28
+msgid "Email address"
+msgstr "གློག་འཕྲིན་ཁ་བྱང་།"
+
+#: app/templates/account/manage.html:27 app/templates/admin/manage_user.html:37
+#: app/templates/macros/dashboard_macros.html:7
+#: app/templates/macros/dashboard_macros.html:30
+msgid "Account type"
+msgstr "ཁ་བྱང་གི་རིགས།"
+
+#: app/templates/account/manage.html:35
+#, fuzzy
+msgid "Account Settings"
+msgstr "ཁ་བྱང་གི་རིགས།"
+
+#: app/templates/account/manage.html:37
+msgid "Manage your account settings and change your login information."
+msgstr ""
+
 #: app/templates/account/register.html:8
 msgid "Create an account"
 msgstr "ཐོ་འགོད།"
 
+#: app/templates/account/reset_password.html:7
+#, fuzzy
+msgid "Reset your password"
+msgstr "གསང་གྲངས་མི་དྲན་ནམ།"
+
+#: app/templates/account/unconfirmed.html:8
+msgid "You need to confirm your account before continuing."
+msgstr ""
+
+#: app/templates/account/unconfirmed.html:15
+msgid "I didn't receive an email!"
+msgstr ""
+
+#: app/templates/account/unconfirmed.html:17
+msgid "No problem! Just click the button below and we'll send you another one."
+msgstr ""
+
+#: app/templates/account/unconfirmed.html:18
+msgid "Resend confirmation email"
+msgstr ""
+
+#: app/templates/account/email/change_email.html:1
+#: app/templates/account/email/confirm.html:1
+#: app/templates/account/email/invite.html:1
+#: app/templates/account/email/reset_password.html:1
+#, fuzzy
+msgid "Dear"
+msgstr "འཚོལ་ཞིབ།"
+
+#: app/templates/account/email/change_email.html:3
+msgid "To confirm your new email address"
+msgstr ""
+
+#: app/templates/account/email/change_email.html:3
+#: app/templates/account/email/confirm.html:5
+#: app/templates/account/email/invite.html:5
+msgid "click here"
+msgstr ""
+
+#: app/templates/account/email/change_email.html:5
+msgid ""
+"Alternatively, you can paste the following link in your browser's address"
+" bar:"
+msgstr ""
+
+#: app/templates/account/email/change_email.html:9
+#: app/templates/account/email/confirm.html:11
+#: app/templates/account/email/invite.html:13
+#: app/templates/account/email/reset_password.html:11
+msgid "Sincerely,"
+msgstr ""
+
+#: app/templates/account/email/change_email.html:13
+#: app/templates/account/email/confirm.html:15
+#: app/templates/account/email/invite.html:17
+#: app/templates/account/email/reset_password.html:15
+#: app/templates/control_lists/email/contact.html:10
+msgid "Note: replies to this email address are not monitored."
+msgstr ""
+
+#: app/templates/account/email/confirm.html:3
+#, fuzzy
+msgid "Welcome to"
+msgstr "Pyrrha ལ་ཕེབས་པར་དགའ་བསུ་ཞུ།"
+
+#: app/templates/account/email/confirm.html:5
+msgid "To confirm your account, please"
+msgstr ""
+
+#: app/templates/account/email/invite.html:3
+msgid "You are invited to join"
+msgstr ""
+
+#: app/templates/account/email/invite.html:5
+msgid "To set your password, please"
+msgstr ""
+
+#: app/templates/account/email/reset_password.html:9
+msgid "If you have not requested a password reset, simply ignore this message."
+msgstr ""
+
+#: app/templates/admin/manage_user.html:41
+#, fuzzy
+msgid "Account status"
+msgstr "ཁ་བྱང་གི་རིགས།"
+
+#: app/templates/admin/manage_user.html:53
+#, fuzzy
+msgid "Back to all users"
+msgstr "སྤྱོད་མཁན་ལ་ཆོག་མཆན་སྤྲོད་པ།"
+
+#: app/templates/admin/manage_user.html:67
+msgid "This action is permanent"
+msgstr ""
+
+#: app/templates/admin/manage_user.html:69
+msgid ""
+"Deleting a user account is not a reversible change. Any information "
+"associated\n"
+"                                with this account will be removed, and "
+"cannot be recovered."
+msgstr ""
+
+#: app/templates/admin/manage_user.html:77
+msgid "I understand that this action cannot be undone."
+msgstr ""
+
+#: app/templates/admin/manage_user.html:79
+msgid ""
+"Delete\n"
+"                        this user"
+msgstr ""
+
+#: app/templates/admin/new_user.html:13
+#: app/templates/admin/registered_users.html:9
+#: app/templates/macros/dashboard_macros.html:158
+msgid "Back to dashboard"
+msgstr "གཙོ་ངོས་སུ་ཕྱིར་ལོག"
+
+#: app/templates/admin/new_user.html:16 app/templates/main/dashboard.html:12
+msgid "Add New User"
+msgstr "སྤྱོད་མཁན་གསར་པ་སྣོན།"
+
+#: app/templates/admin/new_user.html:17 app/templates/main/dashboard.html:13
+msgid "Create a new user account"
+msgstr "སྤྱོད་མཁན་གྱི་ཁ་བྱང་གསར་པ་ཞིག་བཟོ།"
+
+#: app/templates/admin/registered_users.html:12
+#: app/templates/main/dashboard.html:10
+msgid "Registered Users"
+msgstr "ཐོ་འགོད་བྱས་ཟིན་པའི་སྤྱོད་མཁན།"
+
+#: app/templates/admin/registered_users.html:14
+#, fuzzy
+msgid "View and manage currently registered users."
+msgstr "སྐད་མཛོད་སྤྱོད་མཁན་རྣམས་འཆར་སྟོན་དང་དོ་དམ།"
+
 #: app/templates/control_lists/contact.html:8
 msgid "Contact control lists administrators"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་གི་འཛིན་སྐྱོང་བར་འབྲེལ་གཏུགས།"
 
 #: app/templates/control_lists/contact.html:10
 msgid ""
-"This page is built so that you can contact administrators of a control list to propose\n"
-"            new lemma, POS or morphologies accepted values. We have preferred to use email so that discussion could be rich\n"
+"This page is built so that you can contact administrators of a control "
+"list to propose\n"
+"            new lemma, POS or morphologies accepted values. We have "
+"preferred to use email so that discussion could be rich\n"
 "            when needed."
 msgstr ""
-"This page is built so that you can contact administrators of a control list to propose\n"
-"            new lemma, POS or morphologies accepted values. We have preferred to use email so that discussion could be rich\n"
+"This page is built so that you can contact administrators of a control "
+"list to propose\n"
+"            new lemma, POS or morphologies accepted values. We have "
+"preferred to use email so that discussion could be rich\n"
 "            when needed."
 
 #: app/templates/control_lists/control_list.html:10
@@ -54,6 +225,7 @@ msgid "Description"
 msgstr "འགྲེལ་བཤད།"
 
 #: app/templates/control_lists/control_list.html:13
+#: app/templates/control_lists/information_edit.html:23
 #: app/templates/control_lists/information_read.html:15
 msgid "Bibliography"
 msgstr "དཔེ་ཐོ།"
@@ -76,8 +248,8 @@ msgstr "བསྐྱར་འབྲི་བྱེད།"
 
 #: app/templates/control_lists/control_list.html:30
 msgid ""
-"The following pages are made to completely rewrite control lists. Use with "
-"caution !"
+"The following pages are made to completely rewrite control lists. Use "
+"with caution !"
 msgstr ""
 "གཤམ་བཀོད་ཀྱི་ཤོག་ངོས་དག་ནི་སྟངས་འཛིན་ཐོ་གཞུང་ཡོངས་སུ་བསྐྱར་འབྲི་བྱས་པ་ལས་གྲུབ་པ་ཡིན།"
 " བཀོལ་སྤྱོད་སྐབས་གཟབ་ནན་བྱེད་དགོས།"
@@ -103,6 +275,16 @@ msgstr "གཟབ་དགོས་སའི་གནས།"
 msgid "Make public"
 msgstr "སྤྱི་སྤྱོད་ཅན་དུ་བཟོས་པ།"
 
+#: app/templates/control_lists/edit.html:18
+#: app/templates/control_lists/information_edit.html:35
+#: app/templates/main/control_list_switch.html:29
+#: app/templates/main/corpus_custom_dictionary.html:55
+#: app/templates/main/corpus_edit_allowed_values.html:18
+#: app/templates/main/corpus_new.html:211
+#: app/templates/main/corpus_preferences.html:86
+msgid "Submit"
+msgstr "འབུལ་སྤྲོད།"
+
 #: app/templates/control_lists/information_edit.html:11
 msgid "Description of the control list"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་དེའི་འགྲེལ་བཤད།"
@@ -128,26 +310,21 @@ msgstr "སློབ་སྟོན།"
 #: app/templates/control_lists/information_edit.html:30
 msgid ""
 "All the relevant information that might be of help\n"
-"        about how to tag using your control list. Can link to external data and use"
+"        about how to tag using your control list. Can link to external "
+"data and use"
 msgstr ""
 "All the relevant information that might be of help\n"
-"        about how to tag using your control list. Can link to external data and use"
-
-#: app/templates/control_lists/information_edit.html:35
-#: app/templates/main/control_list_switch.html:29
-#: app/templates/main/corpus_custom_dictionary.html:55
-#: app/templates/main/corpus_new.html:198
-#: app/templates/main/corpus_preferences.html:86
-msgid "Submit"
-msgstr "འབུལ་སྤྲོད།"
+"        about how to tag using your control list. Can link to external "
+"data and use"
 
 #: app/templates/control_lists/macros.html:11
 msgid "Manage Lists"
 msgstr "དོ་དམ་ཐོ་གཞུང་།"
 
 #: app/templates/control_lists/macros.html:13
+#: app/templates/macros/tokens_macros.html:514
 #: app/templates/main/corpus_custom_dictionary.html:12
-#: app/templates/main/corpus_info.html:127
+#: app/templates/main/corpus_info.html:129
 #: app/templates/main/tokens_similar_to_token.html:19
 #: app/templates/main/tokens_similar_to_token.html:29
 #: app/templates/main/tokens_similar_to_token.html:52
@@ -155,9 +332,10 @@ msgid "Lemma"
 msgstr "མ་ཚིག"
 
 #: app/templates/control_lists/macros.html:14
-#: app/templates/macros/nav_macros.html:245
+#: app/templates/macros/nav_macros.html:255
+#: app/templates/macros/tokens_macros.html:517
 #: app/templates/main/corpus_custom_dictionary.html:42
-#: app/templates/main/corpus_info.html:153
+#: app/templates/main/corpus_info.html:155
 #: app/templates/main/tokens_similar_to_token.html:22
 #: app/templates/main/tokens_similar_to_token.html:32
 #: app/templates/main/tokens_similar_to_token.html:55
@@ -184,6 +362,39 @@ msgstr "མིང་གསར་བརྗེ།"
 msgid "Edit informations"
 msgstr "གནས་ཚུལ་རྩོམ་སྒྲིག་བྱེད།"
 
+#: app/templates/control_lists/propose_as_public.html:8
+msgid "Propose the list as public list"
+msgstr ""
+
+#: app/templates/control_lists/propose_as_public.html:10
+msgid ""
+"Through this form, you will contact the administrators to ask them for\n"
+"            making this control list public. A public control list can "
+"not be made private again."
+msgstr ""
+
+#: app/templates/control_lists/propose_as_public.html:11
+msgid ""
+"By making this\n"
+"                list public, other users will be able to freely access "
+"this control list. They will be able\n"
+"            to contact the administrators of this list to propose new "
+"allowed values such as lemma."
+msgstr ""
+
+#: app/templates/control_lists/read.html:7
+#, fuzzy
+msgid "Allowed values for"
+msgstr "ཆོག་མཆན་ཐོབ་པའི་ lemma"
+
+#: app/templates/control_lists/read.html:10
+msgid "Readable"
+msgstr ""
+
+#: app/templates/control_lists/read.html:28
+msgid "No allowed value in this category"
+msgstr ""
+
 #: app/templates/control_lists/read_lemma.html:8
 msgid "Control List for Lemmas"
 msgstr "Lemmas ཡི་སྟངས་འཛིན་ཐོ་གཞུང་།"
@@ -220,14 +431,46 @@ msgstr "བཙལ་བྱ།"
 msgid "Show help"
 msgstr "རམ་འདེགས་སྟོན།"
 
+#: app/templates/control_lists/read_lemma.html:45
+#, fuzzy
+msgid "Options :"
+msgstr "མཆན་འགྲེལ།"
+
+#: app/templates/control_lists/read_lemma.html:47
+#: app/templates/main/tokens_search_through_fields.html:64
+msgid "* can be used to match partial words, eg."
+msgstr "* can be used to match partial words, eg."
+
+#: app/templates/control_lists/read_lemma.html:48
+#: app/templates/main/tokens_search_through_fields.html:65
+msgid "! can be used to negate a match, eg."
+msgstr "! can be used to negate a match, eg."
+
+#: app/templates/control_lists/read_lemma.html:49
+#: app/templates/main/tokens_search_through_fields.html:66
+msgid "| can be used to perform an OR operation, eg."
+msgstr "| can be used to perform an OR operation, eg."
+
+#: app/templates/control_lists/read_lemma.html:50
+#, fuzzy
+msgid "To search forms which do not contain 'e', eg."
+msgstr "འདུས་མེད་པའི་འགེངས་ཤོག་འཚོལ་ཕྱིར།"
+
 #: app/templates/control_lists/rename.html:8
 msgid "Rename the control list"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་དེ་ལ་མིང་བསྐྱར་འདོགས་བྱེད།"
 
-#: app/templates/macros/dashboard_macros.html:7
-#: app/templates/macros/dashboard_macros.html:30
-msgid "Account type"
-msgstr "ཁ་བྱང་གི་རིགས།"
+#: app/templates/errors/403.html:5
+msgid "Forbidden: You do not have the right to access this page."
+msgstr ""
+
+#: app/templates/errors/404.html:5
+msgid "Page not found"
+msgstr ""
+
+#: app/templates/errors/500.html:5
+msgid "Internal Server Error"
+msgstr ""
 
 #: app/templates/macros/dashboard_macros.html:9
 msgid "All account types"
@@ -248,10 +491,6 @@ msgstr "དངོས་མིང་།"
 msgid "Last name"
 msgstr "རུས་མིང་།"
 
-#: app/templates/macros/dashboard_macros.html:28
-msgid "Email address"
-msgstr "གློག་འཕྲིན་ཁ་བྱང་།"
-
 #: app/templates/macros/dashboard_macros.html:32
 msgid "Status"
 msgstr "གནས་བབས།"
@@ -288,13 +527,10 @@ msgstr "སྐད་མཛོད།"
 
 #: app/templates/macros/dashboard_macros.html:142
 #: app/templates/macros/nav_macros.html:89
+#: app/templates/main/corpus_new.html:131
 msgid "Control Lists"
 msgstr "གཏན་འབེབ་ཟིན་པའི་ཐོ་གཞུང་།"
 
-#: app/templates/macros/dashboard_macros.html:158
-msgid "Back to dashboard"
-msgstr "གཙོ་ངོས་སུ་ཕྱིར་ལོག"
-
 #: app/templates/macros/dashboard_macros.html:165
 msgid "View and manage"
 msgstr "འཆར་སྟོན་དང་དོ་དམ།"
@@ -334,16 +570,19 @@ msgstr "སྤྱོད་མཁན་ཞིག་ལ་ཆོག་མཆན་
 
 #: app/templates/macros/dashboard_macros.html:226
 msgid ""
-"The user will be able to use the post-correction tools but also grant and remove\n"
+"The user will be able to use the post-correction tools but also grant and"
+" remove\n"
 "                                    accesses to this"
 msgstr ""
-"The user will be able to use the post-correction tools but also grant and remove\n"
+"The user will be able to use the post-correction tools but also grant and"
+" remove\n"
 "                                    accesses to this"
 
 #: app/templates/macros/header_macros.html:2
-#: app/templates/macros/nav_macros.html:182
+#: app/templates/macros/nav_macros.html:192
 #: app/templates/main/tokens_add_row.html:7
 #: app/templates/main/tokens_correct.html:8
+#: app/templates/main/tokens_correct_unallowed.html:8
 #: app/templates/main/tokens_del_row.html:7
 #: app/templates/main/tokens_edit_form.html:7
 #: app/templates/main/tokens_edit_history.html:6
@@ -359,6 +598,11 @@ msgstr "སྐད་མཛོད།"
 msgid "Dashboard"
 msgstr "གཙོ་ངོས།"
 
+#: app/templates/macros/nav_macros.html:37
+#, fuzzy
+msgid "Your Account"
+msgstr "ཐོ་འགོད།"
+
 #: app/templates/macros/nav_macros.html:38
 msgid "Log out"
 msgstr "ཕྱིར་ཐོན།"
@@ -375,105 +619,250 @@ msgstr "ནང་འཛུལ།"
 msgid "New Corpus"
 msgstr "སྐད་མཛོད་གསར་པ།"
 
-#: app/templates/macros/nav_macros.html:103
+#: app/templates/macros/nav_macros.html:113
 msgid "Bug or request ?"
 msgstr "གནད་དོན་ཡིན་ནམ་ཡང་ན་རེ་ཞུ་ཡིན།"
 
-#: app/templates/macros/nav_macros.html:104
+#: app/templates/macros/nav_macros.html:114
 msgid "Help"
 msgstr "རམ་འདེགས།"
 
-#: app/templates/macros/nav_macros.html:140
+#: app/templates/macros/nav_macros.html:150
 msgid "Go to"
 msgstr "དེར་འགྲོ།"
 
-#: app/templates/macros/nav_macros.html:176
+#: app/templates/macros/nav_macros.html:186
 msgid "Menu"
 msgstr "ཐོ།"
 
-#: app/templates/macros/nav_macros.html:185
+#: app/templates/macros/nav_macros.html:195
 msgid "Home"
 msgstr "མདུན་ངོས།"
 
-#: app/templates/macros/nav_macros.html:187
+#: app/templates/macros/nav_macros.html:197
 #: app/templates/main/corpus_preferences.html:5
 msgid "Preferences"
 msgstr "རྒྱུ་ཆ་ཁག"
 
-#: app/templates/macros/nav_macros.html:192
+#: app/templates/macros/nav_macros.html:202
 msgid "Control List"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་།"
 
-#: app/templates/macros/nav_macros.html:195
+#: app/templates/macros/nav_macros.html:205
+#: app/templates/macros/tokens_macros.html:455
 msgid "Edit"
 msgstr "རྩོམ་སྒྲིག"
 
-#: app/templates/macros/nav_macros.html:199
+#: app/templates/macros/nav_macros.html:209
 #: app/templates/main/tokens_similar_to_record.html:14
 msgid "Change"
 msgstr "བསྒྱུར་བཅོས།"
 
-#: app/templates/macros/nav_macros.html:203
+#: app/templates/macros/nav_macros.html:213
 msgid "Corpus Custom Dictionary"
 msgstr "སྐད་མཛོད་འདི་ཉིད་ཀྱི་ཚིག་མཛོད།"
 
-#: app/templates/macros/nav_macros.html:207
+#: app/templates/macros/nav_macros.html:217
 msgid "Text history"
 msgstr "སློབ་དེབ་ཀྱི་ལོ་རྒྱུས།"
 
-#: app/templates/macros/nav_macros.html:211
+#: app/templates/macros/nav_macros.html:221
 msgid "Delete the corpus (2 steps)"
 msgstr "སྐད་མཛོད་དེ་ (2 steps) སུབ།"
 
-#: app/templates/macros/nav_macros.html:217
+#: app/templates/macros/nav_macros.html:227
 msgid "Go to bookmark"
 msgstr "དཔེ་རྟགས་ཡོད་ས་དེར་འགྲོ།"
 
-#: app/templates/macros/nav_macros.html:221
+#: app/templates/macros/nav_macros.html:231
 msgid "Annotations"
 msgstr "མཆན་འགྲེལ།"
 
-#: app/templates/macros/nav_macros.html:224
+#: app/templates/macros/nav_macros.html:234
 #: app/templates/main/tokens_edit_form.html:34
 msgid "Correct"
 msgstr "དག་སྒྲིག"
 
-#: app/templates/macros/nav_macros.html:227
+#: app/templates/macros/nav_macros.html:237
 #: app/templates/main/tokens_search_through_fields.html:70
 msgid "Search"
 msgstr "འཚོལ་ཞིབ།"
 
-#: app/templates/macros/nav_macros.html:230
+#: app/templates/macros/nav_macros.html:240
 msgid "Export"
 msgstr "ཕྱིར་འདྲེན།"
 
-#: app/templates/macros/nav_macros.html:233
+#: app/templates/macros/nav_macros.html:243
 msgid "History"
 msgstr "ལོ་རྒྱུས།"
 
-#: app/templates/macros/nav_macros.html:237
+#: app/templates/macros/nav_macros.html:247
 msgid "Correct unallowed ..."
 msgstr "ཆོག་མཆན་མ་སྤྲད་པ་དག་སྒྲིག"
 
-#: app/templates/macros/nav_macros.html:240
+#: app/templates/macros/nav_macros.html:250
 msgid "lemma"
 msgstr "མ་ཚིག"
 
-#: app/templates/macros/nav_macros.html:250
+#: app/templates/macros/nav_macros.html:260
 msgid "morphology"
 msgstr "ཚིག་གི་གྲུབ་ཆ།"
 
-#: app/templates/macros/nav_macros.html:264
+#: app/templates/macros/nav_macros.html:274
 msgid "Your corpora"
 msgstr "ཁྱེད་ཀྱི་སྐད་མཛོད།"
 
+#: app/templates/macros/tokens_macros.html:16
+#, fuzzy
+msgid "Save checked similar lemma"
+msgstr "མ་ཚིག་གསར་པ་ཉར།"
+
+#: app/templates/macros/tokens_macros.html:18
+#: app/templates/macros/tokens_macros.html:81
+#, fuzzy
+msgid "Save"
+msgstr "ཉར་ཟིན།"
+
+#: app/templates/macros/tokens_macros.html:19
+msgid "Select All"
+msgstr ""
+
+#: app/templates/macros/tokens_macros.html:20
+msgid "Deselect All"
+msgstr ""
+
+#: app/templates/macros/tokens_macros.html:26
+msgid ""
+"An error occured. Contact the administrator\n"
+"        with all the information you can so they can replicate your error."
+msgstr ""
+
+#: app/templates/macros/tokens_macros.html:32
+#, fuzzy
+msgid "Id"
+msgstr "ID"
+
+#: app/templates/macros/tokens_macros.html:34
+#: app/templates/macros/tokens_macros.html:512
+#: app/templates/main/tokens_add_row.html:13
+#: app/templates/main/tokens_del_row.html:13
+#: app/templates/main/tokens_edit_form.html:13
+#: app/templates/main/tokens_search_through_fields.html:19
+#: app/templates/main/tokens_similar_to_token.html:49
+msgid "Form"
+msgstr "འགེངས་ཤོག"
+
+#: app/templates/macros/tokens_macros.html:89
+#, fuzzy
+msgid "Edit the form"
+msgstr "གནས་ཚུལ་རྩོམ་སྒྲིག་བྱེད།"
+
+#: app/templates/macros/tokens_macros.html:90
+#, fuzzy
+msgid "Delete the row"
+msgstr "སུབ།"
+
+#: app/templates/macros/tokens_macros.html:91
+msgid "Add a token after this one"
+msgstr ""
+
+#: app/templates/macros/tokens_macros.html:92
+#, fuzzy
+msgid "Set as bookmark"
+msgstr "དཔེ་རྟགས་ཡོད་ས་དེར་འགྲོ།"
+
+#: app/templates/macros/tokens_macros.html:380
+msgid "Add <code>$val$</code> to custom dictionary of <em>$key$</em>"
+msgstr ""
+
+#: app/templates/macros/tokens_macros.html:454
+#: app/templates/main/tokens_edit_history.html:11
+#: app/templates/main/tokens_similar_to_record.html:17
+msgid "User"
+msgstr "སྤྱོད་མཁན།"
+
+#: app/templates/macros/tokens_macros.html:456
+#: app/templates/macros/tokens_macros.html:522
+#: app/templates/main/tokens_add_row.html:14
+#: app/templates/main/tokens_del_row.html:14
+#: app/templates/main/tokens_edit_form.html:14
+#: app/templates/main/tokens_similar_to_record.html:26
+#: app/templates/main/tokens_similar_to_token.html:50
+msgid "Context"
+msgstr "སྐབས་བསྟུན།"
+
+#: app/templates/macros/tokens_macros.html:458
+#: app/templates/main/tokens_similar_to_record.html:28
+msgid "Old Lemma"
+msgstr "མ་ཚིག་རྙིང་བ།"
+
+#: app/templates/macros/tokens_macros.html:459
+#: app/templates/main/tokens_similar_to_record.html:29
+msgid "Corr Lemma"
+msgstr "Corr མ་ཚིག"
+
+#: app/templates/macros/tokens_macros.html:462
+#: app/templates/main/tokens_similar_to_record.html:32
+msgid "Previous POS"
+msgstr "སྔ་མའི་ POS"
+
+#: app/templates/macros/tokens_macros.html:463
+#: app/templates/main/tokens_similar_to_record.html:33
+msgid "Actual POS"
+msgstr "དངོས་ཡོད་ཀྱི་ POS"
+
+#: app/templates/macros/tokens_macros.html:466
+#: app/templates/main/tokens_similar_to_record.html:36
+msgid "Previous Morph"
+msgstr "སྔ་མའི་ཚིག་གི་གྲུབ་ཆ།"
+
+#: app/templates/macros/tokens_macros.html:467
+#: app/templates/main/tokens_similar_to_record.html:37
+msgid "Actual Morph"
+msgstr "དངོས་ཡོད་ཀྱི་ཚིག་གི་གྲུབ་ཆ།"
+
+#: app/templates/macros/tokens_macros.html:469
+#, fuzzy
+msgid "Similar"
+msgstr "tokens འདྲ་མཚུངས་ཁག"
+
+#: app/templates/macros/tokens_macros.html:520
+#: app/templates/main/corpus_custom_dictionary.html:26
+#: app/templates/main/tokens_similar_to_token.html:25
+#: app/templates/main/tokens_similar_to_token.html:35
+#: app/templates/main/tokens_similar_to_token.html:58
+msgid "Morph"
+msgstr "ཚིག་གི་གྲུབ་ཆ།"
+
+#: app/templates/macros/upload_text.html:4
+msgid "Text file"
+msgstr ""
+
+#: app/templates/macros/upload_text.html:5
+msgid "Upload a text file (.txt) which will be read"
+msgstr ""
+
+#: app/templates/macros/utils.html:3
+msgid "Private"
+msgstr ""
+
+#: app/templates/macros/utils.html:4
+msgid "Sent for publicization"
+msgstr ""
+
+#: app/templates/macros/utils.html:5
+#, fuzzy
+msgid "Public"
+msgstr "སྤྱི་སྤྱོད་ཅན་དུ་བཟོས་པ།"
+
 #: app/templates/main/control_list_switch.html:9
 msgid "Switch control list"
 msgstr "སྟངས་འཛིན་ཐོ་གཞུང་བརྗེ་བ།"
 
 #: app/templates/main/control_list_switch.html:13
-msgid "Current control list: {{ current_control_lists.name }}"
-msgstr "ད་ལྟའི་སྟངས་འཛིན་ཐོ་གཞུང་ནི། {{ current_control_lists.name }}"
+#, fuzzy
+msgid "Current control list:"
+msgstr "སྟངས་འཛིན་ཐོ་གཞུང་དེ་ལ་མིང་བསྐྱར་འདོགས་བྱེད།"
 
 #: app/templates/main/corpus_custom_dictionary.html:6
 msgid "Custom dictionary"
@@ -487,20 +876,17 @@ msgstr "ཚིག་མཛོད་རྣམས།"
 msgid "List of supplementaries lemma for this corpus. One per line"
 msgstr "སྐད་མཛོད་འདིའི་ཁ་གསབ་མ་ཚིག་གི་ཐོ་གཞུང་། རེ་རེ་ཐིག་ཕྲེང་རེ་ཡིན་དགོས།"
 
-#: app/templates/main/corpus_custom_dictionary.html:26
-#: app/templates/main/tokens_similar_to_token.html:25
-#: app/templates/main/tokens_similar_to_token.html:35
-#: app/templates/main/tokens_similar_to_token.html:58
-msgid "Morph"
-msgstr "ཚིག་གི་གྲུབ་ཆ།"
-
 #: app/templates/main/corpus_custom_dictionary.html:34
 msgid ""
-"List of supplementaries morph for this corpus. One per line, a human readable version of the code\n"
-"                    can be given after a tabulation (4 spaces are automatically transformed into tabulation). eg."
+"List of supplementaries morph for this corpus. One per line, a human "
+"readable version of the code\n"
+"                    can be given after a tabulation (4 spaces are "
+"automatically transformed into tabulation). eg."
 msgstr ""
-"List of supplementaries morph for this corpus. One per line, a human readable version of the code\n"
-"                    can be given after a tabulation (4 spaces are automatically transformed into tabulation). eg."
+"List of supplementaries morph for this corpus. One per line, a human "
+"readable version of the code\n"
+"                    can be given after a tabulation (4 spaces are "
+"automatically transformed into tabulation). eg."
 
 #: app/templates/main/corpus_custom_dictionary.html:36
 msgid "Morph=MyMorph    My human readable morph"
@@ -514,6 +900,31 @@ msgstr "བར་ཚེགས་ཀྱིས་བར་ཕྱེས་པའི
 msgid "MyPos,MyPos2"
 msgstr "MyPos,MyPos2"
 
+#: app/templates/main/corpus_delete.html:8
+#, fuzzy
+msgid "Delete the corpus"
+msgstr "སྐད་མཛོད་དེ་ (2 steps) སུབ།"
+
+#: app/templates/main/corpus_edit_allowed_values.html:11
+#, fuzzy
+msgid "Edit allowed"
+msgstr "ཆོག་མཆན་མ་ཐོབ་པ།"
+
+#: app/templates/main/corpus_edit_allowed_values.html:14
+#, fuzzy
+msgid "Allowed"
+msgstr "ཆོག་མཆན་མ་ཐོབ་པ།"
+
+#: app/templates/main/corpus_generate_fixtures.html:10
+#, fuzzy
+msgid "Allowed Lemma"
+msgstr "ཆོག་མཆན་ཐོབ་པའི་ lemma"
+
+#: app/templates/main/corpus_generate_fixtures.html:15
+#: app/templates/main/corpus_new.html:199
+msgid "Allowed POS"
+msgstr "ཆོག་མཆན་ཐོབ་པའི་ཚིག་གཤིས།"
+
 #: app/templates/main/corpus_info.html:15
 msgid "Statistics"
 msgstr "བསྡོམས་རྩིས།"
@@ -547,62 +958,61 @@ msgid "edited"
 msgstr "རྩོམ་སྒྲིག་ཟིན་པ།"
 
 #: app/templates/main/corpus_info.html:57
-msgid "lemma were correct from lemmatization. It means you corrected"
+#, fuzzy, python-format
+msgid ""
+"lemma were correct from lemmatization. It means you corrected %(count)s "
+"lemma."
 msgstr ""
 "མ་ཚིག་རྣམས་མ་ཚིག་གི་གྲུབ་ཚུལ་ནས་དག་སྒྲིག་བྱས་འདུག "
 "འདིའི་ནང་དོན་ནི་ཁྱེད་ཀྱིས་དག་སྒྲིག་བྱས་འདུག་ཅེས་པའི་དོན་ཡིན།"
 
-#: app/templates/main/corpus_info.html:57
-msgid "lemma."
-msgstr "མ་ཚིག"
-
-#: app/templates/main/corpus_info.html:78
-msgid "morphologies were correct from lemmatization. It means you corrected"
+#: app/templates/main/corpus_info.html:79
+#, fuzzy, python-format
+msgid ""
+"morphologies were correct from lemmatization. It means you corrected "
+"%(count)s morphologies."
 msgstr ""
 "ཚིག་གི་གྲུབ་ཆ་རྣམས་མ་ཚིག་གི་གྲུབ་ཚུལ་ནས་དག་སྒྲིག་བྱས་འདུག "
 "འདིའི་ནང་དོན་ནི་ཁྱེད་ཀྱིས་དག་སྒྲིག་བྱས་འདུག་ཅེས་པའི་དོན་ཡིན།"
 
-#: app/templates/main/corpus_info.html:78
-msgid "morphologies."
-msgstr "ཚིག་གི་གྲུབ་ཆ་རྣམས།"
-
-#: app/templates/main/corpus_info.html:101
-msgid "POS were correct from lemmatization. It means you corrected"
+#: app/templates/main/corpus_info.html:102
+#, fuzzy, python-format
+msgid "POS were correct from lemmatization. It means you corrected %(count)s POS."
 msgstr ""
 "POS རྣམས་མ་ཚིག་གི་གྲུབ་ཚུལ་ནས་དག་སྒྲིག་བྱས་འདུག "
 "འདིའི་ནང་དོན་ནི་ཁྱེད་ཀྱིས་དག་སྒྲིག་བྱས་འདུག་ཅེས་པའི་དོན་ཡིན།"
 
-#: app/templates/main/corpus_info.html:101
-msgid "POS."
-msgstr "ཚིག་གཤིས།"
-
-#: app/templates/main/corpus_info.html:107
+#: app/templates/main/corpus_info.html:109
 msgid "Annotations over time"
 msgstr "ཆུ་ཚོད་ལས་བརྒལ་བའི་འགྲེལ་བརྗོད་ཁག"
 
-#: app/templates/main/corpus_info.html:113
+#: app/templates/main/corpus_info.html:115
 msgid "Most common corrections"
 msgstr "ཆེས་སྤྱིར་བཏང་གི་ཕྱོགས་བསྒྲིགས་རྣམས།"
 
-#: app/templates/main/corpus_info.html:129
-#: app/templates/main/corpus_info.html:142
-#: app/templates/main/corpus_info.html:155
+#: app/templates/main/corpus_info.html:122
+msgid "See more"
+msgstr ""
+
+#: app/templates/main/corpus_info.html:131
+#: app/templates/main/corpus_info.html:144
+#: app/templates/main/corpus_info.html:157
 msgid "Count"
 msgstr "གྲངས་བརྩི།"
 
-#: app/templates/main/corpus_info.html:129
-#: app/templates/main/corpus_info.html:142
-#: app/templates/main/corpus_info.html:155
+#: app/templates/main/corpus_info.html:131
+#: app/templates/main/corpus_info.html:144
+#: app/templates/main/corpus_info.html:157
 msgid "Corrected"
 msgstr "དག་སྒྲིག་ཟིན་པ།"
 
-#: app/templates/main/corpus_info.html:129
-#: app/templates/main/corpus_info.html:142
-#: app/templates/main/corpus_info.html:155
+#: app/templates/main/corpus_info.html:131
+#: app/templates/main/corpus_info.html:144
+#: app/templates/main/corpus_info.html:157
 msgid "Original"
 msgstr "མ་ཕྱི།"
 
-#: app/templates/main/corpus_info.html:140
+#: app/templates/main/corpus_info.html:142
 msgid "Morphology"
 msgstr "ཚིག་གི་གྲུབ་ཆ་རྣམས།"
 
@@ -611,6 +1021,7 @@ msgid "Create a new corpus"
 msgstr "སྐད་མཛོད་གསར་བཟོ།"
 
 #: app/templates/main/corpus_new.html:10
+#: app/templates/main/corpus_preferences.html:8
 msgid "Metadata"
 msgstr "གཞི་གྲངས་རྒྱས་པ།"
 
@@ -630,8 +1041,7 @@ msgstr "Left Context"
 #: app/templates/main/corpus_new.html:19
 #: app/templates/main/corpus_preferences.html:41
 msgid "Number of words to display on the left of the word to annotate"
-msgstr ""
-"འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡོན་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
+msgstr "འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡོན་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
 
 #: app/templates/main/corpus_new.html:22
 #: app/templates/main/corpus_preferences.html:45
@@ -641,8 +1051,7 @@ msgstr "Right Context"
 #: app/templates/main/corpus_new.html:24
 #: app/templates/main/corpus_preferences.html:58
 msgid "Number of words to display on the right of the word to annotate"
-msgstr ""
-"འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡས་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
+msgstr "འགྲེལ་བརྗོད་བྱ་རྒྱུའི་མིང་ཚིག་དེའི་གཡས་ངོས་སུ་འཆར་རྒྱུའི་མིང་ཚིག་གི་གྲངས་ཀ"
 
 #: app/templates/main/corpus_new.html:27
 #: app/templates/main/corpus_preferences.html:10
@@ -650,11 +1059,10 @@ msgid "Delimiter token"
 msgstr "Delimiter token"
 
 #: app/templates/main/corpus_new.html:29
-msgid ""
-"[Optional] A specific token that is used as a delimiter between passages."
+msgid "[Optional] A specific token that is used as a delimiter between passages."
 msgstr ""
-"[Optional] བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ "
-"token ཞིག"
+"[Optional] "
+"བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ token ཞིག"
 
 #: app/templates/main/corpus_new.html:33
 #: app/templates/main/corpus_preferences.html:63
@@ -684,36 +1092,56 @@ msgstr ""
 msgid "Example :"
 msgstr "དཔེར་བརྗོད་ནི།"
 
-#: app/templates/main/corpus_new.html:68 app/templates/main/corpus_new.html:80
+#: app/templates/main/corpus_new.html:68 app/templates/main/corpus_new.html:93
 msgid "Lemmatize"
 msgstr "མ་ཚིག་འགར།"
 
 #: app/templates/main/corpus_new.html:70
 msgid ""
-"If your text is not lemmatized, select the language and click on lemmatize."
+"If your text is not lemmatized, select the language and click on "
+"lemmatize."
 msgstr ""
 "གལ་སྲིད་ཁྱེད་ཀྱི་ཡིག་ཆ་མ་ཚིག་བཟོས་མེད་ན། "
 "སྐད་ཡིག་དེ་བདམས་ཏེ་མ་ཚིག་བཟོ་ཟེར་བ་དེའི་སྒང་ལ་སྣུན། "
 "གལ་སྲིད་ཁྱེད་ཀྱི་ཡིག་ཆ་མ་ཚིག་བཟོས་མེད་ན། "
 "སྐད་ཡིག་དེ་བདམས་ཏེ་མ་ཚིག་བཟོ་ཟེར་བ་དེའི་སྒང་ལ་སྣུན།"
 
+#: app/templates/main/corpus_new.html:75
+msgid "Select a service"
+msgstr ""
+
+#: app/templates/main/corpus_new.html:82
+#, python-format
+msgid "%(name) is a lemmatization service provided by %(provider)"
+msgstr ""
+
 #: app/templates/main/corpus_new.html:84
 msgid ""
+"If you only want to lemmatize (and not correct data), you can use their "
+"user interface here:"
+msgstr ""
+
+#: app/templates/main/corpus_new.html:87
+msgid "Please cite tools accordingly. You can use the following APA or Bibtex: "
+msgstr ""
+
+#: app/templates/main/corpus_new.html:97
+msgid ""
 "tokens analyzed... Total is approximately computed and could differ "
 "depending on the language..."
 msgstr ""
 "དབྱེ་ཞིབ་བྱས་ཟིན་པའི་tokens "
 "ཡི་སྤྱི་སྡོམ་ནི་ཧ་ལམ་རྩིས་བརྒྱབ་ཟིན་པ་དང་སྐད་ཡིག་དེར་གཞིགས་ཏེ་ཁྱད་པར་འབྱེད་ཐུབ་་་"
 
-#: app/templates/main/corpus_new.html:87
+#: app/templates/main/corpus_new.html:100
 msgid "Operation finished with success !"
 msgstr "ལས་འགུལ་བདེ་ལེགས་ངང་གྲུབ་སོང་།"
 
-#: app/templates/main/corpus_new.html:87
+#: app/templates/main/corpus_new.html:100
 msgid "tokens analyzed in total."
 msgstr "སྤྱི་བསྡོམས་ཀྱིས་དབྱེ་ཞིབ་བྱས་པའི་ tokens"
 
-#: app/templates/main/corpus_new.html:90
+#: app/templates/main/corpus_new.html:103
 msgid ""
 "Operation failed! Contact the administrator with the corpus you tried to "
 "lemmatize to investigate the issue."
@@ -721,117 +1149,123 @@ msgstr ""
 "ལས་འགུལ་ལམ་ལྷོངས་བྱུང་མ་སོང་། "
 "གནད་དོན་དེར་ཞིབ་འཇུག་བྱེད་པ་ལ་ཁྱེད་ཀྱིས་མ་ཚིག་བཟོ་ཐབས་བྱེད་བཞིན་པའི་སྐད་མཛོད་དེའི་འཛིན་སྐྱོང་བར་འབྲེལ་བ་བྱོས།"
 
-#: app/templates/main/corpus_new.html:98
+#: app/templates/main/corpus_new.html:111
 msgid "Tokenize (beta)"
 msgstr "Tokenize (beta)"
 
-#: app/templates/main/corpus_new.html:100
+#: app/templates/main/corpus_new.html:113
 msgid ""
-"If your text is not tokenized and you do not need to pre-lemmatize it, you "
-"can use this function"
+"If your text is not tokenized and you do not need to pre-lemmatize it, "
+"you can use this function"
 msgstr ""
 "གལ་སྲིད་ཁྱེད་ཀྱི་ཡིག་ཆ་ལ་མ་ཚིག་བཟོས་མེད་པ་དང་ཁྱེད་ལ་དེའི་སྔོན་འགྲོའི་མ་ཚིག་བཟོས་མི་དགོས་པ་ཡིན་ན།"
 " ཁྱེད་ཀྱིས་ཁྱད་ཆོས་འདི་བཀོལ་ཆོག"
 
-#: app/templates/main/corpus_new.html:105
+#: app/templates/main/corpus_new.html:118
 msgid "Remove hyphens (Be careful with this function)"
 msgstr "འཐེན་རྟགས་རྣམས་སུབ། (Be careful with this function)"
 
-#: app/templates/main/corpus_new.html:109
+#: app/templates/main/corpus_new.html:122
 msgid "Keep punctuation"
 msgstr "ཚེག་རྟགས་ཉོར།"
 
-#: app/templates/main/corpus_new.html:113
+#: app/templates/main/corpus_new.html:126
 msgid "Tokenize"
 msgstr "Tokenize"
 
-#: app/templates/main/corpus_new.html:123
+#: app/templates/main/corpus_new.html:136
 msgid "Use an existing control list"
 msgstr "ད་སྔ་ཡོད་བསྡད་པའི་སྟངས་འཛིན་ཐོ་གཞུང་ཞིག་བཀོལ་སྤྱོད་བྱེད།"
 
-#: app/templates/main/corpus_new.html:126
+#: app/templates/main/corpus_new.html:139
 msgid ""
-"By using shared control lists, you ensure that you stick to accepted values in\n"
-"                    the academic community. You will be able to propose new values to the administrators of control lists."
+"By using shared control lists, you ensure that you stick to accepted "
+"values in\n"
+"                    the academic community. You will be able to propose "
+"new values to the administrators of control lists."
 msgstr ""
-"By using shared control lists, you ensure that you stick to accepted values in\n"
-"                    the academic community. You will be able to propose new values to the administrators of control lists."
+"By using shared control lists, you ensure that you stick to accepted "
+"values in\n"
+"                    the academic community. You will be able to propose "
+"new values to the administrators of control lists."
 
-#: app/templates/main/corpus_new.html:148
+#: app/templates/main/corpus_new.html:161
 msgid "Write your own"
 msgstr "ཁྱེད་རང་རང་གིས་ཕྲིས།"
 
-#: app/templates/main/corpus_new.html:150
+#: app/templates/main/corpus_new.html:163
 msgid ""
-"In case the configurations provided do not fit your need, you can create your own configuration.\n"
-"                    You will be able to share it with collaborators, propose it as a canon setting for the whole base of users.\n"
-"                    You can also later add bibliographic information about your settings."
+"In case the configurations provided do not fit your need, you can create "
+"your own configuration.\n"
+"                    You will be able to share it with collaborators, "
+"propose it as a canon setting for the whole base of users.\n"
+"                    You can also later add bibliographic information "
+"about your settings."
 msgstr ""
-"In case the configurations provided do not fit your need, you can create your own configuration.\n"
-"                    You will be able to share it with collaborators, propose it as a canon setting for the whole base of users.\n"
-"                    You can also later add bibliographic information about your settings."
-
-#: app/templates/main/corpus_new.html:157
+"In case the configurations provided do not fit your need, you can create "
+"your own configuration.\n"
+"                    You will be able to share it with collaborators, "
+"propose it as a canon setting for the whole base of users.\n"
+"                    You can also later add bibliographic information "
+"about your settings."
+
+#: app/templates/main/corpus_new.html:170
 msgid "Load a configuration"
 msgstr "གྲུབ་ཆ་གོ་སྒྲིག་ཅིག་ནང་འདྲེན་བྱེད།"
 
-#: app/templates/main/corpus_new.html:158
+#: app/templates/main/corpus_new.html:171
 msgid "This helps kickstarting configuration"
 msgstr "འདིས་གཞི་རྩའི་གྲུབ་ཆའི་སྒྲིག་བཀོད་ལ་རམ་འདེགས་བྱེད།"
 
-#: app/templates/main/corpus_new.html:164
+#: app/templates/main/corpus_new.html:177
 msgid "Ancien Français"
 msgstr "Ancien Français"
 
-#: app/templates/main/corpus_new.html:165
+#: app/templates/main/corpus_new.html:178
 msgid "Latin (LASLA)"
 msgstr "ལཱ་ཏིན། (LASLA)"
 
-#: app/templates/main/corpus_new.html:168
+#: app/templates/main/corpus_new.html:181
 msgid "Load"
 msgstr "ནང་འདྲེན།"
 
-#: app/templates/main/corpus_new.html:173
+#: app/templates/main/corpus_new.html:186
 msgid "Lemma loaded"
 msgstr "ནང་འདྲེན་བྱས་པའི་ Lemma"
 
-#: app/templates/main/corpus_new.html:174
+#: app/templates/main/corpus_new.html:187
 msgid "POS loaded"
 msgstr "ནང་འདྲེན་བྱས་པའི་ཚིག་གཤིས།"
 
-#: app/templates/main/corpus_new.html:175
+#: app/templates/main/corpus_new.html:188
 msgid "Morphology loaded"
 msgstr "ནང་འདྲེན་བྱས་པའི་ Morphology"
 
-#: app/templates/main/corpus_new.html:181
+#: app/templates/main/corpus_new.html:194
 msgid "Allowed lemma"
 msgstr "ཆོག་མཆན་ཐོབ་པའི་ lemma"
 
-#: app/templates/main/corpus_new.html:182
+#: app/templates/main/corpus_new.html:195
 msgid "This should be formatted as a list of lemma separated by new line"
 msgstr ""
 "འདི་ནི་ངེས་པར་དུ་ཐིག་ཕྲེང་གསར་པ་རེས་བར་ཕྱེས་པའི་lemma "
 "ཡི་ཐོ་གཞུང་ཞིག་གི་རྣམ་པ་རུ་ཡོད་དགོས།"
 
-#: app/templates/main/corpus_new.html:186
-msgid "Allowed POS"
-msgstr "ཆོག་མཆན་ཐོབ་པའི་ཚིག་གཤིས།"
-
-#: app/templates/main/corpus_new.html:187
+#: app/templates/main/corpus_new.html:200
 msgid "This should be formatted as a list of POS separated by comma"
 msgstr ""
 "འདི་ནི་ངེས་པར་དུ་བར་ཚེག་གིས་བར་ཕྱེས་པའི་POS "
 "ཡི་ཐོ་གཞུང་ཞིག་གི་རྣམ་པ་རུ་ཡོད་དགོས།"
 
-#: app/templates/main/corpus_new.html:187
+#: app/templates/main/corpus_new.html:200
 msgid "and no space"
 msgstr "དང་བར་སྟོང་མེད་པ།"
 
-#: app/templates/main/corpus_new.html:191
+#: app/templates/main/corpus_new.html:204
 msgid "Allowed Morph (as TSV content)"
 msgstr "ཆོག་མཆན་ཐོབ་པའི་ Morph (as TSV content)"
 
-#: app/templates/main/corpus_new.html:192
+#: app/templates/main/corpus_new.html:205
 msgid ""
 "The TSV should at least have the header : label and could have a "
 "<code>readable</code> column for human"
@@ -844,50 +1278,65 @@ msgid ""
 "[Optional] A specific token that is used as a delimiter\n"
 "                    between passages."
 msgstr ""
-"[Optional]  བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ "
-"token ཞིག"
+"[Optional]  "
+"བརྗོད་བྱ་རེ་རེའི་བར་གྱི་མཚམས་རྟགས་སུ་བཀོལ་རྒྱུའི་དམིགས་བསལ་གྱི་ token ཞིག"
 
 #: app/templates/main/dashboard.html:9
 msgid "User Dashboard"
 msgstr "སྤྱོད་མཁན་གྱི་གཙོ་ངོས།"
 
-#: app/templates/main/dashboard.html:10
-msgid "Registered Users"
-msgstr "ཐོ་འགོད་བྱས་ཟིན་པའི་སྤྱོད་མཁན།"
-
 #: app/templates/main/dashboard.html:11
 msgid "View and manage user accounts"
 msgstr "སྤྱོད་མཁན་གྱི་ཁ་བྱང་འཆར་སྟོན་དང་དོ་དམ།"
 
-#: app/templates/main/dashboard.html:12
-msgid "Add New User"
-msgstr "སྤྱོད་མཁན་གསར་པ་སྣོན།"
-
-#: app/templates/main/dashboard.html:13
-msgid "Create a new user account"
-msgstr "སྤྱོད་མཁན་གྱི་ཁ་བྱང་གསར་པ་ཞིག་བཟོ།"
-
 #: app/templates/main/dashboard.html:14
 msgid "Invite New User"
 msgstr "སྤྱོད་མཁན་གསར་པ་གདན་འདྲེན།"
 
 #: app/templates/main/dashboard.html:15
 msgid "Invites a new user to create their own account"
+msgstr "ཁོ་ཚོ་རང་ཉིད་ཀྱི་ཁ་བྱང་གསར་བཟོའི་ཕྱིར་སྤྱོད་མཁན་གསར་པ་ཞིག་གདན་འདྲེན་བྱེད།"
+
+#: app/templates/main/dashboard_corpus_table.html:5
+msgid "Management of corpora"
+msgstr ""
+
+#: app/templates/main/dashboard_corpus_table.html:10
+msgid "Links"
 msgstr ""
-"ཁོ་ཚོ་རང་ཉིད་ཀྱི་ཁ་བྱང་གསར་བཟོའི་ཕྱིར་སྤྱོད་མཁན་གསར་པ་ཞིག་གདན་འདྲེན་བྱེད།"
+
+#: app/templates/main/dashboard_corpus_table.html:12
+#, fuzzy
+msgid "Corpus name"
+msgstr "སྐད་མཛོད་མིང་།"
+
+#: app/templates/main/dashboard_corpus_table.html:13
+#, fuzzy
+msgid "Last modified"
+msgstr "རུས་མིང་།"
+
+#: app/templates/main/dashboard_corpus_table.html:14
+#, fuzzy
+msgid "Users"
+msgstr "སྤྱོད་མཁན།"
 
 #: app/templates/main/index.html:5 app/templates/main/index_loggedin.html:6
 msgid "Welcome to Pyrrha !"
 msgstr "Pyrrha ལ་ཕེབས་པར་དགའ་བསུ་ཞུ།"
 
+#: app/templates/main/index.html:6
+msgid ""
+"Pyrrha is a simple Python Flask WebApp to fasten the post-correction of "
+"lemmatized and morpho-syntactic tagged corpora."
+msgstr ""
+
 #: app/templates/main/tokens_add_row.html:7
 msgid "Add a token"
 msgstr "token ཞིག་སྣོན།"
 
 #: app/templates/main/tokens_add_row.html:8
 msgid "This page allows you to add a token after the following one"
-msgstr ""
-"ཤོག་ངོས་འདིས་ཁྱེད་ལ་ཡོང་བཞིན་པ་དེའི་རྗེས་སུ་token ཞིག་སྣོན་འཇུག་པ་ཡིན།"
+msgstr "ཤོག་ངོས་འདིས་ཁྱེད་ལ་ཡོང་བཞིན་པ་དེའི་རྗེས་སུ་token ཞིག་སྣོན་འཇུག་པ་ཡིན།"
 
 #: app/templates/main/tokens_add_row.html:12
 #: app/templates/main/tokens_del_row.html:12
@@ -895,22 +1344,6 @@ msgstr ""
 msgid "Token ID"
 msgstr "Token ID"
 
-#: app/templates/main/tokens_add_row.html:13
-#: app/templates/main/tokens_del_row.html:13
-#: app/templates/main/tokens_edit_form.html:13
-#: app/templates/main/tokens_search_through_fields.html:19
-#: app/templates/main/tokens_similar_to_token.html:49
-msgid "Form"
-msgstr "འགེངས་ཤོག"
-
-#: app/templates/main/tokens_add_row.html:14
-#: app/templates/main/tokens_del_row.html:14
-#: app/templates/main/tokens_edit_form.html:14
-#: app/templates/main/tokens_similar_to_record.html:26
-#: app/templates/main/tokens_similar_to_token.html:50
-msgid "Context"
-msgstr "སྐབས་བསྟུན།"
-
 #: app/templates/main/tokens_add_row.html:27
 msgid "Form of the new token to insert"
 msgstr "བར་འཇུག་བྱ་རྒྱུའི་token གསར་པ་དེའི་འགེངས་ཤོག"
@@ -926,15 +1359,12 @@ msgid "Go back to correcting annotations"
 msgstr "ཕྱིར་དག་སྒྲིག་བྱེད་བཞིན་པའི་མཆན་འགྲེལ་ལ་ཕེབས།"
 
 #: app/templates/main/tokens_correct.html:8
+#: app/templates/main/tokens_correct_unallowed.html:8
 #: app/templates/main/tokens_history.html:7
 #: app/templates/main/tokens_similar_to_record.html:10
 msgid "List of tokens"
 msgstr "token ཡི་ཐོ་གཞུང་།"
 
-#: app/templates/main/tokens_correct_unallowed.html:8
-msgid "Corpus {{corpus.name}} - List of tokens"
-msgstr "{{corpus.name}} སྐད་མཛོད་ཀྱི་ཚིག་ཐོ།"
-
 #: app/templates/main/tokens_del_row.html:7
 msgid "Remove a token"
 msgstr "token ཞིག་སུབ།"
@@ -969,11 +1399,6 @@ msgstr "འགེངས་ཤོག་གསར་པ།"
 msgid "History of tokens edition"
 msgstr "tokens ཡི་ཐོན་རིམ་གྱི་ལོ་རྒྱུས།"
 
-#: app/templates/main/tokens_edit_history.html:11
-#: app/templates/main/tokens_similar_to_record.html:17
-msgid "User"
-msgstr "སྤྱོད་མཁན།"
-
 #: app/templates/main/tokens_edit_history.html:11
 #: app/templates/main/tokens_similar_to_record.html:19
 msgid "Date"
@@ -999,46 +1424,10 @@ msgstr "ཕབ་ལེན།"
 msgid "Search tokens within the corpus"
 msgstr "སྐད་མཛོད་ནང་དུ་མ་ཚིག་འཚོལ་བཤེར།"
 
-#: app/templates/main/tokens_search_through_fields.html:64
-msgid "* can be used to match partial words, eg."
-msgstr "* can be used to match partial words, eg."
-
-#: app/templates/main/tokens_search_through_fields.html:65
-msgid "! can be used to negate a match, eg."
-msgstr "! can be used to negate a match, eg."
-
-#: app/templates/main/tokens_search_through_fields.html:66
-msgid "| can be used to perform an OR operation, eg."
-msgstr "| can be used to perform an OR operation, eg."
-
 #: app/templates/main/tokens_search_through_fields.html:67
 msgid "To search forms which do not contain"
 msgstr "འདུས་མེད་པའི་འགེངས་ཤོག་འཚོལ་ཕྱིར།"
 
-#: app/templates/main/tokens_similar_to_record.html:28
-msgid "Old Lemma"
-msgstr "མ་ཚིག་རྙིང་བ།"
-
-#: app/templates/main/tokens_similar_to_record.html:29
-msgid "Corr Lemma"
-msgstr "Corr མ་ཚིག"
-
-#: app/templates/main/tokens_similar_to_record.html:32
-msgid "Previous POS"
-msgstr "སྔ་མའི་ POS"
-
-#: app/templates/main/tokens_similar_to_record.html:33
-msgid "Actual POS"
-msgstr "དངོས་ཡོད་ཀྱི་ POS"
-
-#: app/templates/main/tokens_similar_to_record.html:36
-msgid "Previous Morph"
-msgstr "སྔ་མའི་ཚིག་གི་གྲུབ་ཆ།"
-
-#: app/templates/main/tokens_similar_to_record.html:37
-msgid "Actual Morph"
-msgstr "དངོས་ཡོད་ཀྱི་ཚིག་གི་གྲུབ་ཆ།"
-
 #: app/templates/main/tokens_similar_to_record.html:60
 msgid "Similar matching:"
 msgstr "འདྲ་ཤས་ཆེ་བ་རྣམས་ནི།"
@@ -1069,4 +1458,20 @@ msgstr "མ་ཕྱིའི་ token"
 
 #: app/templates/main/tokens_similar_to_token.html:79
 msgid "Similar matching"
-msgstr "འདྲ་ཤས་ཆེ་བ་རྣམས་ནི།"
\ No newline at end of file
+msgstr "འདྲ་ཤས་ཆེ་བ་རྣམས་ནི།"
+
+#~ msgid "Current control list: {{ current_control_lists.name }}"
+#~ msgstr "ད་ལྟའི་སྟངས་འཛིན་ཐོ་གཞུང་ནི། {{ current_control_lists.name }}"
+
+#~ msgid "lemma."
+#~ msgstr "མ་ཚིག"
+
+#~ msgid "morphologies."
+#~ msgstr "ཚིག་གི་གྲུབ་ཆ་རྣམས།"
+
+#~ msgid "POS."
+#~ msgstr "ཚིག་གཤིས།"
+
+#~ msgid "Corpus {{corpus.name}} - List of tokens"
+#~ msgstr "{{corpus.name}} སྐད་མཛོད་ཀྱི་ཚིག་ཐོ།"
+

From 0acb09dd8c90a74de5ee3b9c1452bd74af8c5aeb Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Mon, 24 Jun 2024 17:05:16 +0200
Subject: [PATCH 06/43] =?UTF-8?q?ajout=20filtres=20modifications=20de=20to?=
 =?UTF-8?q?kens=20+=20d=C3=A9but=20affichage=20+=20templates=20chang=C3=A9?=
 =?UTF-8?q?es?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/control_lists/views.py                    |  28 ++++
 app/main/views/corpus.py                      |  13 +-
 app/main/views/tokens.py                      |  33 -----
 app/models/control_lists.py                   |  12 +-
 app/models/corpus.py                          | 134 +++++++++++-------
 .../control_lists/ignore_filter.html          |  43 ++++++
 app/templates/control_lists/macros.html       |  16 ++-
 app/templates/macros/nav_macros.html          |  20 +--
 app/templates/main/corpus_new.html            |  36 ++++-
 9 files changed, 224 insertions(+), 111 deletions(-)
 create mode 100644 app/templates/control_lists/ignore_filter.html

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 55fe917c..091fd9c3 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -374,3 +374,31 @@ def information_edit(control_list_id, control_list):
 def information_read(control_list_id):
     control_list, is_owner = ControlLists.get_linked_or_404(control_list_id=control_list_id, user=current_user)
     return render_template_with_nav_info('control_lists/information_read.html', control_list=control_list)
+
+
+@control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
+@login_required
+def ignore_terms_filter(control_list_id):
+    current_control_list = ControlLists.query.filter_by(**{"id": control_list_id}).first_or_404()
+    list_filter = []
+    if request.method == "POST":
+        list_filter.append(request.form.get("punct"))
+        list_filter.append(request.form.get("numeral"))
+        list_filter.append(request.form.get('ignore'))
+        list_filter.append(request.form.get('metadata'))
+        filtered_filter = []
+        for el in list_filter:
+            if el != None:
+                filtered_filter.append(el)
+        filter = " ".join(filtered_filter)
+        current_control_list.filter_punct = 'punct' in filter
+        current_control_list.filter_metadata = 'metadata' in filter
+        current_control_list.filter_numeral = 'numeral' in filter
+        current_control_list.filter_ignore='ignore' in filter
+
+        db.session.commit()
+
+        flash('The filters have been updated.', 'success')
+        current_control_list = ControlLists.query.filter_by(**{"id": control_list_id}).first_or_404()
+        return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_control_list)
+    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_control_list)
\ No newline at end of file
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index 922907ab..9d08c37f 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -36,7 +36,6 @@ def _get_available():
         lists[cl.str_public].append(cl)
     return lists
 
-
 @main.route('/corpus/new', methods=["POST", "GET"])
 @login_required
 def corpus_new():
@@ -110,6 +109,18 @@ def error():
                 form_kwargs.update({"word_tokens_dict": tokens, "allowed_lemma": allowed_lemma,
                                     "allowed_POS": allowed_POS, "allowed_morph": allowed_morph})
 
+            list_filter = []
+            if request.form.get("ignoreforms"):
+                list_filter.append(request.form.get("punct"))
+                list_filter.append(request.form.get("numeral"))
+                list_filter.append(request.form.get("ignore"))
+                list_filter.append(request.form.get("metadata"))
+                filtered_filter = []
+                for el in list_filter:
+                    if el != None:
+                        filtered_filter.append(el)
+                filter = " ".join(filtered_filter)
+                form_kwargs.update({"filter":filter})
             try:
                 corpus: Corpus = Corpus.create(**form_kwargs)
                 db.session.add(CorpusUser(corpus=corpus, user=current_user, is_owner=True))
diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index d9be6f5f..83f53069 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -67,38 +67,6 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     )
 
 
-@main.route('/corpus/<corpus_id>/tokens/unallowed/<allowed_type>/correct/filter', methods=['POST'])
-@login_required
-@requires_corpus_access("corpus_id")
-def tokens_correct_unallowed_filter(corpus_id, allowed_type):
-    corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
-    list_filter=[]
-    if request.method=="POST":
-        list_filter.append(request.form.get("punct"))
-        list_filter.append(request.form.get("numeral"))
-        list_filter.append(request.form.get("ignore"))
-        list_filter.append(request.form.get("metadata"))
-        filtered_filter=[]
-        for el in list_filter:
-            if el != None:
-                filtered_filter.append(el)
-        filter = " ".join(filtered_filter)
-        print(filter)
-
-
-    tokens = corpus \
-        .get_unallowed(allowed_type, ignore=filter) \
-        .paginate(
-        page=int_or(request.args.get("page"), 1),
-        per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
-    )
-    return render_template_with_nav_info(
-        'main/tokens_correct_unallowed.html',
-        corpus=corpus,
-        tokens=tokens,
-        allowed_type=allowed_type,
-        changed=corpus.changed(tokens.items))
-
 
 @main.route('/corpus/<int:corpus_id>/tokens/changes/similar/<int:record_id>')
 @login_required
@@ -170,7 +138,6 @@ def tokens_correct_single(corpus_id, token_id):
         )
         if "similar" in corpus.displayed_columns_by_name:
             similar = {
-                "count": change_record.similar_remaining,
                 "link": url_for(".tokens_similar_to_record", corpus_id=corpus_id, record_id=change_record.id)
             }
         else:
diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index 80b3d06a..d6412026 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -43,6 +43,10 @@ class ControlLists(db.Model):
     bibliography = db.Column(db.Text, nullable=True)
     language = db.Column(db.String(10), nullable=True)
     notes = db.Column(db.Text, nullable=True)
+    filter_punct = db.Column(db.Boolean, unique=False, default=True)
+    filter_numeral = db.Column(db.Boolean, unique=False, default=False)
+    filter_metadata = db.Column(db.Boolean, unique=False, default=False)
+    filter_ignore = db.Column(db.Boolean, unique=False, default=False)
 
     # For caching purposes, we record the last time these fields were edited
     #last_lemma_edit = db.Column(db.DateTime, default=datetime.datetime.utcnow)
@@ -240,6 +244,7 @@ def has_list(self, allowed_type):
             ).exists()
         ).scalar()
 
+
     @staticmethod
     def add_default_lists(path=None):
         """ Loads the default lists from the config folder
@@ -255,7 +260,7 @@ def add_default_lists(path=None):
             print("[ControlLists] Adding %s " % data["name"])
             cl = ControlLists(**data, public=PublicationStatus.public)
             db.session.add(cl)
-            db.session.flush()  # Get the AutoIncrement ID
+            db.session.flush()  # Get the AutoIncrement ID/home/jjanes
             configs = [
                 ("lemma.txt", AllowedLemma, read_input_lemma),
                 ("POS.txt", AllowedPOS, read_input_POS),
@@ -303,7 +308,7 @@ class AllowedLemma(db.Model):
     )
 
     @staticmethod
-    def add_batch(allowed_values, control_lists_id, _commit=False):
+    def add_batch(allowed_values, control_lists_id, controlList_filter=None, _commit=False):
         """ Add a batch of allowed values
 
         :param allowed_values: List of dictionary with label and readable keys
@@ -346,6 +351,9 @@ def to_input_format(query):
         )
 
 
+
+
+
 class AllowedPOS(db.Model):
     """ An allowed POS is a POS that is accepted
 
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 1c9a5443..f41f3f10 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -7,6 +7,7 @@
 # PIP Packages
 import unidecode
 import sqlalchemy.exc
+import re
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import backref
 from sqlalchemy import func, literal, not_, or_, and_
@@ -297,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, allowed_type="lemma", ignore=False):
+    def get_unallowed(self, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -317,52 +318,60 @@ def get_unallowed(self, allowed_type="lemma", ignore=False):
         else:
             raise ValueError("Get Allowed value had %s and it's not from the lemma, POS, morph set" % allowed_type)
 
-        if ignore:
+        allowed = db.session.query(cls).filter(
+            cls.control_list == self.control_lists_id,
+            cls.label == prop
+        )
+        custom_dict = db.session.query(CorpusCustomDictionary).filter(
+            CorpusCustomDictionary.corpus == self.id,
+            CorpusCustomDictionary.category == allowed_type,
+            CorpusCustomDictionary.label == prop
+        )
+
+        list_darguments = [
+            WordToken.corpus == self.id,
+            not_(allowed.exists()),
+            not_(custom_dict.exists())
+        ]
+
+        current_control_list = ControlLists.query.filter_by(**{"id": self.control_lists_id}).first_or_404()
+
+        allowed = db.session.query(cls).filter(
+            cls.control_list == self.control_lists_id,
+            cls.label == prop
+        )
+        custom_dict = db.session.query(CorpusCustomDictionary).filter(
+            CorpusCustomDictionary.corpus == self.id,
+            CorpusCustomDictionary.category == allowed_type,
+            CorpusCustomDictionary.label == prop
+        )
+
+        list_darguments = [
+            WordToken.corpus == self.id,
+            not_(allowed.exists()),
+            not_(custom_dict.exists())
+        ]
+
+        dict_filter = {'punct': current_control_list.filter_punct,
+                       'metadata': current_control_list.filter_metadata,
+                       'ignore': current_control_list.filter_ignore,
+                       'numeral': current_control_list.filter_numeral}
+
+        if True in dict_filter.values():
             regex_liste = []
-            if "metadata" in ignore:
+            if dict_filter['metadata']:
                 regex_liste.append(r'^(?!\[[^\]]+:[^\]]*\]$).*')
-            if "ignore" in ignore:
+            if dict_filter['ignore']:
                 regex_liste.append(r'^(?!^\[IGNORE\]$)')
-            if "punct" in ignore:
+            if dict_filter['punct']:
                 regex_liste.append(r"(?!^[^\w\s]$).")
-            if "numeral" in ignore:
+            if dict_filter["numeral"]:
                 regex_liste.append(r'(?!^\d+$).+')
-            regex = "".join(regex_liste)
-            allowed = db.session.query(cls).filter(
-                cls.control_list == self.control_lists_id,
-                cls.label == prop
-            )
-            custom_dict = db.session.query(CorpusCustomDictionary).filter(
-                CorpusCustomDictionary.corpus == self.id,
-                CorpusCustomDictionary.category == allowed_type,
-                CorpusCustomDictionary.label == prop
-            )
-            return db.session.query(WordToken).filter(
-                db.and_(
-                    WordToken.corpus == self.id,
-                    not_(allowed.exists()),
-                    not_(custom_dict.exists()),
-                    WordToken.form.op('~')(regex)
-                )
-            ).order_by(WordToken.order_id)
+            list_darguments.append(WordToken.form.op('~')("".join(regex_liste)))
 
-        else:
-            allowed = db.session.query(cls).filter(
-                cls.control_list == self.control_lists_id,
-                cls.label == prop
-            )
-            custom_dict = db.session.query(CorpusCustomDictionary).filter(
-                CorpusCustomDictionary.corpus == self.id,
-                CorpusCustomDictionary.category == allowed_type,
-                CorpusCustomDictionary.label == prop
-            )
-            return db.session.query(WordToken).filter(
-                db.and_(
-                    WordToken.corpus == self.id,
-                    not_(allowed.exists()),
-                    not_(custom_dict.exists())
-                )
-            ).order_by(WordToken.order_id)
+        return db.session.query(WordToken).filter(
+            db.and_(*list_darguments)
+        ).order_by(WordToken.order_id)
 
     @property
     def tokens_count(self):
@@ -425,7 +434,7 @@ def get_history(self, page=1, limit=100):
     def create(
             name, word_tokens_dict,
             allowed_lemma=None, allowed_POS=None, allowed_morph=None,
-            context_left=None, context_right=None, control_list: ControlLists = None,
+            context_left=None, context_right=None, control_list: ControlLists = None, controlList_filter=None,
             delimiter_token=None, columns=None
     ):
         """ Create a corpus
@@ -449,13 +458,22 @@ def create(
                 db.session.flush()
 
                 if allowed_lemma is not None and len(allowed_lemma) > 0:
-                    AllowedLemma.add_batch(allowed_lemma, control_list.id)
+                    if controlList_filter:
+                        AllowedLemma.add_batch(allowed_lemma, control_list.id, controlList_filter)
+                    else:
+                        AllowedLemma.add_batch(allowed_lemma, control_list.id)
 
                 if allowed_POS is not None and len(allowed_POS) > 0:
-                    AllowedPOS.add_batch(allowed_POS, control_list.id)
+                    if controlList_filter:
+                        AllowedPOS.add_batch(allowed_POS, control_list.id, controlList_filter)
+                    else:
+                        AllowedPOS.add_batch(allowed_POS, control_list.id)
 
                 if allowed_morph is not None and len(allowed_morph) > 0:
-                    AllowedMorph.add_batch(allowed_morph, control_list.id)
+                    if controlList_filter:
+                        AllowedMorph.add_batch(allowed_morp, control_list.id, controlList_filter)
+                    else:
+                        AllowedMorph.add_batch(allowed_morph, control_list.id)
 
             c = Corpus(
                 name=name,
@@ -1127,13 +1145,31 @@ def is_valid(lemma, POS, morph, corpus):
         }
 
         allowed_column = corpus.displayed_columns_by_name
-
+        current_control_list = ControlLists.query.filter_by(**{"id": corpus.control_lists_id}).first_or_404()
+        dict_filter ={'punct': current_control_list.filter_punct,
+                      'metadata': current_control_list.filter_metadata,
+                      'ignore':current_control_list.filter_ignore,
+                      'numeral':current_control_list.filter_numeral}
+        print(dict_filter)
+
+        regex_liste = []
+        if True in dict_filter.values():
+            if dict_filter['metadata']:
+                regex_liste.append(r'(\[[^\]]+:[^\]]*\]$)')
+            if dict_filter['ignore']:
+                regex_liste.append(r'(^\[IGNORE\])')
+            if dict_filter['punct']:
+                regex_liste.append(r"(^[^\w\s]$)")
+            if dict_filter['numeral']:
+                regex_liste.append(r'(^\d+$)')
+        regex = "|".join(regex_liste)
         if lemma is not None \
                 and "lemma" in allowed_column \
                 and allowed_lemma.count() > 0 \
                 and corpus.get_allowed_values("lemma", label=lemma).count() == 0:
-            if not corpus.has_custom_dictionary_value("lemma", lemma):
-                statuses["lemma"] = False
+                if not re.match(regex,lemma):
+                    if not corpus.has_custom_dictionary_value("lemma", lemma):
+                        statuses["lemma"] = False
 
         if POS is not None \
                 and "POS" in allowed_column \
@@ -1148,6 +1184,8 @@ def is_valid(lemma, POS, morph, corpus):
                 and corpus.get_allowed_values("morph", label=morph).count() == 0:
             if not corpus.has_custom_dictionary_value("morph", morph):
                 statuses["morph"] = False
+
+
         return statuses
 
     @staticmethod
diff --git a/app/templates/control_lists/ignore_filter.html b/app/templates/control_lists/ignore_filter.html
new file mode 100644
index 00000000..1679168f
--- /dev/null
+++ b/app/templates/control_lists/ignore_filter.html
@@ -0,0 +1,43 @@
+{% extends 'layouts/base.html' %}
+{% import 'macros/form_macros.html' as f %}
+
+{% block content %}
+    <div class="container">
+        <div class="row">
+            <h2>
+                <i class="fa fa-edit mr-2"></i>{{ _('Change the filters for Control List') }}
+            </h2>
+        </div>
+        <div class="row ">
+            <form method="POST">
+        <ul>
+                <li>
+                    <label>
+                        <input type="checkbox" name="punct" value="punct" {% if current_control_list.filter_punct %}checked{% endif %}>
+                        Punctuation
+                    </label>
+                </li>
+            <li>
+                    <label>
+                        <input type="checkbox" name="numeral" value="numeral" {% if current_control_list.filter_numeral %}checked{% endif %}>
+                        Numeral
+                    </label>
+                </li>
+            <li>
+                    <label>
+                        <input type="checkbox" name="metadata" value="metadata" {% if current_control_list.filter_metadata %}checked{% endif %}>
+                        Metadata
+                    </label>
+                </li>
+            <li>
+                    <label>
+                        <input type="checkbox" name="ignore" value="ignore" {% if current_control_list.filter_ignore %}checked{% endif %}>
+                        Ignore
+                    </label>
+                </li>
+        </ul>
+        <button type="submit">Submit</button>
+    </form>
+        </div>
+    </div>
+{% endblock %}
diff --git a/app/templates/control_lists/macros.html b/app/templates/control_lists/macros.html
index 986a4e8d..9da948ac 100644
--- a/app/templates/control_lists/macros.html
+++ b/app/templates/control_lists/macros.html
@@ -13,6 +13,15 @@
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.read_allowed_values", control_list_id=control_list.id, allowed_type='lemma')}}">{{ _('Lemma') }}</a></li>
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.read_allowed_values", control_list_id=control_list.id, allowed_type='POS')}}">{{ _('POS') }}</a></li>
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.read_allowed_values", control_list_id=control_list.id, allowed_type='morph')}}">{{ _('Morphologies') }}</a></li>
+        {%  if control_list.can_edit() or current_user.is_admin() %}
+      <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.rename", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Rename') }}</a></li>
+      {% endif %}
+      {%  if control_list.can_edit() or current_user.is_admin() %}
+      <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.information_edit", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Edit informations') }}</a></li>
+      {% endif %}
+        {%  if control_list.can_edit() or current_user.is_admin() %}
+      <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.ignore_terms_filter", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Ignore values') }}</a></li>
+      {% endif %}
     </ul>
     {{ _('Others') }}
     <ul class="nav flex-column">
@@ -20,11 +29,6 @@
       {%  if control_list.can_edit() and not current_user.is_admin() %}
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.propose_as_public", control_list_id=control_list.id)}}"><i class="fa fa-share-alt"></i> {{ _('Make public') }}</a></li>
       {% endif %}
-      {%  if control_list.can_edit() or current_user.is_admin() %}
-      <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.rename", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Rename') }}</a></li>
-      {% endif %}
-      {%  if control_list.can_edit() or current_user.is_admin() %}
-      <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.information_edit", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Edit informations') }}</a></li>
-      {% endif %}
+
     </ul>
 {% endmacro -%}
\ No newline at end of file
diff --git a/app/templates/macros/nav_macros.html b/app/templates/macros/nav_macros.html
index ebf4c3ed..b1704398 100644
--- a/app/templates/macros/nav_macros.html
+++ b/app/templates/macros/nav_macros.html
@@ -261,25 +261,7 @@
                 </a>
             {% endif %}
         </div>
-        <div>
-             <div class="btn-group btn-group-sm" role="group">
-                  <a class="btn btn-outline-dark title">{{ _('Correction: Ignore Invalide Lemmas: ') }}</a>
-                 {% set url = request.path %}
-                 {% if 'lemma' in request.path %}
-                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='lemma')}}">
-                 {% endif %}
-                 {% if 'morph' in request.path %}
-                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='morph')}}">
-                 {% endif %}
-                 {% if 'POS' in request.path %}
-                 <form class="btn-group btn-outline-dark title" method="post" action="{{url_for("main.tokens_correct_unallowed_filter", corpus_id=corpus.id, allowed_type='POS')}}">
-                 {% endif %}
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="punct" name="punct" title="Allow punctuation as lemma"> Punctuation</a>
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="numeral" name="numeral" title="Allow numerals as lemma"> Numeral</a>
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="ignore" name="ignore" title="Allow lemma [IGNORE]"> Ignore</a>
-                 <a class="btn btn-outline-dark title"><input type="checkbox" value="metadata" name="metadata" title="Allow metadata such as [METADATA:something] and [REF:1.2.3]"> Metadata</a>
-                 <button type="submit"  class="btn btn-outline-dark title">Submit</button>
-                 </form></div></div>
+
     </div></div>
     </div>
     <hr />
diff --git a/app/templates/main/corpus_new.html b/app/templates/main/corpus_new.html
index 51ea0143..400a2b2c 100644
--- a/app/templates/main/corpus_new.html
+++ b/app/templates/main/corpus_new.html
@@ -134,7 +134,9 @@ <h1>{{ _('Create a new corpus') }}</h1>
     </fieldset>
     <fieldset class="form-fieldset">
         <legend>{{ _('Control Lists') }}</legend>
+
         <div class="row">
+
             <div class="col-md-3">
                 <div class="form-check form-check-inline">
                     <input class="form-check-input" id="checkbox_reuse" type="radio" name="control_list" value="reuse" checked/>
@@ -145,6 +147,8 @@ <h1>{{ _('Create a new corpus') }}</h1>
                     the academic community. You will be able to propose new values to the administrators of control lists.') }}
                 </p>
             </div>
+                    <div>
+
             <div class="col-md-9" id="use_public">
               <select class="form-control" id="control_list_select" name="control_list_select">
                   {%- for class, cls_ in public_control_lists.items() %}
@@ -195,6 +199,7 @@ <h1>{{ _('Create a new corpus') }}</h1>
                         </div>
                     </div>
                 </div>
+
                 <div class="form-group">
                     <label for="allowed_lemma">{{ _('Allowed lemma') }}</label>
                     <small id="allowed_lemma_help" class="form-text text-muted">{{ _('This should be formatted as a list of lemma separated by new line') }}</small>
@@ -211,10 +216,37 @@ <h1>{{ _('Create a new corpus') }}</h1>
                     <textarea aria-describedby="allowed_morph_help" class="form-control" id="allowed_morph" name="allowed_morph">{% if allowed_morph %}{{allowed_morph}}{%endif%}</textarea>
                 </div>
             </div>
+
+
+        </div>
+<div class="form-group row">
+            <div class="col">
+                <label for="ignoreforms" id="ignoreforms" class="form-text text-muted">
+                    Ignore Elements in Control List:
+                </label>
+            </div>
+            <div class="col-md-6">
+              <div class="form-check">
+                <input type="checkbox" class="form-check-input" id="Numeral">
+                <label class="form-check-label" for="hyphens-remove">{{ _('Remove Numeral') }}</label>
+              </div>
+              <div class="form-check">
+                <input type="checkbox" class="form-check-input" id="Punctuation">
+                <label class="form-check-label" for="punct-keep">{{ _('Remove Punctuation') }}</label>
+              </div>
+                <div class="form-check">
+                <input type="checkbox" class="form-check-input" id="Metadata">
+                <label class="form-check-label" for="punct-keep">{{ _('Remove Metadata such as [METADATA:something] or [REF:1.2.3]') }}</label>
+              </div>
+                <div class="form-check">
+                <input type="checkbox" class="form-check-input" id="ignore_ignore">
+                <label class="form-check-label" for="punct-keep">{{ _('Remove [IGNORE]') }}</label>
+              </div>
+            </div>
+        </div>
         </div>
     </fieldset>
-    <button type="submit" id="submit" class="btn btn-primary">{{ _('Submit') }}</button>
-</form>
+<button type="submit" id="submit" class="btn btn-primary">Submit</button>
 <script type="text/javascript">
 
 $(document).ready(function() {

From 04d080c7f934b7fd7b50b607a1279cac04cc863d Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Mon, 24 Jun 2024 17:17:32 +0200
Subject: [PATCH 07/43] suppression anciennes modifications

---
 app/models/control_lists.py |  2 +-
 app/models/corpus.py        | 17 ++++-------------
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index d6412026..023bfcd5 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -308,7 +308,7 @@ class AllowedLemma(db.Model):
     )
 
     @staticmethod
-    def add_batch(allowed_values, control_lists_id, controlList_filter=None, _commit=False):
+    def add_batch(allowed_values, control_lists_id, _commit=False):
         """ Add a batch of allowed values
 
         :param allowed_values: List of dictionary with label and readable keys
diff --git a/app/models/corpus.py b/app/models/corpus.py
index f41f3f10..62ed2c62 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -434,7 +434,7 @@ def get_history(self, page=1, limit=100):
     def create(
             name, word_tokens_dict,
             allowed_lemma=None, allowed_POS=None, allowed_morph=None,
-            context_left=None, context_right=None, control_list: ControlLists = None, controlList_filter=None,
+            context_left=None, context_right=None, control_list: ControlLists = None,
             delimiter_token=None, columns=None
     ):
         """ Create a corpus
@@ -458,22 +458,13 @@ def create(
                 db.session.flush()
 
                 if allowed_lemma is not None and len(allowed_lemma) > 0:
-                    if controlList_filter:
-                        AllowedLemma.add_batch(allowed_lemma, control_list.id, controlList_filter)
-                    else:
-                        AllowedLemma.add_batch(allowed_lemma, control_list.id)
+                    AllowedLemma.add_batch(allowed_lemma, control_list.id)
 
                 if allowed_POS is not None and len(allowed_POS) > 0:
-                    if controlList_filter:
-                        AllowedPOS.add_batch(allowed_POS, control_list.id, controlList_filter)
-                    else:
-                        AllowedPOS.add_batch(allowed_POS, control_list.id)
+                    AllowedPOS.add_batch(allowed_POS, control_list.id)
 
                 if allowed_morph is not None and len(allowed_morph) > 0:
-                    if controlList_filter:
-                        AllowedMorph.add_batch(allowed_morp, control_list.id, controlList_filter)
-                    else:
-                        AllowedMorph.add_batch(allowed_morph, control_list.id)
+                    AllowedMorph.add_batch(allowed_morph, control_list.id)
 
             c = Corpus(
                 name=name,

From 61f595757790ede394b00fbd2afced3901dd7d8d Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 25 Jun 2024 18:11:49 +0200
Subject: [PATCH 08/43] ajout filtres token invalid + lien corpus

---
 app/control_lists/views.py                    | 21 ++++---
 app/main/views/corpus.py                      | 31 +++++++----
 app/main/views/tokens.py                      |  2 +-
 app/models/control_lists.py                   |  9 +--
 app/models/corpus.py                          | 55 +++++++------------
 .../control_lists/ignore_filter.html          | 12 ++--
 app/templates/control_lists/macros.html       |  2 -
 app/templates/main/corpus_new.html            | 46 ++++++++++------
 8 files changed, 91 insertions(+), 87 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 091fd9c3..3e5f9dae 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -7,7 +7,7 @@
 
 
 from app.main.views.utils import render_template_with_nav_info
-from app.models import ControlLists, AllowedLemma, WordToken, User, PublicationStatus, CorpusCustomDictionary
+from app.models import ControlLists, ControlListsUser, AllowedLemma, WordToken, User, PublicationStatus, CorpusCustomDictionary
 from app import db, email
 from ..utils import PyrrhaError
 from ..utils.forms import strip_or_none
@@ -379,7 +379,8 @@ def information_read(control_list_id):
 @control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
 @login_required
 def ignore_terms_filter(control_list_id):
-    current_control_list = ControlLists.query.filter_by(**{"id": control_list_id}).first_or_404()
+    current_controlListUser = ControlListsUser.query.filter_by(**{"control_lists_id":control_list_id,"user_id":current_user.id}).first_or_404()
+    print(current_controlListUser)
     list_filter = []
     if request.method == "POST":
         list_filter.append(request.form.get("punct"))
@@ -391,14 +392,16 @@ def ignore_terms_filter(control_list_id):
             if el != None:
                 filtered_filter.append(el)
         filter = " ".join(filtered_filter)
-        current_control_list.filter_punct = 'punct' in filter
-        current_control_list.filter_metadata = 'metadata' in filter
-        current_control_list.filter_numeral = 'numeral' in filter
-        current_control_list.filter_ignore='ignore' in filter
+        current_controlListUser.filter_punct = 'punct' in filter
+        current_controlListUser.filter_metadata = 'metadata' in filter
+        current_controlListUser.filter_numeral = 'numeral' in filter
+        current_controlListUser.filter_ignore = 'ignore' in filter
 
         db.session.commit()
 
         flash('The filters have been updated.', 'success')
-        current_control_list = ControlLists.query.filter_by(**{"id": control_list_id}).first_or_404()
-        return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_control_list)
-    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_control_list)
\ No newline at end of file
+        current_controlListUser = ControlListsUser.query.filter_by(**{"control_lists_id":control_list_id,"user_id":current_user.id}).first_or_404()
+        return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
+                                             current_control_list=current_controlListUser)
+
+    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)
\ No newline at end of file
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index 9d08c37f..b4cf4098 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -110,23 +110,31 @@ def error():
                                     "allowed_POS": allowed_POS, "allowed_morph": allowed_morph})
 
             list_filter = []
-            if request.form.get("ignoreforms"):
-                list_filter.append(request.form.get("punct"))
-                list_filter.append(request.form.get("numeral"))
-                list_filter.append(request.form.get("ignore"))
-                list_filter.append(request.form.get("metadata"))
-                filtered_filter = []
-                for el in list_filter:
-                    if el != None:
-                        filtered_filter.append(el)
-                filter = " ".join(filtered_filter)
-                form_kwargs.update({"filter":filter})
+            list_filter.append(request.form.get("punct"))
+            list_filter.append(request.form.get("numeral"))
+            list_filter.append(request.form.get("ignore"))
+            list_filter.append(request.form.get("metadata"))
+            filtered_filter = []
+            for el in list_filter:
+                if el != None:
+                    filtered_filter.append(el)
+            filter = " ".join(filtered_filter)
+            print("FILTER "+str(filter))
+
             try:
                 corpus: Corpus = Corpus.create(**form_kwargs)
                 db.session.add(CorpusUser(corpus=corpus, user=current_user, is_owner=True))
                 # Add a link to the control list
                 ControlLists.link(corpus.control_lists_id, current_user.id, is_owner=cl_owner)
                 db.session.commit()
+                current_controlListUser = ControlListsUser.query.filter_by(
+                    **{"control_lists_id": corpus.control_lists_id, "user_id": current_user.id}).first_or_404()
+                print(current_controlListUser)
+                current_controlListUser.filter_punct = 'punct' in filter
+                current_controlListUser.filter_metadata = 'metadata' in filter
+                current_controlListUser.filter_numeral = 'numeral' in filter
+                current_controlListUser.filter_ignore = 'ignore' in filter
+                db.session.commit()
                 flash("New corpus registered", category="success")
             except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
                 db.session.rollback()
@@ -154,6 +162,7 @@ def error():
                 flash(exception, category="error")
                 return error()
             except Exception as e:
+                print(e)
                 db.session.rollback()
                 flash("The corpus cannot be registered. Check your data", category="error")
                 return error()
diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index 83f53069..42065e9b 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -53,7 +53,7 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     """
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
     tokens = corpus\
-        .get_unallowed(allowed_type)\
+        .get_unallowed(current_user, allowed_type)\
         .paginate(
             page=int_or(request.args.get("page"), 1),
             per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index 023bfcd5..49ee3217 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -43,10 +43,6 @@ class ControlLists(db.Model):
     bibliography = db.Column(db.Text, nullable=True)
     language = db.Column(db.String(10), nullable=True)
     notes = db.Column(db.Text, nullable=True)
-    filter_punct = db.Column(db.Boolean, unique=False, default=True)
-    filter_numeral = db.Column(db.Boolean, unique=False, default=False)
-    filter_metadata = db.Column(db.Boolean, unique=False, default=False)
-    filter_ignore = db.Column(db.Boolean, unique=False, default=False)
 
     # For caching purposes, we record the last time these fields were edited
     #last_lemma_edit = db.Column(db.DateTime, default=datetime.datetime.utcnow)
@@ -286,6 +282,11 @@ class ControlListsUser(db.Model):
     user_id = db.Column(db.Integer, db.ForeignKey(User.id), primary_key=True)
     is_owner = db.Column(db.Boolean, default=False)
 
+    filter_punct = db.Column(db.Boolean, unique=False, default=False)
+    filter_numeral = db.Column(db.Boolean, unique=False, default=False)
+    filter_metadata = db.Column(db.Boolean, unique=False, default=False)
+    filter_ignore = db.Column(db.Boolean, unique=False, default=False)
+
     control = db.relationship("ControlLists", backref=backref("control_lists_user", cascade="all, delete-orphan"))
     user = db.relationship(User, backref=backref("control_lists_user", cascade="all, delete-orphan"))
 
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 62ed2c62..d2d60090 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -24,7 +24,7 @@
 
 # Models
 from .user import User
-from .control_lists import ControlLists, AllowedPOS, AllowedMorph, AllowedLemma, PublicationStatus
+from .control_lists import ControlLists, ControlListsUser, AllowedPOS, AllowedMorph, AllowedLemma, PublicationStatus
 
 
 from collections import namedtuple
@@ -298,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, allowed_type="lemma"):
+    def get_unallowed(self, current_user, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -334,29 +334,12 @@ def get_unallowed(self, allowed_type="lemma"):
             not_(custom_dict.exists())
         ]
 
-        current_control_list = ControlLists.query.filter_by(**{"id": self.control_lists_id}).first_or_404()
-
-        allowed = db.session.query(cls).filter(
-            cls.control_list == self.control_lists_id,
-            cls.label == prop
-        )
-        custom_dict = db.session.query(CorpusCustomDictionary).filter(
-            CorpusCustomDictionary.corpus == self.id,
-            CorpusCustomDictionary.category == allowed_type,
-            CorpusCustomDictionary.label == prop
-        )
-
-        list_darguments = [
-            WordToken.corpus == self.id,
-            not_(allowed.exists()),
-            not_(custom_dict.exists())
-        ]
-
-        dict_filter = {'punct': current_control_list.filter_punct,
-                       'metadata': current_control_list.filter_metadata,
-                       'ignore': current_control_list.filter_ignore,
-                       'numeral': current_control_list.filter_numeral}
-
+        current_controlListUser = ControlListsUser.query.filter_by(
+            **{"control_lists_id":self.control_lists_id, "user_id": current_user.id}).first_or_404()
+        dict_filter = {'punct': current_controlListUser.filter_punct,
+                       'metadata': current_controlListUser.filter_metadata,
+                       'ignore': current_controlListUser.filter_ignore,
+                       'numeral': current_controlListUser.filter_numeral}
         if True in dict_filter.values():
             regex_liste = []
             if dict_filter['metadata']:
@@ -364,9 +347,9 @@ def get_unallowed(self, allowed_type="lemma"):
             if dict_filter['ignore']:
                 regex_liste.append(r'^(?!^\[IGNORE\]$)')
             if dict_filter['punct']:
-                regex_liste.append(r"(?!^[^\w\s]$).")
+                regex_liste.append(r"((?!^[^\w\s]$).)")
             if dict_filter["numeral"]:
-                regex_liste.append(r'(?!^\d+$).+')
+                regex_liste.append(r'(^(?!\d+$).+)')
             list_darguments.append(WordToken.form.op('~')("".join(regex_liste)))
 
         return db.session.query(WordToken).filter(
@@ -476,6 +459,8 @@ def create(
             )
             db.session.add(c)
             db.session.commit()
+
+
         except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
             db.session.rollback()
             raise e
@@ -1111,7 +1096,7 @@ def get_like(filter_id, form, group_by, type_like="lemma", allowed_list=False):
         return query
 
     @staticmethod
-    def is_valid(lemma, POS, morph, corpus):
+    def is_valid(lemma, POS, morph, corpus, user_id):
         """ Check if a token is valid for a given corpus
 
         :param lemma: Lemma value of the token to validate
@@ -1136,12 +1121,12 @@ def is_valid(lemma, POS, morph, corpus):
         }
 
         allowed_column = corpus.displayed_columns_by_name
-        current_control_list = ControlLists.query.filter_by(**{"id": corpus.control_lists_id}).first_or_404()
-        dict_filter ={'punct': current_control_list.filter_punct,
-                      'metadata': current_control_list.filter_metadata,
-                      'ignore':current_control_list.filter_ignore,
-                      'numeral':current_control_list.filter_numeral}
-        print(dict_filter)
+        current_controlListUser = ControlListsUser.query.filter_by(
+            **{"control_lists_id": corpus.control_lists_id, "user_id": user_id}).first_or_404()
+        dict_filter = {'punct': current_controlListUser.filter_punct,
+                       'metadata': current_controlListUser.filter_metadata,
+                       'ignore': current_controlListUser.filter_ignore,
+                       'numeral': current_controlListUser.filter_numeral}
 
         regex_liste = []
         if True in dict_filter.values():
@@ -1366,7 +1351,7 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
             raise error
 
         # Check if values are correct regarding allowed values
-        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus)
+        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id)
         if False in list(validity.values()):
             error_msg = "Invalid value in {}".format(
                 ", ".join([key for key in validity.keys() if validity[key] is False])
diff --git a/app/templates/control_lists/ignore_filter.html b/app/templates/control_lists/ignore_filter.html
index 1679168f..83d789a7 100644
--- a/app/templates/control_lists/ignore_filter.html
+++ b/app/templates/control_lists/ignore_filter.html
@@ -1,13 +1,11 @@
 {% extends 'layouts/base.html' %}
-{% import 'macros/form_macros.html' as f %}
+{% import 'macros/nav_macros.html' as nav %}
+
+{% block page_title %}Control List {{current_control_list.name}}{% endblock %}
 
 {% block content %}
-    <div class="container">
-        <div class="row">
-            <h2>
-                <i class="fa fa-edit mr-2"></i>{{ _('Change the filters for Control List') }}
-            </h2>
-        </div>
+<h1>{{current_control_list.name}}</h1>
+<h2>{{ _('Change the filters for Control List') }}</h2>
         <div class="row ">
             <form method="POST">
         <ul>
diff --git a/app/templates/control_lists/macros.html b/app/templates/control_lists/macros.html
index 9da948ac..24d78e1a 100644
--- a/app/templates/control_lists/macros.html
+++ b/app/templates/control_lists/macros.html
@@ -19,9 +19,7 @@
       {%  if control_list.can_edit() or current_user.is_admin() %}
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.information_edit", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Edit informations') }}</a></li>
       {% endif %}
-        {%  if control_list.can_edit() or current_user.is_admin() %}
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.ignore_terms_filter", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Ignore values') }}</a></li>
-      {% endif %}
     </ul>
     {{ _('Others') }}
     <ul class="nav flex-column">
diff --git a/app/templates/main/corpus_new.html b/app/templates/main/corpus_new.html
index 400a2b2c..5080d6a7 100644
--- a/app/templates/main/corpus_new.html
+++ b/app/templates/main/corpus_new.html
@@ -225,24 +225,34 @@ <h1>{{ _('Create a new corpus') }}</h1>
                     Ignore Elements in Control List:
                 </label>
             </div>
-            <div class="col-md-6">
-              <div class="form-check">
-                <input type="checkbox" class="form-check-input" id="Numeral">
-                <label class="form-check-label" for="hyphens-remove">{{ _('Remove Numeral') }}</label>
-              </div>
-              <div class="form-check">
-                <input type="checkbox" class="form-check-input" id="Punctuation">
-                <label class="form-check-label" for="punct-keep">{{ _('Remove Punctuation') }}</label>
-              </div>
-                <div class="form-check">
-                <input type="checkbox" class="form-check-input" id="Metadata">
-                <label class="form-check-label" for="punct-keep">{{ _('Remove Metadata such as [METADATA:something] or [REF:1.2.3]') }}</label>
-              </div>
-                <div class="form-check">
-                <input type="checkbox" class="form-check-input" id="ignore_ignore">
-                <label class="form-check-label" for="punct-keep">{{ _('Remove [IGNORE]') }}</label>
-              </div>
-            </div>
+    <div class="col">
+            <ul>
+                <li>
+                    <label>
+                        <input type="checkbox" name="punct" value="punct">
+                        Punctuation
+                    </label>
+                </li>
+            <li>
+                    <label>
+                        <input type="checkbox" name="numeral" value="numeral">
+                        Numeral
+                    </label>
+                </li>
+            <li>
+                    <label>
+                        <input type="checkbox" name="metadata" value="metadata">
+                        Metadata
+                    </label>
+                </li>
+            <li>
+                    <label>
+                        <input type="checkbox" name="ignore" value="ignore">
+                        Ignore
+                    </label>
+                </li>
+        </ul>
+        </div>
         </div>
         </div>
     </fieldset>

From 9d5844f6cb5e52bcf12f403ad10a6c356c9c87e8 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 26 Jun 2024 16:28:55 +0200
Subject: [PATCH 09/43] correction Ignore values des listes de menus dans
 Control Lists

---
 tests/test_selenium/test_control_lists.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_selenium/test_control_lists.py b/tests/test_selenium/test_control_lists.py
index 68f6b56f..d352b249 100644
--- a/tests/test_selenium/test_control_lists.py
+++ b/tests/test_selenium/test_control_lists.py
@@ -54,7 +54,8 @@ def test_action_as_owner(self):
                 'POS',
                 'Propose changes',
                 'Wauchier',
-                'Rename'
+                'Rename',
+                "Ignore values"
             ]),
             "Full rewrite are limited to control lists users"
         )
@@ -183,7 +184,7 @@ def test_action_as_admin_but_not_owner(self):
         self.assertEqual(
             sorted([link.text.strip() for link in links]),
             sorted(["Guidelines",
-                    "Edit informations", 'Lemma', 'Morphologies', 'POS', 'Propose changes', 'Rename', 'Wauchier']),
+                    "Edit informations", 'Lemma', 'Morphologies', 'POS', 'Propose changes', 'Rename', 'Wauchier', 'Ignore values']),
             "Full rewrite are limited to control lists users"
         )
 
@@ -274,7 +275,7 @@ def test_action_as_user(self):
         self.assertEqual(
             sorted([link.text.strip() for link in links]),
             sorted(['Lemma', "Guidelines",
-                    'Morphologies', 'POS', 'Propose changes', 'Wauchier']),
+                    'Morphologies', 'POS', 'Propose changes', 'Wauchier', 'Ignore values']),
             "Only contacting and reading is possible to users"
         )
 

From f5517e9c4459e291eeec69fd66e0c4d68b8fad98 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Thu, 27 Jun 2024 12:43:53 +0200
Subject: [PATCH 10/43] correction tests corpus_init//fonction get_unallowed

---
 app/main/views/tokens.py                | 3 ++-
 app/models/corpus.py                    | 7 ++++---
 tests/test_models/test_record.py        | 4 ++--
 tests/test_selenium/test_corpus_init.py | 6 +++---
 tests/test_selenium/testexample.csv     | 2 ++
 5 files changed, 13 insertions(+), 9 deletions(-)
 create mode 100644 tests/test_selenium/testexample.csv

diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index 42065e9b..f0563432 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -52,8 +52,9 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     :param allowed_type: Type of allowed value to check agains (lemma, POS, morph)
     """
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
+    user_id = current_user.id
     tokens = corpus\
-        .get_unallowed(current_user, allowed_type)\
+        .get_unallowed(user_id, corpus_id, allowed_type)\
         .paginate(
             page=int_or(request.args.get("page"), 1),
             per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
diff --git a/app/models/corpus.py b/app/models/corpus.py
index d2d60090..ff37a792 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -298,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, current_user, allowed_type="lemma"):
+    def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -333,9 +333,9 @@ def get_unallowed(self, current_user, allowed_type="lemma"):
             not_(allowed.exists()),
             not_(custom_dict.exists())
         ]
-
+        current_corpus = Corpus.query.filter_by(**{"id":corpus_id}).first_or_404()
         current_controlListUser = ControlListsUser.query.filter_by(
-            **{"control_lists_id":self.control_lists_id, "user_id": current_user.id}).first_or_404()
+            **{"control_lists_id":current_corpus.control_lists_id, "user_id": user_id}).first_or_404()
         dict_filter = {'punct': current_controlListUser.filter_punct,
                        'metadata': current_controlListUser.filter_metadata,
                        'ignore': current_controlListUser.filter_ignore,
@@ -1121,6 +1121,7 @@ def is_valid(lemma, POS, morph, corpus, user_id):
         }
 
         allowed_column = corpus.displayed_columns_by_name
+
         current_controlListUser = ControlListsUser.query.filter_by(
             **{"control_lists_id": corpus.control_lists_id, "user_id": user_id}).first_or_404()
         dict_filter = {'punct': current_controlListUser.filter_punct,
diff --git a/tests/test_models/test_record.py b/tests/test_models/test_record.py
index bd658706..a83996ed 100644
--- a/tests/test_models/test_record.py
+++ b/tests/test_models/test_record.py
@@ -91,8 +91,8 @@ def test_similar_lemma_double_change(self):
         token, change_record = WordToken.update(
             user_id=1,
             token_id=1, corpus_id=1,
-            lemma="cil", morph="smn", POS="u"
-        )
+            lemma="cil", morph="smn", POS="u")
+
         self.assertEqual(
             (token.lemma, token.morph, token.POS),
             ("cil", "smn", "u"),
diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 82575100..00f347d4 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -104,7 +104,7 @@ def test_registration_with_full_allowed_lemma(self):
         self.assertEqual(allowed.count(), 21, "There should be 21 allowed token")
 
         # Checking the model
-        self.assertEqual(corpus.get_unallowed("lemma").count(), 0, "There should be no unallowed value")
+        self.assertEqual(corpus.get_unallowed(user_id = 1, corpus_id = 1, allowed_type="lemma").count(), 0, "There should be no unallowed value")
 
     def test_registration_with_partial_allowed_lemma(self):
         """
@@ -151,7 +151,7 @@ def test_registration_with_partial_allowed_lemma(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed("lemma").count(), 22,
+            corpus.get_unallowed(user_id=1, corpus_id=1, allowed_type="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 
@@ -197,7 +197,7 @@ def test_registration_with_allowed_morph(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed("lemma").count(), 22,
+            corpus.get_unallowed(user_id=1, corpus_id=1, allowed_lemma="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 
diff --git a/tests/test_selenium/testexample.csv b/tests/test_selenium/testexample.csv
new file mode 100644
index 00000000..a9ca7664
--- /dev/null
+++ b/tests/test_selenium/testexample.csv
@@ -0,0 +1,2 @@
+form	lemma	POS	morph
+SOIGNORS	seignor	NOMcom	NOMB.=p|GENRE=m|CAS=n

From b5a2355e35e88de5c81799b5daee64165e783926 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Fri, 28 Jun 2024 12:41:19 +0200
Subject: [PATCH 11/43] correction erreurs tests - bug new corpus

---
 app/control_lists/views.py              |  4 +-
 app/main/views/corpus.py                |  3 +-
 app/main/views/tokens.py                |  3 +-
 app/models/corpus.py                    | 52 ++++++++++++++-----------
 tests/test_models/test_record.py        |  4 +-
 tests/test_selenium/base.py             | 10 +++--
 tests/test_selenium/test_corpus_init.py |  4 +-
 7 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 3e5f9dae..c122e344 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -376,7 +376,7 @@ def information_read(control_list_id):
     return render_template_with_nav_info('control_lists/information_read.html', control_list=control_list)
 
 
-@control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
+"""@control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
 @login_required
 def ignore_terms_filter(control_list_id):
     current_controlListUser = ControlListsUser.query.filter_by(**{"control_lists_id":control_list_id,"user_id":current_user.id}).first_or_404()
@@ -404,4 +404,4 @@ def ignore_terms_filter(control_list_id):
         return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
                                              current_control_list=current_controlListUser)
 
-    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)
\ No newline at end of file
+    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)"""
\ No newline at end of file
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index b4cf4098..a932d45c 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -119,7 +119,6 @@ def error():
                 if el != None:
                     filtered_filter.append(el)
             filter = " ".join(filtered_filter)
-            print("FILTER "+str(filter))
 
             try:
                 corpus: Corpus = Corpus.create(**form_kwargs)
@@ -129,7 +128,6 @@ def error():
                 db.session.commit()
                 current_controlListUser = ControlListsUser.query.filter_by(
                     **{"control_lists_id": corpus.control_lists_id, "user_id": current_user.id}).first_or_404()
-                print(current_controlListUser)
                 current_controlListUser.filter_punct = 'punct' in filter
                 current_controlListUser.filter_metadata = 'metadata' in filter
                 current_controlListUser.filter_numeral = 'numeral' in filter
@@ -137,6 +135,7 @@ def error():
                 db.session.commit()
                 flash("New corpus registered", category="success")
             except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
+                print(e)
                 db.session.rollback()
                 flash("The corpus cannot be registered. Check your data", category="error")
                 flash(str(e.orig).lower())
diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index f0563432..5c896c32 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -135,10 +135,11 @@ def tokens_correct_single(corpus_id, token_id):
             token_id=token_id, corpus_id=corpus_id,
             lemma=string_to_none(request.form.get("lemma")),
             POS=string_to_none(request.form.get("POS")),
-            morph=string_to_none(request.form.get("morph"))
+            morph=string_to_none(request.form.get("morph")),
         )
         if "similar" in corpus.displayed_columns_by_name:
             similar = {
+                "count": change_record.similar_remaining,
                 "link": url_for(".tokens_similar_to_record", corpus_id=corpus_id, record_id=change_record.id)
             }
         else:
diff --git a/app/models/corpus.py b/app/models/corpus.py
index ff37a792..3317a3ac 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -333,6 +333,7 @@ def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
             not_(allowed.exists()),
             not_(custom_dict.exists())
         ]
+
         current_corpus = Corpus.query.filter_by(**{"id":corpus_id}).first_or_404()
         current_controlListUser = ControlListsUser.query.filter_by(
             **{"control_lists_id":current_corpus.control_lists_id, "user_id": user_id}).first_or_404()
@@ -1096,7 +1097,7 @@ def get_like(filter_id, form, group_by, type_like="lemma", allowed_list=False):
         return query
 
     @staticmethod
-    def is_valid(lemma, POS, morph, corpus, user_id):
+    def is_valid(lemma, POS, morph, corpus, user_id, filter):
         """ Check if a token is valid for a given corpus
 
         :param lemma: Lemma value of the token to validate
@@ -1121,32 +1122,39 @@ def is_valid(lemma, POS, morph, corpus, user_id):
         }
 
         allowed_column = corpus.displayed_columns_by_name
+        if filter:
+            current_controlListUser = ControlListsUser.query.filter_by(
+                **{"control_lists_id": corpus.control_lists_id, "user_id": user_id}).first_or_404()
+            dict_filter = {'punct': current_controlListUser.filter_punct,
+                           'metadata': current_controlListUser.filter_metadata,
+                           'ignore': current_controlListUser.filter_ignore,
+                           'numeral': current_controlListUser.filter_numeral}
+            regex_liste = []
+            if True in dict_filter.values():
+                if dict_filter['metadata']:
+                    regex_liste.append(r'(\[[^\]]+:[^\]]*\]$)')
+                if dict_filter['ignore']:
+                    regex_liste.append(r'(^\[IGNORE\])')
+                if dict_filter['punct']:
+                    regex_liste.append(r"(^[^\w\s]$)")
+                if dict_filter['numeral']:
+                    regex_liste.append(r'(^\d+$)')
+            regex = "|".join(regex_liste)
 
-        current_controlListUser = ControlListsUser.query.filter_by(
-            **{"control_lists_id": corpus.control_lists_id, "user_id": user_id}).first_or_404()
-        dict_filter = {'punct': current_controlListUser.filter_punct,
-                       'metadata': current_controlListUser.filter_metadata,
-                       'ignore': current_controlListUser.filter_ignore,
-                       'numeral': current_controlListUser.filter_numeral}
-
-        regex_liste = []
-        if True in dict_filter.values():
-            if dict_filter['metadata']:
-                regex_liste.append(r'(\[[^\]]+:[^\]]*\]$)')
-            if dict_filter['ignore']:
-                regex_liste.append(r'(^\[IGNORE\])')
-            if dict_filter['punct']:
-                regex_liste.append(r"(^[^\w\s]$)")
-            if dict_filter['numeral']:
-                regex_liste.append(r'(^\d+$)')
-        regex = "|".join(regex_liste)
         if lemma is not None \
                 and "lemma" in allowed_column \
                 and allowed_lemma.count() > 0 \
                 and corpus.get_allowed_values("lemma", label=lemma).count() == 0:
-                if not re.match(regex,lemma):
+            if filter:
+                if not re.match(regex, lemma):
                     if not corpus.has_custom_dictionary_value("lemma", lemma):
                         statuses["lemma"] = False
+                    else:
+                        if not corpus.has_custom_dictionary_value("lemma", lemma):
+                            statuses["lemma"] = False
+            else:
+                if not corpus.has_custom_dictionary_value("lemma", lemma):
+                    statuses["lemma"] = False
 
         if POS is not None \
                 and "POS" in allowed_column \
@@ -1319,7 +1327,7 @@ def to_input_format(query):
         return csv_file.getvalue()
 
     @staticmethod
-    def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
+    def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None, filter=False):
         """ Update a given token with lemma, POS and morph value
 
         :param user_id: ID of the user who performs the update
@@ -1352,7 +1360,7 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
             raise error
 
         # Check if values are correct regarding allowed values
-        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id)
+        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id, filter=filter)
         if False in list(validity.values()):
             error_msg = "Invalid value in {}".format(
                 ", ".join([key for key in validity.keys() if validity[key] is False])
diff --git a/tests/test_models/test_record.py b/tests/test_models/test_record.py
index a83996ed..5d0695d8 100644
--- a/tests/test_models/test_record.py
+++ b/tests/test_models/test_record.py
@@ -1,4 +1,4 @@
-from app.models import ChangeRecord, WordToken, Corpus, ControlLists
+from app.models import ChangeRecord, WordToken, Corpus, ControlLists, ControlListsUser, CorpusUser
 from .base import TestModels
 import copy
 
@@ -14,6 +14,7 @@
     WordToken(corpus=1, form="Cil", lemma="cel", left_context="_", right_context="_", label_uniform="cel", morph="smn", POS="p"),      # 6
     WordToken(corpus=1, form="Cil", lemma="cel", left_context="_", right_context="_", label_uniform="cel", morph="smn", POS="p"),      # 7
     WordToken(corpus=1, form="Cil", lemma="cel", left_context="_", right_context="_", label_uniform="cel", morph="smn", POS="p"),      # 8
+
 ]
 
 
@@ -57,6 +58,7 @@ def tok_with_id(self, list_of_tokens, _id):
     def test_similar_lemma_single_change(self):
         """ Ensure only similar features are fixed """
         self.load_fixtures()
+
         token, change_record = WordToken.update(
             user_id=1,
             token_id=1, corpus_id=1,
diff --git a/tests/test_selenium/base.py b/tests/test_selenium/base.py
index 19fbe2cc..60156cf0 100644
--- a/tests/test_selenium/base.py
+++ b/tests/test_selenium/base.py
@@ -564,7 +564,7 @@ def edith_nth_row_value(
         if go_to_edit_token_page is None:
             go_to_edit_token_page = self.go_to_edit_token_page(corpus_id)
         go_to_edit_token_page()
-
+        self.driver.save_screenshot("./token_correct_1.png")
         if additional_action_before is not None:
             additional_action_before()
 
@@ -583,14 +583,16 @@ def edith_nth_row_value(
             td = self.element_find_element_by_class_name(row, "token_morph")
         else:
             td = self.element_find_element_by_class_name(row, "token_lemma")
-
+        self.driver.save_screenshot("./token_correct_2.png")
         # Click, clear the td and send a new value
         td.click()
         td.clear()
+        self.driver.save_screenshot("./token_correct_25.png")
         td.send_keys(value)
-
+        self.driver.save_screenshot("./token_correct_3.png")
         # Save
         self.element_find_element_by_css_selector(row, "a.save").click()
+        self.driver.save_screenshot("./token_correct_3.png")
         # It's safer to wait for the AJAX call to be completed
         row = self.driver_find_element_by_id("token_" + id_row + "_row")
 
@@ -631,10 +633,10 @@ def test_edit_token(self):
         self.addCorpus(with_token=True, tokens_up_to=24)
         self.driver.refresh()
         token, status_text, row = self.edith_nth_row_value("un", corpus_id=self.CORPUS_ID)
+        self.driver.save_screenshot('./token_correct_4.png')
         self.assertEqual(token.lemma, "un", "Lemma should have been changed")
         self.assertEqual(status_text, "Save")
         self.assert_saved(row)
-
         self.assertIn("table-changed", row.get_attribute("class"))
         self.driver.refresh()
         row = self.driver_find_element_by_id("token_1_row")
diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 00f347d4..9c40d182 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -27,9 +27,11 @@ def test_registration(self):
         self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
         self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
         self.driver_find_element_by_id("label_checkbox_create").click()
+        self.driver.save_screenshot("./test_registration.png")
         self.driver_find_element_by_id("submit").click()
 
         self.driver.implicitly_wait(15)
+        self.driver.save_screenshot("./test_registration1.png")
         self.assertIn(
             url_for('main.corpus_get', corpus_id=1), self.driver.current_url,
             "Result page is the corpus new page"
@@ -197,7 +199,7 @@ def test_registration_with_allowed_morph(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed(user_id=1, corpus_id=1, allowed_lemma="lemma").count(), 22,
+            corpus.get_unallowed(user_id=1, corpus_id=1, allowed_type="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 

From eaf08228b4e346b0f7bbd63d50f5ae715959496b Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Fri, 28 Jun 2024 12:46:10 +0200
Subject: [PATCH 12/43] suppression commentaire test

---
 app/control_lists/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index c122e344..3e5f9dae 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -376,7 +376,7 @@ def information_read(control_list_id):
     return render_template_with_nav_info('control_lists/information_read.html', control_list=control_list)
 
 
-"""@control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
+@control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
 @login_required
 def ignore_terms_filter(control_list_id):
     current_controlListUser = ControlListsUser.query.filter_by(**{"control_lists_id":control_list_id,"user_id":current_user.id}).first_or_404()
@@ -404,4 +404,4 @@ def ignore_terms_filter(control_list_id):
         return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
                                              current_control_list=current_controlListUser)
 
-    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)"""
\ No newline at end of file
+    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)
\ No newline at end of file

From 797d48420866a3b25d9220b86d882d44d6fd134a Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Mon, 1 Jul 2024 17:09:49 +0200
Subject: [PATCH 13/43] ajouts premiers jets tests

---
 tests/test_selenium/test_corpus_init.py       | 33 +++++++++++++++++--
 .../test_manage_control_lists_user.py         | 14 ++++++++
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 9c40d182..d3253790 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -1,5 +1,5 @@
 from flask import url_for
-from app.models import Corpus, WordToken, AllowedLemma, AllowedMorph, AllowedPOS, ControlLists
+from app.models import Corpus, WordToken, AllowedLemma, AllowedMorph, AllowedPOS, ControlLists, ControlListsUser,User
 from app import db
 from tests.test_selenium.base import TestBase
 from tests.fixtures import PLAINTEXT_CORPORA
@@ -27,11 +27,9 @@ def test_registration(self):
         self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
         self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
         self.driver_find_element_by_id("label_checkbox_create").click()
-        self.driver.save_screenshot("./test_registration.png")
         self.driver_find_element_by_id("submit").click()
 
         self.driver.implicitly_wait(15)
-        self.driver.save_screenshot("./test_registration1.png")
         self.assertIn(
             url_for('main.corpus_get', corpus_id=1), self.driver.current_url,
             "Result page is the corpus new page"
@@ -63,6 +61,7 @@ def test_registration(self):
         self.assertEqual(oir.POS, "VERinf", "It should be correctly saved with POS")
         self.assertEqual(oir.morph, None, "It should be correctly saved with morph")
 
+
     def test_registration_with_full_allowed_lemma(self):
         """
         Test that a user can create a corpus wit allowed lemmas and that this corpus has its data well recorded
@@ -721,3 +720,31 @@ def lemmatizing():
             second_app.join(2)
         finally:
             second_app.close()
+
+    def test_registration_filters(self):
+        self.add_control_lists()
+        target_cl = db.session.query(ControlLists). \
+            filter(ControlLists.name == "Ancien Français - École des Chartes").first()
+
+        # Click register menu link
+        self.driver_find_element_by_id("new_corpus_link").click()
+        self.driver.implicitly_wait(15)
+
+        # Fill in registration form
+        self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
+        self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
+        self.driver_find_element_by_id("label_checkbox_reuse").click()
+        self.driver_find_element_by_id("control_list_select").click()
+        self.driver_find_element_by_id("cl_opt_" + str(target_cl.id)).click()
+        self.driver_find_element_by_id("punct").click()
+
+        self.driver_find_element_by_id("submit").click
+        self.driver.save_screenshot("./test_registration_filter.png")
+        self.driver.implicitly_wait(15)
+
+        #ne fonctionne pas, aller voir direct dans control list ignore si les cases sont cochées
+
+        CL = db.session.query(ControlLists).filter(ControlLists.name == "Ancien Français - École des Chartes").first()
+        CLUser = db.session.query(ControlListsUser).filter(ControlListsUser.control_lists_id == CL.id).first()
+        self.assertEqual(CLUser.filter_punct, True)
+        self.assertEqual(CLUser.filter_numeral, False)
\ No newline at end of file
diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index 8809d1c8..a92a7668 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -199,3 +199,17 @@ def test_corpus_creator_is_owner(self):
 
         el = self.get_ownership_table()
         self.assertTrue(len([e for e in el if e.get_property("checked")]) == 1)
+
+    def test_change_filter(self):
+        self.addControlLists("wauchier")
+        self.driver.save_screenshot("./test_registration_filter1.png")
+        self.go_to_control_lists_management("Wauchier")
+        self.driver.save_screenshot("./test_registration_filter2.png")
+        self.driver_find_element_by_link_text("Ignore values").click()
+        self.driver_find_element_by_id("punct").click()
+
+        self.driver_find_element_by_id("submit").click
+        self.driver.save_screenshot("./test_registration_filter3.png")
+        self.driver.implicitly_wait(15)
+
+        ### tester si punct est toujours cliqué
\ No newline at end of file

From 006fca82ab822c7740752d63946dc77407a9d2a0 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 2 Jul 2024 10:16:25 +0200
Subject: [PATCH 14/43] correction test filter update

---
 .../test_selenium/test_manage_control_lists_user.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index a92a7668..615efd84 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -202,14 +202,17 @@ def test_corpus_creator_is_owner(self):
 
     def test_change_filter(self):
         self.addControlLists("wauchier")
-        self.driver.save_screenshot("./test_registration_filter1.png")
         self.go_to_control_lists_management("Wauchier")
-        self.driver.save_screenshot("./test_registration_filter2.png")
+
         self.driver_find_element_by_link_text("Ignore values").click()
         self.driver_find_element_by_id("punct").click()
+        self.driver_find_element_by_id("submit").click()
 
-        self.driver_find_element_by_id("submit").click
-        self.driver.save_screenshot("./test_registration_filter3.png")
         self.driver.implicitly_wait(15)
 
-        ### tester si punct est toujours cliqué
\ No newline at end of file
+        self.assertEqual(
+            self.driver_find_element_by_css_selector(".alert.alert-success").text.strip(),
+            'The filters have been updated.',
+            "The filters have not been updated."
+        )
+

From 2b11ebc6a23b8c60e488fba314a0ae189c8e21ed Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 2 Jul 2024 15:09:04 +0200
Subject: [PATCH 15/43] correction test registration corpus filter

---
 tests/test_selenium/test_corpus_init.py | 29 ++++++++++++-------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index d3253790..c76232b2 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -722,10 +722,6 @@ def lemmatizing():
             second_app.close()
 
     def test_registration_filters(self):
-        self.add_control_lists()
-        target_cl = db.session.query(ControlLists). \
-            filter(ControlLists.name == "Ancien Français - École des Chartes").first()
-
         # Click register menu link
         self.driver_find_element_by_id("new_corpus_link").click()
         self.driver.implicitly_wait(15)
@@ -733,18 +729,21 @@ def test_registration_filters(self):
         # Fill in registration form
         self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
         self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
-        self.driver_find_element_by_id("label_checkbox_reuse").click()
-        self.driver_find_element_by_id("control_list_select").click()
-        self.driver_find_element_by_id("cl_opt_" + str(target_cl.id)).click()
+        self.driver_find_element_by_id("label_checkbox_create").click()
         self.driver_find_element_by_id("punct").click()
-
-        self.driver_find_element_by_id("submit").click
+        self.driver.save_screenshot("./test_registration_filter0.png")
+        self.driver_find_element_by_id("submit").click()
         self.driver.save_screenshot("./test_registration_filter.png")
         self.driver.implicitly_wait(15)
 
-        #ne fonctionne pas, aller voir direct dans control list ignore si les cases sont cochées
-
-        CL = db.session.query(ControlLists).filter(ControlLists.name == "Ancien Français - École des Chartes").first()
-        CLUser = db.session.query(ControlListsUser).filter(ControlListsUser.control_lists_id == CL.id).first()
-        self.assertEqual(CLUser.filter_punct, True)
-        self.assertEqual(CLUser.filter_numeral, False)
\ No newline at end of file
+        self.driver_find_element_by_link_text("Dashboard").click()
+        controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
+        self.driver.save_screenshot("./test_registration_filter2.png")
+        self.element_find_element_by_partial_link_text(controllists_dashboard, "Wauchier").click()
+        self.driver.save_screenshot("./test_registration_filter3.png")
+        self.driver_find_element_by_link_text("Ignore values").click()
+        self.driver.save_screenshot("./test_registration_filter4.png")
+        punct = self.driver_find_element_by_id("punct")
+        numeral = self.driver_find_element_by_id('numeral')
+        self.assertTrue(punct.get_property("checked"))
+        self.assertFalse(numeral.get_property("checked"))

From ee346469784b1b1e83ffbc7ade2f13483706b605 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 2 Jul 2024 16:11:42 +0200
Subject: [PATCH 16/43] ajout test edit token with filter

---
 tests/test_selenium/test_corpus_init.py   |  5 -----
 tests/test_selenium/test_token_correct.py | 24 +++++++++++++++++++++++
 tests/test_selenium/test_tokens_edit.py   |  1 +
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index c76232b2..49256445 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -731,18 +731,13 @@ def test_registration_filters(self):
         self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
         self.driver_find_element_by_id("label_checkbox_create").click()
         self.driver_find_element_by_id("punct").click()
-        self.driver.save_screenshot("./test_registration_filter0.png")
         self.driver_find_element_by_id("submit").click()
-        self.driver.save_screenshot("./test_registration_filter.png")
         self.driver.implicitly_wait(15)
 
         self.driver_find_element_by_link_text("Dashboard").click()
         controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
-        self.driver.save_screenshot("./test_registration_filter2.png")
         self.element_find_element_by_partial_link_text(controllists_dashboard, "Wauchier").click()
-        self.driver.save_screenshot("./test_registration_filter3.png")
         self.driver_find_element_by_link_text("Ignore values").click()
-        self.driver.save_screenshot("./test_registration_filter4.png")
         punct = self.driver_find_element_by_id("punct")
         numeral = self.driver_find_element_by_id('numeral')
         self.assertTrue(punct.get_property("checked"))
diff --git a/tests/test_selenium/test_token_correct.py b/tests/test_selenium/test_token_correct.py
index 878314cd..f8d1f14d 100644
--- a/tests/test_selenium/test_token_correct.py
+++ b/tests/test_selenium/test_token_correct.py
@@ -99,6 +99,28 @@ def test_edit_morph_with_allowed(self):
         self.assert_saved(row)
         self.assert_token_has_values(token, lemma="mout", POS="ADVgen", morph="NOMB.=s|GENRE=m|CAS=n")
 
+    def test_edit_token_with_filter(self):
+        self.addCorpus(with_token=True)
+        self.driver.save_screenshot("./test_edit_token_filter1.png")
+        self.driver.refresh()
+        # modifier les filtres
+        self.driver_find_element_by_link_text("Dashboard").click()
+        controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
+        self.driver.save_screenshot("./test_edit_token_filter2.png")
+        self.element_find_element_by_partial_link_text(controllists_dashboard, "Wauchier").click()
+        self.driver.save_screenshot("./test_edit_token_filter4.png")
+        self.driver_find_element_by_link_text("Ignore values").click()
+        self.driver.save_screenshot("./test_edit_token_filter5.png")
+        self.driver_find_element_by_id("punct").click()
+        self.driver_find_element_by_id("submit").click()
+        self.driver.save_screenshot("./test_edit_token_filter6.png")
+        self.driver.implicitly_wait(15)
+
+        # ajouter corpus
+        token, status_text, row = self.edith_nth_row_value(",", id_row="1")
+        self.assert_token_has_values(token, lemma=",")
+        self.assert_saved(row)
+
 
 class TestTokensCorrectFloovant(TokenCorrectBase):
     CORPUS = "floovant"
@@ -164,6 +186,8 @@ def test_edit_token_with_same_value(self):
         self.assertNotIn("table-changed", row.get_attribute("class"))
 
 
+
+
 class TestTokensEditTwoCorpora(TokenCorrect2CorporaBase):
     CORPUS = "wauchier"
     CORPUS_ID = "1"
diff --git a/tests/test_selenium/test_tokens_edit.py b/tests/test_selenium/test_tokens_edit.py
index d25bf1a5..64a84681 100644
--- a/tests/test_selenium/test_tokens_edit.py
+++ b/tests/test_selenium/test_tokens_edit.py
@@ -183,3 +183,4 @@ def test_edit_delete(self):
             [("Addition", "oulala", ""), ("Deletion", "", "oulala")],
             "History should be saved"
         )
+

From 0fe965965552ada6250b5bf48ae1279260cbf705 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 9 Jul 2024 15:58:58 +0200
Subject: [PATCH 17/43] modif aggrandissement varchar models corpus + print
 logs

---
 app/main/views/corpus.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index a932d45c..38db5941 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -3,6 +3,7 @@
 import sqlalchemy.exc
 from sqlalchemy import func, distinct, text
 from typing import List
+import logging
 
 
 from app import db
@@ -41,6 +42,9 @@ def _get_available():
 def corpus_new():
     """ Register a new corpus
     """
+    logging.basicConfig(filename='./pyrrha_corpus_creation.log', level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
+    logger = logging.getLogger(__name__)
     lemmatizers = current_app.config.get("LEMMATIZERS", [])
 
     def normal_view():
@@ -92,8 +96,9 @@ def error():
                 tokens = read_input_tokens(request.form.get("tsv"))
                 try:
                     control_list = ControlLists.query.get_or_404(request.form.get("control_list_select"))
-                except Exception:
+                except Exception as e:
                     flash("This control list does not exist", category="error")
+                    logger.error(e)
                     return error()
                 form_kwargs.update({"word_tokens_dict": tokens,
                                     "control_list": control_list})
@@ -139,6 +144,7 @@ def error():
                 db.session.rollback()
                 flash("The corpus cannot be registered. Check your data", category="error")
                 flash(str(e.orig).lower())
+                logger.error(e)
                 if db.session.get_bind().dialect.name == "postgresql":
                     unique_constraint = 'duplicate key value violates unique constraint "corpus_name_key"'
                 else:
@@ -151,6 +157,7 @@ def error():
                 db.session.rollback()
                 flash("At least one line of your corpus is missing a token/form. Check line %s " % exc.line,
                       category="error")
+                logger.error(exc)
                 return error()
             except NoTokensInput:
                 db.session.rollback()
@@ -164,6 +171,7 @@ def error():
                 print(e)
                 db.session.rollback()
                 flash("The corpus cannot be registered. Check your data", category="error")
+                logger.error(e)
                 return error()
             return redirect(url_for(".corpus_get", corpus_id=corpus.id))
 

From f97e4328cccbffb59197e8bf295b4b9000fe4889 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Fri, 12 Jul 2024 17:01:10 +0200
Subject: [PATCH 18/43] =?UTF-8?q?test=20ajout=20cr=C3=A9ation=20user=20pou?=
 =?UTF-8?q?r=20CL=20filter?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_selenium/test_manage_control_lists_user.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index 615efd84..78a0269f 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -201,7 +201,12 @@ def test_corpus_creator_is_owner(self):
         self.assertTrue(len([e for e in el if e.get_property("checked")]) == 1)
 
     def test_change_filter(self):
+        # grant access to foo
+        foo_email = self.add_user("foo", "bar")
         self.addControlLists("wauchier")
+
+        self.addControlListsUser("Wauchier", foo_email, False)
+        self.driver.refresh()
         self.go_to_control_lists_management("Wauchier")
 
         self.driver_find_element_by_link_text("Ignore values").click()

From b8bbd519935f3d2102f73045d6c544d2ff10ed40 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Fri, 12 Jul 2024 17:11:13 +0200
Subject: [PATCH 19/43] test bug control filters - ajout users"

---
 tests/test_selenium/test_corpus_init.py               | 2 ++
 tests/test_selenium/test_manage_control_lists_user.py | 6 ++----
 tests/test_selenium/test_token_correct.py             | 2 ++
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 49256445..965d0c84 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -722,6 +722,8 @@ def lemmatizing():
             second_app.close()
 
     def test_registration_filters(self):
+        self.add_user("foo", "foo")
+        self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
         # Click register menu link
         self.driver_find_element_by_id("new_corpus_link").click()
         self.driver.implicitly_wait(15)
diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index 78a0269f..07834348 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -201,11 +201,9 @@ def test_corpus_creator_is_owner(self):
         self.assertTrue(len([e for e in el if e.get_property("checked")]) == 1)
 
     def test_change_filter(self):
-        # grant access to foo
-        foo_email = self.add_user("foo", "bar")
+        self.add_user("foo", "foo")
+        self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
         self.addControlLists("wauchier")
-
-        self.addControlListsUser("Wauchier", foo_email, False)
         self.driver.refresh()
         self.go_to_control_lists_management("Wauchier")
 
diff --git a/tests/test_selenium/test_token_correct.py b/tests/test_selenium/test_token_correct.py
index f8d1f14d..cd8d9d7f 100644
--- a/tests/test_selenium/test_token_correct.py
+++ b/tests/test_selenium/test_token_correct.py
@@ -100,6 +100,8 @@ def test_edit_morph_with_allowed(self):
         self.assert_token_has_values(token, lemma="mout", POS="ADVgen", morph="NOMB.=s|GENRE=m|CAS=n")
 
     def test_edit_token_with_filter(self):
+        self.add_user("foo", "foo")
+        self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
         self.addCorpus(with_token=True)
         self.driver.save_screenshot("./test_edit_token_filter1.png")
         self.driver.refresh()

From 89e9aae04c375be2dd64d7a2245992932424d3d4 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 27 Aug 2024 11:44:12 +0200
Subject: [PATCH 20/43] tests CL - modification find element by ID > NAME

---
 tests/test_selenium/base.py               | 3 +++
 tests/test_selenium/test_corpus_init.py   | 6 +++---
 tests/test_selenium/test_token_correct.py | 6 +-----
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/tests/test_selenium/base.py b/tests/test_selenium/base.py
index 60156cf0..c35589c7 100644
--- a/tests/test_selenium/base.py
+++ b/tests/test_selenium/base.py
@@ -147,6 +147,9 @@ def element_find_elements_by_tag_name(element, *args, **kwargs):
     def driver_find_element_by_id(self, *args, **kwargs):
         return self.driver.find_element(By.ID, *args, **kwargs)
 
+    def driver_find_element_by_name(self, *args, **kwargs):
+        return self.driver.find_element(By.NAME, *args, **kwargs)
+
     def driver_find_element_by_css_selector(self, *args, **kwargs):
         return self.driver.find_element(By.CSS_SELECTOR, *args, **kwargs)
 
diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 965d0c84..61e49047 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -732,7 +732,7 @@ def test_registration_filters(self):
         self.driver_find_element_by_id("corpusName").send_keys(PLAINTEXT_CORPORA["Wauchier"]["name"])
         self.writeMultiline(self.driver_find_element_by_id("tokens"), PLAINTEXT_CORPORA["Wauchier"]["data"])
         self.driver_find_element_by_id("label_checkbox_create").click()
-        self.driver_find_element_by_id("punct").click()
+        self.driver_find_element_by_name("punct").click()
         self.driver_find_element_by_id("submit").click()
         self.driver.implicitly_wait(15)
 
@@ -740,7 +740,7 @@ def test_registration_filters(self):
         controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
         self.element_find_element_by_partial_link_text(controllists_dashboard, "Wauchier").click()
         self.driver_find_element_by_link_text("Ignore values").click()
-        punct = self.driver_find_element_by_id("punct")
-        numeral = self.driver_find_element_by_id('numeral')
+        punct = self.driver_find_element_by_name("punct")
+        numeral = self.driver_find_element_by_name('numeral')
         self.assertTrue(punct.get_property("checked"))
         self.assertFalse(numeral.get_property("checked"))
diff --git a/tests/test_selenium/test_token_correct.py b/tests/test_selenium/test_token_correct.py
index cd8d9d7f..8a8245bc 100644
--- a/tests/test_selenium/test_token_correct.py
+++ b/tests/test_selenium/test_token_correct.py
@@ -103,17 +103,13 @@ def test_edit_token_with_filter(self):
         self.add_user("foo", "foo")
         self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
         self.addCorpus(with_token=True)
-        self.driver.save_screenshot("./test_edit_token_filter1.png")
         self.driver.refresh()
         # modifier les filtres
         self.driver_find_element_by_link_text("Dashboard").click()
         controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
-        self.driver.save_screenshot("./test_edit_token_filter2.png")
         self.element_find_element_by_partial_link_text(controllists_dashboard, "Wauchier").click()
-        self.driver.save_screenshot("./test_edit_token_filter4.png")
         self.driver_find_element_by_link_text("Ignore values").click()
-        self.driver.save_screenshot("./test_edit_token_filter5.png")
-        self.driver_find_element_by_id("punct").click()
+        self.driver_find_element_by_name("punct").click()
         self.driver_find_element_by_id("submit").click()
         self.driver.save_screenshot("./test_edit_token_filter6.png")
         self.driver.implicitly_wait(15)

From 666ff6d67d61687a3222ed333d59f9b8c2b51e34 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 27 Aug 2024 12:14:54 +0200
Subject: [PATCH 21/43] find element by ID>NAME

---
 tests/test_selenium/test_manage_control_lists_user.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index 07834348..2ef57348 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -208,7 +208,7 @@ def test_change_filter(self):
         self.go_to_control_lists_management("Wauchier")
 
         self.driver_find_element_by_link_text("Ignore values").click()
-        self.driver_find_element_by_id("punct").click()
+        self.driver_find_element_by_name("punct").click()
         self.driver_find_element_by_id("submit").click()
 
         self.driver.implicitly_wait(15)

From 56bac052c023304d115adf2c554c71fee97f5ed2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?=
 <1929830+PonteIneptique@users.noreply.github.com>
Date: Tue, 27 Aug 2024 19:34:08 +0200
Subject: [PATCH 22/43] Correct tests and clean up the way regex are applied
 (#329)

* Fixing tests

* Revert update of status

* Fix an issue where not having a ControlListUser would lead to being unable to apply stuff

* Normalize control list
---
 app/control_lists/views.py                    | 24 +++--
 app/main/views/corpus.py                      | 14 +--
 app/models/control_lists.py                   | 18 +++-
 app/models/corpus.py                          | 98 +++++++++----------
 .../control_lists/ignore_filter.html          |  2 +-
 requirements.txt                              |  1 +
 tests/test_selenium/base.py                   | 26 ++---
 tests/test_selenium/test_token_correct.py     | 20 ++--
 8 files changed, 107 insertions(+), 96 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 3e5f9dae..571412d3 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -379,8 +379,11 @@ def information_read(control_list_id):
 @control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
 @login_required
 def ignore_terms_filter(control_list_id):
-    current_controlListUser = ControlListsUser.query.filter_by(**{"control_lists_id":control_list_id,"user_id":current_user.id}).first_or_404()
-    print(current_controlListUser)
+    current_controlListUser = ControlListsUser.retrieve(
+        user_id=current_user.id,
+        control_list_id=control_list_id
+    ).first_or_404()
+
     list_filter = []
     if request.method == "POST":
         list_filter.append(request.form.get("punct"))
@@ -389,18 +392,21 @@ def ignore_terms_filter(control_list_id):
         list_filter.append(request.form.get('metadata'))
         filtered_filter = []
         for el in list_filter:
-            if el != None:
+            if el is not None:
                 filtered_filter.append(el)
-        filter = " ".join(filtered_filter)
-        current_controlListUser.filter_punct = 'punct' in filter
-        current_controlListUser.filter_metadata = 'metadata' in filter
-        current_controlListUser.filter_numeral = 'numeral' in filter
-        current_controlListUser.filter_ignore = 'ignore' in filter
 
+        current_controlListUser.filter_punct = 'punct' in filtered_filter
+        current_controlListUser.filter_metadata = 'metadata' in filtered_filter
+        current_controlListUser.filter_numeral = 'numeral' in filtered_filter
+        current_controlListUser.filter_ignore = 'ignore' in filtered_filter
+        db.session.add(current_controlListUser)
         db.session.commit()
 
         flash('The filters have been updated.', 'success')
-        current_controlListUser = ControlListsUser.query.filter_by(**{"control_lists_id":control_list_id,"user_id":current_user.id}).first_or_404()
+        current_controlListUser = ControlListsUser.retrieve(
+            user_id=current_user.id,
+            control_list_id=control_list_id
+        ).first()
         return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
                                              current_control_list=current_controlListUser)
 
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index 38db5941..d64cc3ea 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -119,11 +119,7 @@ def error():
             list_filter.append(request.form.get("numeral"))
             list_filter.append(request.form.get("ignore"))
             list_filter.append(request.form.get("metadata"))
-            filtered_filter = []
-            for el in list_filter:
-                if el != None:
-                    filtered_filter.append(el)
-            filter = " ".join(filtered_filter)
+            list_filter = [flt for flt in list_filter if flt]
 
             try:
                 corpus: Corpus = Corpus.create(**form_kwargs)
@@ -133,10 +129,10 @@ def error():
                 db.session.commit()
                 current_controlListUser = ControlListsUser.query.filter_by(
                     **{"control_lists_id": corpus.control_lists_id, "user_id": current_user.id}).first_or_404()
-                current_controlListUser.filter_punct = 'punct' in filter
-                current_controlListUser.filter_metadata = 'metadata' in filter
-                current_controlListUser.filter_numeral = 'numeral' in filter
-                current_controlListUser.filter_ignore = 'ignore' in filter
+                current_controlListUser.filter_punct = 'punct' in list_filter
+                current_controlListUser.filter_metadata = 'metadata' in list_filter
+                current_controlListUser.filter_numeral = 'numeral' in list_filter
+                current_controlListUser.filter_ignore = 'ignore' in list_filter
                 db.session.commit()
                 flash("New corpus registered", category="success")
             except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index 49ee3217..fbaabb32 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -8,9 +8,11 @@
 from collections import Counter
 # PIP Packages
 import unidecode
+import regex as re
 import yaml
+from flask_sqlalchemy.query import Query as FlaskQuery
 from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy.orm import backref
+from sqlalchemy.orm import backref, Query
 from sqlalchemy import literal, case
 from werkzeug.exceptions import BadRequest
 # APP Logic
@@ -290,6 +292,17 @@ class ControlListsUser(db.Model):
     control = db.relationship("ControlLists", backref=backref("control_lists_user", cascade="all, delete-orphan"))
     user = db.relationship(User, backref=backref("control_lists_user", cascade="all, delete-orphan"))
 
+    re_filter_metadata = r'(\[[^\]]+:[^\]]*\]$)'
+    re_filter_ignore = r'(^\[IGNORE\])'
+    re_filter_punct = "(^[^\w\s]$)"
+    re_filter_numeral = r'(^\d+$)'
+
+    @classmethod
+    def retrieve(cls, user_id: int, control_list_id: int) -> FlaskQuery:
+        return cls.query.filter(db.and_(
+            cls.user_id == user_id,
+            cls.control_lists_id == control_list_id
+        ))
 
 class AllowedLemma(db.Model):
     """ An allowed lemma is a lemma that is accepted
@@ -352,9 +365,6 @@ def to_input_format(query):
         )
 
 
-
-
-
 class AllowedPOS(db.Model):
     """ An allowed POS is a POS that is accepted
 
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 3317a3ac..f2f47df6 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -7,7 +7,7 @@
 # PIP Packages
 import unidecode
 import sqlalchemy.exc
-import re
+import regex as re
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import backref
 from sqlalchemy import func, literal, not_, or_, and_
@@ -337,20 +337,19 @@ def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
         current_corpus = Corpus.query.filter_by(**{"id":corpus_id}).first_or_404()
         current_controlListUser = ControlListsUser.query.filter_by(
             **{"control_lists_id":current_corpus.control_lists_id, "user_id": user_id}).first_or_404()
-        dict_filter = {'punct': current_controlListUser.filter_punct,
-                       'metadata': current_controlListUser.filter_metadata,
-                       'ignore': current_controlListUser.filter_ignore,
-                       'numeral': current_controlListUser.filter_numeral}
-        if True in dict_filter.values():
-            regex_liste = []
-            if dict_filter['metadata']:
-                regex_liste.append(r'^(?!\[[^\]]+:[^\]]*\]$).*')
-            if dict_filter['ignore']:
-                regex_liste.append(r'^(?!^\[IGNORE\]$)')
-            if dict_filter['punct']:
-                regex_liste.append(r"((?!^[^\w\s]$).)")
-            if dict_filter["numeral"]:
-                regex_liste.append(r'(^(?!\d+$).+)')
+
+        regex_liste = []
+        if current_controlListUser:
+            if current_controlListUser.filter_metadata:
+                regex_liste.append(ControlListsUser.re_filter_metadata)
+            if current_controlListUser.filter_ignore:
+                regex_liste.append(ControlListsUser.re_filter_ignore)
+            if current_controlListUser.filter_punct:
+                regex_liste.append(ControlListsUser.re_filter_punct)
+            if current_controlListUser.filter_numeral:
+                regex_liste.append(ControlListsUser.re_filter_numeral)
+
+        if regex_liste:
             list_darguments.append(WordToken.form.op('~')("".join(regex_liste)))
 
         return db.session.query(WordToken).filter(
@@ -1097,7 +1096,7 @@ def get_like(filter_id, form, group_by, type_like="lemma", allowed_list=False):
         return query
 
     @staticmethod
-    def is_valid(lemma, POS, morph, corpus, user_id, filter):
+    def is_valid(lemma, POS, morph, corpus, user_id):
         """ Check if a token is valid for a given corpus
 
         :param lemma: Lemma value of the token to validate
@@ -1122,38 +1121,32 @@ def is_valid(lemma, POS, morph, corpus, user_id, filter):
         }
 
         allowed_column = corpus.displayed_columns_by_name
-        if filter:
-            current_controlListUser = ControlListsUser.query.filter_by(
-                **{"control_lists_id": corpus.control_lists_id, "user_id": user_id}).first_or_404()
-            dict_filter = {'punct': current_controlListUser.filter_punct,
-                           'metadata': current_controlListUser.filter_metadata,
-                           'ignore': current_controlListUser.filter_ignore,
-                           'numeral': current_controlListUser.filter_numeral}
+        if lemma and "lemma" in allowed_column and allowed_lemma.count():
+            current_controlListUser = ControlListsUser.retrieve(
+                user_id=user_id, control_list_id=corpus.control_lists_id
+            ).first()
             regex_liste = []
-            if True in dict_filter.values():
-                if dict_filter['metadata']:
-                    regex_liste.append(r'(\[[^\]]+:[^\]]*\]$)')
-                if dict_filter['ignore']:
-                    regex_liste.append(r'(^\[IGNORE\])')
-                if dict_filter['punct']:
-                    regex_liste.append(r"(^[^\w\s]$)")
-                if dict_filter['numeral']:
-                    regex_liste.append(r'(^\d+$)')
-            regex = "|".join(regex_liste)
-
-        if lemma is not None \
-                and "lemma" in allowed_column \
-                and allowed_lemma.count() > 0 \
-                and corpus.get_allowed_values("lemma", label=lemma).count() == 0:
-            if filter:
-                if not re.match(regex, lemma):
-                    if not corpus.has_custom_dictionary_value("lemma", lemma):
-                        statuses["lemma"] = False
-                    else:
-                        if not corpus.has_custom_dictionary_value("lemma", lemma):
-                            statuses["lemma"] = False
-            else:
-                if not corpus.has_custom_dictionary_value("lemma", lemma):
+            if current_controlListUser:
+                if current_controlListUser.filter_metadata:
+                    regex_liste.append(ControlListsUser.re_filter_metadata)
+                if current_controlListUser.filter_ignore:
+                    regex_liste.append(ControlListsUser.re_filter_ignore)
+                if current_controlListUser.filter_punct:
+                    regex_liste.append(ControlListsUser.re_filter_punct)
+                if current_controlListUser.filter_numeral:
+                    regex_liste.append(ControlListsUser.re_filter_numeral)
+
+            ignored_by_regex = False
+
+            for regex in regex_liste:
+                if re.match(regex, lemma) is not None:
+                    ignored_by_regex = True
+
+            if (
+                    not ignored_by_regex and
+                    corpus.has_custom_dictionary_value("lemma", lemma) is False and
+                    corpus.get_allowed_values("lemma", label=lemma).count() == 0
+                ):
                     statuses["lemma"] = False
 
         if POS is not None \
@@ -1170,7 +1163,6 @@ def is_valid(lemma, POS, morph, corpus, user_id, filter):
             if not corpus.has_custom_dictionary_value("morph", morph):
                 statuses["morph"] = False
 
-
         return statuses
 
     @staticmethod
@@ -1327,7 +1319,7 @@ def to_input_format(query):
         return csv_file.getvalue()
 
     @staticmethod
-    def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None, filter=False):
+    def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
         """ Update a given token with lemma, POS and morph value
 
         :param user_id: ID of the user who performs the update
@@ -1358,9 +1350,10 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None, filte
             error = WordToken.NothingChangedError("No value where changed")
             error.msg = "No value where changed"
             raise error
-
+        print(token)
         # Check if values are correct regarding allowed values
-        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id, filter=filter)
+        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id)
+        print(token.POS, validity)
         if False in list(validity.values()):
             error_msg = "Invalid value in {}".format(
                 ", ".join([key for key in validity.keys() if validity[key] is False])
@@ -1370,7 +1363,6 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None, filte
             error.statuses = validity
             error.invalid_columns = [key for key in validity.keys() if validity[key] is False]
             raise error
-
         # Updating
         if not lemma:
             lemma = token.lemma
@@ -1495,7 +1487,7 @@ def get_nearly_similar_to(token, mode):
                     WordToken.id != token.id,
                     *filtering
                 )
-            )
+        )
 
 
 class TokenHistory(db.Model):
diff --git a/app/templates/control_lists/ignore_filter.html b/app/templates/control_lists/ignore_filter.html
index 83d789a7..cf21efa8 100644
--- a/app/templates/control_lists/ignore_filter.html
+++ b/app/templates/control_lists/ignore_filter.html
@@ -34,7 +34,7 @@ <h2>{{ _('Change the filters for Control List') }}</h2>
                     </label>
                 </li>
         </ul>
-        <button type="submit">Submit</button>
+        <button type="submit" id="submit">Submit</button>
     </form>
         </div>
     </div>
diff --git a/requirements.txt b/requirements.txt
index 3bc0c4a6..c244130f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -36,3 +36,4 @@ nose==1.3.7
 selenium>=3.141.0
 Flask-Testing==0.8.1
 mock==5.1.0
+regex
\ No newline at end of file
diff --git a/tests/test_selenium/base.py b/tests/test_selenium/base.py
index c35589c7..bc83252c 100644
--- a/tests/test_selenium/base.py
+++ b/tests/test_selenium/base.py
@@ -5,6 +5,7 @@
 import csv
 import signal
 import logging
+from typing import Union
 import tempfile
 from selenium import webdriver
 from selenium.common.exceptions import NoSuchElementException
@@ -362,8 +363,11 @@ def addControlLists(self, cl_name, *args, **kwargs):
             self.addControlListsUser(cl.name, user_mail, is_owner=owner)
         return cl
 
-    def addControlListsUser(self, cl_name, email, is_owner=False):
-        cl = ControlLists.query.filter(ControlLists.name == cl_name).first()
+    def addControlListsUser(self, cl_name: Union[str, int], email, is_owner=False):
+        if isinstance(cl_name, str):
+            cl = ControlLists.query.filter(ControlLists.name == cl_name).first()
+        else:
+            cl = ControlLists.query.filter(ControlLists.id == cl_name).first()
         user = User.query.filter(User.email == email).first()
         new_clu = ControlListsUser(control_lists_id=cl.id, user_id=user.id, is_owner=is_owner)
         self.db.session.add(new_clu)
@@ -567,7 +571,6 @@ def edith_nth_row_value(
         if go_to_edit_token_page is None:
             go_to_edit_token_page = self.go_to_edit_token_page(corpus_id)
         go_to_edit_token_page()
-        self.driver.save_screenshot("./token_correct_1.png")
         if additional_action_before is not None:
             additional_action_before()
 
@@ -578,7 +581,7 @@ def edith_nth_row_value(
             )
 
         # Take the first row
-        row = self.driver_find_element_by_id("token_" + id_row + "_row")
+        row = self.driver_find_element_by_id(f"token_{id_row}_row")
         # Take the td to edit
         if value_type == "POS":
             td = self.element_find_element_by_class_name(row, "token_pos")
@@ -586,27 +589,27 @@ def edith_nth_row_value(
             td = self.element_find_element_by_class_name(row, "token_morph")
         else:
             td = self.element_find_element_by_class_name(row, "token_lemma")
-        self.driver.save_screenshot("./token_correct_2.png")
         # Click, clear the td and send a new value
         td.click()
         td.clear()
-        self.driver.save_screenshot("./token_correct_25.png")
         td.send_keys(value)
-        self.driver.save_screenshot("./token_correct_3.png")
         # Save
         self.element_find_element_by_css_selector(row, "a.save").click()
-        self.driver.save_screenshot("./token_correct_3.png")
         # It's safer to wait for the AJAX call to be completed
-        row = self.driver_find_element_by_id("token_" + id_row + "_row")
+        row = self.driver_find_element_by_id(f"token_{id_row}_row")
 
+        rel_tr = f"[rel='token_{id_row}_row'] .badge-status"
         WebDriverWait(self.driver, 10).until(
             EC.visibility_of_element_located(
-                (By.CSS_SELECTOR, "[rel='token_{}_row'] .badge-status".format(id_row))
+                (By.CSS_SELECTOR, rel_tr)
             )
         )
 
+        token = self.db.session.get(WordToken, int(id_row))
+
+        self.db.session.refresh(token)
         return (
-            self.db.session.get(WordToken, int(id_row)),
+            token,
             self.element_find_element_by_css_selector(row, "#token_"+id_row+"_row > td a.save").text.strip(),
             row
         )
@@ -636,7 +639,6 @@ def test_edit_token(self):
         self.addCorpus(with_token=True, tokens_up_to=24)
         self.driver.refresh()
         token, status_text, row = self.edith_nth_row_value("un", corpus_id=self.CORPUS_ID)
-        self.driver.save_screenshot('./token_correct_4.png')
         self.assertEqual(token.lemma, "un", "Lemma should have been changed")
         self.assertEqual(status_text, "Save")
         self.assert_saved(row)
diff --git a/tests/test_selenium/test_token_correct.py b/tests/test_selenium/test_token_correct.py
index 8a8245bc..0a4bed63 100644
--- a/tests/test_selenium/test_token_correct.py
+++ b/tests/test_selenium/test_token_correct.py
@@ -1,3 +1,4 @@
+from app.models import ControlListsUser
 from tests.test_selenium.base import TokenCorrectBase, TokenCorrect2CorporaBase
 import selenium
 from sqlalchemy import text
@@ -100,24 +101,27 @@ def test_edit_morph_with_allowed(self):
         self.assert_token_has_values(token, lemma="mout", POS="ADVgen", morph="NOMB.=s|GENRE=m|CAS=n")
 
     def test_edit_token_with_filter(self):
-        self.add_user("foo", "foo")
-        self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
-        self.addCorpus(with_token=True)
+        corpus = self.addCorpus(with_token=True, cl=True, with_allowed_lemma=True)
+        clu = self.addControlListsUser(corpus.control_lists_id, self.app.config["ADMIN_EMAIL"], is_owner=True)
         self.driver.refresh()
+
+        # ajouter corpus
+        token, status_text, row = self.edith_nth_row_value("#", id_row="1")
+        self.assertNotEqual(token.lemma, "#", "Lemma # is forbidden in control list")
+
         # modifier les filtres
         self.driver_find_element_by_link_text("Dashboard").click()
         controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
         self.element_find_element_by_partial_link_text(controllists_dashboard, "Wauchier").click()
-        self.driver_find_element_by_link_text("Ignore values").click()
+        self.driver_find_element_by_partial_link_text("Ignore values").click()
         self.driver_find_element_by_name("punct").click()
         self.driver_find_element_by_id("submit").click()
-        self.driver.save_screenshot("./test_edit_token_filter6.png")
-        self.driver.implicitly_wait(15)
+        self.driver.implicitly_wait(5)
 
         # ajouter corpus
-        token, status_text, row = self.edith_nth_row_value(",", id_row="1")
-        self.assert_token_has_values(token, lemma=",")
+        token, status_text, row = self.edith_nth_row_value("]", value_type="lemma", id_row="1")
         self.assert_saved(row)
+        self.assert_token_has_values(token, lemma="]")
 
 
 class TestTokensCorrectFloovant(TokenCorrectBase):

From a2fb10728d1235a0c3976340b36d10b530e92216 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 28 Aug 2024 14:17:17 +0200
Subject: [PATCH 23/43] changement filtres CLS ControlListUser>controlList

---
 app/control_lists/views.py                    | 28 ++++------
 app/main/views/corpus.py                      | 11 ++--
 app/main/views/tokens.py                      |  2 +-
 app/models/control_lists.py                   | 18 +++---
 app/models/corpus.py                          | 56 +++++++++----------
 .../control_lists/ignore_filter.html          |  1 -
 app/templates/control_lists/macros.html       |  3 +-
 7 files changed, 58 insertions(+), 61 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 571412d3..15ed0ff5 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -379,11 +379,8 @@ def information_read(control_list_id):
 @control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
 @login_required
 def ignore_terms_filter(control_list_id):
-    current_controlListUser = ControlListsUser.retrieve(
-        user_id=current_user.id,
-        control_list_id=control_list_id
-    ).first_or_404()
-
+    current_controlList = ControlLists.query.filter_by(**{"id":control_list_id}).first_or_404()
+    print(current_controlList.filter_punct, current_controlList.filter_ignore)
     list_filter = []
     if request.method == "POST":
         list_filter.append(request.form.get("punct"))
@@ -395,19 +392,18 @@ def ignore_terms_filter(control_list_id):
             if el is not None:
                 filtered_filter.append(el)
 
-        current_controlListUser.filter_punct = 'punct' in filtered_filter
-        current_controlListUser.filter_metadata = 'metadata' in filtered_filter
-        current_controlListUser.filter_numeral = 'numeral' in filtered_filter
-        current_controlListUser.filter_ignore = 'ignore' in filtered_filter
-        db.session.add(current_controlListUser)
+        current_controlList.filter_punct = 'punct' in filtered_filter
+        current_controlList.filter_metadata = 'metadata' in filtered_filter
+        current_controlList.filter_numeral = 'numeral' in filtered_filter
+        current_controlList.filter_ignore = 'ignore' in filtered_filter
+        db.session.add(current_controlList)
         db.session.commit()
 
+
         flash('The filters have been updated.', 'success')
-        current_controlListUser = ControlListsUser.retrieve(
-            user_id=current_user.id,
-            control_list_id=control_list_id
-        ).first()
+        current_controlList = ControlLists.query.filter_by(**{"id":control_list_id}).first_or_404()
+        print(current_controlList.filter_punct, current_controlList.filter_ignore)
         return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
-                                             current_control_list=current_controlListUser)
+                                             current_control_list=current_controlList)
 
-    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)
\ No newline at end of file
+    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlList)
\ No newline at end of file
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index d64cc3ea..a5b1ecfd 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -127,12 +127,11 @@ def error():
                 # Add a link to the control list
                 ControlLists.link(corpus.control_lists_id, current_user.id, is_owner=cl_owner)
                 db.session.commit()
-                current_controlListUser = ControlListsUser.query.filter_by(
-                    **{"control_lists_id": corpus.control_lists_id, "user_id": current_user.id}).first_or_404()
-                current_controlListUser.filter_punct = 'punct' in list_filter
-                current_controlListUser.filter_metadata = 'metadata' in list_filter
-                current_controlListUser.filter_numeral = 'numeral' in list_filter
-                current_controlListUser.filter_ignore = 'ignore' in list_filter
+                current_controlList = ControlLists.query.filter_by(**{"id":corpus.control_lists_id}).first_or_404()
+                current_controlList.filter_punct = 'punct' in list_filter
+                current_controlList.filter_metadata = 'metadata' in list_filter
+                current_controlList.filter_numeral = 'numeral' in list_filter
+                current_controlList.filter_ignore = 'ignore' in list_filter
                 db.session.commit()
                 flash("New corpus registered", category="success")
             except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index 5c896c32..805775a1 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -135,7 +135,7 @@ def tokens_correct_single(corpus_id, token_id):
             token_id=token_id, corpus_id=corpus_id,
             lemma=string_to_none(request.form.get("lemma")),
             POS=string_to_none(request.form.get("POS")),
-            morph=string_to_none(request.form.get("morph")),
+            morph=string_to_none(request.form.get("morph"))
         )
         if "similar" in corpus.displayed_columns_by_name:
             similar = {
diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index fbaabb32..ecf9c340 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -45,6 +45,15 @@ class ControlLists(db.Model):
     bibliography = db.Column(db.Text, nullable=True)
     language = db.Column(db.String(10), nullable=True)
     notes = db.Column(db.Text, nullable=True)
+    filter_punct = db.Column(db.Boolean, unique=False, default=False)
+    filter_numeral = db.Column(db.Boolean, unique=False, default=False)
+    filter_metadata = db.Column(db.Boolean, unique=False, default=False)
+    filter_ignore = db.Column(db.Boolean, unique=False, default=False)
+
+    re_filter_metadata = r'(\[[^\]]+:[^\]]*\]$)'
+    re_filter_ignore = r'(^\[IGNORE\])'
+    re_filter_punct = "(^[^\w\s]$)"
+    re_filter_numeral = r'(^\d+$)'
 
     # For caching purposes, we record the last time these fields were edited
     #last_lemma_edit = db.Column(db.DateTime, default=datetime.datetime.utcnow)
@@ -284,18 +293,11 @@ class ControlListsUser(db.Model):
     user_id = db.Column(db.Integer, db.ForeignKey(User.id), primary_key=True)
     is_owner = db.Column(db.Boolean, default=False)
 
-    filter_punct = db.Column(db.Boolean, unique=False, default=False)
-    filter_numeral = db.Column(db.Boolean, unique=False, default=False)
-    filter_metadata = db.Column(db.Boolean, unique=False, default=False)
-    filter_ignore = db.Column(db.Boolean, unique=False, default=False)
 
     control = db.relationship("ControlLists", backref=backref("control_lists_user", cascade="all, delete-orphan"))
     user = db.relationship(User, backref=backref("control_lists_user", cascade="all, delete-orphan"))
 
-    re_filter_metadata = r'(\[[^\]]+:[^\]]*\]$)'
-    re_filter_ignore = r'(^\[IGNORE\])'
-    re_filter_punct = "(^[^\w\s]$)"
-    re_filter_numeral = r'(^\d+$)'
+
 
     @classmethod
     def retrieve(cls, user_id: int, control_list_id: int) -> FlaskQuery:
diff --git a/app/models/corpus.py b/app/models/corpus.py
index f2f47df6..a3b8bf52 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -335,19 +335,19 @@ def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
         ]
 
         current_corpus = Corpus.query.filter_by(**{"id":corpus_id}).first_or_404()
-        current_controlListUser = ControlListsUser.query.filter_by(
-            **{"control_lists_id":current_corpus.control_lists_id, "user_id": user_id}).first_or_404()
+        current_controlList = ControlLists.query.filter_by(
+            **{"id":current_corpus.control_lists_id}).first_or_404()
 
         regex_liste = []
-        if current_controlListUser:
-            if current_controlListUser.filter_metadata:
-                regex_liste.append(ControlListsUser.re_filter_metadata)
-            if current_controlListUser.filter_ignore:
-                regex_liste.append(ControlListsUser.re_filter_ignore)
-            if current_controlListUser.filter_punct:
-                regex_liste.append(ControlListsUser.re_filter_punct)
-            if current_controlListUser.filter_numeral:
-                regex_liste.append(ControlListsUser.re_filter_numeral)
+        if current_controlList:
+            if current_controlLists.filter_metadata:
+                regex_liste.append(ControlLists.re_filter_metadata)
+            if current_controlList.filter_ignore:
+                regex_liste.append(ControlLists.re_filter_ignore)
+            if current_controlList.filter_punct:
+                regex_liste.append(ControlLists.re_filter_punct)
+            if current_controlList.filter_numeral:
+                regex_liste.append(ControlLists.re_filter_numeral)
 
         if regex_liste:
             list_darguments.append(WordToken.form.op('~')("".join(regex_liste)))
@@ -1096,7 +1096,7 @@ def get_like(filter_id, form, group_by, type_like="lemma", allowed_list=False):
         return query
 
     @staticmethod
-    def is_valid(lemma, POS, morph, corpus, user_id):
+    def is_valid(lemma, POS, morph, corpus):
         """ Check if a token is valid for a given corpus
 
         :param lemma: Lemma value of the token to validate
@@ -1121,20 +1121,22 @@ def is_valid(lemma, POS, morph, corpus, user_id):
         }
 
         allowed_column = corpus.displayed_columns_by_name
-        if lemma and "lemma" in allowed_column and allowed_lemma.count():
-            current_controlListUser = ControlListsUser.retrieve(
-                user_id=user_id, control_list_id=corpus.control_lists_id
-            ).first()
+        print("test")
+        if lemma and "lemma" in allowed_column and corpus.get_allowed_values("lemma", label=lemma).count() == 0:
+            print("test2")
+            current_controlList = ControlLists.query.filter_by(**{"id":corpus.control_lists_id}).first_or_404()
+            print(current_controlList)
             regex_liste = []
-            if current_controlListUser:
-                if current_controlListUser.filter_metadata:
-                    regex_liste.append(ControlListsUser.re_filter_metadata)
-                if current_controlListUser.filter_ignore:
-                    regex_liste.append(ControlListsUser.re_filter_ignore)
-                if current_controlListUser.filter_punct:
-                    regex_liste.append(ControlListsUser.re_filter_punct)
-                if current_controlListUser.filter_numeral:
-                    regex_liste.append(ControlListsUser.re_filter_numeral)
+            if current_controlList:
+                if current_controlList.filter_metadata:
+                    regex_liste.append(ControlLists.re_filter_metadata)
+                if current_controlList.filter_ignore:
+                    regex_liste.append(ControlLists.re_filter_ignore)
+                if current_controlList.filter_punct:
+                    regex_liste.append(ControlLists.re_filter_punct)
+                if current_controlList.filter_numeral:
+                    regex_liste.append(ControlLists.re_filter_numeral)
+            print(regex_liste)
 
             ignored_by_regex = False
 
@@ -1350,10 +1352,8 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
             error = WordToken.NothingChangedError("No value where changed")
             error.msg = "No value where changed"
             raise error
-        print(token)
         # Check if values are correct regarding allowed values
-        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id)
-        print(token.POS, validity)
+        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus)
         if False in list(validity.values()):
             error_msg = "Invalid value in {}".format(
                 ", ".join([key for key in validity.keys() if validity[key] is False])
diff --git a/app/templates/control_lists/ignore_filter.html b/app/templates/control_lists/ignore_filter.html
index cf21efa8..6928a680 100644
--- a/app/templates/control_lists/ignore_filter.html
+++ b/app/templates/control_lists/ignore_filter.html
@@ -36,6 +36,5 @@ <h2>{{ _('Change the filters for Control List') }}</h2>
         </ul>
         <button type="submit" id="submit">Submit</button>
     </form>
-        </div>
     </div>
 {% endblock %}
diff --git a/app/templates/control_lists/macros.html b/app/templates/control_lists/macros.html
index 24d78e1a..f81183a7 100644
--- a/app/templates/control_lists/macros.html
+++ b/app/templates/control_lists/macros.html
@@ -18,8 +18,9 @@
       {% endif %}
       {%  if control_list.can_edit() or current_user.is_admin() %}
       <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.information_edit", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Edit informations') }}</a></li>
+        <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.ignore_terms_filter", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Ignore values') }}</a></li>
       {% endif %}
-      <li class="nav-item"><a class="nav-link" href="{{url_for("control_lists_bp.ignore_terms_filter", control_list_id=control_list.id)}}"><i class="fa fa-edit"></i> {{ _('Ignore values') }}</a></li>
+
     </ul>
     {{ _('Others') }}
     <ul class="nav flex-column">

From 8e0ef9746fc7e51c44e897010e4cd42aa8d3da6d Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 28 Aug 2024 14:54:19 +0200
Subject: [PATCH 24/43] suppression ajout count

---
 app/models/corpus.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/app/models/corpus.py b/app/models/corpus.py
index a3b8bf52..89a1f55e 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -1121,9 +1121,7 @@ def is_valid(lemma, POS, morph, corpus):
         }
 
         allowed_column = corpus.displayed_columns_by_name
-        print("test")
-        if lemma and "lemma" in allowed_column and corpus.get_allowed_values("lemma", label=lemma).count() == 0:
-            print("test2")
+        if lemma and "lemma" in allowed_column and allowed_lemma.count():
             current_controlList = ControlLists.query.filter_by(**{"id":corpus.control_lists_id}).first_or_404()
             print(current_controlList)
             regex_liste = []

From aa1fb7d1569cbd487725b82b64b2ac46c51a722a Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 28 Aug 2024 16:08:05 +0200
Subject: [PATCH 25/43] modifications corpus.id => self.id + get_unallowed
 attributes

---
 app/control_lists/views.py                           |  5 +++--
 app/main/views/tokens.py                             |  2 +-
 app/models/control_lists.py                          |  2 +-
 app/models/corpus.py                                 | 12 ++++--------
 tests/test_selenium/test_control_lists.py            |  2 +-
 tests/test_selenium/test_corpus_init.py              |  6 +++---
 .../test_selenium/test_manage_control_lists_user.py  |  6 +++---
 7 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 15ed0ff5..4e9d4b09 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -378,8 +378,9 @@ def information_read(control_list_id):
 
 @control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
 @login_required
-def ignore_terms_filter(control_list_id):
-    current_controlList = ControlLists.query.filter_by(**{"id":control_list_id}).first_or_404()
+@cl_editable("control_list_id")
+def ignore_terms_filter(control_list_id, control_list):
+    current_controlList = ControlLists.query.filter_by(**{"id":control_list.id}).first_or_404()
     print(current_controlList.filter_punct, current_controlList.filter_ignore)
     list_filter = []
     if request.method == "POST":
diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index 805775a1..f68b0b02 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -54,7 +54,7 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
     user_id = current_user.id
     tokens = corpus\
-        .get_unallowed(user_id, corpus_id, allowed_type)\
+        .get_unallowed(user_id, allowed_type)\
         .paginate(
             page=int_or(request.args.get("page"), 1),
             per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index ecf9c340..795b0157 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -267,7 +267,7 @@ def add_default_lists(path=None):
             print("[ControlLists] Adding %s " % data["name"])
             cl = ControlLists(**data, public=PublicationStatus.public)
             db.session.add(cl)
-            db.session.flush()  # Get the AutoIncrement ID/home/jjanes
+            db.session.flush()  # Get the AutoIncrement ID
             configs = [
                 ("lemma.txt", AllowedLemma, read_input_lemma),
                 ("POS.txt", AllowedPOS, read_input_POS),
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 89a1f55e..d7e01d85 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -298,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
+    def get_unallowed(self, user_id, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -334,13 +334,10 @@ def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
             not_(custom_dict.exists())
         ]
 
-        current_corpus = Corpus.query.filter_by(**{"id":corpus_id}).first_or_404()
-        current_controlList = ControlLists.query.filter_by(
-            **{"id":current_corpus.control_lists_id}).first_or_404()
-
+        current_controlList = self.control_lists
         regex_liste = []
         if current_controlList:
-            if current_controlLists.filter_metadata:
+            if current_controlList.filter_metadata:
                 regex_liste.append(ControlLists.re_filter_metadata)
             if current_controlList.filter_ignore:
                 regex_liste.append(ControlLists.re_filter_ignore)
@@ -1122,8 +1119,7 @@ def is_valid(lemma, POS, morph, corpus):
 
         allowed_column = corpus.displayed_columns_by_name
         if lemma and "lemma" in allowed_column and allowed_lemma.count():
-            current_controlList = ControlLists.query.filter_by(**{"id":corpus.control_lists_id}).first_or_404()
-            print(current_controlList)
+            current_controlList=corpus.control_lists
             regex_liste = []
             if current_controlList:
                 if current_controlList.filter_metadata:
diff --git a/tests/test_selenium/test_control_lists.py b/tests/test_selenium/test_control_lists.py
index d352b249..65da5244 100644
--- a/tests/test_selenium/test_control_lists.py
+++ b/tests/test_selenium/test_control_lists.py
@@ -275,7 +275,7 @@ def test_action_as_user(self):
         self.assertEqual(
             sorted([link.text.strip() for link in links]),
             sorted(['Lemma', "Guidelines",
-                    'Morphologies', 'POS', 'Propose changes', 'Wauchier', 'Ignore values']),
+                    'Morphologies', 'POS', 'Propose changes', 'Wauchier']),
             "Only contacting and reading is possible to users"
         )
 
diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 61e49047..8f347e9c 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -105,7 +105,7 @@ def test_registration_with_full_allowed_lemma(self):
         self.assertEqual(allowed.count(), 21, "There should be 21 allowed token")
 
         # Checking the model
-        self.assertEqual(corpus.get_unallowed(user_id = 1, corpus_id = 1, allowed_type="lemma").count(), 0, "There should be no unallowed value")
+        self.assertEqual(corpus.get_unallowed(user_id = 1, allowed_type="lemma").count(), 0, "There should be no unallowed value")
 
     def test_registration_with_partial_allowed_lemma(self):
         """
@@ -152,7 +152,7 @@ def test_registration_with_partial_allowed_lemma(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed(user_id=1, corpus_id=1, allowed_type="lemma").count(), 22,
+            corpus.get_unallowed(user_id=1, allowed_type="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 
@@ -198,7 +198,7 @@ def test_registration_with_allowed_morph(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed(user_id=1, corpus_id=1, allowed_type="lemma").count(), 22,
+            corpus.get_unallowed(user_id=1, allowed_type="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 
diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index 2ef57348..62fae46a 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -199,11 +199,11 @@ def test_corpus_creator_is_owner(self):
 
         el = self.get_ownership_table()
         self.assertTrue(len([e for e in el if e.get_property("checked")]) == 1)
-
+"""
     def test_change_filter(self):
         self.add_user("foo", "foo")
         self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
-        self.addControlLists("wauchier")
+        self.driver_find_element_by_id("label_checkbox_create").click()
         self.driver.refresh()
         self.go_to_control_lists_management("Wauchier")
 
@@ -218,4 +218,4 @@ def test_change_filter(self):
             'The filters have been updated.',
             "The filters have not been updated."
         )
-
+"""

From dcd28a6f5323d13edcfdbbe56af99df32760fefd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?= <leponteineptique@gmail.com>
Date: Tue, 3 Sep 2024 09:54:19 +0200
Subject: [PATCH 26/43] Adding tests back to control list for changing filter

---
 .../test_manage_control_lists_user.py         | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/tests/test_selenium/test_manage_control_lists_user.py b/tests/test_selenium/test_manage_control_lists_user.py
index 62fae46a..aedc30d4 100644
--- a/tests/test_selenium/test_manage_control_lists_user.py
+++ b/tests/test_selenium/test_manage_control_lists_user.py
@@ -199,11 +199,22 @@ def test_corpus_creator_is_owner(self):
 
         el = self.get_ownership_table()
         self.assertTrue(len([e for e in el if e.get_property("checked")]) == 1)
-"""
+
+
+class TestNotAdmin(TestBase):
+    AUTO_LOG_IN = False
+
+    def go_to_control_lists_management(self, control_lists):
+        self.driver_find_element_by_link_text("Dashboard").click()
+        controllists_dashboard = self.driver_find_element_by_id("control_lists-dashboard")
+        self.element_find_element_by_partial_link_text(controllists_dashboard, control_lists).click()
+
     def test_change_filter(self):
-        self.add_user("foo", "foo")
-        self.login("%s.%s@ppa.fr" % ("foo", "foo"), self.app.config['ADMIN_PASSWORD'])
-        self.driver_find_element_by_id("label_checkbox_create").click()
+        """Check that the owner of a control list can change one of the filters """
+        self.addControlLists("wauchier", no_corpus_user=True)
+        foo_email = self.add_user("foo", "bar")
+        clu = self.addControlListsUser("Wauchier", foo_email, is_owner=True)
+        self.login(foo_email, self.app.config["ADMIN_PASSWORD"])
         self.driver.refresh()
         self.go_to_control_lists_management("Wauchier")
 
@@ -218,4 +229,3 @@ def test_change_filter(self):
             'The filters have been updated.',
             "The filters have not been updated."
         )
-"""

From 41039bdf28c20d82edda333d89be9d8f1479fbe2 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 3 Sep 2024 10:22:32 +0200
Subject: [PATCH 27/43] ajout test base filtre

---
 app/models/corpus.py                      | 11 ++++-------
 tests/test_models/test_record.py          | 20 +++++++++++++++++++-
 tests/test_selenium/test_token_correct.py |  2 ++
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/app/models/corpus.py b/app/models/corpus.py
index d7e01d85..e20e7c97 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -1116,10 +1116,9 @@ def is_valid(lemma, POS, morph, corpus):
             "POS": True,
             "morph": True
         }
-
         allowed_column = corpus.displayed_columns_by_name
-        if lemma and "lemma" in allowed_column and allowed_lemma.count():
-            current_controlList=corpus.control_lists
+        if lemma and "lemma" in allowed_column and allowed_lemma.count()>=0:
+            current_controlList = corpus.control_lists
             regex_liste = []
             if current_controlList:
                 if current_controlList.filter_metadata:
@@ -1130,19 +1129,17 @@ def is_valid(lemma, POS, morph, corpus):
                     regex_liste.append(ControlLists.re_filter_punct)
                 if current_controlList.filter_numeral:
                     regex_liste.append(ControlLists.re_filter_numeral)
-            print(regex_liste)
-
             ignored_by_regex = False
 
             for regex in regex_liste:
                 if re.match(regex, lemma) is not None:
                     ignored_by_regex = True
-
             if (
-                    not ignored_by_regex and
+                    ignored_by_regex is False and
                     corpus.has_custom_dictionary_value("lemma", lemma) is False and
                     corpus.get_allowed_values("lemma", label=lemma).count() == 0
                 ):
+
                     statuses["lemma"] = False
 
         if POS is not None \
diff --git a/tests/test_models/test_record.py b/tests/test_models/test_record.py
index 5d0695d8..6f5df816 100644
--- a/tests/test_models/test_record.py
+++ b/tests/test_models/test_record.py
@@ -1,4 +1,4 @@
-from app.models import ChangeRecord, WordToken, Corpus, ControlLists, ControlListsUser, CorpusUser
+from app.models import ChangeRecord, WordToken, Corpus, ControlLists, ControlListsUser, CorpusUser, Column
 from .base import TestModels
 import copy
 
@@ -6,6 +6,10 @@
 SimilarityFixtures = [
     ControlLists(id=1, name="CL Fixture"),
     Corpus(id=1, name="Fixtures !", control_lists_id=1),
+    Column(heading="Lemma", corpus_id=1),
+    Column(heading="POS", corpus_id=1),
+    Column(heading="Morph", corpus_id=1),
+    Column(heading="Similar", corpus_id=1),
     WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="smn", POS="p"),  # 1
     WordToken(corpus=1, form="Cil", lemma="celle", left_context="_", right_context="_", label_uniform="celle", morph="smn", POS="n"),  # 2
     WordToken(corpus=1, form="Cil", lemma="cil", left_context="_", right_context="_", label_uniform="cil", morph="smn", POS="p"),      # 3
@@ -148,3 +152,17 @@ def test_similar_lemma_double_change(self):
             "Change record should be correct"
         )
         self.assertCountEqual(cr4.changed, ["lemma", "POS"])
+
+    def test_filter_allowed_lemma(self):
+        """ Ensure only similar features are fixed """
+        self.load_fixtures()
+        with self.assertRaises(WordToken.ValidityError):
+            token, change_record = WordToken.update(
+                user_id=1,
+                token_id=1, corpus_id=1,
+                lemma="#", morph="smn", POS="u")
+
+        # faire pareil mais avec les filtres de cochés
+
+
+
diff --git a/tests/test_selenium/test_token_correct.py b/tests/test_selenium/test_token_correct.py
index 0a4bed63..a4b794f8 100644
--- a/tests/test_selenium/test_token_correct.py
+++ b/tests/test_selenium/test_token_correct.py
@@ -284,3 +284,5 @@ def test_correct_delete(self):
         history_row = self.driver_find_elements_by_css_selector("tbody tr.history")
         self.assertEqual(len(history_row), 1, "There should be one record in the history")
 
+
+

From 1c9e66c04285835d141d0039dcc8f8b5149f7bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?= <leponteineptique@gmail.com>
Date: Tue, 3 Sep 2024 10:45:43 +0200
Subject: [PATCH 28/43] Creating combinatory tests

---
 tests/test_cli/test_corpus.py          |  2 +-
 tests/test_models/test_record.py       |  9 -----
 tests/test_models/test_regex_filter.py | 48 ++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 10 deletions(-)
 create mode 100644 tests/test_models/test_regex_filter.py

diff --git a/tests/test_cli/test_corpus.py b/tests/test_cli/test_corpus.py
index f4b11500..deb0a287 100644
--- a/tests/test_cli/test_corpus.py
+++ b/tests/test_cli/test_corpus.py
@@ -279,7 +279,7 @@ def test_corpus_dump_not_found(self):
             )
 
     @nottest
-    def make_test(self, tests, context):
+    def  make_test(self, tests, context):
         with self.app.app_context():
             with self.runner.isolated_filesystem() as f:
                 cur_dir = str(f)
diff --git a/tests/test_models/test_record.py b/tests/test_models/test_record.py
index 6f5df816..87f6f03f 100644
--- a/tests/test_models/test_record.py
+++ b/tests/test_models/test_record.py
@@ -153,15 +153,6 @@ def test_similar_lemma_double_change(self):
         )
         self.assertCountEqual(cr4.changed, ["lemma", "POS"])
 
-    def test_filter_allowed_lemma(self):
-        """ Ensure only similar features are fixed """
-        self.load_fixtures()
-        with self.assertRaises(WordToken.ValidityError):
-            token, change_record = WordToken.update(
-                user_id=1,
-                token_id=1, corpus_id=1,
-                lemma="#", morph="smn", POS="u")
-
         # faire pareil mais avec les filtres de cochés
 
 
diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
new file mode 100644
index 00000000..b75c8297
--- /dev/null
+++ b/tests/test_models/test_regex_filter.py
@@ -0,0 +1,48 @@
+from .test_record import SimilarityFixtures
+from app.models import ChangeRecord, WordToken, Corpus, ControlLists, ControlListsUser, CorpusUser, Column
+from .base import TestModels
+import copy
+from itertools import combinations
+
+
+class TestFilters(TestModels):
+
+    def load_fixtures(self):
+        for fixture in SimilarityFixtures:
+            self.db.session.add(copy.deepcopy(fixture))
+        self.db.session.commit()
+
+    def test_filter_allowed_lemma(self):
+        """ Ensure that when no filters are registered, everything works as intended """
+        self.load_fixtures()
+        with self.assertRaises(WordToken.ValidityError):
+            token, change_record = WordToken.update(
+                user_id=1,
+                token_id=1, corpus_id=1,
+                lemma="#", morph="smn", POS="u")
+
+    def test_combinatory_regex(self):
+        self.load_fixtures()
+        tests = [
+            ("celui", None),
+            ("#", "filter_punct"),
+            ("...", "filter_punct"),
+            ("7", "filter_numeral"),
+            ("75", "filter_numeral")
+        ]
+        situations = ["filter_punct", "filter_numeral"]
+        cl = ControlLists.query.get(1)
+        token = WordToken.query.get(1)
+        corpus = Corpus.query.get(1)
+        for i in range(len(situations)+1):
+            for combi in combinations(situations, i):
+                for filtre in situations:
+                    setattr(cl, filtre, False)
+                for filtre in combi:
+                    setattr(cl, filtre, True)
+                self.db.session.add(cl)
+                self.db.session.commit()
+                self.db.session.refresh(token)
+                self.db.session.refresh(corpus)
+                for category, filtre in tests:
+                    print(WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"])

From dbf15faf3f9822adf4fefc7e39c04db24b738203 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 3 Sep 2024 12:28:46 +0200
Subject: [PATCH 29/43] ajout filtre test combinaison assert + modif filtre
 ponctuation

---
 app/models/control_lists.py            |  2 +-
 tests/test_models/test_record.py       |  2 --
 tests/test_models/test_regex_filter.py | 13 ++++++++++---
 3 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/app/models/control_lists.py b/app/models/control_lists.py
index 795b0157..d006ba05 100644
--- a/app/models/control_lists.py
+++ b/app/models/control_lists.py
@@ -52,7 +52,7 @@ class ControlLists(db.Model):
 
     re_filter_metadata = r'(\[[^\]]+:[^\]]*\]$)'
     re_filter_ignore = r'(^\[IGNORE\])'
-    re_filter_punct = "(^[^\w\s]$)"
+    re_filter_punct = "(^[^\w\s]+$)"
     re_filter_numeral = r'(^\d+$)'
 
     # For caching purposes, we record the last time these fields were edited
diff --git a/tests/test_models/test_record.py b/tests/test_models/test_record.py
index 87f6f03f..9ee7776c 100644
--- a/tests/test_models/test_record.py
+++ b/tests/test_models/test_record.py
@@ -153,7 +153,5 @@ def test_similar_lemma_double_change(self):
         )
         self.assertCountEqual(cr4.changed, ["lemma", "POS"])
 
-        # faire pareil mais avec les filtres de cochés
-
 
 
diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
index b75c8297..b1960311 100644
--- a/tests/test_models/test_regex_filter.py
+++ b/tests/test_models/test_regex_filter.py
@@ -28,9 +28,12 @@ def test_combinatory_regex(self):
             ("#", "filter_punct"),
             ("...", "filter_punct"),
             ("7", "filter_numeral"),
-            ("75", "filter_numeral")
+            ("7578", "filter_numeral"),
+            ("[METADATA:infomation]", "filter_metadata"),
+            ("[12.0:blabla]", "filter_metadata"),
+            ("[IGNORE]", "filter_ignore")
         ]
-        situations = ["filter_punct", "filter_numeral"]
+        situations = ["filter_punct", "filter_numeral", "filter_metadata", "filter_ignore"]
         cl = ControlLists.query.get(1)
         token = WordToken.query.get(1)
         corpus = Corpus.query.get(1)
@@ -45,4 +48,8 @@ def test_combinatory_regex(self):
                 self.db.session.refresh(token)
                 self.db.session.refresh(corpus)
                 for category, filtre in tests:
-                    print(WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"])
+                    validity = WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"]
+                    if filtre in combi:
+                        self.assertTrue(validity, "Filters are not working. Some elements that should not match the regex are being matched.")
+                    else:
+                        self.assertFalse(validity, "Filters are not working. Some elements are not match by the regex filters.")

From 775f7f42038056add5333bce3acd81676916740c Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 3 Sep 2024 15:08:46 +0200
Subject: [PATCH 30/43] ajout filtre none

---
 tests/test_models/test_regex_filter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
index b1960311..1f53968f 100644
--- a/tests/test_models/test_regex_filter.py
+++ b/tests/test_models/test_regex_filter.py
@@ -49,7 +49,7 @@ def test_combinatory_regex(self):
                 self.db.session.refresh(corpus)
                 for category, filtre in tests:
                     validity = WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"]
-                    if filtre in combi:
+                    if filtre and filtre in combi:
                         self.assertTrue(validity, "Filters are not working. Some elements that should not match the regex are being matched.")
                     else:
                         self.assertFalse(validity, "Filters are not working. Some elements are not match by the regex filters.")

From f76e9a1d8bef0fbdcbeaa95c5f5408ab4184e3d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?= <leponteineptique@gmail.com>
Date: Tue, 3 Sep 2024 19:46:36 +0200
Subject: [PATCH 31/43] Fix a condition on lemma

---
 app/models/corpus.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/models/corpus.py b/app/models/corpus.py
index e20e7c97..b7704971 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -1117,7 +1117,11 @@ def is_valid(lemma, POS, morph, corpus):
             "morph": True
         }
         allowed_column = corpus.displayed_columns_by_name
-        if lemma and "lemma" in allowed_column and allowed_lemma.count()>=0:
+        if (lemma  # If we changed the lemma
+                and "lemma" in allowed_column  # And if the lemma is a column known to the project
+                and allowed_lemma.count()  # And if we have a list of accepted lemma,
+        ):
+            # then we check for lemma validity
             current_controlList = corpus.control_lists
             regex_liste = []
             if current_controlList:

From f4e925ceb61551924d9c6afd1f7af754cb6342f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?= <leponteineptique@gmail.com>
Date: Tue, 3 Sep 2024 20:28:18 +0200
Subject: [PATCH 32/43] Better message

---
 tests/test_models/test_regex_filter.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
index 1f53968f..21a49c9c 100644
--- a/tests/test_models/test_regex_filter.py
+++ b/tests/test_models/test_regex_filter.py
@@ -1,15 +1,33 @@
-from .test_record import SimilarityFixtures
-from app.models import ChangeRecord, WordToken, Corpus, ControlLists, ControlListsUser, CorpusUser, Column
+from app.models import ChangeRecord, WordToken, Corpus, ControlLists, Column, AllowedLemma
 from .base import TestModels
 import copy
 from itertools import combinations
 
 
+SimilarityFixtures = [
+    ControlLists(id=1, name="CL Fixture"),
+    Corpus(id=1, name="Fixtures !", control_lists_id=1),
+    Column(heading="Lemma", corpus_id=1),
+    Column(heading="POS", corpus_id=1),
+    Column(heading="Morph", corpus_id=1),
+    Column(heading="Similar", corpus_id=1),
+    WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="smn", POS="p"),  # 1
+    WordToken(corpus=1, form="Cil", lemma="celle", left_context="_", right_context="_", label_uniform="celle", morph="smn", POS="n"),  # 2
+    WordToken(corpus=1, form="Cil", lemma="cil", left_context="_", right_context="_", label_uniform="cil", morph="smn", POS="p"),      # 3
+    WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="mmn", POS="p"),  # 4
+    WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="mmn", POS="n"),  # 5
+    WordToken(corpus=1, form="Cil", lemma="cel", left_context="_", right_context="_", label_uniform="cel", morph="smn", POS="p"),      # 6
+    WordToken(corpus=1, form="Cil", lemma="cel", left_context="_", right_context="_", label_uniform="cel", morph="smn", POS="p"),      # 7
+    WordToken(corpus=1, form="Cil", lemma="cel", left_context="_", right_context="_", label_uniform="cel", morph="smn", POS="p"),      # 8
+    AllowedLemma(id=1, label="celui", label_uniform="celui", control_list=1)
+]
+
 class TestFilters(TestModels):
 
     def load_fixtures(self):
         for fixture in SimilarityFixtures:
             self.db.session.add(copy.deepcopy(fixture))
+            self.db.session.flush()
         self.db.session.commit()
 
     def test_filter_allowed_lemma(self):
@@ -50,6 +68,6 @@ def test_combinatory_regex(self):
                 for category, filtre in tests:
                     validity = WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"]
                     if filtre and filtre in combi:
-                        self.assertTrue(validity, "Filters are not working. Some elements that should not match the regex are being matched.")
+                        self.assertTrue(validity, f"Filters are not working. `{category}` should be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")
                     else:
-                        self.assertFalse(validity, "Filters are not working. Some elements are not match by the regex filters.")
+                        self.assertFalse(validity, f"Filters are not working. `{category}` should not be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")

From 06332f7b0d0e2cc93540d7215c92f72b707818b5 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Mon, 9 Sep 2024 11:36:49 +0200
Subject: [PATCH 33/43] =?UTF-8?q?modif=20test=20regex=20ajout=20condition?=
 =?UTF-8?q?=20sp=C3=A9=20Sans=20test?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tests/test_models/test_regex_filter.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
index 21a49c9c..c8804681 100644
--- a/tests/test_models/test_regex_filter.py
+++ b/tests/test_models/test_regex_filter.py
@@ -67,7 +67,8 @@ def test_combinatory_regex(self):
                 self.db.session.refresh(corpus)
                 for category, filtre in tests:
                     validity = WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"]
-                    if filtre and filtre in combi:
+                    print(combi, category, filtre, validity)
+                    if filtre and filtre in combi or 'celui' in category:
                         self.assertTrue(validity, f"Filters are not working. `{category}` should be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")
                     else:
                         self.assertFalse(validity, f"Filters are not working. `{category}` should not be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")

From d482cf3f8b5496b65053b2612708462a30394116 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 11 Sep 2024 15:51:36 +0200
Subject: [PATCH 34/43] modif filtre metadata sur form et non lemma +
 correction unallowed

---
 app/main/views/tokens.py | 2 +-
 app/models/corpus.py     | 8 +++-----
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index f68b0b02..ef0a0d33 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -54,7 +54,7 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
     user_id = current_user.id
     tokens = corpus\
-        .get_unallowed(user_id, allowed_type)\
+        .get_unallowed(allowed_type)\
         .paginate(
             page=int_or(request.args.get("page"), 1),
             per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
diff --git a/app/models/corpus.py b/app/models/corpus.py
index b7704971..5b48e9fe 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -298,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, user_id, allowed_type="lemma"):
+    def get_unallowed(self, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -338,7 +338,7 @@ def get_unallowed(self, user_id, allowed_type="lemma"):
         regex_liste = []
         if current_controlList:
             if current_controlList.filter_metadata:
-                regex_liste.append(ControlLists.re_filter_metadata)
+                list_darguments.append(WordToken.form.op('!~')(ControlLists.re_filter_metadata))
             if current_controlList.filter_ignore:
                 regex_liste.append(ControlLists.re_filter_ignore)
             if current_controlList.filter_punct:
@@ -347,7 +347,7 @@ def get_unallowed(self, user_id, allowed_type="lemma"):
                 regex_liste.append(ControlLists.re_filter_numeral)
 
         if regex_liste:
-            list_darguments.append(WordToken.form.op('~')("".join(regex_liste)))
+            list_darguments.append(WordToken.lemma.op('!~')("".join(regex_liste)))
 
         return db.session.query(WordToken).filter(
             db.and_(*list_darguments)
@@ -1125,8 +1125,6 @@ def is_valid(lemma, POS, morph, corpus):
             current_controlList = corpus.control_lists
             regex_liste = []
             if current_controlList:
-                if current_controlList.filter_metadata:
-                    regex_liste.append(ControlLists.re_filter_metadata)
                 if current_controlList.filter_ignore:
                     regex_liste.append(ControlLists.re_filter_ignore)
                 if current_controlList.filter_punct:

From 1f4b80a6b06a4f85839162016c1f9e81a1d1d1dd Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 11 Sep 2024 15:51:54 +0200
Subject: [PATCH 35/43] modif test filtres avec metadata

---
 tests/test_models/test_regex_filter.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
index c8804681..210cbac4 100644
--- a/tests/test_models/test_regex_filter.py
+++ b/tests/test_models/test_regex_filter.py
@@ -12,7 +12,7 @@
     Column(heading="Morph", corpus_id=1),
     Column(heading="Similar", corpus_id=1),
     WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="smn", POS="p"),  # 1
-    WordToken(corpus=1, form="Cil", lemma="celle", left_context="_", right_context="_", label_uniform="celle", morph="smn", POS="n"),  # 2
+    WordToken(corpus=1, form="[METADATA:blabla]", lemma="celle", left_context="_", right_context="_", label_uniform="celle", morph="smn", POS="n"),  # 2
     WordToken(corpus=1, form="Cil", lemma="cil", left_context="_", right_context="_", label_uniform="cil", morph="smn", POS="p"),      # 3
     WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="mmn", POS="p"),  # 4
     WordToken(corpus=1, form="Cil", lemma="celui", left_context="_", right_context="_", label_uniform="celui", morph="mmn", POS="n"),  # 5
@@ -47,11 +47,9 @@ def test_combinatory_regex(self):
             ("...", "filter_punct"),
             ("7", "filter_numeral"),
             ("7578", "filter_numeral"),
-            ("[METADATA:infomation]", "filter_metadata"),
-            ("[12.0:blabla]", "filter_metadata"),
             ("[IGNORE]", "filter_ignore")
         ]
-        situations = ["filter_punct", "filter_numeral", "filter_metadata", "filter_ignore"]
+        situations = ["filter_punct", "filter_numeral", "filter_ignore"]
         cl = ControlLists.query.get(1)
         token = WordToken.query.get(1)
         corpus = Corpus.query.get(1)
@@ -72,3 +70,17 @@ def test_combinatory_regex(self):
                         self.assertTrue(validity, f"Filters are not working. `{category}` should be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")
                     else:
                         self.assertFalse(validity, f"Filters are not working. `{category}` should not be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")
+
+    def test_metadata_filter(self):
+        self.load_fixtures()
+        corpus = Corpus.query.get(1)
+        cl = ControlLists.query.get(1)
+        token = corpus.get_unallowed().first()
+        self.assertEqual(token.form, "[METADATA:blabla]", f'Metadata are considered as unallowed token when there is no filter metadata.')
+        setattr(cl, "filter_metadata", True)
+        self.db.session.add(cl)
+        self.db.session.commit()
+        self.db.session.refresh(corpus)
+        token = corpus.get_unallowed().first()
+        self.assertNotEqual(token.form, "[METADATA:blabla]", f'Metadata filter is not working. [METADATA:blabla] should not be considered as unallowed.')
+

From e00e28fad73dc7c108b2491d2604bfb12ad85d19 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 11 Sep 2024 16:03:39 +0200
Subject: [PATCH 36/43] suppression user_id des appels de get_unallowed

---
 tests/test_selenium/test_corpus_init.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_selenium/test_corpus_init.py b/tests/test_selenium/test_corpus_init.py
index 8f347e9c..0a2da034 100644
--- a/tests/test_selenium/test_corpus_init.py
+++ b/tests/test_selenium/test_corpus_init.py
@@ -105,7 +105,7 @@ def test_registration_with_full_allowed_lemma(self):
         self.assertEqual(allowed.count(), 21, "There should be 21 allowed token")
 
         # Checking the model
-        self.assertEqual(corpus.get_unallowed(user_id = 1, allowed_type="lemma").count(), 0, "There should be no unallowed value")
+        self.assertEqual(corpus.get_unallowed(allowed_type="lemma").count(), 0, "There should be no unallowed value")
 
     def test_registration_with_partial_allowed_lemma(self):
         """
@@ -152,7 +152,7 @@ def test_registration_with_partial_allowed_lemma(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed(user_id=1, allowed_type="lemma").count(), 22,
+            corpus.get_unallowed(allowed_type="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 
@@ -198,7 +198,7 @@ def test_registration_with_allowed_morph(self):
 
         # Checking the model
         self.assertEqual(
-            corpus.get_unallowed(user_id=1, allowed_type="lemma").count(), 22,
+            corpus.get_unallowed(allowed_type="lemma").count(), 22,
             "There should be 22 unallowed value as only de saint martin are allowed"
         )
 

From 71a920dae5bb42efc11f53821c12720a623cd007 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 11 Sep 2024 16:38:58 +0200
Subject: [PATCH 37/43] =?UTF-8?q?ajout=20choix=20unallowed=20sqlite=20ou?=
 =?UTF-8?q?=20posgtres=C3=A9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/models/corpus.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/models/corpus.py b/app/models/corpus.py
index 5b48e9fe..7fc6aa0a 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -338,7 +338,10 @@ def get_unallowed(self, allowed_type="lemma"):
         regex_liste = []
         if current_controlList:
             if current_controlList.filter_metadata:
-                list_darguments.append(WordToken.form.op('!~')(ControlLists.re_filter_metadata))
+                try:
+                    list_darguments.append(WordToken.form.op('!~')(ControlLists.re_filter_metadata))
+                except Exception:
+                    list_darguments.append(WordToken.form.op('not regexp')(ControlLists.re_filter_metadata))
             if current_controlList.filter_ignore:
                 regex_liste.append(ControlLists.re_filter_ignore)
             if current_controlList.filter_punct:
@@ -347,7 +350,10 @@ def get_unallowed(self, allowed_type="lemma"):
                 regex_liste.append(ControlLists.re_filter_numeral)
 
         if regex_liste:
-            list_darguments.append(WordToken.lemma.op('!~')("".join(regex_liste)))
+            try:
+                list_darguments.append(WordToken.lemma.op('!~')("".join(regex_liste)))
+            except Exception:
+                list_darguments.append(WordToken.lemma.op('not regexp')("".join(regex_liste)))
 
         return db.session.query(WordToken).filter(
             db.and_(*list_darguments)

From 11caf3927a9305265d08c37061c877c486e85c11 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 11 Sep 2024 17:13:19 +0200
Subject: [PATCH 38/43] ajout diff sqlite/postgres pour filtres get_unallowed

---
 app/models/corpus.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/app/models/corpus.py b/app/models/corpus.py
index 7fc6aa0a..6124d16c 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -39,7 +39,9 @@
 class CorpusUser(db.Model):
     """
         Association proxy that link users to corpora
-        :param corpus_id: a corpus ID
+        :param corpus_id: a corpus                 except:
+
+                    ID
         :param user_id: a user ID
     """
     corpus_id = db.Column(db.Integer, db.ForeignKey("corpus.id", ondelete='CASCADE'), primary_key=True)
@@ -338,10 +340,10 @@ def get_unallowed(self, allowed_type="lemma"):
         regex_liste = []
         if current_controlList:
             if current_controlList.filter_metadata:
-                try:
-                    list_darguments.append(WordToken.form.op('!~')(ControlLists.re_filter_metadata))
-                except Exception:
+                if 'SQLite' in str(db.session.connection().dialect):
                     list_darguments.append(WordToken.form.op('not regexp')(ControlLists.re_filter_metadata))
+                else:
+                    list_darguments.append(WordToken.form.op('!~')(ControlLists.re_filter_metadata))
             if current_controlList.filter_ignore:
                 regex_liste.append(ControlLists.re_filter_ignore)
             if current_controlList.filter_punct:
@@ -350,10 +352,10 @@ def get_unallowed(self, allowed_type="lemma"):
                 regex_liste.append(ControlLists.re_filter_numeral)
 
         if regex_liste:
-            try:
-                list_darguments.append(WordToken.lemma.op('!~')("".join(regex_liste)))
-            except Exception:
+            if 'SQLite' in str(db.session.connection().dialect):
                 list_darguments.append(WordToken.lemma.op('not regexp')("".join(regex_liste)))
+            else:
+                list_darguments.append(WordToken.lemma.op('!~')("".join(regex_liste)))
 
         return db.session.query(WordToken).filter(
             db.and_(*list_darguments)

From 668fd905c0f7f9977eed33debe7f30ec8cc5f3d1 Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Wed, 11 Sep 2024 17:14:16 +0200
Subject: [PATCH 39/43] =?UTF-8?q?d=C3=A9placement=20logging?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/__init__.py          | 4 ++++
 app/main/views/corpus.py | 2 --
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/__init__.py b/app/__init__.py
index bf8a8516..27e47329 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1,4 +1,5 @@
 import os
+import logging
 from flask import Flask, g
 
 from flask_compress import Compress
@@ -64,6 +65,9 @@ def _set_sqlite_case_insensitive_pragma(dbapi_con, connection_record):
     md = Markdown(app, safe_mode=True)
     babel.init_app(app, locale_selector=get_locale)
 
+    logging.basicConfig(filename='./pyrrha_corpus_creation.log', level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
+
     # Register Jinja template functions
     from .main import main as main_blueprint
     app.register_blueprint(main_blueprint)
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index a5b1ecfd..6181434f 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -42,8 +42,6 @@ def _get_available():
 def corpus_new():
     """ Register a new corpus
     """
-    logging.basicConfig(filename='./pyrrha_corpus_creation.log', level=logging.DEBUG,
-                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
     logger = logging.getLogger(__name__)
     lemmatizers = current_app.config.get("LEMMATIZERS", [])
 

From fbabbcf1c5df0ab45d93591269957b2981501d1f Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Thu, 12 Sep 2024 09:36:03 +0200
Subject: [PATCH 40/43] deplacement logging

---
 app/__init__.py          | 8 ++++++--
 app/main/views/corpus.py | 6 +++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/app/__init__.py b/app/__init__.py
index 27e47329..43253728 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -26,6 +26,11 @@
 login_manager.session_protection = 'strong'
 login_manager.login_view = 'account.login'
 
+logging.basicConfig(filename='./pyrrha_corpus_creation.log', level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
+
+logger = logging.getLogger(__name__)
+
 
 def create_app(config_name="dev"):
     """ Create the application """
@@ -65,8 +70,7 @@ def _set_sqlite_case_insensitive_pragma(dbapi_con, connection_record):
     md = Markdown(app, safe_mode=True)
     babel.init_app(app, locale_selector=get_locale)
 
-    logging.basicConfig(filename='./pyrrha_corpus_creation.log', level=logging.DEBUG,
-                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
+
 
     # Register Jinja template functions
     from .main import main as main_blueprint
diff --git a/app/main/views/corpus.py b/app/main/views/corpus.py
index 6181434f..6c78bf6b 100644
--- a/app/main/views/corpus.py
+++ b/app/main/views/corpus.py
@@ -3,7 +3,6 @@
 import sqlalchemy.exc
 from sqlalchemy import func, distinct, text
 from typing import List
-import logging
 
 
 from app import db
@@ -22,7 +21,8 @@
 from .utils import requires_corpus_admin_access, requires_corpus_access
 from ..forms import Delete
 from app.utils import PreferencesUpdateError, PersonalDictionaryError
-
+from app import logger
+import logging
 AUTOCOMPLETE_LIMIT = 20
 
 
@@ -42,7 +42,7 @@ def _get_available():
 def corpus_new():
     """ Register a new corpus
     """
-    logger = logging.getLogger(__name__)
+
     lemmatizers = current_app.config.get("LEMMATIZERS", [])
 
     def normal_view():

From c6c0a1526bdcf9b5b8a9282ddfbb2e9d9e8f4aaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Cl=C3=A9rice?= <leponteineptique@gmail.com>
Date: Tue, 17 Sep 2024 09:42:25 +0200
Subject: [PATCH 41/43] Change the way the control list filter view is shown

---
 app/control_lists/views.py                    | 28 +++++++++++--------
 .../control_lists/ignore_filter.html          | 12 ++++----
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/app/control_lists/views.py b/app/control_lists/views.py
index 4e9d4b09..bcd6fcc1 100644
--- a/app/control_lists/views.py
+++ b/app/control_lists/views.py
@@ -380,8 +380,6 @@ def information_read(control_list_id):
 @login_required
 @cl_editable("control_list_id")
 def ignore_terms_filter(control_list_id, control_list):
-    current_controlList = ControlLists.query.filter_by(**{"id":control_list.id}).first_or_404()
-    print(current_controlList.filter_punct, current_controlList.filter_ignore)
     list_filter = []
     if request.method == "POST":
         list_filter.append(request.form.get("punct"))
@@ -393,18 +391,24 @@ def ignore_terms_filter(control_list_id, control_list):
             if el is not None:
                 filtered_filter.append(el)
 
-        current_controlList.filter_punct = 'punct' in filtered_filter
-        current_controlList.filter_metadata = 'metadata' in filtered_filter
-        current_controlList.filter_numeral = 'numeral' in filtered_filter
-        current_controlList.filter_ignore = 'ignore' in filtered_filter
-        db.session.add(current_controlList)
+        control_list.filter_punct = 'punct' in filtered_filter
+        control_list.filter_metadata = 'metadata' in filtered_filter
+        control_list.filter_numeral = 'numeral' in filtered_filter
+        control_list.filter_ignore = 'ignore' in filtered_filter
+        db.session.add(control_list)
         db.session.commit()
 
 
         flash('The filters have been updated.', 'success')
-        current_controlList = ControlLists.query.filter_by(**{"id":control_list_id}).first_or_404()
-        print(current_controlList.filter_punct, current_controlList.filter_ignore)
-        return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
-                                             current_control_list=current_controlList)
+        db.session.refresh(control_list)
+        return render_template_with_nav_info(
+            'control_lists/ignore_filter.html',
+            control_list_id=control_list_id,
+            control_list=control_list
+        )
 
-    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlList)
\ No newline at end of file
+    return render_template_with_nav_info(
+        'control_lists/ignore_filter.html',
+        control_list_id=control_list_id,
+        control_list=control_list
+    )
\ No newline at end of file
diff --git a/app/templates/control_lists/ignore_filter.html b/app/templates/control_lists/ignore_filter.html
index 6928a680..f68084c8 100644
--- a/app/templates/control_lists/ignore_filter.html
+++ b/app/templates/control_lists/ignore_filter.html
@@ -1,35 +1,35 @@
 {% extends 'layouts/base.html' %}
 {% import 'macros/nav_macros.html' as nav %}
 
-{% block page_title %}Control List {{current_control_list.name}}{% endblock %}
+{% block page_title %}Control List {{control_list.name}}{% endblock %}
 
 {% block content %}
-<h1>{{current_control_list.name}}</h1>
+<h1>{{control_list.name}}</h1>
 <h2>{{ _('Change the filters for Control List') }}</h2>
         <div class="row ">
             <form method="POST">
         <ul>
                 <li>
                     <label>
-                        <input type="checkbox" name="punct" value="punct" {% if current_control_list.filter_punct %}checked{% endif %}>
+                        <input type="checkbox" name="punct" value="punct" {% if control_list.filter_punct %}checked{% endif %}>
                         Punctuation
                     </label>
                 </li>
             <li>
                     <label>
-                        <input type="checkbox" name="numeral" value="numeral" {% if current_control_list.filter_numeral %}checked{% endif %}>
+                        <input type="checkbox" name="numeral" value="numeral" {% if control_list.filter_numeral %}checked{% endif %}>
                         Numeral
                     </label>
                 </li>
             <li>
                     <label>
-                        <input type="checkbox" name="metadata" value="metadata" {% if current_control_list.filter_metadata %}checked{% endif %}>
+                        <input type="checkbox" name="metadata" value="metadata" {% if control_list.filter_metadata %}checked{% endif %}>
                         Metadata
                     </label>
                 </li>
             <li>
                     <label>
-                        <input type="checkbox" name="ignore" value="ignore" {% if current_control_list.filter_ignore %}checked{% endif %}>
+                        <input type="checkbox" name="ignore" value="ignore" {% if control_list.filter_ignore %}checked{% endif %}>
                         Ignore
                     </label>
                 </li>

From db60b6ae6e934c9f152bca37980d883ecd83d8ad Mon Sep 17 00:00:00 2001
From: Juliettejns <juliette.janes@yahoo.fr>
Date: Tue, 17 Sep 2024 16:23:28 +0200
Subject: [PATCH 42/43] modif metadata validity + tests + presentation filtres
 CL dans informations"

---
 app/main/views/tokens.py                      |  1 +
 app/models/corpus.py                          | 73 +++++++++++--------
 .../control_lists/information_read.html       |  8 ++
 app/templates/main/corpus_new.html            | 15 ++--
 tests/test_models/test_record.py              |  4 +-
 tests/test_models/test_regex_filter.py        | 14 +++-
 .../test_selenium/download_temp/wauchier.xml  | 67 +++++++++++++++++
 7 files changed, 140 insertions(+), 42 deletions(-)
 create mode 100644 tests/test_selenium/download_temp/wauchier.xml

diff --git a/app/main/views/tokens.py b/app/main/views/tokens.py
index ef0a0d33..14f8c916 100644
--- a/app/main/views/tokens.py
+++ b/app/main/views/tokens.py
@@ -133,6 +133,7 @@ def tokens_correct_single(corpus_id, token_id):
         token, change_record = WordToken.update(
             user_id=current_user.id,
             token_id=token_id, corpus_id=corpus_id,
+            form = string_to_none(request.form.get("form")),
             lemma=string_to_none(request.form.get("lemma")),
             POS=string_to_none(request.form.get("POS")),
             morph=string_to_none(request.form.get("morph"))
diff --git a/app/models/corpus.py b/app/models/corpus.py
index 6124d16c..ca04369f 100644
--- a/app/models/corpus.py
+++ b/app/models/corpus.py
@@ -1101,7 +1101,7 @@ def get_like(filter_id, form, group_by, type_like="lemma", allowed_list=False):
         return query
 
     @staticmethod
-    def is_valid(lemma, POS, morph, corpus):
+    def is_valid(lemma, POS, morph, corpus, form):
         """ Check if a token is valid for a given corpus
 
         :param lemma: Lemma value of the token to validate
@@ -1125,46 +1125,53 @@ def is_valid(lemma, POS, morph, corpus):
             "morph": True
         }
         allowed_column = corpus.displayed_columns_by_name
-        if (lemma  # If we changed the lemma
+        current_controlList = corpus.control_lists
+        print(form)
+        print(current_controlList.filter_metadata)
+        print(re.match(current_controlList.re_filter_metadata, form))
+        if form and current_controlList.filter_metadata and re.match(current_controlList.re_filter_metadata, form):
+            pass
+        else:
+            if (lemma  # If we changed the lemma
                 and "lemma" in allowed_column  # And if the lemma is a column known to the project
                 and allowed_lemma.count()  # And if we have a list of accepted lemma,
-        ):
-            # then we check for lemma validity
-            current_controlList = corpus.control_lists
-            regex_liste = []
-            if current_controlList:
-                if current_controlList.filter_ignore:
-                    regex_liste.append(ControlLists.re_filter_ignore)
-                if current_controlList.filter_punct:
-                    regex_liste.append(ControlLists.re_filter_punct)
-                if current_controlList.filter_numeral:
-                    regex_liste.append(ControlLists.re_filter_numeral)
-            ignored_by_regex = False
-
-            for regex in regex_liste:
-                if re.match(regex, lemma) is not None:
-                    ignored_by_regex = True
-            if (
-                    ignored_by_regex is False and
-                    corpus.has_custom_dictionary_value("lemma", lemma) is False and
-                    corpus.get_allowed_values("lemma", label=lemma).count() == 0
                 ):
+            # then we check for lemma validity
+            
+                    regex_liste = []
+                    if current_controlList:
+                        if current_controlList.filter_ignore:
+                            regex_liste.append(ControlLists.re_filter_ignore)
+                        if current_controlList.filter_punct:
+                            regex_liste.append(ControlLists.re_filter_punct)
+                        if current_controlList.filter_numeral:
+                            regex_liste.append(ControlLists.re_filter_numeral)
+                    ignored_by_regex = False
+
+                    for regex in regex_liste:
+                        if re.match(regex, lemma) is not None:
+                            ignored_by_regex = True
+                    if (
+                        ignored_by_regex is False and
+                        corpus.has_custom_dictionary_value("lemma", lemma) is False and
+                        corpus.get_allowed_values("lemma", label=lemma).count() == 0
+                        ):
 
-                    statuses["lemma"] = False
+                        statuses["lemma"] = False
 
-        if POS is not None \
+            if POS is not None \
                 and "POS" in allowed_column \
                 and allowed_POS.count() > 0 \
                 and corpus.get_allowed_values("POS", label=POS).count() == 0:
-            if not corpus.has_custom_dictionary_value("POS", POS):
-                statuses["POS"] = False
+                if not corpus.has_custom_dictionary_value("POS", POS):
+                    statuses["POS"] = False
 
-        if morph is not None \
+            if morph is not None \
                 and "morph" in allowed_column \
                 and allowed_morph.count() > 0 \
                 and corpus.get_allowed_values("morph", label=morph).count() == 0:
-            if not corpus.has_custom_dictionary_value("morph", morph):
-                statuses["morph"] = False
+                if not corpus.has_custom_dictionary_value("morph", morph):
+                    statuses["morph"] = False
 
         return statuses
 
@@ -1322,7 +1329,7 @@ def to_input_format(query):
         return csv_file.getvalue()
 
     @staticmethod
-    def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
+    def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None, form=None):
         """ Update a given token with lemma, POS and morph value
 
         :param user_id: ID of the user who performs the update
@@ -1331,6 +1338,8 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
         :type corpus_id: int
         :param token_id: Id of the token
         :type token_id: int
+        :param form: Form
+        :type form: str
         :param lemma: Lemma
         :type lemma: str
         :param POS: PartOfSpeech
@@ -1344,6 +1353,7 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
         corpus = Corpus.query.filter_by(**{"id": corpus_id}).first_or_404()
         token = WordToken.query.filter_by(**{"id": token_id, "corpus": corpus_id}).first_or_404()
         # Strip if things are not None
+        form = strip_or_none(form)
         lemma = strip_or_none(lemma)
         POS = strip_or_none(POS)
         morph = strip_or_none(morph)
@@ -1354,7 +1364,7 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
             error.msg = "No value where changed"
             raise error
         # Check if values are correct regarding allowed values
-        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus)
+        validity = WordToken.is_valid(form=form, lemma=lemma, POS=POS, morph=morph, corpus=corpus)
         if False in list(validity.values()):
             error_msg = "Invalid value in {}".format(
                 ", ".join([key for key in validity.keys() if validity[key] is False])
@@ -1700,6 +1710,7 @@ def apply_changes_to(self, user_id, token_ids):
         ).all():
             apply = {"user_id": user_id, "token_id": token.id, "corpus_id": token.corpus}
             apply.update({attr: val[1] for attr, val in watch.items() if val[0] == getattr(token, attr)})
+            print(apply)
             WordToken.update(**apply)
             changed.append(token)
         return changed
diff --git a/app/templates/control_lists/information_read.html b/app/templates/control_lists/information_read.html
index f15f594b..a2041ef4 100644
--- a/app/templates/control_lists/information_read.html
+++ b/app/templates/control_lists/information_read.html
@@ -16,4 +16,12 @@ <h3>{{ _('Bibliography') }}</h3>
 <p>{{control_list.bibliography | markdown}}</p>
 {% endif %}
 
+<h3>{{ _('Filters') }}</h3>
+<ul>
+  <li>Numeral : {{control_list.filter_numeral}}</li>
+  <li>Punctuation: {{control_list.filter_punct}}</li>
+  <li>Metadata: {{control_list.filter_metadata}}</li>
+  <li>Ignore: {{control_list.filter_ignore}}</li>
+</ul>
+
 {% endblock %}
diff --git a/app/templates/main/corpus_new.html b/app/templates/main/corpus_new.html
index 5080d6a7..ec4c1f7f 100644
--- a/app/templates/main/corpus_new.html
+++ b/app/templates/main/corpus_new.html
@@ -147,7 +147,6 @@ <h1>{{ _('Create a new corpus') }}</h1>
                     the academic community. You will be able to propose new values to the administrators of control lists.') }}
                 </p>
             </div>
-                    <div>
 
             <div class="col-md-9" id="use_public">
               <select class="form-control" id="control_list_select" name="control_list_select">
@@ -215,11 +214,7 @@ <h1>{{ _('Create a new corpus') }}</h1>
                     <small id="allowed_morph_help" class="form-text text-muted">{{ _('The TSV should at least have the header : label and could have a <code>readable</code> column for human') }}</small>
                     <textarea aria-describedby="allowed_morph_help" class="form-control" id="allowed_morph" name="allowed_morph">{% if allowed_morph %}{{allowed_morph}}{%endif%}</textarea>
                 </div>
-            </div>
-
-
-        </div>
-<div class="form-group row">
+                <div class="form-group row">
             <div class="col">
                 <label for="ignoreforms" id="ignoreforms" class="form-text text-muted">
                     Ignore Elements in Control List:
@@ -239,6 +234,10 @@ <h1>{{ _('Create a new corpus') }}</h1>
                         Numeral
                     </label>
                 </li>
+            </ul>
+    </div>
+    <div class="col">
+        <ul>
             <li>
                     <label>
                         <input type="checkbox" name="metadata" value="metadata">
@@ -254,7 +253,11 @@ <h1>{{ _('Create a new corpus') }}</h1>
         </ul>
         </div>
         </div>
+            </div>
+
+
         </div>
+
     </fieldset>
 <button type="submit" id="submit" class="btn btn-primary">Submit</button>
 <script type="text/javascript">
diff --git a/tests/test_models/test_record.py b/tests/test_models/test_record.py
index 9ee7776c..8002a2a7 100644
--- a/tests/test_models/test_record.py
+++ b/tests/test_models/test_record.py
@@ -66,7 +66,7 @@ def test_similar_lemma_single_change(self):
         token, change_record = WordToken.update(
             user_id=1,
             token_id=1, corpus_id=1,
-            lemma="cil", morph="smn", POS="p"
+            form = "Cil", lemma="cil", morph="smn", POS="p"
         )
         self.assertEqual(
             (token.lemma, token.morph, token.POS),
@@ -97,7 +97,7 @@ def test_similar_lemma_double_change(self):
         token, change_record = WordToken.update(
             user_id=1,
             token_id=1, corpus_id=1,
-            lemma="cil", morph="smn", POS="u")
+            form = "Cil", lemma="cil", morph="smn", POS="u")
 
         self.assertEqual(
             (token.lemma, token.morph, token.POS),
diff --git a/tests/test_models/test_regex_filter.py b/tests/test_models/test_regex_filter.py
index 210cbac4..1a12956b 100644
--- a/tests/test_models/test_regex_filter.py
+++ b/tests/test_models/test_regex_filter.py
@@ -37,7 +37,7 @@ def test_filter_allowed_lemma(self):
             token, change_record = WordToken.update(
                 user_id=1,
                 token_id=1, corpus_id=1,
-                lemma="#", morph="smn", POS="u")
+                form = "Cil", lemma="#", morph="smn", POS="u")
 
     def test_combinatory_regex(self):
         self.load_fixtures()
@@ -64,8 +64,7 @@ def test_combinatory_regex(self):
                 self.db.session.refresh(token)
                 self.db.session.refresh(corpus)
                 for category, filtre in tests:
-                    validity = WordToken.is_valid(lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"]
-                    print(combi, category, filtre, validity)
+                    validity = WordToken.is_valid(form = token.form, lemma=category, POS=token.POS, morph=token.morph, corpus=corpus)["lemma"]
                     if filtre and filtre in combi or 'celui' in category:
                         self.assertTrue(validity, f"Filters are not working. `{category}` should be matched by `{filtre}` in {', '.join(combi) or 'absence of filters'}")
                     else:
@@ -83,4 +82,13 @@ def test_metadata_filter(self):
         self.db.session.refresh(corpus)
         token = corpus.get_unallowed().first()
         self.assertNotEqual(token.form, "[METADATA:blabla]", f'Metadata filter is not working. [METADATA:blabla] should not be considered as unallowed.')
+        
+        token = WordToken.query.get(2)
+        validity_lemma = WordToken.is_valid(form = token.form, lemma="blabla", POS=token.POS, morph=token.morph,corpus=corpus)["lemma"]
+        self.assertTrue(validity_lemma, f"Filter metadata is not working for lemma")
+        validity_pos = WordToken.is_valid(form = token.form, lemma=token.lemma, POS="blabla", morph=token.morph,corpus=corpus)["POS"]
+        validity_morph = WordToken.is_valid(form = token.form, lemma=token.lemma, POS=token.POS, morph="blabla",corpus=corpus)["morph"]
+
+
+
 
diff --git a/tests/test_selenium/download_temp/wauchier.xml b/tests/test_selenium/download_temp/wauchier.xml
new file mode 100644
index 00000000..feb858f6
--- /dev/null
+++ b/tests/test_selenium/download_temp/wauchier.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+    <teiHeader>
+        <fileDesc>
+            <titleStmt>
+                <title></title>
+            </titleStmt>
+            <publicationStmt><p></p></publicationStmt>
+            <sourceDesc><p></p></sourceDesc>
+        </fileDesc>
+        
+    </teiHeader>
+    <text xml:lang="fr">
+        <body xml:lang="fro">
+            <div>
+                <ab>
+                
+                        <w xml:id="t1" n="1" lemma="de" type="POS=PRE" >De</w>
+                        <w xml:id="t2" n="2" lemma="saint" type="POS=ADJqua" >seint</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t4" n="4" lemma="martin" type="POS=NOMpro" >Martin</w>
+                        <w xml:id="t5" n="5" lemma="mout" type="POS=ADVgen" >mout</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t7" n="7" lemma="devoir" type="POS=VERcjg" >doit</w>
+                        <w xml:id="t8" n="8" lemma="un" type="POS=PRE" >on</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t10" n="10" lemma="doucement" type="POS=ADVgen" >doucement</w>
+                        <w xml:id="t11" n="11" lemma="et" type="POS=CONcoo" >et</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t13" n="13" lemma="volentiers" type="POS=ADVgen" >volentiers</w>
+                        <w xml:id="t14" n="14" lemma="le" type="POS=DETdef" >le</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t16" n="16" lemma="bien" type="POS=ADVgen" >bien</w>
+                        <w xml:id="t17" n="17" lemma="öir" type="POS=VERinf" >oïr</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t19" n="19" lemma="et" type="POS=CONcoo" >et</w>
+                        <w xml:id="t20" n="20" lemma="entendre" type="POS=VERinf" >entendre</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t22" n="22" lemma="," type="POS=PONfbl" >,</w>
+                        <w xml:id="t23" n="23" lemma="car" type="POS=CONcoo" >car</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t25" n="25" lemma="par" type="POS=PRE" >par</w>
+                        <w xml:id="t26" n="26" lemma="le" type="POS=DETdef" >le</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t28" n="28" lemma="bien" type="POS=ADVgen" >bien</w>
+                        <w xml:id="t29" n="29" lemma="savoir" type="POS=VERinf" >savoir</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t31" n="31" lemma="et" type="POS=CONcoo" >et</w>
+                        <w xml:id="t32" n="32" lemma="retenir" type="POS=VERinf" >retenir</w>
+                        </ab>
+                        <ab>
+                        <w xml:id="t34" n="34" lemma="pöoir" type="POS=VERcjg" >puet</w>
+                        <w xml:id="t35" n="35" lemma="il" type="POS=PROper" >l</w></ab>
+            </div>
+       </body>
+     </text>
+</TEI>
\ No newline at end of file

From 360a560e739e2b253a50d669f36a8fd93f9e69ec Mon Sep 17 00:00:00 2001
From: Juliettejns <56683417+Juliettejns@users.noreply.github.com>
Date: Tue, 17 Sep 2024 16:26:39 +0200
Subject: [PATCH 43/43] Delete tests/test_selenium/download_temp/wauchier.xml

---
 .../test_selenium/download_temp/wauchier.xml  | 67 -------------------
 1 file changed, 67 deletions(-)
 delete mode 100644 tests/test_selenium/download_temp/wauchier.xml

diff --git a/tests/test_selenium/download_temp/wauchier.xml b/tests/test_selenium/download_temp/wauchier.xml
deleted file mode 100644
index feb858f6..00000000
--- a/tests/test_selenium/download_temp/wauchier.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<TEI xmlns="http://www.tei-c.org/ns/1.0">
-    <teiHeader>
-        <fileDesc>
-            <titleStmt>
-                <title></title>
-            </titleStmt>
-            <publicationStmt><p></p></publicationStmt>
-            <sourceDesc><p></p></sourceDesc>
-        </fileDesc>
-        
-    </teiHeader>
-    <text xml:lang="fr">
-        <body xml:lang="fro">
-            <div>
-                <ab>
-                
-                        <w xml:id="t1" n="1" lemma="de" type="POS=PRE" >De</w>
-                        <w xml:id="t2" n="2" lemma="saint" type="POS=ADJqua" >seint</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t4" n="4" lemma="martin" type="POS=NOMpro" >Martin</w>
-                        <w xml:id="t5" n="5" lemma="mout" type="POS=ADVgen" >mout</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t7" n="7" lemma="devoir" type="POS=VERcjg" >doit</w>
-                        <w xml:id="t8" n="8" lemma="un" type="POS=PRE" >on</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t10" n="10" lemma="doucement" type="POS=ADVgen" >doucement</w>
-                        <w xml:id="t11" n="11" lemma="et" type="POS=CONcoo" >et</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t13" n="13" lemma="volentiers" type="POS=ADVgen" >volentiers</w>
-                        <w xml:id="t14" n="14" lemma="le" type="POS=DETdef" >le</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t16" n="16" lemma="bien" type="POS=ADVgen" >bien</w>
-                        <w xml:id="t17" n="17" lemma="öir" type="POS=VERinf" >oïr</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t19" n="19" lemma="et" type="POS=CONcoo" >et</w>
-                        <w xml:id="t20" n="20" lemma="entendre" type="POS=VERinf" >entendre</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t22" n="22" lemma="," type="POS=PONfbl" >,</w>
-                        <w xml:id="t23" n="23" lemma="car" type="POS=CONcoo" >car</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t25" n="25" lemma="par" type="POS=PRE" >par</w>
-                        <w xml:id="t26" n="26" lemma="le" type="POS=DETdef" >le</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t28" n="28" lemma="bien" type="POS=ADVgen" >bien</w>
-                        <w xml:id="t29" n="29" lemma="savoir" type="POS=VERinf" >savoir</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t31" n="31" lemma="et" type="POS=CONcoo" >et</w>
-                        <w xml:id="t32" n="32" lemma="retenir" type="POS=VERinf" >retenir</w>
-                        </ab>
-                        <ab>
-                        <w xml:id="t34" n="34" lemma="pöoir" type="POS=VERcjg" >puet</w>
-                        <w xml:id="t35" n="35" lemma="il" type="POS=PROper" >l</w></ab>
-            </div>
-       </body>
-     </text>
-</TEI>
\ No newline at end of file