From 2845914d44562aa39956386dedde23e5e10316cf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 4 Dec 2021 03:27:23 +0100 Subject: [PATCH 001/436] WIP: foundation for a new portal API to partially replace SSOwat --- bin/yunohost-portal-api | 53 ++++++++++++++++++ share/actionsmap-portal.yml | 51 +++++++++++++++++ src/__init__.py | 20 ++++++- src/authenticators/ldap_ynhuser.py | 59 ++++++++++++++++++++ src/portal.py | 89 ++++++++++++++++++++++++++++++ 5 files changed, 271 insertions(+), 1 deletion(-) create mode 100755 bin/yunohost-portal-api create mode 100644 share/actionsmap-portal.yml create mode 100644 src/authenticators/ldap_ynhuser.py create mode 100644 src/portal.py diff --git a/bin/yunohost-portal-api b/bin/yunohost-portal-api new file mode 100755 index 0000000000..66751e66fe --- /dev/null +++ b/bin/yunohost-portal-api @@ -0,0 +1,53 @@ +#! /usr/bin/python3 +# -*- coding: utf-8 -*- + +import argparse +import yunohost + +# Default server configuration +DEFAULT_HOST = "localhost" +DEFAULT_PORT = 6788 + + +def _parse_api_args(): + """Parse main arguments for the api""" + parser = argparse.ArgumentParser( + add_help=False, + description="Run the YunoHost API to manage your server.", + ) + srv_group = parser.add_argument_group("server configuration") + srv_group.add_argument( + "-h", + "--host", + action="store", + default=DEFAULT_HOST, + help="Host to listen on (default: %s)" % DEFAULT_HOST, + ) + srv_group.add_argument( + "-p", + "--port", + action="store", + default=DEFAULT_PORT, + type=int, + help="Port to listen on (default: %d)" % DEFAULT_PORT, + ) + glob_group = parser.add_argument_group("global arguments") + glob_group.add_argument( + "--debug", + action="store_true", + default=False, + help="Set log level to DEBUG", + ) + glob_group.add_argument( + "--help", + action="help", + help="Show this help message and exit", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + opts = _parse_api_args() + # Run the server + yunohost.portalapi(debug=opts.debug, host=opts.host, port=opts.port) diff --git a/share/actionsmap-portal.yml b/share/actionsmap-portal.yml new file mode 100644 index 0000000000..3d07656ae5 --- /dev/null +++ b/share/actionsmap-portal.yml @@ -0,0 +1,51 @@ +_global: + namespace: yunohost + cookie_name: yunohost.portal + authentication: + api: ldap_ynhuser + cli: null + +portal: + category_help: Portal routes + actions: + + ### portal_me() + me: + action_help: Allow user to fetch their own infos + api: GET /me + + ### portal_apps() + apps: + action_help: Allow users to fetch lit of apps they have access to + api: GET /me/apps + + ### portal_update() + update: + action_help: Allow user to update their infos (display name, mail aliases/forward, password, ...) + api: PUT /me + # FIXME: add args etc + + ### portal_reset_password() + reset_password: + action_help: Allow user to update their infos (display name, mail aliases/forward, ...) + api: PUT /me/reset_password + authentication: + # FIXME: to be implemented ? + api: reset_password_token + # FIXME: add args etc + + ### portal_register() + register: + action_help: Allow user to register using an invite token or ??? + api: POST /me + authentication: + # FIXME: to be implemented ? + api: register_invite_token + # FIXME: add args etc + + ### portal_public() + public: + action_help: Allow anybody to list public apps and other infos regarding the public portal + api: GET /public + authentication: + api: null diff --git a/src/__init__.py b/src/__init__.py index b9dcd93d9c..aaeea7751f 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -53,6 +53,20 @@ def is_installed_api(): sys.exit(ret) +def portalapi(debug, host, port): + + # FIXME : is this the logdir we want ? (yolo to work around permission issue) + init_logging(interface="portalapi", debug=debug, logdir="/var/log") + + ret = moulinette.api( + host=host, + port=port, + actionsmap="/usr/share/yunohost/actionsmap-portal.yml", + locales_dir="/usr/share/yunohost/locales/" + ) + sys.exit(ret) + + def check_command_is_valid_before_postinstall(args): allowed_if_not_postinstalled = [ @@ -125,6 +139,10 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "level": "DEBUG" if debug else "INFO", "class": "moulinette.interfaces.api.APIQueueHandler", }, + "portalapi": { + "level": "DEBUG" if debug else "INFO", + "class": "moulinette.interfaces.api.APIQueueHandler", + }, "file": { "class": "logging.FileHandler", "formatter": "precise", @@ -151,7 +169,7 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun } # Logging configuration for CLI (or any other interface than api...) # - if interface != "api": + if interface not in ["api", "portalapi"]: configure_logging(logging_configuration) # Logging configuration for API # diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py new file mode 100644 index 0000000000..50dca3cc92 --- /dev/null +++ b/src/authenticators/ldap_ynhuser.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +import logging +import ldap +import ldap.sasl + +from moulinette import m18n +from moulinette.authentication import BaseAuthenticator +from yunohost.utils.error import YunohostError + +logger = logging.getLogger("yunohostportal.authenticators.ldap_ynhuser") + +URI = "ldap://localhost:389" +USERDN = "uid={username},ou=users,dc=yunohost,dc=org" + + +class Authenticator(BaseAuthenticator): + + name = "ldap_ynhuser" + + def _authenticate_credentials(self, credentials=None): + + # FIXME ':' should a legit char in the password ? shall we encode the password as base64 or something idk + if ":" not in credentials or len(credentials.split(":")) != 2: + raise YunohostError("invalid_credentials_format") + + username, password = credentials.split(":") + + def _reconnect(): + con = ldap.ldapobject.ReconnectLDAPObject( + URI, retry_max=2, retry_delay=0.5 + ) + con.simple_bind_s(USERDN.format(username=username), password) + return con + + try: + con = _reconnect() + except ldap.INVALID_CREDENTIALS: + raise YunohostError("invalid_password") + except ldap.SERVER_DOWN: + logger.warning(m18n.n("ldap_server_down")) + + # Check that we are indeed logged in with the expected identity + try: + # whoami_s return dn:..., then delete these 3 characters + who = con.whoami_s()[3:] + except Exception as e: + logger.warning("Error during ldap authentication process: %s", e) + raise + else: + if who != USERDN.format(username=username): + raise YunohostError( + "Not logged with the appropriate identity ?!", + raw_msg=True, + ) + finally: + # Free the connection, we don't really need it to keep it open as the point is only to check authentication... + if con: + con.unbind_s() diff --git a/src/portal.py b/src/portal.py new file mode 100644 index 0000000000..4a2b449b27 --- /dev/null +++ b/src/portal.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2021 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" + +# from moulinette import Moulinette, m18n +from moulinette.utils.log import getActionLogger + +from yunohost.utils.error import YunohostValidationError + +logger = getActionLogger("yunohostportal.user") + + +def me(): + """ + Get user informations + + Keyword argument: + username -- Username to get informations + + """ + + username = None # FIXME : this info should come from the authentication layer + + from yunohost.utils.ldap import _get_ldap_interface + + ldap = _get_ldap_interface() + + user_attrs = ["cn", "mail", "uid", "maildrop", "givenName", "sn", "mailuserquota"] + + filter = "uid=" + username + result = ldap.search("ou=users,dc=yunohost,dc=org", filter, user_attrs) + + if result: + user = result[0] + else: + raise YunohostValidationError("user_unknown", user=username) + + result_dict = { + "username": user["uid"][0], + "fullname": user["cn"][0], + "firstname": user["givenName"][0], + "lastname": user["sn"][0], + "mail": user["mail"][0], + "mail-aliases": [], + "mail-forward": [], + } + + if len(user["mail"]) > 1: + result_dict["mail-aliases"] = user["mail"][1:] + + if len(user["maildrop"]) > 1: + result_dict["mail-forward"] = user["maildrop"][1:] + + if "mailuserquota" in user: + pass + # FIXME + # result_dict["mailbox-quota"] = { + # "limit": userquota if is_limited else m18n.n("unlimit"), + # "use": storage_use, + # } + + # FIXME : should also parse "permission" key in ldap maybe ? + # and list of groups / memberof ? + # (in particular to have e.g. the mail / xmpp / ssh / ... perms) + + return result_dict + + +def apps(username): + return {"foo": "bar"} + # FIXME: should list available apps and corresponding infos ? + # from /etc/ssowat/conf.json ? From 1efb50c7abc0141b0f6325ae155a653ed9027ff0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 25 Dec 2021 15:44:14 +0100 Subject: [PATCH 002/436] Iterate on new portal API design: nginx config, cookie format, be able to open a non-root ldap session, --- conf/nginx/plain/yunohost_sso.conf.inc | 4 +- conf/nginx/yunohost_api.conf.inc | 20 +++++++ debian/control | 3 +- share/actionsmap-portal.yml | 3 +- src/authenticators/ldap_ynhuser.py | 83 +++++++++++++++++++++++++- src/portal.py | 12 ++-- src/utils/ldap.py | 38 ++++++++---- 7 files changed, 141 insertions(+), 22 deletions(-) diff --git a/conf/nginx/plain/yunohost_sso.conf.inc b/conf/nginx/plain/yunohost_sso.conf.inc index 308e5a9a4b..9844406793 100644 --- a/conf/nginx/plain/yunohost_sso.conf.inc +++ b/conf/nginx/plain/yunohost_sso.conf.inc @@ -2,6 +2,6 @@ rewrite ^/yunohost/sso$ /yunohost/sso/ permanent; location /yunohost/sso/ { - # This is an empty location, only meant to avoid other locations - # from matching /yunohost/sso, such that it's correctly handled by ssowat + alias /usr/share/ssowat/portal/; + index index.html; } diff --git a/conf/nginx/yunohost_api.conf.inc b/conf/nginx/yunohost_api.conf.inc index c9ae34f820..3a463c23bf 100644 --- a/conf/nginx/yunohost_api.conf.inc +++ b/conf/nginx/yunohost_api.conf.inc @@ -23,3 +23,23 @@ location = /yunohost/api/error/502 { add_header Content-Type text/plain; internal; } + +location /yunohost/portalapi/ { + proxy_read_timeout 3600s; + proxy_pass http://127.0.0.1:6788/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $http_host; + + # Custom 502 error page + error_page 502 /yunohost/portalapi/error/502; +} + + +# Yunohost admin output complete 502 error page, so use only plain text. +location = /yunohost/portalapi/error/502 { + return 502 '502 - Bad Gateway'; + add_header Content-Type text/plain; + internal; +} diff --git a/debian/control b/debian/control index 31204a180c..8a9d841c5f 100644 --- a/debian/control +++ b/debian/control @@ -14,7 +14,8 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 - , python3-ldap, python3-zeroconf, python3-lexicon, + , python3-ldap, python3-zeroconf, python3-lexicon + , python3-jwt , python-is-python3 , nginx, nginx-extras (>=1.18) , apt, apt-transport-https, apt-utils, dirmngr diff --git a/share/actionsmap-portal.yml b/share/actionsmap-portal.yml index 3d07656ae5..761d5a6ceb 100644 --- a/share/actionsmap-portal.yml +++ b/share/actionsmap-portal.yml @@ -1,9 +1,10 @@ _global: namespace: yunohost - cookie_name: yunohost.portal authentication: api: ldap_ynhuser cli: null + lock: false + cache: false portal: category_help: Portal routes diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 50dca3cc92..28b8c49fd6 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -1,12 +1,17 @@ # -*- coding: utf-8 -*- +import jwt import logging import ldap import ldap.sasl +import datetime from moulinette import m18n from moulinette.authentication import BaseAuthenticator -from yunohost.utils.error import YunohostError +from moulinette.utils.text import random_ascii +from yunohost.utils.error import YunohostError, YunohostAuthenticationError + +session_secret = random_ascii() logger = logging.getLogger("yunohostportal.authenticators.ldap_ynhuser") @@ -57,3 +62,79 @@ def _reconnect(): # Free the connection, we don't really need it to keep it open as the point is only to check authentication... if con: con.unbind_s() + + + + + + + + # FIXME FIXME FIXME : the password is to be encrypted to not expose it in the JWT cookie which is only signed and base64 encoded but not encrypted + + + + + + + + + + return {"user": username, "password": password} + + def set_session_cookie(self, infos): + + from bottle import response + + assert isinstance(infos, dict) + + # This allows to generate a new session id or keep the existing one + current_infos = self.get_session_cookie(raise_if_no_session_exists=False) + new_infos = { + "id": current_infos["id"], + # See https://pyjwt.readthedocs.io/en/latest/usage.html#registered-claim-names + # for explanations regarding nbf, exp + "nbf": int(datetime.datetime.now().timestamp()), + "exp": int(datetime.datetime.now().timestamp()) + (7 * 24 * 3600) # One week validity + } + new_infos.update(infos) + + response.set_cookie( + "yunohost.portal", + jwt.encode(new_infos, session_secret, algorithm="HS256").decode(), + secure=True, + httponly=True, + path="/", + # samesite="strict", # Bottle 0.12 doesn't support samesite, to be added in next versions + # FIXME : add Expire clause + ) + + def get_session_cookie(self, raise_if_no_session_exists=True): + + from bottle import request + + try: + token = request.get_cookie("yunohost.portal", default="").encode() + infos = jwt.decode(token, session_secret, algorithms="HS256", options={"require": ["id", "user", "exp", "nbf"]}) + except Exception: + if not raise_if_no_session_exists: + return {"id": random_ascii()} + raise YunohostAuthenticationError("unable_authenticate") + + if not infos and raise_if_no_session_exists: + raise YunohostAuthenticationError("unable_authenticate") + + if "id" not in infos: + infos["id"] = random_ascii() + + # FIXME: Here, maybe we want to re-authenticate the session via the authenticator + # For example to check that the username authenticated is still in the admin group... + + return infos + + @staticmethod + def delete_session_cookie(self): + + from bottle import response + + response.set_cookie("yunohost.portal", "", max_age=-1) + response.delete_cookie("yunohost.portal") diff --git a/src/portal.py b/src/portal.py index 4a2b449b27..2eaa59dd45 100644 --- a/src/portal.py +++ b/src/portal.py @@ -22,12 +22,14 @@ # from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger +from yunohost.authenticators.ldap_ynhuser import Authenticator as Auth +from yunohost.utils.ldap import LDAPInterface from yunohost.utils.error import YunohostValidationError logger = getActionLogger("yunohostportal.user") -def me(): +def portal_me(): """ Get user informations @@ -36,11 +38,13 @@ def me(): """ - username = None # FIXME : this info should come from the authentication layer + import pdb; pdb.set_trace() - from yunohost.utils.ldap import _get_ldap_interface + auth = Auth().get_session_cookie() + username = auth["user"] + password = auth["password"] - ldap = _get_ldap_interface() + ldap = LDAPInterface(username, password) user_attrs = ["cn", "mail", "uid", "maildrop", "givenName", "sn", "mailuserquota"] diff --git a/src/utils/ldap.py b/src/utils/ldap.py index 651d09f75a..852fa89c28 100644 --- a/src/utils/ldap.py +++ b/src/utils/ldap.py @@ -42,7 +42,7 @@ def _get_ldap_interface(): global _ldap_interface if _ldap_interface is None: - _ldap_interface = LDAPInterface() + _ldap_interface = LDAPInterface(user="root") return _ldap_interface @@ -71,22 +71,34 @@ def _destroy_ldap_interface(): atexit.register(_destroy_ldap_interface) +URI = "ldapi://%2Fvar%2Frun%2Fslapd%2Fldapi" +BASEDN = "dc=yunohost,dc=org" +ROOTDN = "gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" +USERDN = "uid={username},ou=users,dc=yunohost,dc=org" + class LDAPInterface: - def __init__(self): - logger.debug("initializing ldap interface") - self.uri = "ldapi://%2Fvar%2Frun%2Fslapd%2Fldapi" - self.basedn = "dc=yunohost,dc=org" - self.rootdn = "gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth" + def __init__(self, user="root", password=None): + + if user == "root": + logger.debug("initializing root ldap interface") + self.userdn = ROOTDN + self._connect = lambda con: con.sasl_non_interactive_bind_s("EXTERNAL") + else: + logger.debug("initializing user ldap interface") + self.userdn = USERDN.format(username=user) + self._connect = lambda con: con.simple_bind_s(self.userdn, password) + self.connect() def connect(self): + def _reconnect(): con = ldap.ldapobject.ReconnectLDAPObject( - self.uri, retry_max=10, retry_delay=0.5 + URI, retry_max=10, retry_delay=0.5 ) - con.sasl_non_interactive_bind_s("EXTERNAL") + self._connect(con) return con try: @@ -113,7 +125,7 @@ def _reconnect(): logger.warning("Error during ldap authentication process: %s", e) raise else: - if who != self.rootdn: + if who != self.userdn: raise MoulinetteError("Not logged in with the expected userdn ?!") else: self.con = con @@ -139,7 +151,7 @@ def search(self, base=None, filter="(objectClass=*)", attrs=["dn"]): """ if not base: - base = self.basedn + base = BASEDN try: result = self.con.search_s(base, ldap.SCOPE_SUBTREE, filter, attrs) @@ -184,7 +196,7 @@ def add(self, rdn, attr_dict): Boolean | MoulinetteError """ - dn = rdn + "," + self.basedn + dn = f"{rdn},{BASEDN}" ldif = modlist.addModlist(attr_dict) for i, (k, v) in enumerate(ldif): if isinstance(v, list): @@ -215,7 +227,7 @@ def remove(self, rdn): Boolean | MoulinetteError """ - dn = rdn + "," + self.basedn + dn = f"{rdn},{BASEDN}" try: self.con.delete_s(dn) except Exception as e: @@ -240,7 +252,7 @@ def update(self, rdn, attr_dict, new_rdn=False): Boolean | MoulinetteError """ - dn = rdn + "," + self.basedn + dn = f"{rdn},{BASEDN}" actual_entry = self.search(base=dn, attrs=None) ldif = modlist.modifyModlist(actual_entry[0], attr_dict, ignore_oldexistent=1) From 62808152eecf4fd5da659e1785b64460e9775a7f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 26 Dec 2021 16:31:05 +0100 Subject: [PATCH 003/436] Cookie handling for the new portal API --- src/app.py | 3 +++ src/authenticators/ldap_ynhuser.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 27cf7caecc..9db7a4e4e2 100644 --- a/src/app.py +++ b/src/app.py @@ -1323,6 +1323,7 @@ def app_ssowatconf(): "public": True, "uris": [domain + "/yunohost/admin" for domain in domains] + [domain + "/yunohost/api" for domain in domains] + + [domain + "/yunohost/portalapi" for domain in domains] + [ "re:^[^/]*/%.well%-known/ynh%-diagnosis/.*$", "re:^[^/]*/%.well%-known/acme%-challenge/.*$", @@ -1368,6 +1369,8 @@ def app_ssowatconf(): } conf_dict = { + "cookie_secret_file": "/etc/yunohost/.ssowat_cookie_secret", + "cookie_name": "yunohost.portal", "portal_domain": main_domain, "portal_path": "/yunohost/sso/", "additional_headers": { diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 28b8c49fd6..0e51d1925e 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -11,7 +11,9 @@ from moulinette.utils.text import random_ascii from yunohost.utils.error import YunohostError, YunohostAuthenticationError -session_secret = random_ascii() +# FIXME : we shall generate this somewhere if it doesnt exists yet +# FIXME : fix permissions +session_secret = open("/etc/yunohost/.ssowat_cookie_secret").read() logger = logging.getLogger("yunohostportal.authenticators.ldap_ynhuser") From 45baaead3689703b257c8ce16aee763a3a67c9ba Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 26 Dec 2021 18:22:33 +0100 Subject: [PATCH 004/436] Fix typo + unused import --- src/authenticators/ldap_admin.py | 1 - src/authenticators/ldap_ynhuser.py | 1 - src/tools.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/authenticators/ldap_admin.py b/src/authenticators/ldap_admin.py index 7f96165cbc..872dd3c8de 100644 --- a/src/authenticators/ldap_admin.py +++ b/src/authenticators/ldap_admin.py @@ -120,7 +120,6 @@ def get_session_cookie(self, raise_if_no_session_exists=True): return infos - @staticmethod def delete_session_cookie(self): from bottle import response diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 0e51d1925e..fe2a657f55 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -133,7 +133,6 @@ def get_session_cookie(self, raise_if_no_session_exists=True): return infos - @staticmethod def delete_session_cookie(self): from bottle import response diff --git a/src/tools.py b/src/tools.py index b66d20811b..467e33cfad 100644 --- a/src/tools.py +++ b/src/tools.py @@ -33,7 +33,7 @@ from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output, call_async_output +from moulinette.utils.process import call_async_output from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm from yunohost.app import ( From bd564e6a536f0b0b54c09fdd687a23f76372a59a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Dec 2021 12:44:20 +0100 Subject: [PATCH 005/436] Add systemd conf for new service yunohost-portal-api --- conf/yunohost/yunohost-portal-api.service | 14 ++++++++++++++ debian/postinst | 2 ++ hooks/conf_regen/01-yunohost | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 conf/yunohost/yunohost-portal-api.service diff --git a/conf/yunohost/yunohost-portal-api.service b/conf/yunohost/yunohost-portal-api.service new file mode 100644 index 0000000000..0ba6e8b3dd --- /dev/null +++ b/conf/yunohost/yunohost-portal-api.service @@ -0,0 +1,14 @@ +[Unit] +Description=YunoHost Portal API +After=network.target + +[Service] +User=ynh-portal +Type=simple +ExecStart=/usr/bin/yunohost-portal-api +Restart=always +RestartSec=5 +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/debian/postinst b/debian/postinst index e93845e88f..c62926a303 100644 --- a/debian/postinst +++ b/debian/postinst @@ -29,6 +29,8 @@ do_configure() { yunohost diagnosis run --force fi + systemctl restart yunohost-portal-api + # Trick to let yunohost handle the restart of the API, # to prevent the webadmin from cutting the branch it's sitting on if systemctl is-enabled yunohost-api --quiet diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 14840e2f19..597595231e 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -56,7 +56,10 @@ do_init_regen() { chown root:root /var/cache/yunohost chmod 700 /var/cache/yunohost + getent passwd ynh-portal &>/dev/null || useradd --no-create-home --shell /usr/sbin/nologin --system --user-group ynh-portal + cp yunohost-api.service /etc/systemd/system/yunohost-api.service + cp yunohost-portal-api.service /etc/systemd/system/yunohost-portal-api.service cp yunohost-firewall.service /etc/systemd/system/yunohost-firewall.service cp yunoprompt.service /etc/systemd/system/yunoprompt.service @@ -64,6 +67,10 @@ do_init_regen() { systemctl enable yunohost-api.service systemctl start yunohost-api.service + + systemctl enable yunohost-portal-api.service + systemctl start yunohost-portal-api.service + # Yunohost-firewall is enabled only during postinstall, not init, not 100% sure why cp dpkg-origins /etc/dpkg/origins/yunohost @@ -152,6 +159,7 @@ HandleLidSwitchExternalPower=ignore EOF cp yunohost-api.service ${pending_dir}/etc/systemd/system/yunohost-api.service + cp yunohost-portal-api.service ${pending_dir}/etc/systemd/system/yunohost-portal-api.service cp yunohost-firewall.service ${pending_dir}/etc/systemd/system/yunohost-firewall.service cp yunoprompt.service ${pending_dir}/etc/systemd/system/yunoprompt.service @@ -169,6 +177,13 @@ EOF do_post_regen() { regen_conf_files=$1 + getent passwd ynh-portal &>/dev/null || useradd --no-create-home --shell /usr/sbin/nologin --system --user-group ynh-portal + if [ ! -e /etc/yunohost/.ssowat_cookie_secret ]; then + dd if=/dev/urandom bs=1 count=1000 2>/dev/null | tr --complement --delete 'A-Za-z0-9' | head -c 64 > /etc/yunohost/.ssowat_cookie_secret + fi + chown ynh-portal:root /etc/yunohost/.ssowat_cookie_secret + chmod 400 /etc/yunohost/.ssowat_cookie_secret + ###################### # Enfore permissions # ###################### @@ -225,10 +240,12 @@ do_post_regen() { systemctl daemon-reload systemctl restart ntp } + [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload [[ ! "$regen_conf_files" =~ "login.conf.d/ynh-override.conf" ]] || systemctl daemon-reload [[ ! "$regen_conf_files" =~ "yunohost-firewall.service" ]] || systemctl daemon-reload [[ ! "$regen_conf_files" =~ "yunohost-api.service" ]] || systemctl daemon-reload + [[ ! "$regen_conf_files" =~ "yunohost-portal-api.service" ]] || systemctl daemon-reload if [[ "$regen_conf_files" =~ "yunoprompt.service" ]]; then systemctl daemon-reload @@ -241,6 +258,9 @@ do_post_regen() { systemctl $action proc-hidepid --quiet --now fi + systemctl enable yunohost-portal-api.service --quiet + systemctl is-active yunohost-portal-api --quiet || systemctl start yunohost-portal-api.service + # Change dpkg vendor # see https://wiki.debian.org/Derivatives/Guidelines#Vendor if readlink -f /etc/dpkg/origins/default | grep -q debian; From 76eba6fc88814c1900cb94a3477eb3fea9a93801 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 27 Dec 2021 13:05:11 +0100 Subject: [PATCH 006/436] Fix log permission issue for yunohost-portal-api --- hooks/conf_regen/01-yunohost | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 597595231e..4dfd599127 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -184,6 +184,10 @@ do_post_regen() { chown ynh-portal:root /etc/yunohost/.ssowat_cookie_secret chmod 400 /etc/yunohost/.ssowat_cookie_secret + touch /var/log/yunohost-portalapi.log + chown ynh-portal:root /var/log/yunohost-portalapi.log + chmod 600 /var/log/yunohost-portalapi.log + ###################### # Enfore permissions # ###################### From 22681a4f241c074feaa628b5c8d28716d65812e9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Feb 2023 18:50:31 +0100 Subject: [PATCH 007/436] Kill the old 'unprotected/protected/skipped' setting hell --- helpers/setting | 18 ++------ src/app.py | 115 ------------------------------------------------ 2 files changed, 3 insertions(+), 130 deletions(-) diff --git a/helpers/setting b/helpers/setting index a2cf3a93d5..a89f72091c 100644 --- a/helpers/setting +++ b/helpers/setting @@ -18,11 +18,7 @@ ynh_app_setting_get() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key - else - ynh_app_setting "get" "$app" "$key" - fi + ynh_app_setting "get" "$app" "$key" } # Set an application setting @@ -45,11 +41,7 @@ ynh_app_setting_set() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key -v $value - else - ynh_app_setting "set" "$app" "$key" "$value" - fi + ynh_app_setting "set" "$app" "$key" "$value" } # Delete an application setting @@ -70,11 +62,7 @@ ynh_app_setting_delete() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then - yunohost app setting $app $key -d - else - ynh_app_setting "delete" "$app" "$key" - fi + ynh_app_setting "delete" "$app" "$key" } # Small "hard-coded" interface to avoid calling "yunohost app" directly each diff --git a/src/app.py b/src/app.py index f17c469294..65402a0246 100644 --- a/src/app.py +++ b/src/app.py @@ -1492,119 +1492,6 @@ def app_setting(app, key, value=None, delete=False): """ app_settings = _get_app_settings(app) or {} - # - # Legacy permission setting management - # (unprotected, protected, skipped_uri/regex) - # - - is_legacy_permission_setting = any( - key.startswith(word + "_") for word in ["unprotected", "protected", "skipped"] - ) - - if is_legacy_permission_setting: - from yunohost.permission import ( - user_permission_list, - user_permission_update, - permission_create, - permission_delete, - permission_url, - ) - - permissions = user_permission_list(full=True, apps=[app])["permissions"] - key_ = key.split("_")[0] - permission_name = f"{app}.legacy_{key_}_uris" - permission = permissions.get(permission_name) - - # GET - if value is None and not delete: - return ( - ",".join(permission.get("uris", []) + permission["additional_urls"]) - if permission - else None - ) - - # DELETE - if delete: - # If 'is_public' setting still exists, we interpret this as - # coming from a legacy app (because new apps shouldn't manage the - # is_public state themselves anymore...) - # - # In that case, we interpret the request for "deleting - # unprotected/skipped" setting as willing to make the app - # private - if ( - "is_public" in app_settings - and "visitors" in permissions[app + ".main"]["allowed"] - ): - if key.startswith("unprotected_") or key.startswith("skipped_"): - user_permission_update(app + ".main", remove="visitors") - - if permission: - permission_delete(permission_name) - - # SET - else: - urls = value - # If the request is about the root of the app (/), ( = the vast majority of cases) - # we interpret this as a change for the main permission - # (i.e. allowing/disallowing visitors) - if urls == "/": - if key.startswith("unprotected_") or key.startswith("skipped_"): - permission_url(app + ".main", url="/", sync_perm=False) - user_permission_update(app + ".main", add="visitors") - else: - user_permission_update(app + ".main", remove="visitors") - else: - urls = urls.split(",") - if key.endswith("_regex"): - urls = ["re:" + url for url in urls] - - if permission: - # In case of new regex, save the urls, to add a new time in the additional_urls - # In case of new urls, we do the same thing but inversed - if key.endswith("_regex"): - # List of urls to save - current_urls_or_regex = [ - url - for url in permission["additional_urls"] - if not url.startswith("re:") - ] - else: - # List of regex to save - current_urls_or_regex = [ - url - for url in permission["additional_urls"] - if url.startswith("re:") - ] - - new_urls = urls + current_urls_or_regex - # We need to clear urls because in the old setting the new setting override the old one and dont just add some urls - permission_url(permission_name, clear_urls=True, sync_perm=False) - permission_url(permission_name, add_url=new_urls) - else: - from yunohost.utils.legacy import legacy_permission_label - - # Let's create a "special" permission for the legacy settings - permission_create( - permission=permission_name, - # FIXME find a way to limit to only the user allowed to the main permission - allowed=["all_users"] - if key.startswith("protected_") - else ["all_users", "visitors"], - url=None, - additional_urls=urls, - auth_header=not key.startswith("skipped_"), - label=legacy_permission_label(app, key.split("_")[0]), - show_tile=False, - protected=True, - ) - - return - - # - # Regular setting management - # - # GET if value is None and not delete: return app_settings.get(key, None) @@ -1616,8 +1503,6 @@ def app_setting(app, key, value=None, delete=False): # SET else: - if key in ["redirected_urls", "redirected_regex"]: - value = yaml.safe_load(value) app_settings[key] = value _set_app_settings(app, app_settings) From 04eadd715c8723e43113461b92c7493a76760e02 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 28 Feb 2023 18:55:47 +0100 Subject: [PATCH 008/436] Kill old --package option and .ini for PHP FPM config --- helpers/php | 71 ++--------------------------------------------------- 1 file changed, 2 insertions(+), 69 deletions(-) diff --git a/helpers/php b/helpers/php index 417dbbc610..1b28b32f7a 100644 --- a/helpers/php +++ b/helpers/php @@ -15,7 +15,7 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # # ----------------------------------------------------------------------------- # -# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--package=packages] [--dedicated_service] +# usage 2: ynh_add_fpm_config [--phpversion=7.X] --usage=usage --footprint=footprint [--dedicated_service] # | arg: -v, --phpversion= - Version of PHP to use. # | arg: -f, --footprint= - Memory footprint of the service (low/medium/high). # low - Less than 20 MB of RAM by pool. @@ -30,7 +30,6 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} # medium - Low usage, few people or/and publicly accessible. # high - High usage, frequently visited website. # -# | arg: -p, --package= - Additionnal PHP packages to install for a specific version of PHP # | arg: -d, --dedicated_service - Use a dedicated PHP-FPM service instead of the common one. # # @@ -60,16 +59,14 @@ ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. local legacy_args=vtufpd - local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service) + local -A args_array=([v]=phpversion= [t]=use_template [u]=usage= [f]=footprint= [d]=dedicated_service) local phpversion local use_template local usage local footprint - local package local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" - package=${package:-} # The default behaviour is to use the template. use_template="${use_template:-1}" @@ -103,13 +100,6 @@ ynh_add_fpm_config() { fi fi - # Legacy args (packager should just list their php dependency as regular apt dependencies... - if [ -n "$package" ]; then - # Install the additionnal packages from the default repository - ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future" - ynh_install_app_dependencies "$package" - fi - if [ $dedicated_service -eq 1 ]; then local fpm_service="${app}-phpfpm" local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" @@ -197,11 +187,6 @@ pm.process_idle_timeout = 10s local finalphpconf="$fpm_config_dir/pool.d/$app.conf" ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" - if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]; then - ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." - ynh_add_config --template="php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini" - fi - if [ $dedicated_service -eq 1 ]; then # Create a dedicated php-fpm.conf for the service local globalphpconf=$fpm_config_dir/php-fpm-$app.conf @@ -272,9 +257,6 @@ ynh_remove_fpm_config() { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - if [ -e $fpm_config_dir/conf.d/20-$app.ini ]; then - ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" - fi if [ $dedicated_service -eq 1 ]; then # Remove the dedicated service PHP-FPM service for the app @@ -286,55 +268,6 @@ ynh_remove_fpm_config() { elif ynh_package_is_installed --package="php${phpversion}-fpm"; then ynh_systemd_action --service_name=$fpm_service --action=reload fi - - # If the PHP version used is not the default version for YunoHost - # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script - # (we don't actually care about its value, we just check its not empty hence it exists) - if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - # Remove app dependencies ... but ideally should happen via an explicit call from packager - ynh_remove_app_dependencies - fi -} - -# Install another version of PHP. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_install_php --phpversion=phpversion [--package=packages] -# | arg: -v, --phpversion= - Version of PHP to install. -# | arg: -p, --package= - Additionnal PHP packages to install -# -# Requires YunoHost version 3.8.1 or higher. -ynh_install_php() { - # Declare an array to define the options of this helper. - local legacy_args=vp - local -A args_array=([v]=phpversion= [p]=package=) - local phpversion - local package - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - package=${package:-} - - if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]; then - ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" - fi - - ynh_install_app_dependencies "$package" -} - -# Remove the specific version of PHP used by the app. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_remove_php -# -# Requires YunoHost version 3.8.1 or higher. -ynh_remove_php () { - ynh_remove_app_dependencies } # Define the values to configure PHP-FPM From 0ab7c952f160b4113fe8506a355e26523c2fd23b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 May 2023 20:11:10 +0200 Subject: [PATCH 009/436] bookworm/debian: adapt control file to bookworm --- debian/control | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/debian/control b/debian/control index 0258eaac77..3674a62a48 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: yunohost Section: utils Priority: extra Maintainer: YunoHost Contributors -Build-Depends: debhelper (>=9), debhelper-compat (= 13), dh-python, python3-all (>= 3.7), python3-yaml, python3-jinja2 +Build-Depends: debhelper (>=9), debhelper-compat (= 13), dh-python, python3-all (>= 3.11), python3-yaml, python3-jinja2 Standards-Version: 3.9.6 Homepage: https://yunohost.org/ @@ -14,9 +14,9 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 - , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon, + , python3-ldap, python3-zeroconf (>=0.47), python3-lexicon, , python-is-python3 - , nginx, nginx-extras (>=1.18) + , nginx, nginx-extras (>=1.22) , apt, apt-transport-https, apt-utils, dirmngr , openssh-server, iptables, fail2ban, bind9-dnsutils , openssl, ca-certificates, netcat-openbsd, iproute2 @@ -32,23 +32,18 @@ Depends: ${python3:Depends}, ${misc:Depends} Recommends: yunohost-admin , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog - , php7.4-common, php7.4-fpm, php7.4-ldap, php7.4-intl - , mariadb-server, php7.4-mysql - , php7.4-gd, php7.4-curl, php-php-gettext - , python3-pip , unattended-upgrades , libdbd-ldap-perl, libnet-dns-perl - , metronome (>=3.14.0) Conflicts: iptables-persistent , apache2 , bind9 - , nginx-extras (>= 1.19) - , openssl (>= 1.1.1o-0) - , slapd (>= 2.4.58) - , dovecot-core (>= 1:2.3.14) - , redis-server (>= 5:6.1) - , fail2ban (>= 0.11.3) - , iptables (>= 1.8.8) + , nginx-extras (>= 1.23) + , openssl (>= 3.1) + , slapd (>= 2.6) + , dovecot-core (>= 1:2.4) + , redis-server (>= 5:7.1) + , fail2ban (>= 1.1) + , iptables (>= 1.8.10) Description: manageable and configured self-hosting server YunoHost aims to make self-hosting accessible to everyone. It configures an email, Web and IM server alongside a LDAP base. It also provides From 37eac5e1214da02cb1994ba98c5d71c66e17fc4f Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 4 May 2023 20:31:14 +0200 Subject: [PATCH 010/436] Update changelog for 12.0.0 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 64fc2ff23d..9434b72fdc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +yunohost (12.0.0) unstable; urgency=low + + - Tmp changelog to prepare Bookworm + + -- Alexandre Aubin Thu, 04 May 2023 20:30:19 +0200 + yunohost (11.1.18) stable; urgency=low - appsv2: always set an 'app' setting equal to app id to be able to use __APP__ in markdown templates ([#1645](https://github.com/yunohost/yunohost/pull/1645)) From 224f1b17306b031e9b9bc6fc551043c96ebb469c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 00:09:09 +0200 Subject: [PATCH 011/436] firewall: fix upnpc.discover() behavior that somehow now trigger an exception when cant talk to upnp device --- src/firewall.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/firewall.py b/src/firewall.py index 310d263c61..d6e4b53174 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -402,7 +402,13 @@ def firewall_upnp(action="status", no_refresh=False): # Discover UPnP device(s) logger.debug("discovering UPnP devices...") - nb_dev = upnpc.discover() + try: + nb_dev = upnpc.discover() + except Exception as e: + logger.warning("Failed to find any UPnP device on the network") + nb_dev = -1 + enabled = False + logger.debug("found %d UPnP device(s)", int(nb_dev)) if nb_dev < 1: logger.error(m18n.n("upnp_dev_not_found")) From 859f9c05a5185883d02ed9a8105d08de16ce69b5 Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 5 May 2023 16:37:33 +0200 Subject: [PATCH 012/436] Update test.gitlab-ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix tests for bookworm, don't try this at home² --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index b0ffd3db59..b59686bb3e 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,7 +1,7 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb - - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" + - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22 --break-system-packages" # for bookworm .test-stage: stage: test From 30bd0e05b2cdc259641e937c1c3f43eeb86ff9f5 Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 5 May 2023 16:38:53 +0200 Subject: [PATCH 013/436] Update test.gitlab-ci.yml oopsie --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index b59686bb3e..da048897cd 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,7 +1,7 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb - - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22 --break-system-packages" # for bookworm + - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: stage: test From 191520ebddc7b0171fc36d552cc0aafcf3354e92 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 19:22:37 +0200 Subject: [PATCH 014/436] mysql: dirty/ugly patch to autoinstall mariadb-server when calling ynhmysql_setup_db when it's not installed --- helpers/mysql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helpers/mysql b/helpers/mysql index a5290f7942..c9629537b0 100644 --- a/helpers/mysql +++ b/helpers/mysql @@ -210,6 +210,9 @@ ynh_mysql_setup_db() { # If $db_pwd is not provided, use new_db_pwd instead for db_pwd db_pwd="${db_pwd:-$new_db_pwd}" + # Dirty patch for super-legacy apps + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; apt install mariadb-server --no-install-recommends -y; } + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd } From 1fb3965e51a033a37d7abbb04386293009f98fa7 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 20:10:52 +0200 Subject: [PATCH 015/436] improve previous commit, use ynh_apt to have non-interactive apt etc --- helpers/mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/mysql b/helpers/mysql index c9629537b0..eade5804ec 100644 --- a/helpers/mysql +++ b/helpers/mysql @@ -211,7 +211,7 @@ ynh_mysql_setup_db() { db_pwd="${db_pwd:-$new_db_pwd}" # Dirty patch for super-legacy apps - dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; apt install mariadb-server --no-install-recommends -y; } + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd From 7fc7d188ad59af01cff710326cdc94e852b1c9b5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 20:11:16 +0200 Subject: [PATCH 016/436] Fix XMPP stuff that may not exists --- hooks/backup/27-data_xmpp | 4 ++-- hooks/restore/27-data_xmpp | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hooks/backup/27-data_xmpp b/hooks/backup/27-data_xmpp index 2cd93e02b1..078184ed8c 100644 --- a/hooks/backup/27-data_xmpp +++ b/hooks/backup/27-data_xmpp @@ -9,5 +9,5 @@ source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/xmpp" -ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" -ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" +ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" --not_mandatory +ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" --not_mandatory diff --git a/hooks/restore/27-data_xmpp b/hooks/restore/27-data_xmpp index 02a4c6703a..f07ac6a333 100644 --- a/hooks/restore/27-data_xmpp +++ b/hooks/restore/27-data_xmpp @@ -1,4 +1,11 @@ backup_dir="$1/data/xmpp" -cp -a $backup_dir/var_lib_metronome/. /var/lib/metronome -cp -a $backup_dir/var_xmpp-upload/. /var/xmpp-upload +if [[ -e $backup_dir/var_lib_metronome/ ]] +then + cp -a $backup_dir/var_lib_metronome/. /var/lib/metronome +fi + +if [[ -e $backup_dir/var_xmpp-upload ]] +then + cp -a $backup_dir/var_xmpp-upload/. /var/xmpp-upload +fi From 09012989351bcd08b2fba77fabd6c79503467d18 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 5 May 2023 20:18:19 +0200 Subject: [PATCH 017/436] bookworm: add php7.x -> php8.2 autopatch --- src/app.py | 10 ++++----- src/backup.py | 2 +- src/utils/legacy.py | 52 ++++++++++++++++++++++++++++----------------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/app.py b/src/app.py index 59e58cde1b..c19f654f34 100644 --- a/src/app.py +++ b/src/app.py @@ -2012,7 +2012,7 @@ def _get_app_settings(app): ): settings["path"] = "/" + settings["path"].strip("/") _set_app_settings(app, settings) - + # Make the app id available as $app too settings["app"] = app @@ -3044,10 +3044,10 @@ def _assert_system_is_sane_for_app(manifest, when): services = manifest.get("services", []) - # Some apps use php-fpm, php5-fpm or php7.x-fpm which is now php7.4-fpm + # Some apps use php-fpm, php5-fpm or php7.x-fpm which is now php8.2-fpm def replace_alias(service): - if service in ["php-fpm", "php5-fpm", "php7.0-fpm", "php7.3-fpm"]: - return "php7.4-fpm" + if service in ["php-fpm", "php5-fpm", "php7.0-fpm", "php7.3-fpm", "php7.4-fpm"]: + return "php8.2-fpm" else: return service @@ -3056,7 +3056,7 @@ def replace_alias(service): # We only check those, mostly to ignore "custom" services # (added by apps) and because those are the most popular # services - service_filter = ["nginx", "php7.4-fpm", "mysql", "postfix"] + service_filter = ["nginx", "php8.2-fpm", "mysql", "postfix"] services = [str(s) for s in services if s in service_filter] if "nginx" not in services: diff --git a/src/backup.py b/src/backup.py index ce1e8ba2c1..a23a8d8e0d 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1204,7 +1204,7 @@ def restore(self): def _patch_legacy_php_versions_in_csv_file(self): """ - Apply dirty patch to redirect php5 and php7.0 files to php7.4 + Apply dirty patch to redirect php5 and php7.x files to php8.2 """ from yunohost.utils.legacy import LEGACY_PHP_VERSION_REPLACEMENTS diff --git a/src/utils/legacy.py b/src/utils/legacy.py index 82507d64d3..226a8c929d 100644 --- a/src/utils/legacy.py +++ b/src/utils/legacy.py @@ -163,32 +163,45 @@ def translate_legacy_default_app_in_ssowant_conf_json_persistent(): LEGACY_PHP_VERSION_REPLACEMENTS = [ - ("/etc/php5", "/etc/php/7.4"), - ("/etc/php/7.0", "/etc/php/7.4"), - ("/etc/php/7.3", "/etc/php/7.4"), - ("/var/run/php5-fpm", "/var/run/php/php7.4-fpm"), - ("/var/run/php/php7.0-fpm", "/var/run/php/php7.4-fpm"), - ("/var/run/php/php7.3-fpm", "/var/run/php/php7.4-fpm"), - ("php5", "php7.4"), - ("php7.0", "php7.4"), - ("php7.3", "php7.4"), - ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="7.4"'), + ("/etc/php5", "/etc/php/8.2"), + ("/etc/php/7.0", "/etc/php/8.2"), + ("/etc/php/7.3", "/etc/php/8.2"), + ("/etc/php/7.4", "/etc/php/8.2"), + ("/var/run/php5-fpm", "/var/run/php/php8.2-fpm"), + ("/var/run/php/php7.0-fpm", "/var/run/php/php8.2-fpm"), + ("/var/run/php/php7.3-fpm", "/var/run/php/php8.2-fpm"), + ("/var/run/php/php7.4-fpm", "/var/run/php/php8.2-fpm"), + ("php5", "php8.2"), + ("php7.0", "php8.2"), + ("php7.3", "php8.2"), + ("php7.4", "php8.2"), + ('YNH_PHP_VERSION="7.3"', 'YNH_PHP_VERSION="8.2"'), + ('YNH_PHP_VERSION="7.4"', 'YNH_PHP_VERSION="8.2"'), ( 'phpversion="${phpversion:-7.0}"', - 'phpversion="${phpversion:-7.4}"', + 'phpversion="${phpversion:-8.2}"', ), # Many helpers like the composer ones use 7.0 by default ... ( 'phpversion="${phpversion:-7.3}"', + 'phpversion="${phpversion:-8.2}"', + ), # Many helpers like the composer ones use 7.0 by default ... + ( 'phpversion="${phpversion:-7.4}"', + 'phpversion="${phpversion:-8.2}"', ), # Many helpers like the composer ones use 7.0 by default ... ( '"$phpversion" == "7.0"', - '$(bc <<< "$phpversion >= 7.4") -eq 1', + '$(bc <<< "$phpversion >= 8.2") -eq 1', ), # patch ynh_install_php to refuse installing/removing php <= 7.3 ( '"$phpversion" == "7.3"', - '$(bc <<< "$phpversion >= 7.4") -eq 1', + '$(bc <<< "$phpversion >= 8.2") -eq 1', ), # patch ynh_install_php to refuse installing/removing php <= 7.3 + ( + '"$phpversion" == "7.4"', + '$(bc <<< "$phpversion >= 8.2") -eq 1', + ), # patch ynh_install_php to refuse installing/removing php <= 7.3 + ] @@ -217,15 +230,16 @@ def _patch_legacy_php_versions(app_folder): def _patch_legacy_php_versions_in_settings(app_folder): settings = read_yaml(os.path.join(app_folder, "settings.yml")) - if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm"]: - settings["fpm_config_dir"] = "/etc/php/7.4/fpm" - if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm"]: - settings["fpm_service"] = "php7.4-fpm" - if settings.get("phpversion") in ["7.0", "7.3"]: - settings["phpversion"] = "7.4" + if settings.get("fpm_config_dir") in ["/etc/php/7.0/fpm", "/etc/php/7.3/fpm", "/etc/php/7.4/fpm"]: + settings["fpm_config_dir"] = "/etc/php/8.2/fpm" + if settings.get("fpm_service") in ["php7.0-fpm", "php7.3-fpm", "php7.4-fpm"]: + settings["fpm_service"] = "php8.2-fpm" + if settings.get("phpversion") in ["7.0", "7.3", "7.4"]: + settings["phpversion"] = "8.2" # We delete these checksums otherwise the file will appear as manually modified list_to_remove = [ + "checksum__etc_php_7.4_fpm_pool", "checksum__etc_php_7.3_fpm_pool", "checksum__etc_php_7.0_fpm_pool", "checksum__etc_nginx_conf.d", From 8fb225f3ad12d26d8caeef5182af74327f51b95c Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 19:16:21 +0000 Subject: [PATCH 018/436] remove some legacy pre-bullseye workarounds --- bin/yunomdns | 8 ++------ src/app.py | 14 -------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/bin/yunomdns b/bin/yunomdns index 1bdcf88caf..da9233045f 100755 --- a/bin/yunomdns +++ b/bin/yunomdns @@ -132,12 +132,8 @@ def main() -> bool: ) continue - # Only broadcast IPv4 because IPv6 is buggy ... because we ain't using python3-ifaddr >= 0.1.7 - # Buster only ships 0.1.6 - # Bullseye ships 0.1.7 - # To be re-enabled once we're on bullseye... - # ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] - ips: List[str] = interfaces[interface]["ipv4"] + # Broadcast IPv4 and IPv6 + ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] # If at least one IP is listed if not ips: diff --git a/src/app.py b/src/app.py index c19f654f34..944c1b66fa 100644 --- a/src/app.py +++ b/src/app.py @@ -1999,20 +1999,6 @@ def _get_app_settings(app): logger.error(m18n.n("app_not_correctly_installed", app=app)) return {} - # Stupid fix for legacy bullshit - # In the past, some setups did not have proper normalization for app domain/path - # Meaning some setups (as of January 2021) still have path=/foobar/ (with a trailing slash) - # resulting in stupid issue unless apps using ynh_app_normalize_path_stuff - # So we yolofix the settings if such an issue is found >_> - # A simple call to `yunohost app list` (which happens quite often) should be enough - # to migrate all app settings ... so this can probably be removed once we're past Bullseye... - if settings.get("path") != "/" and ( - settings.get("path", "").endswith("/") - or not settings.get("path", "/").startswith("/") - ): - settings["path"] = "/" + settings["path"].strip("/") - _set_app_settings(app, settings) - # Make the app id available as $app too settings["app"] = app From 1135cf1b62338a1371e1fd4d8ddd74ec165b9506 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 21:23:23 +0000 Subject: [PATCH 019/436] php-cli is needed for ynhtest_config.sh --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index da048897cd..777174e48d 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: From ea7bdb62ed3d37b3054abcf52b8e5ac10d159ee2 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 22:24:43 +0000 Subject: [PATCH 020/436] mariadb-client needed by tests/test_app_resources.py --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 777174e48d..013f45ee1b 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: From de3dd9436c26514d4edfa725c67c6514c486ee65 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 23:40:17 +0000 Subject: [PATCH 021/436] create migration : s/bullseye/bookworm s/buster/bullseye yolo, needs some more cleanup --- locales/en.json | 15 +- src/migrations/0027_migrate_to_bookworm.py | 546 +++++++++++++++++++++ 2 files changed, 560 insertions(+), 1 deletion(-) create mode 100644 src/migrations/0027_migrate_to_bookworm.py diff --git a/locales/en.json b/locales/en.json index 4dcb00ee69..ea51b41842 100644 --- a/locales/en.json +++ b/locales/en.json @@ -572,7 +572,20 @@ "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuild_apps}", "migration_0024_rebuild_python_venv_failed": "Failed to rebuild the Python virtualenv for {app}. The app may not work as long as this is not resolved. You should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.", "migration_0024_rebuild_python_venv_in_progress": "Now attempting to rebuild the Python virtualenv for `{app}`", - "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x", + "migration_0027_cleaning_up": "Cleaning up cache and packages not useful anymore...", + "migration_0027_hgjghjghjgeneral_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.", + "migration_0027_main_upgrade": "Starting main upgrade...", + "migration_0027_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", + "migration_0027_not_buster2": "The current Debian distribution is not Bullseye! If you already ran the Bullseye->Bookworm migration, then this error is symptomatic of the fact that the migration procedure was not 100% succesful (otherwise YunoHost would have flagged it as completed). It is recommended to investigate what happened with the support team, who will need the **full** log of the `migration, which can be found in Tools > Logs in the webadmin.", + "migration_0027_not_enough_free_space": "Free space is pretty low in /var/! You should have at least 1GB free to run this migration.", + "migration_0027_patch_yunohost_conflicts": "Applying patch to workaround conflict issue...", + "migration_0027_patching_sources_list": "Patching the sources.lists...", + "migration_0027_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", + "migration_0027_start": "Starting migration to Bookworm", + "migration_0027_still_on_buster_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Bullseye", + "migration_0027_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bookworm.", + "migration_0027_yunohost_upgrade": "Starting YunoHost core upgrade...", + "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bookworm and YunoHost 12", "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4", "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13", "migration_description_0024_rebuild_python_venv": "Repair Python app after bullseye migration", diff --git a/src/migrations/0027_migrate_to_bookworm.py b/src/migrations/0027_migrate_to_bookworm.py new file mode 100644 index 0000000000..ef5510db3e --- /dev/null +++ b/src/migrations/0027_migrate_to_bookworm.py @@ -0,0 +1,546 @@ +import glob +import os + +from moulinette import m18n +from yunohost.utils.error import YunohostError +from moulinette.utils.log import getActionLogger +from moulinette.utils.process import check_output, call_async_output +from moulinette.utils.filesystem import read_file, rm, write_to_file + +from yunohost.tools import ( + Migration, + tools_update, + tools_upgrade, + _apt_log_line_is_relevant, +) +from yunohost.app import unstable_apps +from yunohost.regenconf import manually_modified_files, _force_clear_hashes +from yunohost.utils.system import ( + free_space_in_directory, + get_ynh_package_version, + _list_upgradable_apt_packages, +) +from yunohost.service import _get_services, _save_services + +logger = getActionLogger("yunohost.migration") + +N_CURRENT_DEBIAN = 10 +N_CURRENT_YUNOHOST = 4 + +VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bookworm_upgrade.txt" + + +def _get_all_venvs(dir, level=0, maxlevel=3): + """ + Returns the list of all python virtual env directories recursively + + Arguments: + dir - the directory to scan in + maxlevel - the depth of the recursion + level - do not edit this, used as an iterator + """ + if not os.path.exists(dir): + return [] + + result = [] + # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth + for file in os.listdir(dir): + path = os.path.join(dir, file) + if os.path.isdir(path): + activatepath = os.path.join(path, "bin", "activate") + if os.path.isfile(activatepath): + content = read_file(activatepath) + if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): + result.append(path) + continue + if level < maxlevel: + result += _get_all_venvs(path, level=level + 1) + return result + + +def _backup_pip_freeze_for_python_app_venvs(): + """ + Generate a requirements file for all python virtual env located inside /opt/ and /var/www/ + """ + + venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") + for venv in venvs: + # Generate a requirements file from venv + os.system( + f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" + ) + + +class MyMigration(Migration): + "Upgrade the system to Debian Bookworm and Yunohost 11.x" + + mode = "manual" + + def run(self): + self.check_assertions() + + logger.info(m18n.n("migration_0021_start")) + + # + # Add new apt .deb signing key + # + + new_apt_key = "https://forge.yunohost.org/yunohost_bookworm.asc" + check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") + + # + # Patch sources.list + # + logger.info(m18n.n("migration_0021_patching_sources_list")) + self.patch_apt_sources_list() + + # Stupid OVH has some repo configured which dont work with bookworm and break apt ... + os.system("sudo rm -f /etc/apt/sources.list.d/ovh-*.list") + + # Force add sury if it's not there yet + # This is to solve some weird issue with php-common breaking php7.3-common, + # hence breaking many php7.3-deps + # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) + # Adding it there shouldnt be a big deal - Yunohost 11.x does add it + # through its regen conf anyway. + if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): + open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( + "deb https://packages.sury.org/php/ bookworm main" + ) + + # Add Sury key even if extra_php_version.list was already there, + # because some old system may be using an outdated key not valid for Bookworm + # and that'll block the migration + os.system( + 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' + ) + + # Remove legacy, duplicated sury entry if it exists + if os.path.exists("/etc/apt/sources.list.d/sury.list"): + os.system("rm -rf /etc/apt/sources.list.d/sury.list") + + # + # Get requirements of the different venvs from python apps + # + + _backup_pip_freeze_for_python_app_venvs() + + # + # Run apt update + # + + tools_update(target="system") + + # Tell libc6 it's okay to restart system stuff during the upgrade + os.system( + "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" + ) + + # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... + # c.f. https://manpages.debian.org/bookworm/init-system-helpers/deb-systemd-invoke.1p.en.html + # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz + # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... + # and also invoke-rc.d ... + write_to_file( + "/usr/sbin/policy-rc.d", + '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', + ) + os.system("chmod +x /usr/sbin/policy-rc.d") + + # Don't send an email to root about the postgresql migration. It should be handled automatically after. + os.system( + "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" + ) + + # + # Patch yunohost conflicts + # + logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) + + self.patch_yunohost_conflicts() + + # + # Specific tweaking to get rid of custom my.cnf and use debian's default one + # (my.cnf is actually a symlink to mariadb.cnf) + # + + _force_clear_hashes(["/etc/mysql/my.cnf"]) + rm("/etc/mysql/mariadb.cnf", force=True) + rm("/etc/mysql/my.cnf", force=True) + ret = self.apt_install( + "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" + ) + if ret != 0: + raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) + + # + # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl + # + if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): + os.system( + "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" + ) + rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) + + # + # /home/yunohost.conf -> /var/cache/yunohost/regenconf + # + if os.path.exists("/home/yunohost.conf"): + os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") + rm("/home/yunohost.conf", recursive=True, force=True) + + # Remove legacy postgresql service record added by helpers, + # will now be dynamically handled by the core in bookworm + services = _get_services() + if "postgresql" in services: + del services["postgresql"] + _save_services(services) + + # + # Critical fix for RPI otherwise network is down after rebooting + # https://forum.yunohost.org/t/20652 + # + if os.system("systemctl | grep -q dhcpcd") == 0: + logger.info("Applying fix for DHCPCD ...") + os.system("mkdir -p /etc/systemd/system/dhcpcd.service.d") + write_to_file( + "/etc/systemd/system/dhcpcd.service.d/wait.conf", + "[Service]\nExecStart=\nExecStart=/usr/sbin/dhcpcd -w", + ) + + # + # Main upgrade + # + logger.info(m18n.n("migration_0021_main_upgrade")) + + apps_packages = self.get_apps_equivs_packages() + self.hold(apps_packages) + tools_upgrade(target="system", allow_yunohost_upgrade=False) + + if self.debian_major_version() == N_CURRENT_DEBIAN: + raise YunohostError("migration_0021_still_on_bullseye_after_main_upgrade") + + # Force explicit install of php8.2fpm and other old 'default' dependencies + # that are now only in Recommends + # + # Also, we need to install php8.2 equivalents of other php7.4 dependencies. + # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration + # to autoupgrade Nextcloud to 8.2, it will need the php8.2-zip to work. + # The following list is based on an ad-hoc analysis of php deps found in the + # app ecosystem, with a known equivalent on php8.2. + # + # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages + # with the proper list of dependencies, and the dependencies install this way + # will get flagged as 'manually installed'. + # + # We'll probably want to do something during the Bookworm->Bookworm migration to re-flag + # these as 'auto' so they get autoremoved if not needed anymore. + # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use + # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) + + php74packages_suffixes = [ + "apcu", + "bcmath", + "bz2", + "dom", + "gmp", + "igbinary", + "imagick", + "imap", + "mbstring", + "memcached", + "mysqli", + "mysqlnd", + "pgsql", + "redis", + "simplexml", + "soap", + "sqlite3", + "ssh2", + "tidy", + "xml", + "xmlrpc", + "xsl", + "zip", + ] + + cmd = ( + "apt show '*-ynh-deps' 2>/dev/null" + " | grep Depends" + f" | grep -o -E \"php7.4-({'|'.join(php73packages_suffixes)})\"" + " | sort | uniq" + " | sed 's/php7.4/php8.2/g'" + " || true" + ) + + basephp82packages_to_install = [ + "php8.2-fpm", + "php8.2-common", + "php8.2-ldap", + "php8.2-intl", + "php8.2-mysql", + "php8.2-gd", + "php8.2-curl", + "php-php-gettext", + ] + + php74packages_to_install = basephp74packages_to_install + [ + f.strip() for f in check_output(cmd).split("\n") if f.strip() + ] + + ret = self.apt_install( + f"{' '.join(php74packages_to_install)} " + "$(dpkg --list | grep ynh-deps | awk '{print $2}') " + "-o Dpkg::Options::='--force-confmiss'" + ) + if ret != 0: + raise YunohostError( + "Failed to force the install of php dependencies ?", raw_msg=True + ) + + # Clean the mess + logger.info(m18n.n("migration_0021_cleaning_up")) + os.system( + "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt autoremove --assume-yes" + ) + os.system("apt clean --assume-yes") + + # + # Stupid hack for stupid dnsmasq not picking up its new init.d script then breaking everything ... + # https://forum.yunohost.org/t/20676 + # + if os.path.exists("/etc/init.d/dnsmasq.dpkg-dist"): + logger.info("Copying new version for /etc/init.d/dnsmasq ...") + os.system("cp /etc/init.d/dnsmasq.dpkg-dist /etc/init.d/dnsmasq") + + # + # Yunohost upgrade + # + logger.info(m18n.n("migration_0021_yunohost_upgrade")) + + self.unhold(apps_packages) + + cmd = "LC_ALL=C" + cmd += " DEBIAN_FRONTEND=noninteractive" + cmd += " APT_LISTCHANGES_FRONTEND=none" + cmd += " apt dist-upgrade " + cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" + cmd += " | grep -q 'ynh-deps'" + + logger.info("Simulating upgrade...") + if os.system(cmd) == 0: + raise YunohostError( + "The upgrade cannot be completed, because some app dependencies would need to be removed?", + raw_msg=True, + ) + + postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" + postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" + postupgradecmds += "echo 'Restarting nginx...' >&2\n" + postupgradecmds += "systemctl restart nginx\n" + + tools_upgrade(target="system", postupgradecmds=postupgradecmds) + + def debian_major_version(self): + # The python module "platform" and lsb_release are not reliable because + # on some setup, they may still return Release=9 even after upgrading to + # bullseye ... (Apparently this is related to OVH overriding some stuff + # with /etc/lsb-release for instance -_-) + # Instead, we rely on /etc/os-release which should be the raw info from + # the distribution... + return int( + check_output( + "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" + ) + ) + + def yunohost_major_version(self): + return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) + + def check_assertions(self): + # Be on bullseye (10.x) and yunohost 4.x + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be > 9.x but yunohost package + # would still be in 3.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + try: + # Here we try to find the previous migration log, which should be somewhat recent and be at least 10k (we keep the biggest one) + maybe_previous_migration_log_id = check_output( + "cd /var/log/yunohost/categories/operation && find -name '*migrate*.log' -size +10k -mtime -100 -exec ls -s {} \\; | sort -n | tr './' ' ' | awk '{print $2}' | tail -n 1" + ) + if maybe_previous_migration_log_id: + logger.info( + f"NB: the previous migration log id seems to be {maybe_previous_migration_log_id}. You can share it with the support team with : sudo yunohost log share {maybe_previous_migration_log_id}" + ) + except Exception: + # Yeah it's not that important ... it's to simplify support ... + pass + + raise YunohostError("migration_0021_not_bullseye2") + + # Have > 1 Go free space on /var/ ? + if free_space_in_directory("/var/") / (1024**3) < 1.0: + raise YunohostError("migration_0021_not_enough_free_space") + + # Have > 70 MB free space on /var/ ? + # FIXME: Create a way to ignore this check, on some system 70M is enough... + if free_space_in_directory("/boot/") / (1024**2) < 70.0: + raise YunohostError( + "/boot/ has less than 70MB available. This will probably trigger a crash during the upgrade because a new kernel needs to be installed. Please look for advice on the forum on how to remove old, unused kernels to free up some space in /boot/.", + raw_msg=True, + ) + + # Check system is up to date + # (but we don't if 'bookworm' is already in the sources.list ... + # which means maybe a previous upgrade crashed and we're re-running it) + if os.path.exists("/etc/apt/sources.list") and " bookworm " not in read_file( + "/etc/apt/sources.list" + ): + tools_update(target="system") + upgradable_system_packages = list(_list_upgradable_apt_packages()) + upgradable_system_packages = [ + package["name"] for package in upgradable_system_packages + ] + upgradable_system_packages = set(upgradable_system_packages) + # Lime2 have hold packages to avoid ethernet instability + # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df + lime2_hold_packages = set( + [ + "armbian-firmware", + "armbian-bsp-cli-lime2", + "linux-dtb-current-sunxi", + "linux-image-current-sunxi", + "linux-u-boot-lime2-current", + "linux-image-next-sunxi", + ] + ) + if upgradable_system_packages - lime2_hold_packages: + raise YunohostError("migration_0021_system_not_fully_up_to_date") + + @property + def disclaimer(self): + # Avoid having a super long disclaimer + uncessary check if we ain't + # on bullseye / yunohost 4.x anymore + # NB : we do both check to cover situations where the upgrade crashed + # in the middle and debian version could be >= 10.x but yunohost package + # would still be in 4.x... + if ( + not self.debian_major_version() == N_CURRENT_DEBIAN + and not self.yunohost_major_version() == N_CURRENT_YUNOHOST + ): + return None + + # Get list of problematic apps ? I.e. not official or community+working + problematic_apps = unstable_apps() + problematic_apps = "".join(["\n - " + app for app in problematic_apps]) + + # Manually modified files ? (c.f. yunohost service regen-conf) + modified_files = manually_modified_files() + modified_files = "".join(["\n - " + f for f in modified_files]) + + message = m18n.n("migration_0021_general_warning") + + message = ( + "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n" + + message + ) + + if problematic_apps: + message += "\n\n" + m18n.n( + "migration_0021_problematic_apps_warning", + problematic_apps=problematic_apps, + ) + + if modified_files: + message += "\n\n" + m18n.n( + "migration_0021_modified_files", manually_modified_files=modified_files + ) + + return message + + def patch_apt_sources_list(self): + sources_list = glob.glob("/etc/apt/sources.list.d/*.list") + if os.path.exists("/etc/apt/sources.list"): + sources_list.append("/etc/apt/sources.list") + + # This : + # - replace single 'bullseye' occurence by 'bulleye' + # - comments lines containing "backports" + for f in sources_list: + command = ( + f"sed -i {f} " + "-e 's@ bullseye @ bookworm @g' " + "-e '/backports/ s@^#*@#@' " + "-e 's@ bullseye-@ bookworm-@g' " + ) + os.system(command) + + def get_apps_equivs_packages(self): + command = ( + "dpkg --get-selections" + " | grep -v deinstall" + " | awk '{print $1}'" + " | { grep 'ynh-deps$' || true; }" + ) + + output = check_output(command) + + return output.split("\n") if output else [] + + def hold(self, packages): + for package in packages: + os.system(f"apt-mark hold {package}") + + def unhold(self, packages): + for package in packages: + os.system(f"apt-mark unhold {package}") + + def apt_install(self, cmd): + def is_relevant(line): + return "Reading database ..." not in line.rstrip() + + callbacks = ( + lambda l: logger.info("+ " + l.rstrip() + "\r") + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip() + "\r"), + lambda l: logger.warning(l.rstrip()) + if _apt_log_line_is_relevant(l) + else logger.debug(l.rstrip()), + ) + + cmd = ( + "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + + cmd + ) + + logger.debug("Running: %s" % cmd) + + return call_async_output(cmd, callbacks, shell=True) + + def patch_yunohost_conflicts(self): + # + # This is a super dirty hack to remove the conflicts from yunohost's debian/control file + # Those conflicts are there to prevent mistakenly upgrading critical packages + # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly + # using backports etc. + # + # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status + # + + # We only patch the conflict if we're on yunohost 4.x + if self.yunohost_major_version() != N_CURRENT_YUNOHOST: + return + + conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() + if conflicts: + # We want to keep conflicting with apache/bind9 tho + new_conflicts = "Conflicts: apache2, bind9" + + command = ( + f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" + ) + logger.debug(f"Running: {command}") + os.system(command) From 2dbe34c0301a4bc666d3175d998ea68c70bb705c Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 23:44:46 +0000 Subject: [PATCH 022/436] aaand mariadb-server ofc... --- .gitlab/ci/test.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 013f45ee1b..c5f1ee13ce 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client mariadb-server - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: From f617287eb2e74103ef4f54dfe3d52a810e51514e Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Fri, 5 May 2023 23:58:30 +0000 Subject: [PATCH 023/436] woops thanks codeql i guess --- src/migrations/0027_migrate_to_bookworm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrations/0027_migrate_to_bookworm.py b/src/migrations/0027_migrate_to_bookworm.py index ef5510db3e..85e2235aff 100644 --- a/src/migrations/0027_migrate_to_bookworm.py +++ b/src/migrations/0027_migrate_to_bookworm.py @@ -267,7 +267,7 @@ def run(self): cmd = ( "apt show '*-ynh-deps' 2>/dev/null" " | grep Depends" - f" | grep -o -E \"php7.4-({'|'.join(php73packages_suffixes)})\"" + f" | grep -o -E \"php7.4-({'|'.join(php74packages_suffixes)})\"" " | sort | uniq" " | sed 's/php7.4/php8.2/g'" " || true" @@ -284,7 +284,7 @@ def run(self): "php-php-gettext", ] - php74packages_to_install = basephp74packages_to_install + [ + php74packages_to_install = basephp82packages_to_install + [ f.strip() for f in check_output(cmd).split("\n") if f.strip() ] From 691ce5eace8f8a0f107d607318d7fc02da5bcb56 Mon Sep 17 00:00:00 2001 From: Tagadda <36127788+Tagadda@users.noreply.github.com> Date: Sat, 13 May 2023 14:37:32 +0000 Subject: [PATCH 024/436] fix: python3.11 now supports Possessive Quantifiers regex --- src/tests/test_appurl.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tests/test_appurl.py b/src/tests/test_appurl.py index 351bb4e833..8e5b14d34f 100644 --- a/src/tests/test_appurl.py +++ b/src/tests/test_appurl.py @@ -202,10 +202,6 @@ def test_normalize_permission_path_with_bad_regex(): ) # Full Regex - with pytest.raises(YunohostError): - _validate_and_sanitize_permission_url( - "re:" + maindomain + "/yolo?+/", maindomain + "/path", "test_permission" - ) with pytest.raises(YunohostError): _validate_and_sanitize_permission_url( "re:" + maindomain + "/yolo[1-9]**/", From 482f8f3443a37f5644a7be9c4743c453981659f9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 17:37:11 +0200 Subject: [PATCH 025/436] backup: fix again backup hook for xmpp when data folder dont exist --- hooks/backup/27-data_xmpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/backup/27-data_xmpp b/hooks/backup/27-data_xmpp index 078184ed8c..253aacdc21 100644 --- a/hooks/backup/27-data_xmpp +++ b/hooks/backup/27-data_xmpp @@ -9,5 +9,5 @@ source /usr/share/yunohost/helpers # Backup destination backup_dir="${1}/data/xmpp" -ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" --not_mandatory -ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" --not_mandatory +[[ ! -d /var/lib/metronome ]] || ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" --not_mandatory +[[ ! -d /var/xmpp-upload ]] || ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" --not_mandatory From 031c641b77cae7139c4be33f819e2c23cb0570ad Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 18:51:18 +0200 Subject: [PATCH 026/436] tests: fix spy on m18n.n which in some cases doesnt work anymore ... not sure why, bit confused ... --- src/tests/conftest.py | 13 ++++-- src/tests/test_apps.py | 40 ++++++++--------- src/tests/test_backuprestore.py | 78 ++++++++++++++++---------------- src/tests/test_ldapauth.py | 2 +- src/tests/test_permission.py | 80 ++++++++++++++++----------------- src/tests/test_regenconf.py | 4 +- src/tests/test_user-group.py | 44 +++++++++--------- 7 files changed, 133 insertions(+), 128 deletions(-) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 393c335644..abba2ee195 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,5 +1,6 @@ import os import pytest +from unittest.mock import Mock import moulinette from moulinette import m18n, Moulinette @@ -23,11 +24,15 @@ def get_test_apps_dir(): @contextmanager -def message(mocker, key, **kwargs): - mocker.spy(m18n, "n") +def message(key, **kwargs): + m = Mock(wraps=m18n.n) + old_m18n = m18n.n + m18n.n = m yield - m18n.n.assert_any_call(key, **kwargs) - + try: + m.assert_any_call(key, **kwargs) + finally: + m18n.n = old_m18n @contextmanager def raiseYunohostError(mocker, key, **kwargs): diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index 747eb5dcdb..a0c4315310 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -392,9 +392,9 @@ def test_legacy_app_install_private(secondary_domain): assert app_is_not_installed(secondary_domain, "legacy_app") -def test_legacy_app_install_unknown_domain(mocker): +def test_legacy_app_install_unknown_domain(): with pytest.raises(YunohostError): - with message(mocker, "app_argument_invalid"): + with message("app_argument_invalid"): install_legacy_app("whatever.nope", "/legacy") assert app_is_not_installed("whatever.nope", "legacy_app") @@ -421,12 +421,12 @@ def test_legacy_app_install_multiple_instances(secondary_domain): assert app_is_not_installed(secondary_domain, "legacy_app__2") -def test_legacy_app_install_path_unavailable(mocker, secondary_domain): +def test_legacy_app_install_path_unavailable(secondary_domain): # These will be removed in teardown install_legacy_app(secondary_domain, "/legacy") with pytest.raises(YunohostError): - with message(mocker, "app_location_unavailable"): + with message("app_location_unavailable"): install_legacy_app(secondary_domain, "/") assert app_is_installed(secondary_domain, "legacy_app") @@ -442,19 +442,19 @@ def test_legacy_app_install_with_nginx_down(mocker, secondary_domain): install_legacy_app(secondary_domain, "/legacy") -def test_legacy_app_failed_install(mocker, secondary_domain): +def test_legacy_app_failed_install(secondary_domain): # This will conflict with the folder that the app # attempts to create, making the install fail mkdir("/var/www/legacy_app/", 0o750) with pytest.raises(YunohostError): - with message(mocker, "app_install_script_failed"): + with message("app_install_script_failed"): install_legacy_app(secondary_domain, "/legacy") assert app_is_not_installed(secondary_domain, "legacy_app") -def test_legacy_app_failed_remove(mocker, secondary_domain): +def test_legacy_app_failed_remove(secondary_domain): install_legacy_app(secondary_domain, "/legacy") # The remove script runs with set -eu and attempt to remove this @@ -486,52 +486,52 @@ def test_full_domain_app_with_conflicts(mocker, secondary_domain): install_full_domain_app(secondary_domain) -def test_systemfuckedup_during_app_install(mocker, secondary_domain): +def test_systemfuckedup_during_app_install(secondary_domain): with pytest.raises(YunohostError): - with message(mocker, "app_install_failed"): - with message(mocker, "app_action_broke_system"): + with message("app_install_failed"): + with message("app_action_broke_system"): install_break_yo_system(secondary_domain, breakwhat="install") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_remove(mocker, secondary_domain): +def test_systemfuckedup_during_app_remove(secondary_domain): install_break_yo_system(secondary_domain, breakwhat="remove") with pytest.raises(YunohostError): - with message(mocker, "app_action_broke_system"): - with message(mocker, "app_removed"): + with message("app_action_broke_system"): + with message("app_removed"): app_remove("break_yo_system") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_install_and_remove(mocker, secondary_domain): +def test_systemfuckedup_during_app_install_and_remove(secondary_domain): with pytest.raises(YunohostError): - with message(mocker, "app_install_failed"): - with message(mocker, "app_action_broke_system"): + with message("app_install_failed"): + with message("app_action_broke_system"): install_break_yo_system(secondary_domain, breakwhat="everything") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain): +def test_systemfuckedup_during_app_upgrade(secondary_domain): install_break_yo_system(secondary_domain, breakwhat="upgrade") with pytest.raises(YunohostError): - with message(mocker, "app_action_broke_system"): + with message("app_action_broke_system"): app_upgrade( "break_yo_system", file=os.path.join(get_test_apps_dir(), "break_yo_system_ynh"), ) -def test_failed_multiple_app_upgrade(mocker, secondary_domain): +def test_failed_multiple_app_upgrade(secondary_domain): install_legacy_app(secondary_domain, "/legacy") install_break_yo_system(secondary_domain, breakwhat="upgrade") with pytest.raises(YunohostError): - with message(mocker, "app_not_upgraded"): + with message("app_not_upgraded"): app_upgrade( ["break_yo_system", "legacy_app"], file={ diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 413d444703..a2dcfe8fbd 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -236,10 +236,10 @@ def add_archive_system_from_4p2(): # -def test_backup_only_ldap(mocker): +def test_backup_only_ldap(): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=["conf_ldap"], apps=None) archives = backup_list()["archives"] @@ -253,7 +253,7 @@ def test_backup_only_ldap(mocker): def test_backup_system_part_that_does_not_exists(mocker): # Create the backup - with message(mocker, "backup_hook_unknown", hook="doesnt_exist"): + with message("backup_hook_unknown", hook="doesnt_exist"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=["doesnt_exist"], apps=None) @@ -263,10 +263,10 @@ def test_backup_system_part_that_does_not_exists(mocker): # -def test_backup_and_restore_all_sys(mocker): +def test_backup_and_restore_all_sys(): name = random_ascii(8) # Create the backup - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] @@ -284,7 +284,7 @@ def test_backup_and_restore_all_sys(mocker): assert not os.path.exists("/etc/ssowat/conf.json") # Restore the backup - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore(name=archives[0], force=True, system=[], apps=None) # Check ssowat conf is back @@ -297,17 +297,17 @@ def test_backup_and_restore_all_sys(mocker): @pytest.mark.with_system_archive_from_4p2 -def test_restore_system_from_Ynh4p2(monkeypatch, mocker): +def test_restore_system_from_Ynh4p2(monkeypatch): name = random_ascii(8) # Backup current system - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] assert len(archives) == 2 # Restore system archive from 3.8 try: - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( name=backup_list()["archives"][1], system=[], apps=None, force=True ) @@ -336,7 +336,7 @@ def custom_hook_exec(name, *args, **kwargs): # with the expected error message key monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) - with message(mocker, "backup_app_failed", app="backup_recommended_app"): + with message("backup_app_failed", app="backup_recommended_app"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["backup_recommended_app"]) @@ -363,7 +363,7 @@ def custom_free_space_in_directory(dirpath): def test_backup_app_not_installed(mocker): assert not _is_installed("wordpress") - with message(mocker, "unbackup_app", app="wordpress"): + with message("unbackup_app", app="wordpress"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["wordpress"]) @@ -375,14 +375,14 @@ def test_backup_app_with_no_backup_script(mocker): assert not os.path.exists(backup_script) with message( - mocker, "backup_with_no_backup_script_for_app", app="backup_recommended_app" + "backup_with_no_backup_script_for_app", app="backup_recommended_app" ): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["backup_recommended_app"]) @pytest.mark.with_backup_recommended_app_installed -def test_backup_app_with_no_restore_script(mocker): +def test_backup_app_with_no_restore_script(): restore_script = "/etc/yunohost/apps/backup_recommended_app/scripts/restore" os.system("rm %s" % restore_script) assert not os.path.exists(restore_script) @@ -391,16 +391,16 @@ def test_backup_app_with_no_restore_script(mocker): # user... with message( - mocker, "backup_with_no_restore_script_for_app", app="backup_recommended_app" + "backup_with_no_restore_script_for_app", app="backup_recommended_app" ): backup_create(system=None, apps=["backup_recommended_app"]) @pytest.mark.clean_opt_dir -def test_backup_with_different_output_directory(mocker): +def test_backup_with_different_output_directory(): name = random_ascii(8) # Create the backup - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create( system=["conf_ynh_settings"], apps=None, @@ -420,10 +420,10 @@ def test_backup_with_different_output_directory(mocker): @pytest.mark.clean_opt_dir -def test_backup_using_copy_method(mocker): +def test_backup_using_copy_method(): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create( system=["conf_ynh_settings"], apps=None, @@ -442,8 +442,8 @@ def test_backup_using_copy_method(mocker): @pytest.mark.with_wordpress_archive_from_4p2 @pytest.mark.with_custom_domain("yolo.test") -def test_restore_app_wordpress_from_Ynh4p2(mocker): - with message(mocker, "restore_complete"): +def test_restore_app_wordpress_from_Ynh4p2(): + with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) @@ -461,7 +461,7 @@ def custom_hook_exec(name, *args, **kwargs): assert not _is_installed("wordpress") - with message(mocker, "app_restore_script_failed"): + with message("app_restore_script_failed"): with raiseYunohostError(mocker, "restore_nothings_done"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] @@ -494,7 +494,7 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("wordpress") assert not _is_installed("yoloswag") - with message(mocker, "backup_archive_app_not_found", app="yoloswag"): + with message("backup_archive_app_not_found", app="yoloswag"): with raiseYunohostError(mocker, "restore_nothings_done"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["yoloswag"] @@ -509,7 +509,7 @@ def test_restore_app_not_in_backup(mocker): def test_restore_app_already_installed(mocker): assert not _is_installed("wordpress") - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) @@ -525,22 +525,22 @@ def test_restore_app_already_installed(mocker): @pytest.mark.with_legacy_app_installed -def test_backup_and_restore_legacy_app(mocker): - _test_backup_and_restore_app(mocker, "legacy_app") +def test_backup_and_restore_legacy_app(): + _test_backup_and_restore_app("legacy_app") @pytest.mark.with_backup_recommended_app_installed -def test_backup_and_restore_recommended_app(mocker): - _test_backup_and_restore_app(mocker, "backup_recommended_app") +def test_backup_and_restore_recommended_app(): + _test_backup_and_restore_app("backup_recommended_app") @pytest.mark.with_backup_recommended_app_installed_with_ynh_restore -def test_backup_and_restore_with_ynh_restore(mocker): - _test_backup_and_restore_app(mocker, "backup_recommended_app") +def test_backup_and_restore_with_ynh_restore(): + _test_backup_and_restore_app("backup_recommended_app") @pytest.mark.with_permission_app_installed -def test_backup_and_restore_permission_app(mocker): +def test_backup_and_restore_permission_app(): res = user_permission_list(full=True)["permissions"] assert "permissions_app.main" in res assert "permissions_app.admin" in res @@ -554,7 +554,7 @@ def test_backup_and_restore_permission_app(mocker): assert res["permissions_app.admin"]["allowed"] == ["alice"] assert res["permissions_app.dev"]["allowed"] == [] - _test_backup_and_restore_app(mocker, "permissions_app") + _test_backup_and_restore_app("permissions_app") res = user_permission_list(full=True)["permissions"] assert "permissions_app.main" in res @@ -570,10 +570,10 @@ def test_backup_and_restore_permission_app(mocker): assert res["permissions_app.dev"]["allowed"] == [] -def _test_backup_and_restore_app(mocker, app): +def _test_backup_and_restore_app(app): # Create a backup of this app name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=None, apps=[app]) archives = backup_list()["archives"] @@ -590,7 +590,7 @@ def _test_backup_and_restore_app(mocker, app): assert app + ".main" not in user_permission_list()["permissions"] # Restore the app - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore(system=None, name=archives[0], apps=[app]) assert app_is_installed(app) @@ -631,19 +631,19 @@ def test_restore_archive_with_bad_archive(mocker): clean_tmp_backup_directory() -def test_restore_archive_with_custom_hook(mocker): +def test_restore_archive_with_custom_hook(): custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore") os.system("touch %s/99-yolo" % custom_restore_hook_folder) # Backup with custom hook system name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] assert len(archives) == 1 # Restore system with custom hook - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( name=backup_list()["archives"][0], system=[], apps=None, force=True ) @@ -651,7 +651,7 @@ def test_restore_archive_with_custom_hook(mocker): os.system("rm %s/99-yolo" % custom_restore_hook_folder) -def test_backup_binds_are_readonly(mocker, monkeypatch): +def test_backup_binds_are_readonly(monkeypatch): def custom_mount_and_backup(self): self._organize_files() @@ -676,5 +676,5 @@ def custom_mount_and_backup(self): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[]) diff --git a/src/tests/test_ldapauth.py b/src/tests/test_ldapauth.py index 9e3ae36cc3..73cb09d274 100644 --- a/src/tests/test_ldapauth.py +++ b/src/tests/test_ldapauth.py @@ -59,7 +59,7 @@ def test_authenticate_with_wrong_password(): assert expected_msg in str(exception) -def test_authenticate_server_down(mocker): +def test_authenticate_server_down(): os.system("systemctl stop slapd && sleep 5") LDAPAuth().authenticate_credentials(credentials="alice:Yunohost") diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 10bd018d23..4ab333584d 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -435,8 +435,8 @@ def test_permission_list(): # -def test_permission_create_main(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_main(): + with message("permission_created", permission="site.main"): permission_create("site.main", allowed=["all_users"], protected=False) res = user_permission_list(full=True)["permissions"] @@ -446,8 +446,8 @@ def test_permission_create_main(mocker): assert res["site.main"]["protected"] is False -def test_permission_create_extra(mocker): - with message(mocker, "permission_created", permission="site.test"): +def test_permission_create_extra(): + with message("permission_created", permission="site.test"): permission_create("site.test") res = user_permission_list(full=True)["permissions"] @@ -466,8 +466,8 @@ def test_permission_create_with_specific_user(): assert res["site.test"]["allowed"] == ["alice"] -def test_permission_create_with_tile_management(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_tile_management(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -483,8 +483,8 @@ def test_permission_create_with_tile_management(mocker): assert res["site.main"]["show_tile"] is False -def test_permission_create_with_tile_management_with_main_default_value(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_tile_management_with_main_default_value(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -500,8 +500,8 @@ def test_permission_create_with_tile_management_with_main_default_value(mocker): assert res["site.main"]["show_tile"] is True -def test_permission_create_with_tile_management_with_not_main_default_value(mocker): - with message(mocker, "permission_created", permission="wiki.api"): +def test_permission_create_with_tile_management_with_not_main_default_value(): + with message("permission_created", permission="wiki.api"): _permission_create_with_dummy_app( "wiki.api", allowed=["all_users"], @@ -517,8 +517,8 @@ def test_permission_create_with_tile_management_with_not_main_default_value(mock assert res["wiki.api"]["show_tile"] is True -def test_permission_create_with_urls_management_without_url(mocker): - with message(mocker, "permission_created", permission="wiki.api"): +def test_permission_create_with_urls_management_without_url(): + with message("permission_created", permission="wiki.api"): _permission_create_with_dummy_app( "wiki.api", allowed=["all_users"], domain=maindomain, path="/site" ) @@ -530,8 +530,8 @@ def test_permission_create_with_urls_management_without_url(mocker): assert res["wiki.api"]["auth_header"] is True -def test_permission_create_with_urls_management_simple_domain(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_urls_management_simple_domain(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -553,8 +553,8 @@ def test_permission_create_with_urls_management_simple_domain(mocker): @pytest.mark.other_domains(number=2) -def test_permission_create_with_urls_management_multiple_domain(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_urls_management_multiple_domain(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -575,14 +575,14 @@ def test_permission_create_with_urls_management_multiple_domain(mocker): assert res["site.main"]["auth_header"] is True -def test_permission_delete(mocker): - with message(mocker, "permission_deleted", permission="wiki.main"): +def test_permission_delete(): + with message("permission_deleted", permission="wiki.main"): permission_delete("wiki.main", force=True) res = user_permission_list()["permissions"] assert "wiki.main" not in res - with message(mocker, "permission_deleted", permission="blog.api"): + with message("permission_deleted", permission="blog.api"): permission_delete("blog.api", force=False) res = user_permission_list()["permissions"] @@ -625,8 +625,8 @@ def test_permission_delete_main_without_force(mocker): # user side functions -def test_permission_add_group(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_add_group(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", add="alice") res = user_permission_list(full=True)["permissions"] @@ -634,8 +634,8 @@ def test_permission_add_group(mocker): assert set(res["wiki.main"]["corresponding_users"]) == {"alice", "bob"} -def test_permission_remove_group(mocker): - with message(mocker, "permission_updated", permission="blog.main"): +def test_permission_remove_group(): + with message("permission_updated", permission="blog.main"): user_permission_update("blog.main", remove="alice") res = user_permission_list(full=True)["permissions"] @@ -643,8 +643,8 @@ def test_permission_remove_group(mocker): assert res["blog.main"]["corresponding_users"] == [] -def test_permission_add_and_remove_group(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_add_and_remove_group(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", add="alice", remove="all_users") res = user_permission_list(full=True)["permissions"] @@ -652,9 +652,9 @@ def test_permission_add_and_remove_group(mocker): assert res["wiki.main"]["corresponding_users"] == ["alice"] -def test_permission_add_group_already_allowed(mocker): +def test_permission_add_group_already_allowed(): with message( - mocker, "permission_already_allowed", permission="blog.main", group="alice" + "permission_already_allowed", permission="blog.main", group="alice" ): user_permission_update("blog.main", add="alice") @@ -663,9 +663,9 @@ def test_permission_add_group_already_allowed(mocker): assert res["blog.main"]["corresponding_users"] == ["alice"] -def test_permission_remove_group_already_not_allowed(mocker): +def test_permission_remove_group_already_not_allowed(): with message( - mocker, "permission_already_disallowed", permission="blog.main", group="bob" + "permission_already_disallowed", permission="blog.main", group="bob" ): user_permission_update("blog.main", remove="bob") @@ -674,8 +674,8 @@ def test_permission_remove_group_already_not_allowed(mocker): assert res["blog.main"]["corresponding_users"] == ["alice"] -def test_permission_reset(mocker): - with message(mocker, "permission_updated", permission="blog.main"): +def test_permission_reset(): + with message("permission_updated", permission="blog.main"): user_permission_reset("blog.main") res = user_permission_list(full=True)["permissions"] @@ -693,42 +693,42 @@ def test_permission_reset_idempotency(): assert set(res["blog.main"]["corresponding_users"]) == {"alice", "bob"} -def test_permission_change_label(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_change_label(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", label="New Wiki") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["label"] == "New Wiki" -def test_permission_change_label_with_same_value(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_change_label_with_same_value(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", label="Wiki") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["label"] == "Wiki" -def test_permission_switch_show_tile(mocker): +def test_permission_switch_show_tile(): # Note that from the actionmap the value is passed as string, not as bool # Try with lowercase - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="false") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["show_tile"] is False # Try with uppercase - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="TRUE") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["show_tile"] is True -def test_permission_switch_show_tile_with_same_value(mocker): +def test_permission_switch_show_tile_with_same_value(): # Note that from the actionmap the value is passed as string, not as bool - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="True") res = user_permission_list(full=True)["permissions"] diff --git a/src/tests/test_regenconf.py b/src/tests/test_regenconf.py index 8dda1a7f25..3966ef2911 100644 --- a/src/tests/test_regenconf.py +++ b/src/tests/test_regenconf.py @@ -87,7 +87,7 @@ def test_ssh_conf_unmanaged(): assert SSHD_CONFIG in _get_conf_hashes("ssh") -def test_ssh_conf_unmanaged_and_manually_modified(mocker): +def test_ssh_conf_unmanaged_and_manually_modified(): _force_clear_hashes([SSHD_CONFIG]) os.system("echo ' ' >> %s" % SSHD_CONFIG) @@ -98,7 +98,7 @@ def test_ssh_conf_unmanaged_and_manually_modified(mocker): assert SSHD_CONFIG in _get_conf_hashes("ssh") assert SSHD_CONFIG in manually_modified_files() - with message(mocker, "regenconf_need_to_explicitly_specify_ssh"): + with message("regenconf_need_to_explicitly_specify_ssh"): regen_conf(force=True) assert SSHD_CONFIG in _get_conf_hashes("ssh") diff --git a/src/tests/test_user-group.py b/src/tests/test_user-group.py index eececb8279..57f9ffa3fe 100644 --- a/src/tests/test_user-group.py +++ b/src/tests/test_user-group.py @@ -91,8 +91,8 @@ def test_list_groups(): # -def test_create_user(mocker): - with message(mocker, "user_created"): +def test_create_user(): + with message("user_created"): user_create("albert", maindomain, "test123Ynh", fullname="Albert Good") group_res = user_group_list()["groups"] @@ -102,8 +102,8 @@ def test_create_user(mocker): assert "albert" in group_res["all_users"]["members"] -def test_del_user(mocker): - with message(mocker, "user_deleted"): +def test_del_user(): + with message("user_deleted"): user_delete("alice") group_res = user_group_list()["groups"] @@ -112,7 +112,7 @@ def test_del_user(mocker): assert "alice" not in group_res["all_users"]["members"] -def test_import_user(mocker): +def test_import_user(): import csv from io import StringIO @@ -157,7 +157,7 @@ def test_import_user(mocker): } ) csv_io.seek(0) - with message(mocker, "user_import_success"): + with message("user_import_success"): user_import(csv_io, update=True, delete=True) group_res = user_group_list()["groups"] @@ -171,7 +171,7 @@ def test_import_user(mocker): assert "alice" not in group_res["dev"]["members"] -def test_export_user(mocker): +def test_export_user(): result = user_export() should_be = ( "username;firstname;lastname;password;mail;mail-alias;mail-forward;mailbox-quota;groups\r\n" @@ -182,8 +182,8 @@ def test_export_user(mocker): assert result == should_be -def test_create_group(mocker): - with message(mocker, "group_created", group="adminsys"): +def test_create_group(): + with message("group_created", group="adminsys"): user_group_create("adminsys") group_res = user_group_list()["groups"] @@ -192,8 +192,8 @@ def test_create_group(mocker): assert group_res["adminsys"]["members"] == [] -def test_del_group(mocker): - with message(mocker, "group_deleted", group="dev"): +def test_del_group(): + with message("group_deleted", group="dev"): user_group_delete("dev") group_res = user_group_list()["groups"] @@ -262,46 +262,46 @@ def test_del_group_that_does_not_exist(mocker): # -def test_update_user(mocker): - with message(mocker, "user_updated"): +def test_update_user(): + with message("user_updated"): user_update("alice", firstname="NewName", lastname="NewLast") info = user_info("alice") assert info["fullname"] == "NewName NewLast" - with message(mocker, "user_updated"): + with message("user_updated"): user_update("alice", fullname="New2Name New2Last") info = user_info("alice") assert info["fullname"] == "New2Name New2Last" -def test_update_group_add_user(mocker): - with message(mocker, "group_updated", group="dev"): +def test_update_group_add_user(): + with message("group_updated", group="dev"): user_group_update("dev", add=["bob"]) group_res = user_group_list()["groups"] assert set(group_res["dev"]["members"]) == {"alice", "bob"} -def test_update_group_add_user_already_in(mocker): - with message(mocker, "group_user_already_in_group", user="bob", group="apps"): +def test_update_group_add_user_already_in(): + with message("group_user_already_in_group", user="bob", group="apps"): user_group_update("apps", add=["bob"]) group_res = user_group_list()["groups"] assert group_res["apps"]["members"] == ["bob"] -def test_update_group_remove_user(mocker): - with message(mocker, "group_updated", group="apps"): +def test_update_group_remove_user(): + with message("group_updated", group="apps"): user_group_update("apps", remove=["bob"]) group_res = user_group_list()["groups"] assert group_res["apps"]["members"] == [] -def test_update_group_remove_user_not_already_in(mocker): - with message(mocker, "group_user_not_in_group", user="jack", group="apps"): +def test_update_group_remove_user_not_already_in(): + with message("group_user_not_in_group", user="jack", group="apps"): user_group_update("apps", remove=["jack"]) group_res = user_group_list()["groups"] From 1af88b0c55d5d3cadf8e758496254a3143abdf41 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 19:04:32 +0200 Subject: [PATCH 027/436] ci: force tox install during lint tasks --- .gitlab/ci/lint.gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 7a8fbf1fb0..6e33af4080 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -1,3 +1,6 @@ +.install_tox: &install_tox + - pip3 install -U tox --break-system-packages + ######################################## # LINTER ######################################## @@ -8,6 +11,8 @@ lint39: image: "before-install" needs: [] allow_failure: true + before_script: + - *install_tox script: - tox -e py39-lint @@ -15,6 +20,8 @@ invalidcode39: stage: lint image: "before-install" needs: [] + before_script: + - *install_tox script: - tox -e py39-invalidcode @@ -22,6 +29,8 @@ mypy: stage: lint image: "before-install" needs: [] + before_script: + - *install_tox script: - tox -e py39-mypy @@ -29,6 +38,8 @@ black: stage: lint image: "before-install" needs: [] + before_script: + - *install_tox before_script: - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" From bed9ecc09e057af2fca2788672aa426a2ca1dd24 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 15 May 2023 22:02:40 +0200 Subject: [PATCH 028/436] py39->py311 in tox --- .gitlab/ci/lint.gitlab-ci.yml | 12 ++++++------ tox.ini | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 6e33af4080..349819a90b 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -6,7 +6,7 @@ ######################################## # later we must fix lint and format-check jobs and remove "allow_failure" -lint39: +lint311: stage: lint image: "before-install" needs: [] @@ -14,16 +14,16 @@ lint39: before_script: - *install_tox script: - - tox -e py39-lint + - tox -e py311-lint -invalidcode39: +invalidcode311: stage: lint image: "before-install" needs: [] before_script: - *install_tox script: - - tox -e py39-invalidcode + - tox -e py311-invalidcode mypy: stage: lint @@ -32,7 +32,7 @@ mypy: before_script: - *install_tox script: - - tox -e py39-mypy + - tox -e py311-mypy black: stage: lint @@ -49,7 +49,7 @@ black: script: # create a local branch that will overwrite distant one - git checkout -b "ci-format-${CI_COMMIT_REF_NAME}" --no-track - - tox -e py39-black-run + - tox -e py311-black-run - '[ $(git diff | wc -l) != 0 ] || exit 0' # stop if there is nothing to commit - git commit -am "[CI] Format code with Black" || true - git push -f origin "ci-format-${CI_COMMIT_REF_NAME}":"ci-format-${CI_COMMIT_REF_NAME}" diff --git a/tox.ini b/tox.ini index 49c78959d2..58c443f25d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,15 @@ [tox] -envlist = py39-{lint,invalidcode},py39-black-{run,check} +envlist = py311-{lint,invalidcode},py311-black-{run,check} [testenv] skip_install=True deps = - py39-{lint,invalidcode}: flake8 - py39-black-{run,check}: black - py39-mypy: mypy >= 0.900 + py311-{lint,invalidcode}: flake8 + py311-black-{run,check}: black + py311-mypy: mypy >= 0.900 commands = - py39-lint: flake8 src doc maintenance tests --ignore E402,E501,E203,W503,E741 --exclude src/vendor - py39-invalidcode: flake8 src bin maintenance --exclude src/tests,src/vendor --select F,E722,W605 - py39-black-check: black --check --diff bin src doc maintenance tests - py39-black-run: black bin src doc maintenance tests - py39-mypy: mypy --ignore-missing-import --install-types --non-interactive --follow-imports silent src/ --exclude (acme_tiny|migrations) + py311-lint: flake8 src doc maintenance tests --ignore E402,E501,E203,W503,E741 --exclude src/vendor + py311-invalidcode: flake8 src bin maintenance --exclude src/tests,src/vendor --select F,E722,W605 + py311-black-check: black --check --diff bin src doc maintenance tests + py311-black-run: black bin src doc maintenance tests + py311-mypy: mypy --ignore-missing-import --install-types --non-interactive --follow-imports silent src/ --exclude (acme_tiny|migrations) From 734db1994c30faac1d9d14ab40d32a275245f567 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Tue, 16 May 2023 11:30:56 +0200 Subject: [PATCH 029/436] ci: don't install any package with pip, it's supposed to be preinstalled --- .gitlab/ci/lint.gitlab-ci.yml | 11 ----------- .gitlab/ci/test.gitlab-ci.yml | 1 - 2 files changed, 12 deletions(-) diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 349819a90b..65b74ddca2 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -1,6 +1,3 @@ -.install_tox: &install_tox - - pip3 install -U tox --break-system-packages - ######################################## # LINTER ######################################## @@ -11,8 +8,6 @@ lint311: image: "before-install" needs: [] allow_failure: true - before_script: - - *install_tox script: - tox -e py311-lint @@ -20,8 +15,6 @@ invalidcode311: stage: lint image: "before-install" needs: [] - before_script: - - *install_tox script: - tox -e py311-invalidcode @@ -29,8 +22,6 @@ mypy: stage: lint image: "before-install" needs: [] - before_script: - - *install_tox script: - tox -e py311-mypy @@ -38,8 +29,6 @@ black: stage: lint image: "before-install" needs: [] - before_script: - - *install_tox before_script: - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index c5f1ee13ce..a49fc13b77 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,7 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client mariadb-server - - pip3 install -U mock pip pytest pytest-cov pytest-mock pytest-sugar requests-mock tox ansi2html black jinja2 "packaging<22" --break-system-packages # for bookworm .test-stage: stage: test From 5564f7dc12fa5ac8036c683e2a3976d5f0327f14 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 16 May 2023 15:11:51 +0200 Subject: [PATCH 030/436] tests: fix remaining funky mocker.spy --- src/tests/test_app_catalog.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/tests/test_app_catalog.py b/src/tests/test_app_catalog.py index f7363dabef..40daf58736 100644 --- a/src/tests/test_app_catalog.py +++ b/src/tests/test_app_catalog.py @@ -5,6 +5,8 @@ import glob import shutil +from .conftest import message + from moulinette import m18n from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml @@ -258,13 +260,12 @@ def test_apps_catalog_load_with_conflicts_between_lists(mocker): assert "bar" in app_dict.keys() -def test_apps_catalog_load_with_oudated_api_version(mocker): +def test_apps_catalog_load_with_outdated_api_version(): # Initialize ... _initialize_apps_catalog_system() # Update with requests_mock.Mocker() as m: - mocker.spy(m18n, "n") m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) _update_apps_catalog() @@ -282,10 +283,8 @@ def test_apps_catalog_load_with_oudated_api_version(mocker): with requests_mock.Mocker() as m: # Mock the server response with a dummy apps catalog m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) - - mocker.spy(m18n, "n") - app_dict = _load_apps_catalog()["apps"] - m18n.n.assert_any_call("apps_catalog_update_success") + with message("apps_catalog_update_success"): + app_dict = _load_apps_catalog()["apps"] assert "foo" in app_dict.keys() assert "bar" in app_dict.keys() From 5b9721eb23c4086930dd16c1d2745688350731cd Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 16 May 2023 15:17:03 +0200 Subject: [PATCH 031/436] tests: fix bad regex permission test because python3.11 now accepts ++ quantifier, so change the 'bad regex' trick --- src/tests/test_permission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 4ab333584d..8620e96119 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -806,7 +806,7 @@ def test_permission_main_url_regex(): def test_permission_main_url_bad_regex(mocker): with raiseYunohostError(mocker, "invalid_regex"): - permission_url("blog.main", url="re:/[a-z]++reboy/.*") + permission_url("blog.main", url="re:/[a-z]+++reboy/.*") @pytest.mark.other_domains(number=1) @@ -837,7 +837,7 @@ def test_permission_add_additional_regex(): def test_permission_add_additional_bad_regex(mocker): with raiseYunohostError(mocker, "invalid_regex"): - permission_url("blog.main", add_url=["re:/[a-z]++reboy/.*"]) + permission_url("blog.main", add_url=["re:/[a-z]+++reboy/.*"]) def test_permission_remove_additional_url(): From 3b754859230410f90967cb9ab454b1c8d1d363c5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 16 May 2023 15:21:27 +0200 Subject: [PATCH 032/436] tests: somehow using 'Domain' as http header aint supported anymore, gotta use Host --- src/tests/test_changeurl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_changeurl.py b/src/tests/test_changeurl.py index 04cb4a1a9e..b8ca20355f 100644 --- a/src/tests/test_changeurl.py +++ b/src/tests/test_changeurl.py @@ -39,7 +39,7 @@ def check_changeurl_app(path): assert appmap[maindomain][path]["id"] == "change_url_app" r = requests.get( - "https://127.0.0.1%s/" % path, headers={"domain": maindomain}, verify=False + "https://127.0.0.1%s/" % path, headers={"Host": maindomain}, verify=False ) assert r.status_code == 200 From 85b08e44c9ee03151cae1c35ef20902ffdb7ddd4 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 22 May 2023 15:29:21 +0200 Subject: [PATCH 033/436] ci: preinstall more package --- .gitlab/ci/doc.gitlab-ci.yml | 1 - .gitlab/ci/lint.gitlab-ci.yml | 1 - .gitlab/ci/test.gitlab-ci.yml | 2 +- .gitlab/ci/translation.gitlab-ci.yml | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml index 4f6ea6ba18..183d153a4e 100644 --- a/.gitlab/ci/doc.gitlab-ci.yml +++ b/.gitlab/ci/doc.gitlab-ci.yml @@ -7,7 +7,6 @@ generate-helpers-doc: image: "before-install" needs: [] before_script: - - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" - git config --global user.name "$GITHUB_USER" script: diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index 65b74ddca2..1eeb71eab2 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -30,7 +30,6 @@ black: image: "before-install" needs: [] before_script: - - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" - git config --global user.name "$GITHUB_USER" - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index a49fc13b77..2c6e1717d1 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,6 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 - - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb php8.2-cli mariadb-client mariadb-server + - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb .test-stage: stage: test diff --git a/.gitlab/ci/translation.gitlab-ci.yml b/.gitlab/ci/translation.gitlab-ci.yml index 83db2b5a45..387860e40e 100644 --- a/.gitlab/ci/translation.gitlab-ci.yml +++ b/.gitlab/ci/translation.gitlab-ci.yml @@ -16,7 +16,6 @@ autofix-translated-strings: image: "before-install" needs: [] before_script: - - apt-get update -y && apt-get install git hub -y - git config --global user.email "yunohost@yunohost.org" - git config --global user.name "$GITHUB_USER" - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo From 23eaf609da112fda99e76d544d25a7634b685188 Mon Sep 17 00:00:00 2001 From: ElderTek Date: Thu, 25 May 2023 00:00:07 +0400 Subject: [PATCH 034/436] remove deprecated --- share/actionsmap.yml | 11 ----------- src/domain.py | 4 ---- 2 files changed, 15 deletions(-) diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 58787790c7..107853c339 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -537,17 +537,6 @@ domain: full: --force help: Do not ask confirmation to remove apps action: store_true - - - ### domain_dns_conf() - dns-conf: - deprecated: true - action_help: Generate sample DNS configuration for a domain - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain ### domain_maindomain() main-domain: diff --git a/src/domain.py b/src/domain.py index 4f96d08c4d..a2d570b4bf 100644 --- a/src/domain.py +++ b/src/domain.py @@ -718,10 +718,6 @@ def domain_cert_renew(domain_list, force=False, no_checks=False, email=False): return certificate_renew(domain_list, force, no_checks, email) -def domain_dns_conf(domain): - return domain_dns_suggest(domain) - - def domain_dns_suggest(domain): from yunohost.dns import domain_dns_suggest From 78cd79ec480c5e4643b792ed9fcb8dd36fb882cd Mon Sep 17 00:00:00 2001 From: Kayou Date: Mon, 5 Jun 2023 10:11:50 +0200 Subject: [PATCH 035/436] Update debian/changelog --- debian/changelog | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 69418598cb..bc06c7e423 100644 --- a/debian/changelog +++ b/debian/changelog @@ -15,7 +15,6 @@ yunohost (11.1.20) stable; urgency=low Thanks to all contributors <3 ! (axolotle, Éric Gaspar, Ilya, Jose Riha, Neko Nekowazarashi, Yann Autissier) -- Alexandre Aubin Sat, 20 May 2023 18:57:26 +0200 ->>>>>>> origin/dev yunohost (11.1.19) stable; urgency=low From 8728b2030cea9c83f3c65e77406cc2524ca4dfd8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 14:55:28 +0200 Subject: [PATCH 036/436] Remove migrations/0027_migrate_to_bookworm for now because it's triggering errors on the CI, at least half of it should be reworked, and it should be in a separated PR to target dev(=bullseye) --- src/migrations/0027_migrate_to_bookworm.py | 546 --------------------- 1 file changed, 546 deletions(-) delete mode 100644 src/migrations/0027_migrate_to_bookworm.py diff --git a/src/migrations/0027_migrate_to_bookworm.py b/src/migrations/0027_migrate_to_bookworm.py deleted file mode 100644 index 85e2235aff..0000000000 --- a/src/migrations/0027_migrate_to_bookworm.py +++ /dev/null @@ -1,546 +0,0 @@ -import glob -import os - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file, rm, write_to_file - -from yunohost.tools import ( - Migration, - tools_update, - tools_upgrade, - _apt_log_line_is_relevant, -) -from yunohost.app import unstable_apps -from yunohost.regenconf import manually_modified_files, _force_clear_hashes -from yunohost.utils.system import ( - free_space_in_directory, - get_ynh_package_version, - _list_upgradable_apt_packages, -) -from yunohost.service import _get_services, _save_services - -logger = getActionLogger("yunohost.migration") - -N_CURRENT_DEBIAN = 10 -N_CURRENT_YUNOHOST = 4 - -VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bookworm_upgrade.txt" - - -def _get_all_venvs(dir, level=0, maxlevel=3): - """ - Returns the list of all python virtual env directories recursively - - Arguments: - dir - the directory to scan in - maxlevel - the depth of the recursion - level - do not edit this, used as an iterator - """ - if not os.path.exists(dir): - return [] - - result = [] - # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth - for file in os.listdir(dir): - path = os.path.join(dir, file) - if os.path.isdir(path): - activatepath = os.path.join(path, "bin", "activate") - if os.path.isfile(activatepath): - content = read_file(activatepath) - if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): - result.append(path) - continue - if level < maxlevel: - result += _get_all_venvs(path, level=level + 1) - return result - - -def _backup_pip_freeze_for_python_app_venvs(): - """ - Generate a requirements file for all python virtual env located inside /opt/ and /var/www/ - """ - - venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") - for venv in venvs: - # Generate a requirements file from venv - os.system( - f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" - ) - - -class MyMigration(Migration): - "Upgrade the system to Debian Bookworm and Yunohost 11.x" - - mode = "manual" - - def run(self): - self.check_assertions() - - logger.info(m18n.n("migration_0021_start")) - - # - # Add new apt .deb signing key - # - - new_apt_key = "https://forge.yunohost.org/yunohost_bookworm.asc" - check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") - - # - # Patch sources.list - # - logger.info(m18n.n("migration_0021_patching_sources_list")) - self.patch_apt_sources_list() - - # Stupid OVH has some repo configured which dont work with bookworm and break apt ... - os.system("sudo rm -f /etc/apt/sources.list.d/ovh-*.list") - - # Force add sury if it's not there yet - # This is to solve some weird issue with php-common breaking php7.3-common, - # hence breaking many php7.3-deps - # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) - # Adding it there shouldnt be a big deal - Yunohost 11.x does add it - # through its regen conf anyway. - if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): - open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( - "deb https://packages.sury.org/php/ bookworm main" - ) - - # Add Sury key even if extra_php_version.list was already there, - # because some old system may be using an outdated key not valid for Bookworm - # and that'll block the migration - os.system( - 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' - ) - - # Remove legacy, duplicated sury entry if it exists - if os.path.exists("/etc/apt/sources.list.d/sury.list"): - os.system("rm -rf /etc/apt/sources.list.d/sury.list") - - # - # Get requirements of the different venvs from python apps - # - - _backup_pip_freeze_for_python_app_venvs() - - # - # Run apt update - # - - tools_update(target="system") - - # Tell libc6 it's okay to restart system stuff during the upgrade - os.system( - "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" - ) - - # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... - # c.f. https://manpages.debian.org/bookworm/init-system-helpers/deb-systemd-invoke.1p.en.html - # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz - # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... - # and also invoke-rc.d ... - write_to_file( - "/usr/sbin/policy-rc.d", - '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', - ) - os.system("chmod +x /usr/sbin/policy-rc.d") - - # Don't send an email to root about the postgresql migration. It should be handled automatically after. - os.system( - "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" - ) - - # - # Patch yunohost conflicts - # - logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) - - self.patch_yunohost_conflicts() - - # - # Specific tweaking to get rid of custom my.cnf and use debian's default one - # (my.cnf is actually a symlink to mariadb.cnf) - # - - _force_clear_hashes(["/etc/mysql/my.cnf"]) - rm("/etc/mysql/mariadb.cnf", force=True) - rm("/etc/mysql/my.cnf", force=True) - ret = self.apt_install( - "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) - - # - # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl - # - if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): - os.system( - "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" - ) - rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) - - # - # /home/yunohost.conf -> /var/cache/yunohost/regenconf - # - if os.path.exists("/home/yunohost.conf"): - os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") - rm("/home/yunohost.conf", recursive=True, force=True) - - # Remove legacy postgresql service record added by helpers, - # will now be dynamically handled by the core in bookworm - services = _get_services() - if "postgresql" in services: - del services["postgresql"] - _save_services(services) - - # - # Critical fix for RPI otherwise network is down after rebooting - # https://forum.yunohost.org/t/20652 - # - if os.system("systemctl | grep -q dhcpcd") == 0: - logger.info("Applying fix for DHCPCD ...") - os.system("mkdir -p /etc/systemd/system/dhcpcd.service.d") - write_to_file( - "/etc/systemd/system/dhcpcd.service.d/wait.conf", - "[Service]\nExecStart=\nExecStart=/usr/sbin/dhcpcd -w", - ) - - # - # Main upgrade - # - logger.info(m18n.n("migration_0021_main_upgrade")) - - apps_packages = self.get_apps_equivs_packages() - self.hold(apps_packages) - tools_upgrade(target="system", allow_yunohost_upgrade=False) - - if self.debian_major_version() == N_CURRENT_DEBIAN: - raise YunohostError("migration_0021_still_on_bullseye_after_main_upgrade") - - # Force explicit install of php8.2fpm and other old 'default' dependencies - # that are now only in Recommends - # - # Also, we need to install php8.2 equivalents of other php7.4 dependencies. - # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration - # to autoupgrade Nextcloud to 8.2, it will need the php8.2-zip to work. - # The following list is based on an ad-hoc analysis of php deps found in the - # app ecosystem, with a known equivalent on php8.2. - # - # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages - # with the proper list of dependencies, and the dependencies install this way - # will get flagged as 'manually installed'. - # - # We'll probably want to do something during the Bookworm->Bookworm migration to re-flag - # these as 'auto' so they get autoremoved if not needed anymore. - # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use - # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) - - php74packages_suffixes = [ - "apcu", - "bcmath", - "bz2", - "dom", - "gmp", - "igbinary", - "imagick", - "imap", - "mbstring", - "memcached", - "mysqli", - "mysqlnd", - "pgsql", - "redis", - "simplexml", - "soap", - "sqlite3", - "ssh2", - "tidy", - "xml", - "xmlrpc", - "xsl", - "zip", - ] - - cmd = ( - "apt show '*-ynh-deps' 2>/dev/null" - " | grep Depends" - f" | grep -o -E \"php7.4-({'|'.join(php74packages_suffixes)})\"" - " | sort | uniq" - " | sed 's/php7.4/php8.2/g'" - " || true" - ) - - basephp82packages_to_install = [ - "php8.2-fpm", - "php8.2-common", - "php8.2-ldap", - "php8.2-intl", - "php8.2-mysql", - "php8.2-gd", - "php8.2-curl", - "php-php-gettext", - ] - - php74packages_to_install = basephp82packages_to_install + [ - f.strip() for f in check_output(cmd).split("\n") if f.strip() - ] - - ret = self.apt_install( - f"{' '.join(php74packages_to_install)} " - "$(dpkg --list | grep ynh-deps | awk '{print $2}') " - "-o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - raise YunohostError( - "Failed to force the install of php dependencies ?", raw_msg=True - ) - - # Clean the mess - logger.info(m18n.n("migration_0021_cleaning_up")) - os.system( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt autoremove --assume-yes" - ) - os.system("apt clean --assume-yes") - - # - # Stupid hack for stupid dnsmasq not picking up its new init.d script then breaking everything ... - # https://forum.yunohost.org/t/20676 - # - if os.path.exists("/etc/init.d/dnsmasq.dpkg-dist"): - logger.info("Copying new version for /etc/init.d/dnsmasq ...") - os.system("cp /etc/init.d/dnsmasq.dpkg-dist /etc/init.d/dnsmasq") - - # - # Yunohost upgrade - # - logger.info(m18n.n("migration_0021_yunohost_upgrade")) - - self.unhold(apps_packages) - - cmd = "LC_ALL=C" - cmd += " DEBIAN_FRONTEND=noninteractive" - cmd += " APT_LISTCHANGES_FRONTEND=none" - cmd += " apt dist-upgrade " - cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" - cmd += " | grep -q 'ynh-deps'" - - logger.info("Simulating upgrade...") - if os.system(cmd) == 0: - raise YunohostError( - "The upgrade cannot be completed, because some app dependencies would need to be removed?", - raw_msg=True, - ) - - postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" - postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" - postupgradecmds += "echo 'Restarting nginx...' >&2\n" - postupgradecmds += "systemctl restart nginx\n" - - tools_upgrade(target="system", postupgradecmds=postupgradecmds) - - def debian_major_version(self): - # The python module "platform" and lsb_release are not reliable because - # on some setup, they may still return Release=9 even after upgrading to - # bullseye ... (Apparently this is related to OVH overriding some stuff - # with /etc/lsb-release for instance -_-) - # Instead, we rely on /etc/os-release which should be the raw info from - # the distribution... - return int( - check_output( - "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" - ) - ) - - def yunohost_major_version(self): - return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) - - def check_assertions(self): - # Be on bullseye (10.x) and yunohost 4.x - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be > 9.x but yunohost package - # would still be in 3.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - try: - # Here we try to find the previous migration log, which should be somewhat recent and be at least 10k (we keep the biggest one) - maybe_previous_migration_log_id = check_output( - "cd /var/log/yunohost/categories/operation && find -name '*migrate*.log' -size +10k -mtime -100 -exec ls -s {} \\; | sort -n | tr './' ' ' | awk '{print $2}' | tail -n 1" - ) - if maybe_previous_migration_log_id: - logger.info( - f"NB: the previous migration log id seems to be {maybe_previous_migration_log_id}. You can share it with the support team with : sudo yunohost log share {maybe_previous_migration_log_id}" - ) - except Exception: - # Yeah it's not that important ... it's to simplify support ... - pass - - raise YunohostError("migration_0021_not_bullseye2") - - # Have > 1 Go free space on /var/ ? - if free_space_in_directory("/var/") / (1024**3) < 1.0: - raise YunohostError("migration_0021_not_enough_free_space") - - # Have > 70 MB free space on /var/ ? - # FIXME: Create a way to ignore this check, on some system 70M is enough... - if free_space_in_directory("/boot/") / (1024**2) < 70.0: - raise YunohostError( - "/boot/ has less than 70MB available. This will probably trigger a crash during the upgrade because a new kernel needs to be installed. Please look for advice on the forum on how to remove old, unused kernels to free up some space in /boot/.", - raw_msg=True, - ) - - # Check system is up to date - # (but we don't if 'bookworm' is already in the sources.list ... - # which means maybe a previous upgrade crashed and we're re-running it) - if os.path.exists("/etc/apt/sources.list") and " bookworm " not in read_file( - "/etc/apt/sources.list" - ): - tools_update(target="system") - upgradable_system_packages = list(_list_upgradable_apt_packages()) - upgradable_system_packages = [ - package["name"] for package in upgradable_system_packages - ] - upgradable_system_packages = set(upgradable_system_packages) - # Lime2 have hold packages to avoid ethernet instability - # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df - lime2_hold_packages = set( - [ - "armbian-firmware", - "armbian-bsp-cli-lime2", - "linux-dtb-current-sunxi", - "linux-image-current-sunxi", - "linux-u-boot-lime2-current", - "linux-image-next-sunxi", - ] - ) - if upgradable_system_packages - lime2_hold_packages: - raise YunohostError("migration_0021_system_not_fully_up_to_date") - - @property - def disclaimer(self): - # Avoid having a super long disclaimer + uncessary check if we ain't - # on bullseye / yunohost 4.x anymore - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be >= 10.x but yunohost package - # would still be in 4.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - return None - - # Get list of problematic apps ? I.e. not official or community+working - problematic_apps = unstable_apps() - problematic_apps = "".join(["\n - " + app for app in problematic_apps]) - - # Manually modified files ? (c.f. yunohost service regen-conf) - modified_files = manually_modified_files() - modified_files = "".join(["\n - " + f for f in modified_files]) - - message = m18n.n("migration_0021_general_warning") - - message = ( - "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n" - + message - ) - - if problematic_apps: - message += "\n\n" + m18n.n( - "migration_0021_problematic_apps_warning", - problematic_apps=problematic_apps, - ) - - if modified_files: - message += "\n\n" + m18n.n( - "migration_0021_modified_files", manually_modified_files=modified_files - ) - - return message - - def patch_apt_sources_list(self): - sources_list = glob.glob("/etc/apt/sources.list.d/*.list") - if os.path.exists("/etc/apt/sources.list"): - sources_list.append("/etc/apt/sources.list") - - # This : - # - replace single 'bullseye' occurence by 'bulleye' - # - comments lines containing "backports" - for f in sources_list: - command = ( - f"sed -i {f} " - "-e 's@ bullseye @ bookworm @g' " - "-e '/backports/ s@^#*@#@' " - "-e 's@ bullseye-@ bookworm-@g' " - ) - os.system(command) - - def get_apps_equivs_packages(self): - command = ( - "dpkg --get-selections" - " | grep -v deinstall" - " | awk '{print $1}'" - " | { grep 'ynh-deps$' || true; }" - ) - - output = check_output(command) - - return output.split("\n") if output else [] - - def hold(self, packages): - for package in packages: - os.system(f"apt-mark hold {package}") - - def unhold(self, packages): - for package in packages: - os.system(f"apt-mark unhold {package}") - - def apt_install(self, cmd): - def is_relevant(line): - return "Reading database ..." not in line.rstrip() - - callbacks = ( - lambda l: logger.info("+ " + l.rstrip() + "\r") - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip() + "\r"), - lambda l: logger.warning(l.rstrip()) - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip()), - ) - - cmd = ( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " - + cmd - ) - - logger.debug("Running: %s" % cmd) - - return call_async_output(cmd, callbacks, shell=True) - - def patch_yunohost_conflicts(self): - # - # This is a super dirty hack to remove the conflicts from yunohost's debian/control file - # Those conflicts are there to prevent mistakenly upgrading critical packages - # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly - # using backports etc. - # - # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status - # - - # We only patch the conflict if we're on yunohost 4.x - if self.yunohost_major_version() != N_CURRENT_YUNOHOST: - return - - conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() - if conflicts: - # We want to keep conflicting with apache/bind9 tho - new_conflicts = "Conflicts: apache2, bind9" - - command = ( - f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" - ) - logger.debug(f"Running: {command}") - os.system(command) From c4c353843c6fc147d1cbaa92e21edc3b09dda702 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 14:56:21 +0200 Subject: [PATCH 037/436] Unused vars, unhappy linter gods --- src/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/firewall.py b/src/firewall.py index d6e4b53174..ccd7e6d886 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -404,7 +404,7 @@ def firewall_upnp(action="status", no_refresh=False): logger.debug("discovering UPnP devices...") try: nb_dev = upnpc.discover() - except Exception as e: + except Exception: logger.warning("Failed to find any UPnP device on the network") nb_dev = -1 enabled = False From 194eb9c6c7d6684a2802311bd6afcece7336d7d6 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 20:14:46 +0200 Subject: [PATCH 038/436] conf: Update ciphers for nginx, postfix, dovecot --- conf/dovecot/dovecot.conf | 7 +++---- conf/nginx/security.conf.inc | 10 +++++----- conf/postfix/main.cf | 10 +++++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/conf/dovecot/dovecot.conf b/conf/dovecot/dovecot.conf index e614c37966..14d4075635 100644 --- a/conf/dovecot/dovecot.conf +++ b/conf/dovecot/dovecot.conf @@ -13,9 +13,8 @@ protocols = imap sieve {% if pop3_enabled == "True" %}pop3{% endif %} mail_plugins = $mail_plugins quota notify push_notification ############################################################################### - -# generated 2020-08-18, Mozilla Guideline v5.6, Dovecot 2.3.4, OpenSSL 1.1.1d, intermediate configuration -# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4&config=intermediate&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Dovecot 2.3.19, OpenSSL 3.0.9, intermediate configuration +# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.19&config=intermediate&openssl=3.0.9&guideline=5.7 ssl = required @@ -32,7 +31,7 @@ ssl_dh = = 1024 bits smtpd_tls_dh1024_param_file = /usr/share/yunohost/ffdhe2048.pem -tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305 {% else %} -# generated 2020-08-18, Mozilla Guideline v5.6, Postfix 3.4.14, OpenSSL 1.1.1d, modern configuration -# https://ssl-config.mozilla.org/#server=postfix&version=3.4.14&config=modern&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Postfix 3.7.5, OpenSSL 3.0.9, modern configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.7.5&config=modern&openssl=3.0.9&guideline=5.7 smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 From 81b96ad6d45c16eeae0bbcd4774bec139b63ca08 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 13 Jun 2023 21:30:20 +0200 Subject: [PATCH 039/436] tests: tmp tweaks to adapt for removed deprecated features --- src/tests/test_backuprestore.py | 5 ++++- src/tests/test_permission.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 873deec7d2..eb59d4feaf 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -439,7 +439,8 @@ def test_backup_using_copy_method(): # App restore # # - +# FIXME : switch to a backup from 11.x +@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_4p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_wordpress_from_Ynh4p2(): @@ -504,6 +505,8 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") +# FIXME : switch to a backup from 11.x +@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_4p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 8620e96119..dc91217458 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -1131,7 +1131,7 @@ def test_permission_app_propagation_on_ssowat(): def test_permission_legacy_app_propagation_on_ssowat(): app_install( os.path.join(get_test_apps_dir(), "legacy_app_ynh"), - args="domain=%s&domain_2=%s&path=%s&is_public=1" + args="domain=%s&domain_2=%s&path=%s&is_public=0" % (maindomain, other_domains[0], "/legacy"), force=True, ) @@ -1139,12 +1139,12 @@ def test_permission_legacy_app_propagation_on_ssowat(): # App is configured as public by default using the legacy unprotected_uri mechanics # It should automatically be migrated during the install res = user_permission_list(full=True)["permissions"] - assert "visitors" in res["legacy_app.main"]["allowed"] + assert "visitors" not in res["legacy_app.main"]["allowed"] assert "all_users" in res["legacy_app.main"]["allowed"] app_webroot = "https://%s/legacy" % maindomain - assert can_access_webpage(app_webroot, logged_as=None) + assert not can_access_webpage(app_webroot, logged_as=None) assert can_access_webpage(app_webroot, logged_as="alice") # Try to update the permission and check that permissions are still consistent From f6ab380730c7b687dcf8b82985c229d7b0c880ba Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 14 Jun 2023 01:34:42 +0200 Subject: [PATCH 040/436] helpers/php: Default PHP version in bookworm is now 8.2 --- helpers/php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/php b/helpers/php index 1b28b32f7a..7e8d35d6e9 100644 --- a/helpers/php +++ b/helpers/php @@ -1,6 +1,6 @@ #!/bin/bash -readonly YNH_DEFAULT_PHP_VERSION=7.4 +readonly YNH_DEFAULT_PHP_VERSION=8.2 # Declare the actual PHP version to use. # A packager willing to use another version of PHP can override the variable into its _common.sh. YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} From 8ac48ee09e7d3d77e3b636e6700c5b9a26dedc93 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 14 Jun 2023 08:04:16 +0200 Subject: [PATCH 041/436] Drop deprecated firstname/lastname in user_create/update + also drop old deprecated cert- commands --- share/actionsmap.yml | 76 +----------------------------------- src/domain.py | 3 +- src/tests/test_user-group.py | 8 +--- src/user.py | 45 ++++++--------------- 4 files changed, 17 insertions(+), 115 deletions(-) diff --git a/share/actionsmap.yml b/share/actionsmap.yml index e02f5c1d0b..444533a1d0 100644 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -70,26 +70,10 @@ user: help: The full name of the user. For example 'Camille Dupont' extra: ask: ask_fullname - required: False + required: True pattern: &pattern_fullname - !!str ^([^\W_]{1,30}[ ,.'-]{0,3})+$ - "pattern_fullname" - -f: - full: --firstname - help: Deprecated. Use --fullname instead. - extra: - required: False - pattern: &pattern_firstname - - !!str ^([^\W\d_]{1,30}[ ,.'-]{0,3})+$ - - "pattern_firstname" - -l: - full: --lastname - help: Deprecated. Use --fullname instead. - extra: - required: False - pattern: &pattern_lastname - - !!str ^([^\W\d_]{1,30}[ ,.'-]{0,3})+$ - - "pattern_lastname" -p: full: --password help: User password @@ -147,16 +131,6 @@ user: help: The full name of the user. For example 'Camille Dupont' extra: pattern: *pattern_fullname - -f: - full: --firstname - help: Deprecated. Use --fullname instead. - extra: - pattern: *pattern_firstname - -l: - full: --lastname - help: Deprecated. Use --fullname instead. - extra: - pattern: *pattern_lastname -m: full: --mail extra: @@ -551,54 +525,6 @@ domain: extra: pattern: *pattern_domain - ### certificate_status() - cert-status: - deprecated: true - action_help: List status of current certificates (all by default). - arguments: - domain_list: - help: Domains to check - nargs: "*" - --full: - help: Show more details - action: store_true - - ### certificate_install() - cert-install: - deprecated: true - action_help: Install Let's Encrypt certificates for given domains (all by default). - arguments: - domain_list: - help: Domains for which to install the certificates - nargs: "*" - --force: - help: Install even if current certificate is not self-signed - action: store_true - --no-checks: - help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to install. (Not recommended) - action: store_true - --self-signed: - help: Install self-signed certificate instead of Let's Encrypt - action: store_true - - ### certificate_renew() - cert-renew: - deprecated: true - action_help: Renew the Let's Encrypt certificates for given domains (all by default). - arguments: - domain_list: - help: Domains for which to renew the certificates - nargs: "*" - --force: - help: Ignore the validity threshold (30 days) - action: store_true - --email: - help: Send an email to root with logs if some renewing fails - action: store_true - --no-checks: - help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to renew. (Not recommended) - action: store_true - ### domain_url_available() url-available: hide_in_help: True diff --git a/src/domain.py b/src/domain.py index a2d570b4bf..8347308901 100644 --- a/src/domain.py +++ b/src/domain.py @@ -154,11 +154,12 @@ def domain_info(domain): from yunohost.app import app_info from yunohost.dns import _get_registar_settings + from yunohost.certificate import certificate_status _assert_domain_exists(domain) registrar, _ = _get_registar_settings(domain) - certificate = domain_cert_status([domain], full=True)["certificates"][domain] + certificate = certificate_status([domain], full=True)["certificates"][domain] apps = [] for app in _installed_apps(): diff --git a/src/tests/test_user-group.py b/src/tests/test_user-group.py index 57f9ffa3fe..f347fc9bcc 100644 --- a/src/tests/test_user-group.py +++ b/src/tests/test_user-group.py @@ -263,12 +263,6 @@ def test_del_group_that_does_not_exist(mocker): def test_update_user(): - with message("user_updated"): - user_update("alice", firstname="NewName", lastname="NewLast") - - info = user_info("alice") - assert info["fullname"] == "NewName NewLast" - with message("user_updated"): user_update("alice", fullname="New2Name New2Last") @@ -315,7 +309,7 @@ def test_update_group_remove_user_not_already_in(): def test_update_user_that_doesnt_exist(mocker): with raiseYunohostError(mocker, "user_unknown"): - user_update("doesnt_exist", firstname="NewName", lastname="NewLast") + user_update("doesnt_exist", fullname="Foo Bar") def test_update_group_that_doesnt_exist(mocker): diff --git a/src/user.py b/src/user.py index 00876854ec..9627a37cb2 100644 --- a/src/user.py +++ b/src/user.py @@ -141,33 +141,20 @@ def user_create( domain, password, fullname=None, - firstname=None, - lastname=None, mailbox_quota="0", admin=False, from_import=False, loginShell=None, ): - if firstname or lastname: - logger.warning( - "Options --firstname / --lastname of 'yunohost user create' are deprecated. We recommend using --fullname instead." - ) - if not fullname or not fullname.strip(): - if not firstname.strip(): - raise YunohostValidationError( - "You should specify the fullname of the user using option -F" - ) - lastname = ( - lastname or " " - ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... - fullname = f"{firstname} {lastname}".strip() - else: - fullname = fullname.strip() - firstname = fullname.split()[0] - lastname = ( - " ".join(fullname.split()[1:]) or " " - ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + raise YunohostValidationError( + "You should specify the fullname of the user using option -F" + ) + fullname = fullname.strip() + firstname = fullname.split()[0] + lastname = ( + " ".join(fullname.split()[1:]) or " " + ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... from yunohost.domain import domain_list, _get_maindomain, _assert_domain_exists from yunohost.hook import hook_callback @@ -364,8 +351,6 @@ def user_delete(operation_logger, username, purge=False, from_import=False): def user_update( operation_logger, username, - firstname=None, - lastname=None, mail=None, change_password=None, add_mailforward=None, @@ -377,17 +362,15 @@ def user_update( fullname=None, loginShell=None, ): - if firstname or lastname: - logger.warning( - "Options --firstname / --lastname of 'yunohost user create' are deprecated. We recommend using --fullname instead." - ) - if fullname and fullname.strip(): fullname = fullname.strip() firstname = fullname.split()[0] lastname = ( " ".join(fullname.split()[1:]) or " " ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + else: + firstname = None + lastname = None from yunohost.domain import domain_list from yunohost.app import app_ssowatconf @@ -884,8 +867,7 @@ def update(new_infos, old_infos=False): user_update( new_infos["username"], - firstname=new_infos["firstname"], - lastname=new_infos["lastname"], + fullname=(new_infos["firstname"] + " " + new_infos["lastname"]).strip(), change_password=new_infos["password"], mailbox_quota=new_infos["mailbox-quota"], mail=new_infos["mail"], @@ -930,8 +912,7 @@ def update(new_infos, old_infos=False): user["password"], user["mailbox-quota"], from_import=True, - firstname=user["firstname"], - lastname=user["lastname"], + fullname=(user["firstname"] + " " + user["lastname"]).strip(), ) update(user) result["created"] += 1 From a673b3ed420723054c0aef6761cdf8c03fa77b6e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 14 Jun 2023 10:28:58 +0200 Subject: [PATCH 042/436] Postgresql is now version 15 --- helpers/postgresql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/postgresql b/helpers/postgresql index 796a362145..8ff63df436 100644 --- a/helpers/postgresql +++ b/helpers/postgresql @@ -1,7 +1,7 @@ #!/bin/bash PSQL_ROOT_PWD_FILE=/etc/yunohost/psql -PSQL_VERSION=13 +PSQL_VERSION=15 # Open a connection as a user # From ced6d5c975caa90fa51ff5506dc5a82ad992f0bf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 18 Jun 2023 15:45:44 +0200 Subject: [PATCH 043/436] apps: fix version.parse now refusing to parse legacy version numbers --- src/app.py | 65 ++++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/src/app.py b/src/app.py index 55d351a84b..70c0657dff 100644 --- a/src/app.py +++ b/src/app.py @@ -241,8 +241,8 @@ def _app_upgradable(app_infos): # Determine upgradability app_in_catalog = app_infos.get("from_catalog") - installed_version = version.parse(app_infos.get("version", "0~ynh0")) - version_in_catalog = version.parse( + installed_version = _parse_app_version(app_infos.get("version", "0~ynh0")) + version_in_catalog = _parse_app_version( app_infos.get("from_catalog", {}).get("manifest", {}).get("version", "0~ynh0") ) @@ -257,25 +257,7 @@ def _app_upgradable(app_infos): ): return "bad_quality" - # If the app uses the standard version scheme, use it to determine - # upgradability - if "~ynh" in str(installed_version) and "~ynh" in str(version_in_catalog): - if installed_version < version_in_catalog: - return "yes" - else: - return "no" - - # Legacy stuff for app with old / non-standard version numbers... - - # In case there is neither update_time nor install_time, we assume the app can/has to be upgraded - if not app_infos["from_catalog"].get("lastUpdate") or not app_infos[ - "from_catalog" - ].get("git"): - return "url_required" - - settings = app_infos["settings"] - local_update_time = settings.get("update_time", settings.get("install_time", 0)) - if app_infos["from_catalog"]["lastUpdate"] > local_update_time: + if installed_version < version_in_catalog: return "yes" else: return "no" @@ -620,9 +602,11 @@ def app_upgrade( # Manage upgrade type and avoid any upgrade if there is nothing to do upgrade_type = "UNKNOWN" # Get current_version and new version - app_new_version = version.parse(manifest.get("version", "?")) - app_current_version = version.parse(app_dict.get("version", "?")) - if "~ynh" in str(app_current_version) and "~ynh" in str(app_new_version): + app_new_version_raw = manifest.get("version", "?") + app_current_version_raw = app_dict.get("version", "?") + app_new_version = _parse_app_version(app_new_version_raw) + app_current_version = _parse_app_version(app_current_version_raw) + if "~ynh" in str(app_current_version_raw) and "~ynh" in str(app_new_version_raw): if app_current_version >= app_new_version and not force: # In case of upgrade from file or custom repository # No new version available @@ -642,10 +626,10 @@ def app_upgrade( upgrade_type = "UPGRADE_FORCED" else: app_current_version_upstream, app_current_version_pkg = str( - app_current_version + app_current_version_raw ).split("~ynh") app_new_version_upstream, app_new_version_pkg = str( - app_new_version + app_new_version_raw ).split("~ynh") if app_current_version_upstream == app_new_version_upstream: upgrade_type = "UPGRADE_PACKAGE" @@ -675,7 +659,7 @@ def app_upgrade( settings = _get_app_settings(app_instance_name) notifications = _filter_and_hydrate_notifications( manifest["notifications"]["PRE_UPGRADE"], - current_version=app_current_version, + current_version=app_current_version_raw, data=settings, ) _display_notifications(notifications, force=force) @@ -732,8 +716,8 @@ def app_upgrade( env_dict_more = { "YNH_APP_UPGRADE_TYPE": upgrade_type, - "YNH_APP_MANIFEST_VERSION": str(app_new_version), - "YNH_APP_CURRENT_VERSION": str(app_current_version), + "YNH_APP_MANIFEST_VERSION": str(app_new_version_raw), + "YNH_APP_CURRENT_VERSION": str(app_current_version_raw), } if manifest["packaging_format"] < 2: @@ -916,7 +900,7 @@ def app_upgrade( settings = _get_app_settings(app_instance_name) notifications = _filter_and_hydrate_notifications( manifest["notifications"]["POST_UPGRADE"], - current_version=app_current_version, + current_version=app_current_version_raw, data=settings, ) if Moulinette.interface.type == "cli": @@ -1924,6 +1908,20 @@ def _set_app_settings(app, settings): yaml.safe_dump(settings, f, default_flow_style=False) +def _parse_app_version(v): + + if v == "?": + return (0,0) + + try: + if "~" in v: + return (version.parse(v.split("~")[0]), int(v.split("~")[1].replace("ynh", ""))) + else: + return (version.parse(v), 0) + except Exception as e: + raise YunohostError(f"Failed to parse app version '{v}' : {e}", raw_msg=True) + + def _get_manifest_of_app(path): "Get app manifest stored in json or in toml" @@ -3020,12 +3018,7 @@ def _notification_is_dismissed(name, settings): def _filter_and_hydrate_notifications(notifications, current_version=None, data={}): def is_version_more_recent_than_current_version(name, current_version): current_version = str(current_version) - # Boring code to handle the fact that "0.1 < 9999~ynh1" is False - - if "~" in name: - return version.parse(name) > version.parse(current_version) - else: - return version.parse(name) > version.parse(current_version.split("~")[0]) + return _parse_app_version(name) > _parse_app_version(current_version) return { # Should we render the markdown maybe? idk From 8a865dadddbbe82a565a55524261f21c8510fa1b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 19 Jun 2023 16:04:31 +0200 Subject: [PATCH 044/436] apps: add YNH_DEFAULT_PHP_VERSION in app's dict as a boring workaround/fix for apps using YNH_DEFAULT_PHP_VERSION in _common.sh *before* sourcing helpers ... --- src/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app.py b/src/app.py index 70c0657dff..4d11b47fb3 100644 --- a/src/app.py +++ b/src/app.py @@ -2792,6 +2792,7 @@ def _make_environment_for_app_script( app_id, app_instance_nb = _parse_app_instance_name(app) env_dict = { + "YNH_DEFAULT_PHP_VERSION": "8.2", "YNH_APP_ID": app_id, "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), From cab7667dcca4b44663c7ec6d5b939a00ba3bda4e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 4 Jul 2023 19:48:55 +0200 Subject: [PATCH 045/436] misc: more boring irrelevant postgresql warnings to filter out --- src/hook.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hook.py b/src/hook.py index 4b07d1c17b..6c9c84a008 100644 --- a/src/hook.py +++ b/src/hook.py @@ -359,6 +359,7 @@ def is_relevant_warning(msg): r"Removing obsolete dictionary files", r"Creating new PostgreSQL cluster", r"/usr/lib/postgresql/13/bin/initdb", + r"/usr/lib/postgresql/15/bin/initdb", r"The files belonging to this database system will be owned by user", r"This user must also own the server process.", r"The database cluster will be initialized with locale", @@ -366,6 +367,7 @@ def is_relevant_warning(msg): r"The default text search configuration will be set to", r"Data page checksums are disabled.", r"fixing permissions on existing directory /var/lib/postgresql/13/main ... ok", + r"fixing permissions on existing directory /var/lib/postgresql/15/main ... ok", r"creating subdirectories \.\.\. ok", r"selecting dynamic .* \.\.\. ", r"selecting default .* \.\.\. ", From 2f2ff6eb190cd9a825e88d91b3101f2e3fba6c96 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 15:58:59 +0200 Subject: [PATCH 046/436] Simplify fpm add config helper (Bookworm) (#1685) * Simplify ynh_add_fpm_config helper * helpers: drop dedicated_service option in ynh_add_fpm_config --- helpers/php | 97 ++++++----------------------------------------------- 1 file changed, 11 insertions(+), 86 deletions(-) diff --git a/helpers/php b/helpers/php index 5cfe521f1d..21fb20e277 100644 --- a/helpers/php +++ b/helpers/php @@ -70,12 +70,11 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vufd - local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [d]=dedicated_service) + local legacy_args=vuf + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint=) local phpversion local usage local footprint - local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" @@ -103,8 +102,6 @@ ynh_add_fpm_config() { fi fi - # Do not use a dedicated service by default - dedicated_service=${dedicated_service:-0} # Set the default PHP-FPM version by default if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then @@ -128,38 +125,16 @@ ynh_add_fpm_config() { fi fi - if [ $dedicated_service -eq 1 ]; then - ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future" - local fpm_service="${app}-phpfpm" - local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" - else - local fpm_service="php${phpversion}-fpm" - local fpm_config_dir="/etc/php/$phpversion/fpm" - fi + local fpm_service="php${phpversion}-fpm" + local fpm_config_dir="/etc/php/$phpversion/fpm" # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion - # Migrate from mutual PHP service to dedicated one. - if [ $dedicated_service -eq 1 ]; then - local old_fpm_config_dir="/etc/php/$phpversion/fpm" - # If a config file exist in the common pool, move it. - if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]; then - ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." - # Create a backup of the old file before migration - ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" - # Remove the old PHP config file - ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" - # Reload PHP to release the socket and allow the dedicated service to use it - ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload - fi - fi - if [ $autogenconf == "false" ]; then # Usage 1, use the template in conf/php-fpm.conf local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" @@ -212,51 +187,13 @@ pm.process_idle_timeout = 10s local finalphpconf="$fpm_config_dir/pool.d/$app.conf" ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" - if [ $dedicated_service -eq 1 ]; then - # Create a dedicated php-fpm.conf for the service - local globalphpconf=$fpm_config_dir/php-fpm-$app.conf - - echo "[global] -pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid -error_log = /var/log/php/fpm-php.__APP__.log -syslog.ident = php-fpm-__APP__ -include = __FINALPHPCONF__ -" >$YNH_APP_BASEDIR/conf/php-fpm-$app.conf - - ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf" - - # Create a config for a dedicated PHP-FPM service for the app - echo "[Unit] -Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__ -After=network.target - -[Service] -Type=notify -PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid -ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__ -ExecReload=/bin/kill -USR2 \$MAINPID - -[Install] -WantedBy=multi-user.target -" >$YNH_APP_BASEDIR/conf/$fpm_service - - # Create this dedicated PHP-FPM service - ynh_add_systemd_config --service=$fpm_service --template=$fpm_service - # Integrate the service in YunoHost admin panel - yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app" - # Configure log rotate - ynh_use_logrotate --logfile=/var/log/php - # Restart the service, as this service is either stopped or only for this app - ynh_systemd_action --service_name=$fpm_service --action=restart - else - # Validate that the new php conf doesn't break php-fpm entirely - if ! php-fpm${phpversion} --test 2>/dev/null; then - php-fpm${phpversion} --test || true - ynh_secure_remove --file="$finalphpconf" - ynh_die --message="The new configuration broke php-fpm?" - fi - ynh_systemd_action --service_name=$fpm_service --action=reload + # Validate that the new php conf doesn't break php-fpm entirely + if ! php-fpm${phpversion} --test 2>/dev/null; then + php-fpm${phpversion} --test || true + ynh_secure_remove --file="$finalphpconf" + ynh_die --message="The new configuration broke php-fpm?" fi + ynh_systemd_action --service_name=$fpm_service --action=reload } # Remove the dedicated PHP-FPM config @@ -267,8 +204,6 @@ WantedBy=multi-user.target ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) - local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) - dedicated_service=${dedicated_service:-0} # Get the version of PHP used by this app local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) @@ -282,17 +217,7 @@ ynh_remove_fpm_config() { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - - if [ $dedicated_service -eq 1 ]; then - # Remove the dedicated service PHP-FPM service for the app - ynh_remove_systemd_config --service=$fpm_service - # Remove the global PHP-FPM conf - ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" - # Remove the service from the list of services known by YunoHost - yunohost service remove $fpm_service - elif ynh_package_is_installed --package="php${phpversion}-fpm"; then - ynh_systemd_action --service_name=$fpm_service --action=reload - fi + ynh_systemd_action --service_name=$fpm_service --action=reload } # Define the values to configure PHP-FPM From 9a5080ea16b36e465587d4f0c9b2d531e9dfc6ce Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 17:49:25 +0200 Subject: [PATCH 047/436] portalapi: fix split or user/password in auth code --- src/authenticators/ldap_ynhuser.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index fe2a657f55..c8ba4ecf2e 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -27,11 +27,10 @@ class Authenticator(BaseAuthenticator): def _authenticate_credentials(self, credentials=None): - # FIXME ':' should a legit char in the password ? shall we encode the password as base64 or something idk - if ":" not in credentials or len(credentials.split(":")) != 2: - raise YunohostError("invalid_credentials_format") - - username, password = credentials.split(":") + try: + username, password = credentials.split(":", 1) + except ValueError: + raise YunohostError("invalid_credentials") def _reconnect(): con = ldap.ldapobject.ReconnectLDAPObject( From f1200b81dc605a5320bfd5124b43c0ee02946d14 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 18:10:32 +0200 Subject: [PATCH 048/436] apt: always add yarn repo because it's annoying to have to deal with an extra repo in each nodejs app just to install a single package.. --- hooks/conf_regen/10-apt | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/hooks/conf_regen/10-apt b/hooks/conf_regen/10-apt index 93ff053b8d..72c0773b94 100755 --- a/hooks/conf_regen/10-apt +++ b/hooks/conf_regen/10-apt @@ -23,19 +23,33 @@ Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" for package in $packages_to_refuse_from_sury; do echo " Package: $package -Pin: origin \"packages.sury.org\" +Pin: origin \"packages.sury.org\" Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" done + # Add yarn + echo "deb https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" + + # Ban everything from Yarn except Yarn + echo " +Package: * +Pin: origin \"dl.yarnpkg.com\" +Pin-Priority: -1 + +Package: yarn +Pin: origin \"dl.yarnpkg.com\" +Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/yarn" + + # Ban apache2, bind9 echo " # PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE -# You are probably reading this file because you tried to install apache2 or +# You are probably reading this file because you tried to install apache2 or # bind9. These 2 packages conflict with YunoHost. # Installing apache2 will break nginx and break the entire YunoHost ecosystem -# on your server, therefore don't remove those lines! +# on your server, therefore don't remove those lines! # You have been warned. @@ -69,6 +83,12 @@ do_post_regen() { wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg" fi + # Similar to Sury + if [[ ! -s /etc/apt/trusted.gpg.d/yarn.gpg ]] + then + wget --timeout 900 --quiet "https://dl.yarnpkg.com/debian/pubkey.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/yarn.gpg" + fi + # Make sure php7.4 is the default version when using php in cli if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION then From 236e85eece3f4d6fa479c9d57de41710fc4c736c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 18:12:08 +0200 Subject: [PATCH 049/436] apt: add signed-by clause to sury and yarn repo --- hooks/conf_regen/10-apt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/conf_regen/10-apt b/hooks/conf_regen/10-apt index 72c0773b94..725fdde82e 100755 --- a/hooks/conf_regen/10-apt +++ b/hooks/conf_regen/10-apt @@ -11,7 +11,7 @@ do_pre_regen() { # Add sury mkdir -p ${pending_dir}/etc/apt/sources.list.d/ - echo "deb https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list" + echo "deb [signed-by=/etc/apt/trusted.gpg.d/extra_php_version.gpg] https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list" # Ban some packages from sury echo " @@ -28,7 +28,7 @@ Pin-Priority: -1" >>"${pending_dir}/etc/apt/preferences.d/extra_php_version" done # Add yarn - echo "deb https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" + echo "deb [signed-by=/etc/apt/trusted.gpg.d/yarn.gpg] https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" # Ban everything from Yarn except Yarn echo " From 6c6dd318fb8c5a31fe64fbeb78ed1d7304dd8a8c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 11 Jul 2023 22:39:22 +0200 Subject: [PATCH 050/436] portalapi: implement encrypted password storage in the user's cookie using AES256 --- debian/control | 2 +- hooks/conf_regen/01-yunohost | 3 +- src/authenticators/ldap_ynhuser.py | 73 ++++++++++++++++++++++-------- src/portal.py | 13 ++---- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/debian/control b/debian/control index 8880867a2f..df9a6d2bdd 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon, - , python3-jwt + , python3-cryptography, python3-jwt , python-is-python3 , nginx, nginx-extras (>=1.18) , apt, apt-transport-https, apt-utils, dirmngr diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 7bd835f8fc..4d53997a58 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -177,7 +177,8 @@ do_post_regen() { getent passwd ynh-portal &>/dev/null || useradd --no-create-home --shell /usr/sbin/nologin --system --user-group ynh-portal if [ ! -e /etc/yunohost/.ssowat_cookie_secret ]; then - dd if=/dev/urandom bs=1 count=1000 2>/dev/null | tr --complement --delete 'A-Za-z0-9' | head -c 64 > /etc/yunohost/.ssowat_cookie_secret + # NB: we need this to be exactly 32 char long, because it is later used as a key for AES256 + dd if=/dev/urandom bs=1 count=1000 2>/dev/null | tr --complement --delete 'A-Za-z0-9' | head -c 32 > /etc/yunohost/.ssowat_cookie_secret fi chown ynh-portal:root /etc/yunohost/.ssowat_cookie_secret chmod 400 /etc/yunohost/.ssowat_cookie_secret diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index c8ba4ecf2e..407277acf4 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -5,6 +5,13 @@ import ldap import ldap.sasl import datetime +import base64 +import os + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.backends import default_backend + from moulinette import m18n from moulinette.authentication import BaseAuthenticator @@ -13,13 +20,52 @@ # FIXME : we shall generate this somewhere if it doesnt exists yet # FIXME : fix permissions -session_secret = open("/etc/yunohost/.ssowat_cookie_secret").read() +session_secret = open("/etc/yunohost/.ssowat_cookie_secret").read().strip() logger = logging.getLogger("yunohostportal.authenticators.ldap_ynhuser") URI = "ldap://localhost:389" USERDN = "uid={username},ou=users,dc=yunohost,dc=org" +# We want to save the password in the cookie, but we should do so in an encrypted fashion +# This is needed because the SSO later needs to possibly inject the Basic Auth header +# which includes the user's password +# It's also needed because we need to be able to open LDAP sessions, authenticated as the user, +# which requires the user's password +# +# To do so, we use AES-256-CBC. As it's a block encryption algorithm, it requires an IV, +# which we need to keep around for decryption on SSOwat'side. +# +# session_secret is used as the encryption key, which implies it must be exactly 32-char long (256/8) +# +# The result is a string formatted as | +# For example: ctl8kk5GevYdaA5VZ2S88Q==|yTAzCx0Gd1+MCit4EQl9lA== +def encrypt(data): + + alg = algorithms.AES(session_secret.encode()) + iv = os.urandom(int(alg.block_size / 8)) + + E = Cipher(alg, modes.CBC(iv), default_backend()).encryptor() + p = padding.PKCS7(alg.block_size).padder() + data_padded = p.update(data.encode()) + p.finalize() + data_enc = E.update(data_padded) + E.finalize() + data_enc_b64 = base64.b64encode(data_enc).decode() + iv_b64 = base64.b64encode(iv).decode() + return data_enc_b64 + "|" + iv_b64 + +def decrypt(data_enc_and_iv_b64): + + data_enc_b64, iv_b64 = data_enc_and_iv_b64.split("|") + data_enc = base64.b64decode(data_enc_b64) + iv = base64.b64decode(iv_b64) + + alg = algorithms.AES(session_secret.encode()) + D = Cipher(alg, modes.CBC(iv), default_backend()).decryptor() + p = padding.PKCS7(alg.block_size).unpadder() + data_padded = D.update(data_enc) + data = p.update(data_padded) + p.finalize() + return data.decode() + class Authenticator(BaseAuthenticator): @@ -64,23 +110,7 @@ def _reconnect(): if con: con.unbind_s() - - - - - - - # FIXME FIXME FIXME : the password is to be encrypted to not expose it in the JWT cookie which is only signed and base64 encoded but not encrypted - - - - - - - - - - return {"user": username, "password": password} + return {"user": username, "pwd": encrypt(password)} def set_session_cookie(self, infos): @@ -101,7 +131,7 @@ def set_session_cookie(self, infos): response.set_cookie( "yunohost.portal", - jwt.encode(new_infos, session_secret, algorithm="HS256").decode(), + jwt.encode(new_infos, session_secret, algorithm="HS256"), secure=True, httponly=True, path="/", @@ -109,7 +139,7 @@ def set_session_cookie(self, infos): # FIXME : add Expire clause ) - def get_session_cookie(self, raise_if_no_session_exists=True): + def get_session_cookie(self, raise_if_no_session_exists=True, decrypt_pwd=False): from bottle import request @@ -127,6 +157,9 @@ def get_session_cookie(self, raise_if_no_session_exists=True): if "id" not in infos: infos["id"] = random_ascii() + if decrypt_pwd: + infos["pwd"] = decrypt(infos["pwd"]) + # FIXME: Here, maybe we want to re-authenticate the session via the authenticator # For example to check that the username authenticated is still in the admin group... diff --git a/src/portal.py b/src/portal.py index 2eaa59dd45..6aa7cba6d0 100644 --- a/src/portal.py +++ b/src/portal.py @@ -32,24 +32,17 @@ def portal_me(): """ Get user informations - - Keyword argument: - username -- Username to get informations - """ - import pdb; pdb.set_trace() - - auth = Auth().get_session_cookie() + auth = Auth().get_session_cookie(decrypt_pwd=True) username = auth["user"] - password = auth["password"] - ldap = LDAPInterface(username, password) + ldap = LDAPInterface(username, auth["pwd"]) user_attrs = ["cn", "mail", "uid", "maildrop", "givenName", "sn", "mailuserquota"] filter = "uid=" + username - result = ldap.search("ou=users,dc=yunohost,dc=org", filter, user_attrs) + result = ldap.search("ou=users", filter, user_attrs) if result: user = result[0] From 2c0f49cef3fbf044e89ea5b43933c06dbbf8d956 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jul 2023 04:44:03 +0200 Subject: [PATCH 051/436] portalapi: add groups and apps list in infos returned by GET /me --- src/portal.py | 61 +++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/portal.py b/src/portal.py index 6aa7cba6d0..6a51f33d4e 100644 --- a/src/portal.py +++ b/src/portal.py @@ -19,14 +19,14 @@ """ -# from moulinette import Moulinette, m18n from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_json from yunohost.authenticators.ldap_ynhuser import Authenticator as Auth from yunohost.utils.ldap import LDAPInterface from yunohost.utils.error import YunohostValidationError -logger = getActionLogger("yunohostportal.user") +logger = getActionLogger("portal") def portal_me(): @@ -39,48 +39,43 @@ def portal_me(): ldap = LDAPInterface(username, auth["pwd"]) - user_attrs = ["cn", "mail", "uid", "maildrop", "givenName", "sn", "mailuserquota"] + user_attrs = ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] - filter = "uid=" + username - result = ldap.search("ou=users", filter, user_attrs) + result = ldap.search("ou=users", f"uid={username}", user_attrs) if result: user = result[0] else: raise YunohostValidationError("user_unknown", user=username) + groups = [g.replace("cn=", "").replace(",ou=groups,dc=yunohost,dc=org", "") for g in user["memberOf"]] + groups = [g for g in groups if g not in [username, "all_users"]] + + permissions = [p.replace("cn=", "").replace(",ou=permission,dc=yunohost,dc=org", "") for p in user["permission"]] + + ssowat_conf = read_json("/etc/ssowat/conf.json") + apps = { + perm.replace(".main", ""): {"label": infos["label"], "url": infos["uris"][0]} + for perm, infos in ssowat_conf["permissions"].items() + if perm in permissions and infos["show_tile"] and username in infos["users"] + } + result_dict = { - "username": user["uid"][0], + "username": username, "fullname": user["cn"][0], - "firstname": user["givenName"][0], - "lastname": user["sn"][0], "mail": user["mail"][0], - "mail-aliases": [], - "mail-forward": [], + "mail-aliases": user["mail"][1:], + "mail-forward": user["maildrop"][1:], + "groups": groups, + "apps": apps } - if len(user["mail"]) > 1: - result_dict["mail-aliases"] = user["mail"][1:] - - if len(user["maildrop"]) > 1: - result_dict["mail-forward"] = user["maildrop"][1:] - - if "mailuserquota" in user: - pass - # FIXME - # result_dict["mailbox-quota"] = { - # "limit": userquota if is_limited else m18n.n("unlimit"), - # "use": storage_use, - # } - - # FIXME : should also parse "permission" key in ldap maybe ? - # and list of groups / memberof ? - # (in particular to have e.g. the mail / xmpp / ssh / ... perms) + # FIXME / TODO : add mail quota status ? + # result_dict["mailbox-quota"] = { + # "limit": userquota if is_limited else m18n.n("unlimit"), + # "use": storage_use, + # } + # Could use : doveadm -c /dev/null -f flow quota recalc -u johndoe + # But this requires to be in the mail group ... return result_dict - - -def apps(username): - return {"foo": "bar"} - # FIXME: should list available apps and corresponding infos ? - # from /etc/ssowat/conf.json ? From 5e1d69a2cb09c8a125f6a6cce76a54c42eed9338 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jul 2023 18:55:33 +0200 Subject: [PATCH 052/436] portalapi: harden systemd service configuration --- conf/yunohost/yunohost-portal-api.service | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/conf/yunohost/yunohost-portal-api.service b/conf/yunohost/yunohost-portal-api.service index 0ba6e8b3dd..006af00809 100644 --- a/conf/yunohost/yunohost-portal-api.service +++ b/conf/yunohost/yunohost-portal-api.service @@ -4,11 +4,45 @@ After=network.target [Service] User=ynh-portal +Group=ynh-portal Type=simple ExecStart=/usr/bin/yunohost-portal-api Restart=always RestartSec=5 TimeoutStopSec=30 +# Sandboxing options to harden security +# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html +NoNewPrivileges=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK +RestrictNamespaces=yes +RestrictRealtime=yes +DevicePolicy=closed +ProtectClock=yes +ProtectHostname=yes +ProtectProc=invisible +ProtectSystem=full +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +LockPersonality=yes +SystemCallArchitectures=native +SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap @cpu-emulation @privileged + +# Denying access to capabilities that should not be relevant +# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html +CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD +CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE +CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT +CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK +CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM +CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG +CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE +CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW +CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG + + [Install] WantedBy=multi-user.target From 5104c2a79f698bd359adc4d7c32ab9232c9c05b0 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jul 2023 19:11:32 +0200 Subject: [PATCH 053/436] portalapi: add CORS headers ... though gotta revisit this later, I don't know what I'm doing --- conf/nginx/yunohost_api.conf.inc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/conf/nginx/yunohost_api.conf.inc b/conf/nginx/yunohost_api.conf.inc index b4567e0b80..a3a23f1a9c 100644 --- a/conf/nginx/yunohost_api.conf.inc +++ b/conf/nginx/yunohost_api.conf.inc @@ -25,12 +25,25 @@ location = /yunohost/api/error/502 { } location /yunohost/portalapi/ { - proxy_read_timeout 3600s; + + # FIXME FIXME FIXME : we should think about what we really want here ... + more_set_headers "Access-Control-Allow-Origin: $http_origin"; + more_set_headers "Access-Control-Allow-Methods: GET, HEAD, POST, OPTIONS, DELETE"; + more_set_headers "Access-Control-Allow-Headers: Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With"; + more_set_headers "Access-Control-Allow-Credentials: true"; + + if ($request_method = 'OPTIONS') { + more_set_headers "Content-Type: text/plain; charset=utf-8"; + more_set_headers "Content-Length: 0"; + return 204; + } + + proxy_read_timeout 5s; proxy_pass http://127.0.0.1:6788/; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_set_header Host $http_host; + proxy_set_header Host $http; # Custom 502 error page error_page 502 /yunohost/portalapi/error/502; From 0cb673c12504d386ec7341ff199440a216f8ffd4 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 14 Jul 2023 19:35:05 +0200 Subject: [PATCH 054/436] portalapi: woopsies --- conf/nginx/yunohost_api.conf.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/nginx/yunohost_api.conf.inc b/conf/nginx/yunohost_api.conf.inc index a3a23f1a9c..8133624b38 100644 --- a/conf/nginx/yunohost_api.conf.inc +++ b/conf/nginx/yunohost_api.conf.inc @@ -43,7 +43,7 @@ location /yunohost/portalapi/ { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_set_header Host $http; + proxy_set_header Host $host; # Custom 502 error page error_page 502 /yunohost/portalapi/error/502; From f4dfb560068b1f952de86653427ded5cbc1206bb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 15 Jul 2023 16:01:03 +0200 Subject: [PATCH 055/436] portal refactoring: the 'yunohost tile' thingy won't work anymore, gotta discuss what we want to do exactly --- conf/nginx/plain/yunohost_panel.conf.inc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/conf/nginx/plain/yunohost_panel.conf.inc b/conf/nginx/plain/yunohost_panel.conf.inc index 16a6e6b29e..69ca48a626 100644 --- a/conf/nginx/plain/yunohost_panel.conf.inc +++ b/conf/nginx/plain/yunohost_panel.conf.inc @@ -1,8 +1,12 @@ -# Insert YunoHost button + portal overlay -sub_filter ''; -sub_filter_once on; -# Apply to other mime types than text/html -sub_filter_types application/xhtml+xml; -# Prevent YunoHost panel files from being blocked by specific app rules -location ~ (ynh_portal.js|ynh_overlay.css|ynh_userinfo.json|ynhtheme/custom_portal.js|ynhtheme/custom_overlay.css) { -} +# This is some old code that worked with the old portal +# We need to rethink wether we want to keep something similar, +# or drop the feature + +# # Insert YunoHost button + portal overlay +# sub_filter ''; +# sub_filter_once on; +# # Apply to other mime types than text/html +# sub_filter_types application/xhtml+xml; +# # Prevent YunoHost panel files from being blocked by specific app rules +# location ~ (ynh_portal.js|ynh_overlay.css|ynh_userinfo.json|ynhtheme/custom_portal.js|ynhtheme/custom_overlay.css) { +# } From ec96558c8126a60bf45e14583cff0ce428ae578d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 15 Jul 2023 20:07:18 +0200 Subject: [PATCH 056/436] portalapi: add FIXMEs about auth layer --- src/authenticators/ldap_ynhuser.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 407277acf4..e8cfaf108c 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -88,6 +88,7 @@ def _reconnect(): try: con = _reconnect() except ldap.INVALID_CREDENTIALS: + # FIXME FIXME FIXME : this should be properly logged and caught by Fail2ban ! ! ! ! ! ! ! raise YunohostError("invalid_password") except ldap.SERVER_DOWN: logger.warning(m18n.n("ldap_server_down")) @@ -125,7 +126,7 @@ def set_session_cookie(self, infos): # See https://pyjwt.readthedocs.io/en/latest/usage.html#registered-claim-names # for explanations regarding nbf, exp "nbf": int(datetime.datetime.now().timestamp()), - "exp": int(datetime.datetime.now().timestamp()) + (7 * 24 * 3600) # One week validity + "exp": int(datetime.datetime.now().timestamp()) + (7 * 24 * 3600) # One week validity # FIXME : does it mean the session suddenly expires after a week ? Can we somehow auto-renew it at every usage or something ? } new_infos.update(infos) @@ -149,6 +150,7 @@ def get_session_cookie(self, raise_if_no_session_exists=True, decrypt_pwd=False) except Exception: if not raise_if_no_session_exists: return {"id": random_ascii()} + # FIXME FIXME FIXME : we might also want this to be caught by fail2ban ? Idk ... raise YunohostAuthenticationError("unable_authenticate") if not infos and raise_if_no_session_exists: @@ -160,8 +162,9 @@ def get_session_cookie(self, raise_if_no_session_exists=True, decrypt_pwd=False) if decrypt_pwd: infos["pwd"] = decrypt(infos["pwd"]) - # FIXME: Here, maybe we want to re-authenticate the session via the authenticator - # For example to check that the username authenticated is still in the admin group... + # FIXME : maybe check expiration here ? Or is it already done in jwt.decode ? + + # FIXME: also a valid cookie ain't everything ... i.e. maybe we should validate that the user still exists return infos From 4561f900df20897cffef8633e89e492f45551465 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 15 Jul 2023 21:20:15 +0200 Subject: [PATCH 057/436] portal refactoring: update ssowat conf format with a dict mapping domains to portal urls. For now, let's have one portal per main/parent domain (which is anyway imposed by cookie management unless we reintroduce complex cross-domain authentication...) --- src/app.py | 14 ++++---------- src/domain.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/app.py b/src/app.py index 069134798f..31108fde50 100644 --- a/src/app.py +++ b/src/app.py @@ -1712,7 +1712,7 @@ def app_ssowatconf(): """ - from yunohost.domain import domain_list, _get_maindomain, domain_config_get + from yunohost.domain import domain_list, _get_maindomain, domain_config_get, _get_domain_portal_dict from yunohost.permission import user_permission_list from yunohost.settings import settings_get @@ -1740,6 +1740,8 @@ def app_ssowatconf(): ], } } + + # FIXME : what's the reason we do this only for the maindomain ? x_X redirected_regex = { main_domain + r"/yunohost[\/]?$": "https://" + main_domain + "/yunohost/sso/" } @@ -1808,17 +1810,9 @@ def app_ssowatconf(): "cookie_secret_file": "/etc/yunohost/.ssowat_cookie_secret", "cookie_name": "yunohost.portal", "theme": settings_get("misc.portal.portal_theme"), - "portal_domain": main_domain, - "portal_path": "/yunohost/sso/", - "additional_headers": { - "Auth-User": "uid", - "Remote-User": "uid", - "Name": "cn", - "Email": "mail", - }, - "domains": domains, "redirected_urls": redirected_urls, "redirected_regex": redirected_regex, + "domain_portal_urls": _get_domain_portal_dict(), "permissions": permissions, } diff --git a/src/domain.py b/src/domain.py index 4f96d08c4d..ecb1cc5ea3 100644 --- a/src/domain.py +++ b/src/domain.py @@ -99,6 +99,26 @@ def cmp_domain(domain): return domain_list_cache +def _get_domain_portal_dict(): + + domains = _get_domains() + out = OrderedDict() + + for domain in domains: + + parent = None + + # Use the topest parent domain if any + for d in out.keys(): + if domain.endswith(f".{d}"): + parent = d + break + + out[domain] = f'{parent or domain}/yunohost/sso' + + return dict(out) + + def domain_list(exclude_subdomains=False, tree=False, features=[]): """ List domains From ae37b5fc248c600e9be6f1c404c3383a8a0e258d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 17 Jul 2023 19:47:24 +0200 Subject: [PATCH 058/436] portalapi: Add new yunohost-portal-api to yunohost services --- conf/yunohost/services.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/yunohost/services.yml b/conf/yunohost/services.yml index 45621876ee..693793465d 100644 --- a/conf/yunohost/services.yml +++ b/conf/yunohost/services.yml @@ -51,6 +51,9 @@ ssh: test_conf: sshd -t needs_exposed_ports: [22] category: admin +yunohost-portal-api: + log: /var/log/yunohost-portal-api.log + category: userportal yunohost-api: log: /var/log/yunohost/yunohost-api.log category: admin From 2ece3b65f6c6e51cdbb240bdb24cb74d8c21802b Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 18 Jul 2023 00:19:16 +0200 Subject: [PATCH 059/436] Moulinette logging is an unecessarily complex mess, episode 57682 --- src/__init__.py | 13 +++---------- src/app.py | 4 ++-- src/app_catalog.py | 4 ++-- src/backup.py | 4 ++-- src/certificate.py | 5 ++--- src/diagnosis.py | 3 ++- src/dns.py | 5 ++--- src/domain.py | 4 ++-- src/dyndns.py | 4 ++-- src/firewall.py | 4 ++-- src/hook.py | 4 ++-- src/log.py | 3 +-- src/migrations/0021_migrate_to_bullseye.py | 4 ++-- src/migrations/0022_php73_to_php74_pools.py | 5 ++--- src/migrations/0023_postgresql_11_to_13.py | 4 ++-- src/migrations/0024_rebuild_python_venv.py | 4 ++-- .../0025_global_settings_to_configpanel.py | 4 ++-- src/migrations/0026_new_admins_group.py | 4 ++-- src/permission.py | 4 ++-- src/regenconf.py | 14 +++++++------- src/service.py | 5 ++--- src/settings.py | 4 ++-- src/tools.py | 4 ++-- src/user.py | 4 ++-- src/utils/configpanel.py | 4 ++-- src/utils/form.py | 4 ++-- src/utils/legacy.py | 5 +++-- src/utils/resources.py | 4 ++-- 28 files changed, 62 insertions(+), 72 deletions(-) diff --git a/src/__init__.py b/src/__init__.py index d13d61089f..e23b622199 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -115,17 +115,11 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "version": 1, "disable_existing_loggers": True, "formatters": { - "console": { - "format": "%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" + "tty-debug": { + "format": "%(relativeCreated)-4d %(level_with_color)s %(message)s" }, - "tty-debug": {"format": "%(relativeCreated)-4d %(fmessage)s"}, "precise": { - "format": "%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" - }, - }, - "filters": { - "action": { - "()": "moulinette.utils.log.ActionFilter", + "format": "%(asctime)-15s %(levelname)-8s %(name)s.%(funcName)s - %(message)s" }, }, "handlers": { @@ -142,7 +136,6 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "class": "logging.FileHandler", "formatter": "precise", "filename": logfile, - "filters": ["action"], }, }, "loggers": { diff --git a/src/app.py b/src/app.py index 75d3362413..483d34153c 100644 --- a/src/app.py +++ b/src/app.py @@ -28,9 +28,9 @@ import copy from typing import List, Tuple, Dict, Any, Iterator, Optional from packaging import version +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import run_commands, check_output from moulinette.utils.filesystem import ( read_file, @@ -71,7 +71,7 @@ APPS_CATALOG_LOGOS, ) -logger = getActionLogger("yunohost.app") +logger = getLogger("yunohost.app") APPS_SETTING_PATH = "/etc/yunohost/apps/" APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs" diff --git a/src/app_catalog.py b/src/app_catalog.py index 9fb6628457..2a50a0f823 100644 --- a/src/app_catalog.py +++ b/src/app_catalog.py @@ -19,9 +19,9 @@ import os import re import hashlib +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.network import download_json from moulinette.utils.filesystem import ( read_json, @@ -34,7 +34,7 @@ from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError -logger = getActionLogger("yunohost.app_catalog") +logger = getLogger("yunohost.app_catalog") APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" APPS_CATALOG_LOGOS = "/usr/share/yunohost/applogos" diff --git a/src/backup.py b/src/backup.py index ce1e8ba2c1..505e071a10 100644 --- a/src/backup.py +++ b/src/backup.py @@ -30,10 +30,10 @@ from collections import OrderedDict from functools import reduce from packaging import version +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.utils.text import random_ascii -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, mkdir, @@ -84,7 +84,7 @@ CONF_MARGIN_SPACE_SIZE = 10 # IN MB POSTINSTALL_ESTIMATE_SPACE_SIZE = 5 # In MB MB_ALLOWED_TO_ORGANIZE = 10 -logger = getActionLogger("yunohost.backup") +logger = getLogger("yunohost.backup") class BackupRestoreTargetsManager: diff --git a/src/certificate.py b/src/certificate.py index 76d3f32b70..c3aaacc87d 100644 --- a/src/certificate.py +++ b/src/certificate.py @@ -21,11 +21,10 @@ import shutil import subprocess from glob import glob - +from logging import getLogger from datetime import datetime from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, chown, chmod from moulinette.utils.process import check_output @@ -38,7 +37,7 @@ from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger -logger = getActionLogger("yunohost.certmanager") +logger = getLogger("yunohost.certmanager") CERT_FOLDER = "/etc/yunohost/certs/" TMP_FOLDER = "/var/www/.well-known/acme-challenge-private/" diff --git a/src/diagnosis.py b/src/diagnosis.py index 02047c0011..be3208b02f 100644 --- a/src/diagnosis.py +++ b/src/diagnosis.py @@ -21,6 +21,7 @@ import time import glob from importlib import import_module +from logging import getLogger from moulinette import m18n, Moulinette from moulinette.utils import log @@ -33,7 +34,7 @@ from yunohost.utils.error import YunohostError, YunohostValidationError -logger = log.getActionLogger("yunohost.diagnosis") +logger = getLogger("yunohost.diagnosis") DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" DIAGNOSIS_CONFIG_FILE = "/etc/yunohost/diagnosis.yml" diff --git a/src/dns.py b/src/dns.py index e25d0f3ec4..9a081e228a 100644 --- a/src/dns.py +++ b/src/dns.py @@ -19,12 +19,11 @@ import os import re import time - +from logging import getLogger from difflib import SequenceMatcher from collections import OrderedDict from moulinette import m18n, Moulinette -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, write_to_file, read_toml, mkdir from yunohost.domain import ( @@ -42,7 +41,7 @@ from yunohost.log import is_unit_operation from yunohost.hook import hook_callback -logger = getActionLogger("yunohost.domain") +logger = getLogger("yunohost.domain") DOMAIN_REGISTRAR_LIST_PATH = "/usr/share/yunohost/registrar_list.toml" diff --git a/src/domain.py b/src/domain.py index 8fc9799cd4..2d36fdfa42 100644 --- a/src/domain.py +++ b/src/domain.py @@ -20,10 +20,10 @@ import time from typing import List, Optional from collections import OrderedDict +from logging import getLogger from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm from yunohost.app import ( @@ -39,7 +39,7 @@ from yunohost.utils.dns import is_yunohost_dyndns_domain from yunohost.log import is_unit_operation -logger = getActionLogger("yunohost.domain") +logger = getLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" diff --git a/src/dyndns.py b/src/dyndns.py index a3afd655fd..5c9e2a36ea 100644 --- a/src/dyndns.py +++ b/src/dyndns.py @@ -22,10 +22,10 @@ import base64 import subprocess import hashlib +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import write_to_file, rm, chown, chmod from moulinette.utils.network import download_json @@ -36,7 +36,7 @@ from yunohost.log import is_unit_operation from yunohost.regenconf import regen_conf -logger = getActionLogger("yunohost.dyndns") +logger = getLogger("yunohost.dyndns") DYNDNS_PROVIDER = "dyndns.yunohost.org" DYNDNS_DNS_AUTH = ["ns0.yunohost.org", "ns1.yunohost.org"] diff --git a/src/firewall.py b/src/firewall.py index 392678fe11..9375aded0e 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -19,16 +19,16 @@ import os import yaml import miniupnpc +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils import process -from moulinette.utils.log import getActionLogger FIREWALL_FILE = "/etc/yunohost/firewall.yml" UPNP_CRON_JOB = "/etc/cron.d/yunohost-firewall-upnp" -logger = getActionLogger("yunohost.firewall") +logger = getLogger("yunohost.firewall") def firewall_allow( diff --git a/src/hook.py b/src/hook.py index 4b07d1c17b..60abb98d46 100644 --- a/src/hook.py +++ b/src/hook.py @@ -23,16 +23,16 @@ import mimetypes from glob import iglob from importlib import import_module +from logging import getLogger from moulinette import m18n, Moulinette from yunohost.utils.error import YunohostError, YunohostValidationError -from moulinette.utils import log from moulinette.utils.filesystem import read_yaml, cp HOOK_FOLDER = "/usr/share/yunohost/hooks/" CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/" -logger = log.getActionLogger("yunohost.hook") +logger = getLogger("yunohost.hook") def hook_add(app, file): diff --git a/src/log.py b/src/log.py index 5ab918e765..13683d8ef7 100644 --- a/src/log.py +++ b/src/log.py @@ -32,10 +32,9 @@ from moulinette.core import MoulinetteError from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.system import get_ynh_package_version -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, read_yaml -logger = getActionLogger("yunohost.log") +logger = getLogger("yunohost.log") CATEGORIES_PATH = "/var/log/yunohost/categories/" OPERATIONS_PATH = "/var/log/yunohost/categories/operation/" diff --git a/src/migrations/0021_migrate_to_bullseye.py b/src/migrations/0021_migrate_to_bullseye.py index f320577e12..eee8cc667d 100644 --- a/src/migrations/0021_migrate_to_bullseye.py +++ b/src/migrations/0021_migrate_to_bullseye.py @@ -1,9 +1,9 @@ import glob import os +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output, call_async_output from moulinette.utils.filesystem import read_file, rm, write_to_file @@ -22,7 +22,7 @@ ) from yunohost.service import _get_services, _save_services -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") N_CURRENT_DEBIAN = 10 N_CURRENT_YUNOHOST = 4 diff --git a/src/migrations/0022_php73_to_php74_pools.py b/src/migrations/0022_php73_to_php74_pools.py index dc428e5042..23ecc5b897 100644 --- a/src/migrations/0022_php73_to_php74_pools.py +++ b/src/migrations/0022_php73_to_php74_pools.py @@ -1,15 +1,14 @@ import os import glob from shutil import copy2 - -from moulinette.utils.log import getActionLogger +from logging import getLogger from yunohost.app import _is_installed from yunohost.utils.legacy import _patch_legacy_php_versions_in_settings from yunohost.tools import Migration from yunohost.service import _run_service_command -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") OLDPHP_POOLS = "/etc/php/7.3/fpm/pool.d" NEWPHP_POOLS = "/etc/php/7.4/fpm/pool.d" diff --git a/src/migrations/0023_postgresql_11_to_13.py b/src/migrations/0023_postgresql_11_to_13.py index 6d37ffa740..086b17af7c 100644 --- a/src/migrations/0023_postgresql_11_to_13.py +++ b/src/migrations/0023_postgresql_11_to_13.py @@ -1,15 +1,15 @@ import subprocess import time import os +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError -from moulinette.utils.log import getActionLogger from yunohost.tools import Migration from yunohost.utils.system import free_space_in_directory, space_used_by_directory -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") class MyMigration(Migration): diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0024_rebuild_python_venv.py index 01a229b878..5b77e69a0f 100644 --- a/src/migrations/0024_rebuild_python_venv.py +++ b/src/migrations/0024_rebuild_python_venv.py @@ -1,14 +1,14 @@ import os +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from yunohost.tools import Migration, tools_migrations_state from moulinette.utils.filesystem import rm -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt" diff --git a/src/migrations/0025_global_settings_to_configpanel.py b/src/migrations/0025_global_settings_to_configpanel.py index 3a88184617..76289e6088 100644 --- a/src/migrations/0025_global_settings_to_configpanel.py +++ b/src/migrations/0025_global_settings_to_configpanel.py @@ -1,13 +1,13 @@ import os +from logging import getLogger from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_json, write_to_yaml from yunohost.tools import Migration from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") SETTINGS_PATH = "/etc/yunohost/settings.yml" OLD_SETTINGS_PATH = "/etc/yunohost/settings.json" diff --git a/src/migrations/0026_new_admins_group.py b/src/migrations/0026_new_admins_group.py index 43f10a7b66..30237e7201 100644 --- a/src/migrations/0026_new_admins_group.py +++ b/src/migrations/0026_new_admins_group.py @@ -1,8 +1,8 @@ -from moulinette.utils.log import getActionLogger +from logging import getLogger from yunohost.tools import Migration -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") ################################################### # Tools used also for restoration diff --git a/src/permission.py b/src/permission.py index 72975561fd..7ec6f17bc7 100644 --- a/src/permission.py +++ b/src/permission.py @@ -20,13 +20,13 @@ import copy import grp import random +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation -logger = getActionLogger("yunohost.user") +logger = getLogger("yunohost.user") SYSTEM_PERMS = ["mail", "xmpp", "sftp", "ssh"] diff --git a/src/regenconf.py b/src/regenconf.py index 74bbdb17c2..f180368ad6 100644 --- a/src/regenconf.py +++ b/src/regenconf.py @@ -20,12 +20,12 @@ import yaml import shutil import hashlib - +from logging import getLogger from difflib import unified_diff from datetime import datetime from moulinette import m18n -from moulinette.utils import log, filesystem +from moulinette.utils.filesystem import mkdir from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError @@ -37,7 +37,7 @@ PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, "pending") REGEN_CONF_FILE = "/etc/yunohost/regenconf.yml" -logger = log.getActionLogger("yunohost.regenconf") +logger = getLogger("yunohost.regenconf") # FIXME : those ain't just services anymore ... what are we supposed to do with this ... @@ -102,7 +102,7 @@ def regen_conf( for name in names: shutil.rmtree(os.path.join(PENDING_CONF_DIR, name), ignore_errors=True) else: - filesystem.mkdir(PENDING_CONF_DIR, 0o755, True) + mkdir(PENDING_CONF_DIR, 0o755, True) # Execute hooks for pre-regen # element 2 and 3 with empty string is because of legacy... @@ -111,7 +111,7 @@ def regen_conf( def _pre_call(name, priority, path, args): # create the pending conf directory for the category category_pending_path = os.path.join(PENDING_CONF_DIR, name) - filesystem.mkdir(category_pending_path, 0o755, True, uid="root") + mkdir(category_pending_path, 0o755, True, uid="root") # return the arguments to pass to the script return pre_args + [ @@ -622,7 +622,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): backup_dir = os.path.dirname(backup_path) if not os.path.isdir(backup_dir): - filesystem.mkdir(backup_dir, 0o755, True) + mkdir(backup_dir, 0o755, True) shutil.copy2(system_conf, backup_path) logger.debug( @@ -637,7 +637,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): system_dir = os.path.dirname(system_conf) if not os.path.isdir(system_dir): - filesystem.mkdir(system_dir, 0o755, True) + mkdir(system_dir, 0o755, True) shutil.copyfile(new_conf, system_conf) logger.debug(m18n.n("regenconf_file_updated", conf=system_conf)) diff --git a/src/service.py b/src/service.py index 47bc1903ae..6bb61d8419 100644 --- a/src/service.py +++ b/src/service.py @@ -21,14 +21,13 @@ import time import yaml import subprocess - +from logging import getLogger from glob import glob from datetime import datetime from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.process import check_output -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, append_to_file, @@ -42,7 +41,7 @@ SERVICES_CONF = "/etc/yunohost/services.yml" SERVICES_CONF_BASE = "/usr/share/yunohost/conf/yunohost/services.yml" -logger = getActionLogger("yunohost.service") +logger = getLogger("yunohost.service") def service_add( diff --git a/src/settings.py b/src/settings.py index 6690ab3fd0..e2f34bda9a 100644 --- a/src/settings.py +++ b/src/settings.py @@ -18,18 +18,18 @@ # import os import subprocess +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.configpanel import ConfigPanel from yunohost.utils.form import BaseOption -from moulinette.utils.log import getActionLogger from yunohost.regenconf import regen_conf from yunohost.firewall import firewall_reload from yunohost.log import is_unit_operation from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings -logger = getActionLogger("yunohost.settings") +logger = getLogger("yunohost.settings") SETTINGS_PATH = "/etc/yunohost/settings.yml" diff --git a/src/tools.py b/src/tools.py index cd48f00ee1..23edf10043 100644 --- a/src/tools.py +++ b/src/tools.py @@ -24,9 +24,9 @@ from importlib import import_module from packaging import version from typing import List +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm, chown @@ -55,7 +55,7 @@ MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml" -logger = getActionLogger("yunohost.tools") +logger = getLogger("yunohost.tools") def tools_versions(): diff --git a/src/user.py b/src/user.py index 00876854ec..780797a613 100644 --- a/src/user.py +++ b/src/user.py @@ -25,9 +25,9 @@ import string import subprocess import copy +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError, YunohostValidationError @@ -35,7 +35,7 @@ from yunohost.log import is_unit_operation from yunohost.utils.system import binary_to_human -logger = getActionLogger("yunohost.user") +logger = getLogger("yunohost.user") FIELDS_FOR_IMPORT = { "username": r"^[a-z0-9_]+$", diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 42a030cbcb..56e0719564 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -22,11 +22,11 @@ import urllib.parse from collections import OrderedDict from typing import Union +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize from moulinette.utils.filesystem import mkdir, read_toml, read_yaml, write_to_yaml -from moulinette.utils.log import getActionLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.form import ( OPTIONS, @@ -40,7 +40,7 @@ ) from yunohost.utils.i18n import _value_for_locale -logger = getActionLogger("yunohost.configpanel") +logger = getLogger("yunohost.configpanel") CONFIG_PANEL_VERSION_SUPPORTED = 1.0 diff --git a/src/utils/form.py b/src/utils/form.py index 1ca03373e7..e2e01ca129 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -25,16 +25,16 @@ import urllib.parse from enum import Enum from typing import Any, Callable, Dict, List, Literal, Mapping, Optional, Union +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize from moulinette.utils.filesystem import read_file, write_to_file -from moulinette.utils.log import getActionLogger from yunohost.log import OperationLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.i18n import _value_for_locale -logger = getActionLogger("yunohost.form") +logger = getLogger("yunohost.form") Context = dict[str, Any] diff --git a/src/utils/legacy.py b/src/utils/legacy.py index 82507d64d3..8b44fb3fbb 100644 --- a/src/utils/legacy.py +++ b/src/utils/legacy.py @@ -19,8 +19,9 @@ import os import re import glob +from logging import getLogger + from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, write_to_file, @@ -32,7 +33,7 @@ from yunohost.utils.error import YunohostValidationError -logger = getActionLogger("yunohost.legacy") +logger = getLogger("yunohost.utils.legacy") LEGACY_PERMISSION_LABEL = { ("nextcloud", "skipped"): "api", # .well-known diff --git a/src/utils/resources.py b/src/utils/resources.py index 60a5f44f61..5e3f2c5693 100644 --- a/src/utils/resources.py +++ b/src/utils/resources.py @@ -23,11 +23,11 @@ import tempfile import subprocess from typing import Dict, Any, List, Union +from logging import getLogger from moulinette import m18n from moulinette.utils.text import random_ascii from moulinette.utils.process import check_output -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import mkdir, chown, chmod, write_to_file from moulinette.utils.filesystem import ( rm, @@ -35,7 +35,7 @@ from yunohost.utils.system import system_arch from yunohost.utils.error import YunohostError, YunohostValidationError -logger = getActionLogger("yunohost.app_resources") +logger = getLogger("yunohost.utils.resources") class AppResourceManager: From 704e42a6af530c4816a50c9cb93655a14ed4cbe8 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jul 2023 19:13:00 +0200 Subject: [PATCH 060/436] portalapi: fix cookie not being deleted because maxage=-1 or something --- src/authenticators/ldap_ynhuser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index e8cfaf108c..9702693ed4 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -172,5 +172,5 @@ def delete_session_cookie(self): from bottle import response - response.set_cookie("yunohost.portal", "", max_age=-1) + response.set_cookie("yunohost.portal", "") response.delete_cookie("yunohost.portal") From 09c5a4cfb91cdede55956bb2e3cf747a0a2c6e18 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 29 Jul 2023 19:15:30 +0200 Subject: [PATCH 061/436] admin and portalapi: propagate new configurable CORS mechanism from moulinette --- conf/nginx/yunohost_api.conf.inc | 12 ------------ src/__init__.py | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/conf/nginx/yunohost_api.conf.inc b/conf/nginx/yunohost_api.conf.inc index 8133624b38..9cb4ff00da 100644 --- a/conf/nginx/yunohost_api.conf.inc +++ b/conf/nginx/yunohost_api.conf.inc @@ -26,18 +26,6 @@ location = /yunohost/api/error/502 { location /yunohost/portalapi/ { - # FIXME FIXME FIXME : we should think about what we really want here ... - more_set_headers "Access-Control-Allow-Origin: $http_origin"; - more_set_headers "Access-Control-Allow-Methods: GET, HEAD, POST, OPTIONS, DELETE"; - more_set_headers "Access-Control-Allow-Headers: Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With"; - more_set_headers "Access-Control-Allow-Credentials: true"; - - if ($request_method = 'OPTIONS') { - more_set_headers "Content-Type: text/plain; charset=utf-8"; - more_set_headers "Content-Length: 0"; - return 204; - } - proxy_read_timeout 5s; proxy_pass http://127.0.0.1:6788/; proxy_http_version 1.1; diff --git a/src/__init__.py b/src/__init__.py index 146485d2df..99f3739bf3 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -50,6 +50,13 @@ def cli(debug, quiet, output_as, timeout, args, parser): def api(debug, host, port): + + allowed_cors_origins = [] + allowed_cors_origins_file = "/etc/yunohost/.admin-api-allowed-cors-origins" + + if os.path.exists(allowed_cors_origins_file): + allowed_cors_origins = open(allowed_cors_origins_file).read().strip().split(",") + init_logging(interface="api", debug=debug) def is_installed_api(): @@ -64,12 +71,19 @@ def is_installed_api(): actionsmap="/usr/share/yunohost/actionsmap.yml", locales_dir="/usr/share/yunohost/locales/", routes={("GET", "/installed"): is_installed_api}, + allowed_cors_origins=allowed_cors_origins, ) sys.exit(ret) def portalapi(debug, host, port): + allowed_cors_origins = [] + allowed_cors_origins_file = "/etc/yunohost/.portal-api-allowed-cors-origins" + + if os.path.exists(allowed_cors_origins_file): + allowed_cors_origins = open(allowed_cors_origins_file).read().strip().split(",") + # FIXME : is this the logdir we want ? (yolo to work around permission issue) init_logging(interface="portalapi", debug=debug, logdir="/var/log") @@ -77,7 +91,8 @@ def portalapi(debug, host, port): host=host, port=port, actionsmap="/usr/share/yunohost/actionsmap-portal.yml", - locales_dir="/usr/share/yunohost/locales/" + locales_dir="/usr/share/yunohost/locales/", + allowed_cors_origins=allowed_cors_origins, ) sys.exit(ret) From 5fd1850f19c020115298d7595fc0212a8fb0b9be Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 Jul 2023 23:53:04 +0200 Subject: [PATCH 062/436] Add dependency to new yunohost-portal debian package --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index df9a6d2bdd..121ac3f6fa 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Package: yunohost Essential: yes Architecture: all Depends: ${python3:Depends}, ${misc:Depends} - , moulinette (>= 11.1), ssowat (>= 11.1) + , moulinette (>= 11.1), ssowat (>= 11.1), yunohost-portal (>= 11.1) , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 From afd7b37ebcabfd53e6fbfe2dc64503aab25d5f1a Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sun, 30 Jul 2023 23:53:43 +0200 Subject: [PATCH 063/436] Tweak nginx portal conf to serve html/css/js/assets from /usr/share/yunohost/portal, similar to webadmin --- conf/nginx/plain/yunohost_sso.conf.inc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/nginx/plain/yunohost_sso.conf.inc b/conf/nginx/plain/yunohost_sso.conf.inc index 9844406793..578a75e52e 100644 --- a/conf/nginx/plain/yunohost_sso.conf.inc +++ b/conf/nginx/plain/yunohost_sso.conf.inc @@ -2,6 +2,16 @@ rewrite ^/yunohost/sso$ /yunohost/sso/ permanent; location /yunohost/sso/ { - alias /usr/share/ssowat/portal/; + alias /usr/share/yunohost/portal/; + default_type text/html; index index.html; + try_files $uri $uri/ /index.html; + + location = /yunohost/sso/index.html { + etag off; + expires off; + more_set_headers "Cache-Control: no-store, no-cache, must-revalidate"; + } + + more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self'; object-src 'none'; img-src 'self' data:;"; } From ca6eb2cbaf8e65abc95c78e2223a24b36c9005e3 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 1 Aug 2023 15:15:52 +0200 Subject: [PATCH 064/436] lint --- src/portal.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/portal.py b/src/portal.py index 6a51f33d4e..f102f15153 100644 --- a/src/portal.py +++ b/src/portal.py @@ -48,10 +48,16 @@ def portal_me(): else: raise YunohostValidationError("user_unknown", user=username) - groups = [g.replace("cn=", "").replace(",ou=groups,dc=yunohost,dc=org", "") for g in user["memberOf"]] + groups = [ + g.replace("cn=", "").replace(",ou=groups,dc=yunohost,dc=org", "") + for g in user["memberOf"] + ] groups = [g for g in groups if g not in [username, "all_users"]] - permissions = [p.replace("cn=", "").replace(",ou=permission,dc=yunohost,dc=org", "") for p in user["permission"]] + permissions = [ + p.replace("cn=", "").replace(",ou=permission,dc=yunohost,dc=org", "") + for p in user["permission"] + ] ssowat_conf = read_json("/etc/ssowat/conf.json") apps = { @@ -67,7 +73,7 @@ def portal_me(): "mail-aliases": user["mail"][1:], "mail-forward": user["maildrop"][1:], "groups": groups, - "apps": apps + "apps": apps, } # FIXME / TODO : add mail quota status ? From c3a4b7dabb9946bc6ed1d12aa7ba50fc85c255cb Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 1 Aug 2023 15:18:48 +0200 Subject: [PATCH 065/436] add _get_user_infos helper --- src/portal.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/portal.py b/src/portal.py index f102f15153..7e67ff35d4 100644 --- a/src/portal.py +++ b/src/portal.py @@ -29,24 +29,25 @@ logger = getActionLogger("portal") -def portal_me(): - """ - Get user informations - """ - +def _get_user_infos(user_attrs: list[str]): auth = Auth().get_session_cookie(decrypt_pwd=True) username = auth["user"] - ldap = LDAPInterface(username, auth["pwd"]) + result = ldap.search("ou=users", f"uid={username}", user_attrs) + if not result: + raise YunohostValidationError("user_unknown", user=username) - user_attrs = ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] + return username, result[0], ldap - result = ldap.search("ou=users", f"uid={username}", user_attrs) - if result: - user = result[0] - else: - raise YunohostValidationError("user_unknown", user=username) +def portal_me(): + """ + Get user informations + """ + + username, user, ldap = _get_user_infos( + ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] + ) groups = [ g.replace("cn=", "").replace(",ou=groups,dc=yunohost,dc=org", "") From c9092b2aadd82f3bc7c57ce626248f6a9983fba6 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 1 Aug 2023 15:29:09 +0200 Subject: [PATCH 066/436] add portal_update to update user infos --- share/actionsmap-portal.yml | 28 +++++++++++- src/portal.py | 85 +++++++++++++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 5 deletions(-) diff --git a/share/actionsmap-portal.yml b/share/actionsmap-portal.yml index 761d5a6ceb..268a420b32 100644 --- a/share/actionsmap-portal.yml +++ b/share/actionsmap-portal.yml @@ -23,8 +23,32 @@ portal: ### portal_update() update: action_help: Allow user to update their infos (display name, mail aliases/forward, password, ...) - api: PUT /me - # FIXME: add args etc + api: PUT /update + arguments: + -F: + full: --fullname + help: The full name of the user. For example 'Camille Dupont' + extra: + pattern: &pattern_fullname + - !!str ^([^\W_]{1,30}[ ,.'-]{0,3})+$ + - "pattern_fullname" + --mailforward: + help: Mailforward addresses to add + nargs: "*" + metavar: MAIL + extra: + pattern: &pattern_email_forward + - !!str ^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ + - "pattern_email_forward" + --mailalias: + help: Mail aliases to add + nargs: "*" + metavar: MAIL + extra: + pattern: &pattern_email + - !!str ^[\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ + - "pattern_email" + ### portal_reset_password() reset_password: diff --git a/src/portal.py b/src/portal.py index 7e67ff35d4..2e234ec733 100644 --- a/src/portal.py +++ b/src/portal.py @@ -18,16 +18,19 @@ along with this program; if not, see http://www.gnu.org/licenses """ +from typing import Union from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_json from yunohost.authenticators.ldap_ynhuser import Authenticator as Auth from yunohost.utils.ldap import LDAPInterface -from yunohost.utils.error import YunohostValidationError +from yunohost.utils.error import YunohostError, YunohostValidationError logger = getActionLogger("portal") +ADMIN_ALIASES = ["root", "admin", "admins", "webmaster", "postmaster", "abuse"] + def _get_user_infos(user_attrs: list[str]): auth = Auth().get_session_cookie(decrypt_pwd=True) @@ -71,8 +74,8 @@ def portal_me(): "username": username, "fullname": user["cn"][0], "mail": user["mail"][0], - "mail-aliases": user["mail"][1:], - "mail-forward": user["maildrop"][1:], + "mailalias": user["mail"][1:], + "mailforward": user["maildrop"][1:], "groups": groups, "apps": apps, } @@ -86,3 +89,79 @@ def portal_me(): # But this requires to be in the mail group ... return result_dict + + +def portal_update( + fullname: Union[str, None] = None, + mailforward: Union[list[str], None] = None, + mailalias: Union[list[str], None] = None, +): + from yunohost.domain import domain_list + + domains = domain_list()["domains"] + username, current_user, ldap = _get_user_infos( + ["givenName", "sn", "cn", "mail", "maildrop", "memberOf"] + ) + new_attr_dict = {} + + if fullname is not None and fullname != current_user["cn"]: + fullname = fullname.strip() + firstname = fullname.split()[0] + lastname = ( + " ".join(fullname.split()[1:]) or " " + ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + new_attr_dict["givenName"] = [firstname] # TODO: Validate + new_attr_dict["sn"] = [lastname] # TODO: Validate + new_attr_dict["cn"] = new_attr_dict["displayName"] = [ + (firstname + " " + lastname).strip() + ] + + if mailalias is not None: + mailalias = [mail.strip() for mail in mailalias if mail and mail.strip()] + # keep first current mail unaltered + mails = [current_user["mail"][0]] + + for index, mail in enumerate(mailalias): + if mail in current_user["mail"]: + if mail != current_user["mail"][0]: + mails.append(mail) + continue # already in mails, skip validation + + local_part, domain = mail.strip().split("@") + if local_part in ADMIN_ALIASES: + raise YunohostValidationError( + "mail_unavailable", path="mailalias", index=index + ) + + try: + ldap.validate_uniqueness({"mail": mail}) + except Exception as e: + raise YunohostError("user_update_failed", user=username, error=e) + + if domain not in domains: + raise YunohostError("mail_domain_unknown", domain=domain) + + mails.append(mail) + + new_attr_dict["mail"] = mails + + if mailforward is not None: + new_attr_dict["maildrop"] = [current_user["maildrop"][0]] + [ + mail.strip() + for mail in mailforward + if mail and mail.strip() and mail != current_user["maildrop"][0] + ] + + try: + ldap.update(f"uid={username},ou=users", new_attr_dict) + except Exception as e: + raise YunohostError("user_update_failed", user=username, error=e) + + # FIXME: Here we could want to trigger "post_user_update" hook but hooks has to + # be run as root + + return { + "fullname": new_attr_dict["cn"][0], + "mailalias": new_attr_dict["mail"][1:], + "mailforward": new_attr_dict["maildrop"][1:], + } From db1670ca5d8cec856cda93ed3b5e06d4c03e6dfe Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 1 Aug 2023 16:28:25 +0200 Subject: [PATCH 067/436] add temp portal_update_password --- share/actionsmap-portal.yml | 11 +++++++++++ src/portal.py | 24 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/share/actionsmap-portal.yml b/share/actionsmap-portal.yml index 268a420b32..2673fc2264 100644 --- a/share/actionsmap-portal.yml +++ b/share/actionsmap-portal.yml @@ -49,6 +49,17 @@ portal: - !!str ^[\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ - "pattern_email" + ### portal_update_password() + update_password: + action_help: Allow user to change their password + api: PUT /me/update_password + arguments: + -c: + full: --current + help: Current password + -p: + full: --password + help: New password to set ### portal_reset_password() reset_password: diff --git a/src/portal.py b/src/portal.py index 2e234ec733..fa2d1a30a8 100644 --- a/src/portal.py +++ b/src/portal.py @@ -26,6 +26,11 @@ from yunohost.authenticators.ldap_ynhuser import Authenticator as Auth from yunohost.utils.ldap import LDAPInterface from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.password import ( + assert_password_is_compatible, + assert_password_is_strong_enough, +) +from yunohost.user import _hash_user_password logger = getActionLogger("portal") @@ -165,3 +170,22 @@ def portal_update( "mailalias": new_attr_dict["mail"][1:], "mailforward": new_attr_dict["maildrop"][1:], } + + +def portal_update_password(current: str, password: str): + username, current_user, ldap = _get_user_infos(["userPassword", "memberOf"]) + is_admin = "cn=admins,ou=groups,dc=yunohost,dc=org" in current_user["memberOf"] + + # FIXME: Verify current password ? + + # Ensure compatibility and sufficiently complex password + assert_password_is_compatible(password) + assert_password_is_strong_enough("admin" if is_admin else "user", password) + + try: + ldap.update( + f"uid={username},ou=users", + {"userPassword": [_hash_user_password(password)]}, + ) + except Exception as e: + raise YunohostError("user_update_failed", user=username, error=e) From 6f8b3fd57feebbbf987a02605fd842f9324ff646 Mon Sep 17 00:00:00 2001 From: selfhoster1312 Date: Sun, 13 Aug 2023 23:11:31 +0200 Subject: [PATCH 068/436] Handle both cookies in the same way (please let me logout) --- src/authenticators/ldap_admin.py | 4 ++-- src/authenticators/ldap_ynhuser.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/authenticators/ldap_admin.py b/src/authenticators/ldap_admin.py index b1b550bc0c..155e84127d 100644 --- a/src/authenticators/ldap_admin.py +++ b/src/authenticators/ldap_admin.py @@ -138,6 +138,7 @@ def set_session_cookie(self, infos): secure=True, secret=session_secret, httponly=True, + path="/" # samesite="strict", # Bottle 0.12 doesn't support samesite, to be added in next versions ) @@ -172,5 +173,4 @@ def get_session_cookie(self, raise_if_no_session_exists=True): def delete_session_cookie(self): from bottle import response - response.set_cookie("yunohost.admin", "", max_age=-1) - response.delete_cookie("yunohost.admin") + response.delete_cookie("yunohost.admin", path="/") diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 9702693ed4..08138f1b5a 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -172,5 +172,4 @@ def delete_session_cookie(self): from bottle import response - response.set_cookie("yunohost.portal", "") - response.delete_cookie("yunohost.portal") + response.delete_cookie("yunohost.portal", path="/") From 101b5704c491de6df80f236a10c4fc865fdc9963 Mon Sep 17 00:00:00 2001 From: selfhoster1312 Date: Tue, 15 Aug 2023 12:23:56 +0200 Subject: [PATCH 069/436] Serialize the JWT token to a cookie string instead of failing --- src/authenticators/ldap_ynhuser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 08138f1b5a..2add68cab4 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -132,7 +132,7 @@ def set_session_cookie(self, infos): response.set_cookie( "yunohost.portal", - jwt.encode(new_infos, session_secret, algorithm="HS256"), + jwt.encode(new_infos, session_secret, algorithm="HS256").decode(), secure=True, httponly=True, path="/", From 26d4d9420c6e129e3de869f85cd308e087d25d5b Mon Sep 17 00:00:00 2001 From: selfhoster1312 Date: Tue, 15 Aug 2023 14:12:08 +0200 Subject: [PATCH 070/436] Allow inline scripts for yunohost-portal (nginx CSP) --- conf/nginx/plain/yunohost_sso.conf.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/nginx/plain/yunohost_sso.conf.inc b/conf/nginx/plain/yunohost_sso.conf.inc index 578a75e52e..fb5406cfc8 100644 --- a/conf/nginx/plain/yunohost_sso.conf.inc +++ b/conf/nginx/plain/yunohost_sso.conf.inc @@ -13,5 +13,5 @@ location /yunohost/sso/ { more_set_headers "Cache-Control: no-store, no-cache, must-revalidate"; } - more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self'; object-src 'none'; img-src 'self' data:;"; + more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; object-src 'none'; img-src 'self' data:;"; } From 8f0f85b7221a6f6577d1391916ade88a78506d17 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 29 Aug 2023 15:24:52 +0200 Subject: [PATCH 071/436] merge update_password with update --- share/actionsmap-portal.yml | 29 ++++++++------ src/portal.py | 78 ++++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 47 deletions(-) diff --git a/share/actionsmap-portal.yml b/share/actionsmap-portal.yml index 2673fc2264..6b02a061d1 100644 --- a/share/actionsmap-portal.yml +++ b/share/actionsmap-portal.yml @@ -25,8 +25,7 @@ portal: action_help: Allow user to update their infos (display name, mail aliases/forward, password, ...) api: PUT /update arguments: - -F: - full: --fullname + --fullname: help: The full name of the user. For example 'Camille Dupont' extra: pattern: &pattern_fullname @@ -48,18 +47,24 @@ portal: pattern: &pattern_email - !!str ^[\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ - "pattern_email" - - ### portal_update_password() - update_password: - action_help: Allow user to change their password - api: PUT /me/update_password - arguments: - -c: - full: --current + --currentpassword: help: Current password - -p: - full: --password + nargs: "?" + --newpassword: help: New password to set + nargs: "?" + + ### portal_update_password() + # update_password: + # action_help: Allow user to change their password + # api: PUT /me/update_password + # arguments: + # -c: + # full: --current + # help: Current password + # -p: + # full: --password + # help: New password to set ### portal_reset_password() reset_password: diff --git a/src/portal.py b/src/portal.py index fa2d1a30a8..e9c2b3f483 100644 --- a/src/portal.py +++ b/src/portal.py @@ -20,17 +20,17 @@ """ from typing import Union -from moulinette.utils.log import getActionLogger +import ldap from moulinette.utils.filesystem import read_json - -from yunohost.authenticators.ldap_ynhuser import Authenticator as Auth -from yunohost.utils.ldap import LDAPInterface +from moulinette.utils.log import getActionLogger +from yunohost.authenticators.ldap_ynhuser import URI, USERDN, Authenticator as Auth +from yunohost.user import _hash_user_password from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.ldap import LDAPInterface from yunohost.utils.password import ( assert_password_is_compatible, assert_password_is_strong_enough, ) -from yunohost.user import _hash_user_password logger = getActionLogger("portal") @@ -40,12 +40,12 @@ def _get_user_infos(user_attrs: list[str]): auth = Auth().get_session_cookie(decrypt_pwd=True) username = auth["user"] - ldap = LDAPInterface(username, auth["pwd"]) - result = ldap.search("ou=users", f"uid={username}", user_attrs) + ldap_interface = LDAPInterface(username, auth["pwd"]) + result = ldap_interface.search("ou=users", f"uid={username}", user_attrs) if not result: raise YunohostValidationError("user_unknown", user=username) - return username, result[0], ldap + return username, result[0], ldap_interface def portal_me(): @@ -53,7 +53,7 @@ def portal_me(): Get user informations """ - username, user, ldap = _get_user_infos( + username, user, _ = _get_user_infos( ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] ) @@ -100,11 +100,13 @@ def portal_update( fullname: Union[str, None] = None, mailforward: Union[list[str], None] = None, mailalias: Union[list[str], None] = None, + currentpassword: Union[str, None] = None, + newpassword: Union[str, None] = None, ): from yunohost.domain import domain_list domains = domain_list()["domains"] - username, current_user, ldap = _get_user_infos( + username, current_user, ldap_interface = _get_user_infos( ["givenName", "sn", "cn", "mail", "maildrop", "memberOf"] ) new_attr_dict = {} @@ -128,23 +130,25 @@ def portal_update( for index, mail in enumerate(mailalias): if mail in current_user["mail"]: - if mail != current_user["mail"][0]: + if mail != current_user["mail"][0] and mail not in mails: mails.append(mail) continue # already in mails, skip validation - local_part, domain = mail.strip().split("@") + local_part, domain = mail.split("@") if local_part in ADMIN_ALIASES: raise YunohostValidationError( - "mail_unavailable", path="mailalias", index=index + "mail_unavailable", path=f"mailalias[{index}]" ) try: - ldap.validate_uniqueness({"mail": mail}) + ldap_interface.validate_uniqueness({"mail": mail}) except Exception as e: raise YunohostError("user_update_failed", user=username, error=e) if domain not in domains: - raise YunohostError("mail_domain_unknown", domain=domain) + raise YunohostValidationError( + "mail_domain_unknown", domain=domain, path=f"mailalias[{index}]" + ) mails.append(mail) @@ -157,8 +161,31 @@ def portal_update( if mail and mail.strip() and mail != current_user["maildrop"][0] ] + if newpassword: + # Check that current password is valid + try: + con = ldap.ldapobject.ReconnectLDAPObject(URI, retry_max=0) + con.simple_bind_s(USERDN.format(username=username), currentpassword) + except ldap.INVALID_CREDENTIALS: + raise YunohostValidationError("invalid_password", path="currentpassword") + finally: + # Free the connection, we don't really need it to keep it open as the point is only to check authentication... + if con: + con.unbind_s() + + # Ensure compatibility and sufficiently complex password + try: + assert_password_is_compatible(newpassword) + is_admin = "cn=admins,ou=groups,dc=yunohost,dc=org" in current_user["memberOf"] + assert_password_is_strong_enough("admin" if is_admin else "user", newpassword) + except YunohostValidationError as e: + raise YunohostValidationError(e.key, path="newpassword") + + Auth().delete_session_cookie() + new_attr_dict["userPassword"] = [_hash_user_password(newpassword)] + try: - ldap.update(f"uid={username},ou=users", new_attr_dict) + ldap_interface.update(f"uid={username},ou=users", new_attr_dict) except Exception as e: raise YunohostError("user_update_failed", user=username, error=e) @@ -170,22 +197,3 @@ def portal_update( "mailalias": new_attr_dict["mail"][1:], "mailforward": new_attr_dict["maildrop"][1:], } - - -def portal_update_password(current: str, password: str): - username, current_user, ldap = _get_user_infos(["userPassword", "memberOf"]) - is_admin = "cn=admins,ou=groups,dc=yunohost,dc=org" in current_user["memberOf"] - - # FIXME: Verify current password ? - - # Ensure compatibility and sufficiently complex password - assert_password_is_compatible(password) - assert_password_is_strong_enough("admin" if is_admin else "user", password) - - try: - ldap.update( - f"uid={username},ou=users", - {"userPassword": [_hash_user_password(password)]}, - ) - except Exception as e: - raise YunohostError("user_update_failed", user=username, error=e) From 0645d18e677ad399d36d1ca23327097af2043504 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 4 Sep 2023 16:19:07 +0200 Subject: [PATCH 072/436] add host as session cookie info --- src/authenticators/ldap_ynhuser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 2add68cab4..331cf9e259 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -115,7 +115,7 @@ def _reconnect(): def set_session_cookie(self, infos): - from bottle import response + from bottle import response, request assert isinstance(infos, dict) @@ -126,7 +126,8 @@ def set_session_cookie(self, infos): # See https://pyjwt.readthedocs.io/en/latest/usage.html#registered-claim-names # for explanations regarding nbf, exp "nbf": int(datetime.datetime.now().timestamp()), - "exp": int(datetime.datetime.now().timestamp()) + (7 * 24 * 3600) # One week validity # FIXME : does it mean the session suddenly expires after a week ? Can we somehow auto-renew it at every usage or something ? + "exp": int(datetime.datetime.now().timestamp()) + (7 * 24 * 3600), # One week validity # FIXME : does it mean the session suddenly expires after a week ? Can we somehow auto-renew it at every usage or something ? + "host": request.get_header('host'), } new_infos.update(infos) From 5562b61db07de881f72405198156eecc80a0be7b Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 4 Sep 2023 16:20:29 +0200 Subject: [PATCH 073/436] add 'list_portal' AppOption modifier to add portal as a possible choice --- src/utils/configpanel.py | 1 + src/utils/form.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 42a030cbcb..02454bd0bb 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -382,6 +382,7 @@ def _get_config_panel(self): "filter", "readonly", "enabled", + "list_portal", # "confirm", # TODO: to ask confirmation before running an action ], "defaults": {}, diff --git a/src/utils/form.py b/src/utils/form.py index 1ca03373e7..64155d8e2c 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -914,6 +914,7 @@ def __init__(self, question): super().__init__(question) self.filter = question.get("filter", None) + self.list_portal = question.get("list_portal", False) apps = app_list(full=True)["apps"] @@ -929,6 +930,8 @@ def _app_display(app): return app["label"] + domain_path_or_id self.choices = {"_none": "---"} + if self.list_portal: + self.choices["portal_public_apps"] = "Portal" self.choices.update({app["id"]: _app_display(app) for app in apps}) From a1a47e5221809751949f7bf17cf7c63148028164 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 4 Sep 2023 16:21:50 +0200 Subject: [PATCH 074/436] update config_domain.toml with portal options --- share/config_domain.toml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/share/config_domain.toml b/share/config_domain.toml index 82ef90c323..b6a0b51a7a 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -4,11 +4,34 @@ i18n = "domain_config" [feature] name = "Features" + [feature.portal] + name = "Portal" + + [feature.portal.show_other_domains_apps] + type = "boolean" + default = 1 + + [feature.portal.portal_title] + type = "string" + default = "YunoHost" + + [feature.portal.portal_logo] + type = "string" + default = "" + + [feature.portal.portal_theme] + type = "select" + choices = ["system", "light", "dark", "cupcake", "bumblebee", "emerald", "corporate", "synthwave", "retro", "cyberpunk", "valentine", "halloween", "garden", "forest", "aqua", "lofi", "pastel", "fantasy", "wireframe", "black", "luxury", "dracula", "cmyk", "autumn", "business", "acid", "lemonade", "night", "coffee", "winter"] + default = "system" + + # FIXME link to GCU + [feature.app] [feature.app.default_app] type = "app" filter = "is_webapp" default = "_none" + list_portal = true [feature.mail] From 20d21b57e047863c421d5369cb710d33f7e53c0d Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 4 Sep 2023 16:24:01 +0200 Subject: [PATCH 075/436] wip: save portal configpanel options in separate file .portal.yml --- src/domain.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/domain.py b/src/domain.py index 5f0e76d281..9ddb67a769 100644 --- a/src/domain.py +++ b/src/domain.py @@ -737,6 +737,29 @@ def _apply(self): ): app_ssowatconf() + portal_options = [ + "default_app", + "show_other_domains_apps", + "portal_title", + "portal_logo", + "portal_theme", + ] + if any( + option in self.future_values + and self.future_values[option] != self.values[option] + for option in portal_options + ): + # Portal options are also saved in a `domain.portal.yml` file + # that can be read by the portal API. + # FIXME remove those from the config panel saved values? + portal_values = { + option: self.future_values[option] for option in portal_options + } + # FIXME config file should be readable by the portal entity + write_to_yaml( + f"{DOMAIN_SETTINGS_DIR}/{self.entity}.portal.yml", portal_values + ) + stuff_to_regen_conf = [] if ( "xmpp" in self.future_values From 2136db32b6d7da0d711bc441588d872ca49a2ee0 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 4 Sep 2023 16:27:06 +0200 Subject: [PATCH 076/436] return domain from _get_user_infos --- src/portal.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/portal.py b/src/portal.py index e9c2b3f483..cdb6fb9ddc 100644 --- a/src/portal.py +++ b/src/portal.py @@ -18,7 +18,7 @@ along with this program; if not, see http://www.gnu.org/licenses """ -from typing import Union +from typing import Any, Union import ldap from moulinette.utils.filesystem import read_json @@ -37,7 +37,9 @@ ADMIN_ALIASES = ["root", "admin", "admins", "webmaster", "postmaster", "abuse"] -def _get_user_infos(user_attrs: list[str]): +def _get_user_infos( + user_attrs: list[str], +) -> tuple[str, str, dict[str, Any], LDAPInterface]: auth = Auth().get_session_cookie(decrypt_pwd=True) username = auth["user"] ldap_interface = LDAPInterface(username, auth["pwd"]) @@ -45,15 +47,14 @@ def _get_user_infos(user_attrs: list[str]): if not result: raise YunohostValidationError("user_unknown", user=username) - return username, result[0], ldap_interface + return username, auth["host"], result[0], ldap_interface def portal_me(): """ Get user informations """ - - username, user, _ = _get_user_infos( + username, domain, user, _ = _get_user_infos( ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] ) @@ -106,7 +107,7 @@ def portal_update( from yunohost.domain import domain_list domains = domain_list()["domains"] - username, current_user, ldap_interface = _get_user_infos( + username, domain, current_user, ldap_interface = _get_user_infos( ["givenName", "sn", "cn", "mail", "maildrop", "memberOf"] ) new_attr_dict = {} @@ -176,8 +177,12 @@ def portal_update( # Ensure compatibility and sufficiently complex password try: assert_password_is_compatible(newpassword) - is_admin = "cn=admins,ou=groups,dc=yunohost,dc=org" in current_user["memberOf"] - assert_password_is_strong_enough("admin" if is_admin else "user", newpassword) + is_admin = ( + "cn=admins,ou=groups,dc=yunohost,dc=org" in current_user["memberOf"] + ) + assert_password_is_strong_enough( + "admin" if is_admin else "user", newpassword + ) except YunohostValidationError as e: raise YunohostValidationError(e.key, path="newpassword") From bfedf144b30e5f295bde724e2db0c69c1f85dbd6 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 4 Sep 2023 16:31:58 +0200 Subject: [PATCH 077/436] add settings getter + /public route to get settings and public apps --- src/portal.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/portal.py b/src/portal.py index cdb6fb9ddc..61d0c301ad 100644 --- a/src/portal.py +++ b/src/portal.py @@ -18,10 +18,11 @@ along with this program; if not, see http://www.gnu.org/licenses """ +from pathlib import Path from typing import Any, Union import ldap -from moulinette.utils.filesystem import read_json +from moulinette.utils.filesystem import read_json, read_yaml from moulinette.utils.log import getActionLogger from yunohost.authenticators.ldap_ynhuser import URI, USERDN, Authenticator as Auth from yunohost.user import _hash_user_password @@ -50,6 +51,56 @@ def _get_user_infos( return username, auth["host"], result[0], ldap_interface +def _get_portal_settings(domain: Union[str, None] = None): + from yunohost.domain import DOMAIN_SETTINGS_DIR + + if not domain: + from bottle import request + + domain = request.get_header("host") + + if Path(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml").exists(): + settings = read_yaml(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml") + else: + settings = { + "public": False, + "portal_logo": "", + "portal_theme": "system", + "portal_title": "YunoHost", + "show_other_domains_apps": 1, + } + + settings["domain"] = domain + + return settings + + +def portal_public(): + settings = _get_portal_settings() + settings["apps"] = {} + settings["public"] = settings.pop("default_app") == "portal_public_apps" + + if settings["public"]: + ssowat_conf = read_json("/etc/ssowat/conf.json") + settings["apps"] = { + perm.replace(".main", ""): { + "label": infos["label"], + "url": infos["uris"][0], + } + for perm, infos in ssowat_conf["permissions"].items() + if infos["show_tile"] and infos["public"] + } + + if not settings["show_other_domains_apps"]: + settings["apps"] = { + name: data + for name, data in settings["apps"].items() + if settings["domain"] in data["url"] + } + + return settings + + def portal_me(): """ Get user informations @@ -76,6 +127,10 @@ def portal_me(): if perm in permissions and infos["show_tile"] and username in infos["users"] } + settings = _get_portal_settings(domain=domain) + if not settings["show_other_domains_apps"]: + apps = {name: data for name, data in apps.items() if domain in data["url"]} + result_dict = { "username": username, "fullname": user["cn"][0], From c641f099c5ae8162a91a3e020c16aa944a1df221 Mon Sep 17 00:00:00 2001 From: axolotle Date: Thu, 7 Sep 2023 17:57:08 +0200 Subject: [PATCH 078/436] add temp messy file handling for portal custom logo --- share/config_domain.toml | 3 +-- src/domain.py | 46 +++++++++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/share/config_domain.toml b/share/config_domain.toml index b6a0b51a7a..27493b4e75 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -16,8 +16,7 @@ name = "Features" default = "YunoHost" [feature.portal.portal_logo] - type = "string" - default = "" + type = "file" [feature.portal.portal_theme] type = "select" diff --git a/src/domain.py b/src/domain.py index 9ddb67a769..a476112b29 100644 --- a/src/domain.py +++ b/src/domain.py @@ -728,15 +728,6 @@ def _apply(self): other_app=app_map(raw=True)[self.entity]["/"]["id"], ) - super()._apply() - - # Reload ssowat if default app changed - if ( - "default_app" in self.future_values - and self.future_values["default_app"] != self.values["default_app"] - ): - app_ssowatconf() - portal_options = [ "default_app", "show_other_domains_apps", @@ -755,11 +746,46 @@ def _apply(self): portal_values = { option: self.future_values[option] for option in portal_options } - # FIXME config file should be readable by the portal entity + if portal_values["portal_logo"].startswith("/tmp/ynh_filequestion_"): + # FIXME rework this whole mess + # currently only handling API sent images, need to adapt FileOption + # to handle file extensions and file saving since "bind" is only + # done in bash helpers which are not executed in domain config + if "portal_logo[name]" in self.args or self.values["portal_logo"]: + import mimetypes + import base64 + + if "portal_logo[name]" in self.args: + # FIXME choose where to save the file + filepath = os.path.join("/tmp", self.args["portal_logo[name]"]) + # move the temp file created by FileOption with proper name and extension + os.rename(self.new_values["portal_logo"], filepath) + mimetype = mimetypes.guess_type(filepath) + else: + # image has already been saved, do not overwrite it with the empty temp file created by the FileOption + filepath = self.values["portal_logo"] + mimetype = mimetypes.guess_type(filepath) + + # save the proper path to config panel settings + self.new_values["portal_logo"] = filepath + # save the base64 content with mimetype to portal settings + with open(filepath, "rb") as f: + portal_values["portal_logo"] = mimetype[0] + ":" + base64.b64encode(f.read()).decode("utf-8") + + # FIXME config file should be readable by non-root portal entity write_to_yaml( f"{DOMAIN_SETTINGS_DIR}/{self.entity}.portal.yml", portal_values ) + super()._apply() + + # Reload ssowat if default app changed + if ( + "default_app" in self.future_values + and self.future_values["default_app"] != self.values["default_app"] + ): + app_ssowatconf() + stuff_to_regen_conf = [] if ( "xmpp" in self.future_values From 8eb2e72282ea6bcb2aab3901ffc412d42b273e4a Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 15:13:20 +0200 Subject: [PATCH 079/436] Update Fail2ban jail.conf file from https://sources.debian.org/src/fail2ban/1.0.2-2/config/jail.conf/ --- conf/fail2ban/jail.conf | 301 ++++++++++++++++++++++++++++------------ 1 file changed, 213 insertions(+), 88 deletions(-) diff --git a/conf/fail2ban/jail.conf b/conf/fail2ban/jail.conf index bd522c4ba3..fe8db527db 100644 --- a/conf/fail2ban/jail.conf +++ b/conf/fail2ban/jail.conf @@ -18,7 +18,7 @@ # See man 5 jail.conf for details. # # [DEFAULT] -# bantime = 3600 +# bantime = 1h # # [sshd] # enabled = true @@ -44,10 +44,52 @@ before = paths-debian.conf # MISCELLANEOUS OPTIONS # -# "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not -# ban a host which matches an address in this list. Several addresses can be -# defined using space (and/or comma) separator. -ignoreip = 127.0.0.1/8 +# "bantime.increment" allows to use database for searching of previously banned ip's to increase a +# default ban time using special formula, default it is banTime * 1, 2, 4, 8, 16, 32... +#bantime.increment = true + +# "bantime.rndtime" is the max number of seconds using for mixing with random time +# to prevent "clever" botnets calculate exact time IP can be unbanned again: +#bantime.rndtime = + +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) +#bantime.maxtime = + +# "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, +# default value of factor is 1 and with default value of formula, the ban time +# grows by 1, 2, 4, 8, 16 ... +#bantime.factor = 1 + +# "bantime.formula" used by default to calculate next value of ban time, default value below, +# the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... +#bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor +# +# more aggressive example of formula has the same values only for factor "2.0 / 2.885385" : +#bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) + +# "bantime.multipliers" used to calculate next value of ban time instead of formula, corresponding +# previously ban count and given "bantime.factor" (for multipliers default is 1); +# following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, +# always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours +#bantime.multipliers = 1 2 4 8 16 32 64 +# following example can be used for small initial ban time (bantime=60) - it grows more aggressive at begin, +# for bantime=60 the multipliers are minutes and equal: 1 min, 5 min, 30 min, 1 hour, 5 hour, 12 hour, 1 day, 2 day +#bantime.multipliers = 1 5 30 60 300 720 1440 2880 + +# "bantime.overalljails" (if true) specifies the search of IP in the database will be executed +# cross over all jails, if false (default), only current jail of the ban IP will be searched +#bantime.overalljails = false + +# -------------------- + +# "ignoreself" specifies whether the local resp. own IP addresses should be ignored +# (default is true). Fail2ban will not ban a host which matches such addresses. +#ignoreself = true + +# "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban +# will not ban a host which matches an address in this list. Several addresses +# can be defined using space (and/or comma) separator. +#ignoreip = 127.0.0.1/8 ::1 # External command that will take an tagged arguments to ignore, e.g. , # and return true if the IP is to be ignored. False otherwise. @@ -56,14 +98,17 @@ ignoreip = 127.0.0.1/8 ignorecommand = # "bantime" is the number of seconds that a host is banned. -bantime = 600 +bantime = 10m # A host is banned if it has generated "maxretry" during the last "findtime" # seconds. -findtime = 600 +findtime = 10m # "maxretry" is the number of failures before a host get banned. -maxretry = 10 +maxretry = 5 + +# "maxmatches" is the number of matches stored in ticket (resolvable via tag in actions). +maxmatches = %(maxretry)s # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling", "systemd" and "auto". @@ -113,10 +158,13 @@ logencoding = auto enabled = false +# "mode" defines the mode of the filter (see corresponding filter implementation for more info). +mode = normal + # "filter" defines the filter to use by the jail. # By default jails have names matching their filter name # -filter = %(__name__)s +filter = %(__name__)s[mode=%(mode)s] # @@ -130,7 +178,7 @@ filter = %(__name__)s destemail = root@localhost # Sender email address used solely for some actions -sender = root@localhost +sender = root@ # E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the # mailing. Change mta configuration parameter to mail if you want to @@ -140,8 +188,8 @@ mta = sendmail # Default protocol protocol = tcp -# Specify chain where jumps would need to be added in iptables-* actions -chain = INPUT +# Specify chain where jumps would need to be added in ban-actions expecting parameter chain +chain = # Ports to be banned # Usually should be overridden in a particular jail @@ -161,51 +209,53 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] +action_mwl = %(action_)s + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath=%(logpath)s, port="%(port)s"] +action_xarf = %(action_)s + xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] + +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +# action = %(action_)s +# apprise # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # -# See the IMPORTANT note in action.d/blocklist_de.conf for when to -# use this action. Create a file jail.d/blocklist_de.local containing -# [Init] -# blocklist_de_apikey = {api key from registration] +# See the IMPORTANT note in action.d/blocklist_de.conf for when to use this action. +# Specify expected parameters in file action.d/blocklist_de.local or if the interpolation +# `action_blocklist_de` used for the action, set value of `blocklist_de_apikey` +# in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in +# corresponding jail.d/my-jail.local file). # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. +# Report ban via abuseipdb.com. # -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. +# See action.d/abuseipdb.conf for usage example and details. # -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] +action_abuseipdb = abuseipdb # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local @@ -223,15 +273,10 @@ action = %(action_)s [sshd] -port = ssh -logpath = %(sshd_log)s -backend = %(sshd_backend)s - - -[sshd-ddos] -# This jail corresponds to the standard configuration in Fail2ban. -# The mail-whois action send a notification e-mail with a whois request -# in the body. +# To use more aggressive sshd modes set filter parameter "mode" in jail.local: +# normal (default), ddos, extra or aggressive (combines all). +# See "tests/files/logs/sshd" or "filter.d/sshd.conf" for usage example and details. +#mode = normal port = ssh logpath = %(sshd_log)s backend = %(sshd_backend)s @@ -265,7 +310,7 @@ logpath = %(apache_error_log)s # for email addresses. The mail outputs are buffered. port = http,https logpath = %(apache_access_log)s -bantime = 172800 +bantime = 48h maxretry = 1 @@ -301,7 +346,7 @@ maxretry = 2 port = http,https logpath = %(apache_access_log)s maxretry = 1 -ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot +ignorecommand = %(fail2ban_confpath)s/filter.d/ignorecommands/apache-fakegooglebot [apache-modsecurity] @@ -321,12 +366,15 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log +# To use more aggressive http-auth modes set filter parameter "mode" in jail.local: +# normal (default), aggressive (combines all), auth or fallback +# See "tests/files/logs/nginx-http-auth" or "filter.d/nginx-http-auth.conf" for usage example and details. [nginx-http-auth] - +# mode = normal port = http,https logpath = %(nginx_error_log)s @@ -342,8 +390,10 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_error_log)s -maxretry = 2 +[nginx-bad-request] +port = http,https +logpath = %(nginx_access_log)s # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year @@ -377,6 +427,8 @@ logpath = %(lighttpd_error_log)s port = http,https logpath = %(roundcube_errors_log)s +# Use following line in your jail.local if roundcube logs to journal. +#backend = %(syslog_backend)s [openwebmail] @@ -426,11 +478,13 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login port = 2812 logpath = /var/log/monit + /var/log/monit.log [webmin-auth] @@ -513,27 +567,29 @@ logpath = %(vsftpd_log)s # ASSP SMTP Proxy Jail [assp] -port = smtp,submission +port = smtp,465,submission logpath = /root/path/to/assp/logs/maillog.txt [courier-smtp] -port = smtp,submission +port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix] - -port = smtp,submission -logpath = %(postfix_log)s -backend = %(postfix_backend)s +# To use another modes set filter parameter "mode" in jail.local: +mode = more +port = smtp,465,submission +logpath = %(postfix_log)s +backend = %(postfix_backend)s [postfix-rbl] -port = smtp,submission +filter = postfix[mode=rbl] +port = smtp,465,submission logpath = %(postfix_log)s backend = %(postfix_backend)s maxretry = 1 @@ -541,14 +597,17 @@ maxretry = 1 [sendmail-auth] -port = submission,smtp +port = submission,465,smtp logpath = %(syslog_mail)s backend = %(syslog_backend)s [sendmail-reject] - -port = smtp,submission +# To use more aggressive modes set filter parameter "mode" in jail.local: +# normal (default), extra or aggressive +# See "tests/files/logs/sendmail-reject" or "filter.d/sendmail-reject.conf" for usage example and details. +#mode = normal +port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s @@ -556,7 +615,7 @@ backend = %(syslog_backend)s [qmail-rbl] filter = qmail -port = smtp,submission +port = smtp,465,submission logpath = /service/qmail/log/main/current @@ -564,14 +623,14 @@ logpath = /service/qmail/log/main/current # but can be set by syslog_facility in the dovecot configuration. [dovecot] -port = pop3,pop3s,imap,imaps,submission,sieve +port = pop3,pop3s,imap,imaps,submission,465,sieve logpath = %(dovecot_log)s backend = %(dovecot_backend)s [sieve] -port = smtp,submission +port = smtp,465,submission logpath = %(dovecot_log)s backend = %(dovecot_backend)s @@ -583,20 +642,21 @@ logpath = %(solidpop3d_log)s [exim] - -port = smtp,submission +# see filter.d/exim.conf for further modes supported from filter: +#mode = normal +port = smtp,465,submission logpath = %(exim_main_log)s [exim-spam] -port = smtp,submission +port = smtp,465,submission logpath = %(exim_main_log)s [kerio] -port = imap,smtp,imaps +port = imap,smtp,imaps,465 logpath = /opt/kerio/mailserver/store/logs/security.log @@ -607,14 +667,15 @@ logpath = /opt/kerio/mailserver/store/logs/security.log [courier-auth] -port = smtp,submission,imaps,pop3,pop3s +port = smtp,465,submission,imap,imaps,pop3,pop3s logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix-sasl] -port = smtp,submission,imap,imaps,pop3,pop3s +filter = postfix[mode=auth] +port = smtp,465,submission,imap,imaps,pop3,pop3s # You might consider monitoring /var/log/mail.warn instead if you are # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. @@ -631,7 +692,7 @@ backend = %(syslog_backend)s [squirrelmail] -port = smtp,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks +port = smtp,465,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log @@ -684,8 +745,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -696,9 +757,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -706,16 +766,22 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 +# enable adminlog; it will log to a file inside znc's directory by default. +[znc-adminlog] + +port = 6667 +logpath = /var/lib/znc/moddata/adminlog/znc.log + + # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld] or # equivalent section: -# log-warning = 2 +# log-warnings = 2 # # for syslog (daemon facility) # [mysqld_safe] @@ -731,6 +797,14 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +[mssql-auth] +# Default configuration for Microsoft SQL Server for Linux +# See the 'mssql-conf' manpage how to change logpath or port +logpath = /var/opt/mssql/log/errorlog +port = 1433 +filter = mssql-auth + + # Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] # change port when running with "--shardsvr" or "--configsvr" runtime operation @@ -749,8 +823,8 @@ logpath = /var/log/mongodb/mongodb.log logpath = /var/log/fail2ban.log banaction = %(banaction_allports)s -bantime = 604800 ; 1 week -findtime = 86400 ; 1 day +bantime = 1w +findtime = 1d # Generic filter for PAM. Has to be used with action which bans all @@ -786,11 +860,31 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] + +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log + +[bitwarden] +port = http,https +logpath = /home/*/bwdata/logs/identity/Identity/log.txt + +[centreon] +port = http,https +logpath = /var/log/centreon/login.log # consider low maxretry and a long bantime # nobody except your own Nagios server should ever probe nrpe @@ -824,7 +918,9 @@ filter = apache-pass[knocking_url="%(knocking_url)s"] logpath = %(apache_access_log)s blocktype = RETURN returntype = DROP -bantime = 3600 +action = %(action_)s[blocktype=%(blocktype)s, returntype=%(returntype)s, + actionstart_on_demand=false, actionrepair_on_unban=true] +bantime = 1h maxretry = 1 findtime = 1 @@ -832,8 +928,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log @@ -851,5 +947,34 @@ logpath = /var/log/haproxy.log [slapd] port = ldap,ldaps -filter = slapd logpath = /var/log/slapd.log + +[domino-smtp] +port = smtp,ssmtp +logpath = /home/domino01/data/IBM_TECHNICAL_SUPPORT/console.log + +[phpmyadmin-syslog] +port = http,https +logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s + + +[zoneminder] +# Zoneminder HTTP/HTTPS web interface auth +# Logs auth failures to apache2 error log +port = http,https +logpath = %(apache_error_log)s + +[traefik-auth] +# to use 'traefik-auth' filter you have to configure your Traefik instance, +# see `filter.d/traefik-auth.conf` for details and service example. +port = http,https +logpath = /var/log/traefik/access.log + +[scanlogd] +logpath = %(syslog_local0)s +banaction = %(banaction_allports)s + +[monitorix] +port = 8080 +logpath = /var/log/monitorix-httpd From d0b65d56614a04c44ae917a4872485f39ced2132 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 15:17:25 +0200 Subject: [PATCH 080/436] revert important variables in fail2ban jail.conf --- conf/fail2ban/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/fail2ban/jail.conf b/conf/fail2ban/jail.conf index fe8db527db..2ffc71c288 100644 --- a/conf/fail2ban/jail.conf +++ b/conf/fail2ban/jail.conf @@ -178,7 +178,7 @@ filter = %(__name__)s[mode=%(mode)s] destemail = root@localhost # Sender email address used solely for some actions -sender = root@ +sender = root@localhost # E-mail action. Since 0.8.1 Fail2Ban uses sendmail MTA for the # mailing. Change mta configuration parameter to mail if you want to @@ -189,7 +189,7 @@ mta = sendmail protocol = tcp # Specify chain where jumps would need to be added in ban-actions expecting parameter chain -chain = +chain = INPUT # Ports to be banned # Usually should be overridden in a particular jail From 2bd3dd2bba93ec414b368d308d7684ca1ca3591b Mon Sep 17 00:00:00 2001 From: Kayou Date: Fri, 8 Sep 2023 22:31:08 +0200 Subject: [PATCH 081/436] set maxretry to 10 --- conf/fail2ban/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/fail2ban/jail.conf b/conf/fail2ban/jail.conf index 2ffc71c288..8b6a2ec6db 100644 --- a/conf/fail2ban/jail.conf +++ b/conf/fail2ban/jail.conf @@ -105,7 +105,7 @@ bantime = 10m findtime = 10m # "maxretry" is the number of failures before a host get banned. -maxretry = 5 +maxretry = 10 # "maxmatches" is the number of matches stored in ticket (resolvable via tag in actions). maxmatches = %(maxretry)s From e77e9a0a9a804241159468b0024fbcf3e6801a40 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 23:13:38 +0200 Subject: [PATCH 082/436] backup/restore tests from 11.2 --- src/tests/test_backuprestore.py | 52 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index eb59d4feaf..4a59e75745 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -49,8 +49,8 @@ def setup_function(function): for m in function.__dict__.get("pytestmark", []) } - if "with_wordpress_archive_from_4p2" in markers: - add_archive_wordpress_from_4p2() + if "with_wordpress_archive_from_11p2" in markers: + add_archive_wordpress_from_11p2() assert len(backup_list()["archives"]) == 1 if "with_legacy_app_installed" in markers: @@ -72,8 +72,8 @@ def setup_function(function): ) assert app_is_installed("backup_recommended_app") - if "with_system_archive_from_4p2" in markers: - add_archive_system_from_4p2() + if "with_system_archive_from11p2" in markers: + add_archive_system_from_11p2() assert len(backup_list()["archives"]) == 1 if "with_permission_app_installed" in markers: @@ -148,7 +148,7 @@ def app_is_installed(app): def backup_test_dependencies_are_met(): # Dummy test apps (or backup archives) assert os.path.exists( - os.path.join(get_test_apps_dir(), "backup_wordpress_from_4p2") + os.path.join(get_test_apps_dir(), "backup_wordpress_from_11p2") ) assert os.path.exists(os.path.join(get_test_apps_dir(), "legacy_app_ynh")) assert os.path.exists( @@ -211,23 +211,23 @@ def install_app(app, path, additionnal_args=""): ) -def add_archive_wordpress_from_4p2(): +def add_archive_wordpress_from_11p2(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_wordpress_from_4p2/backup.tar") - + " /home/yunohost.backup/archives/backup_wordpress_from_4p2.tar" + + os.path.join(get_test_apps_dir(), "backup_wordpress_from_11p2/backup.tar") + + " /home/yunohost.backup/archives/backup_wordpress_from_11p2.tar" ) -def add_archive_system_from_4p2(): +def add_archive_system_from_11p2(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_system_from_4p2/backup.tar") - + " /home/yunohost.backup/archives/backup_system_from_4p2.tar" + + os.path.join(get_test_apps_dir(), "backup_system_from_11p2/backup.tar") + + " /home/yunohost.backup/archives/backup_system_from_11p2.tar" ) @@ -292,12 +292,12 @@ def test_backup_and_restore_all_sys(): # -# System restore from 3.8 # +# System restore from 11.2 # # -@pytest.mark.with_system_archive_from_4p2 -def test_restore_system_from_Ynh4p2(monkeypatch): +@pytest.mark.with_system_archive_from_11p2 +def test_restore_system_from_Ynh11p2(monkeypatch): name = random_ascii(8) # Backup current system with message("backup_created", name=name): @@ -305,7 +305,7 @@ def test_restore_system_from_Ynh4p2(monkeypatch): archives = backup_list()["archives"] assert len(archives) == 2 - # Restore system archive from 3.8 + # Restore system archive from 11.2 try: with message("restore_complete"): backup_restore( @@ -439,18 +439,17 @@ def test_backup_using_copy_method(): # App restore # # -# FIXME : switch to a backup from 11.x @pytest.mark.skip -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") -def test_restore_app_wordpress_from_Ynh4p2(): +def test_restore_app_wordpress_from_Ynh11p2(): with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_script_failure_handling(monkeypatch, mocker): def custom_hook_exec(name, *args, **kwargs): @@ -471,7 +470,7 @@ def custom_hook_exec(name, *args, **kwargs): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_app_not_enough_free_space(monkeypatch, mocker): def custom_free_space_in_directory(dirpath): return 0 @@ -490,7 +489,7 @@ def custom_free_space_in_directory(dirpath): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_app_not_in_backup(mocker): assert not _is_installed("wordpress") assert not _is_installed("yoloswag") @@ -505,9 +504,8 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") -# FIXME : switch to a backup from 11.x @pytest.mark.skip -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): assert not _is_installed("wordpress") @@ -619,17 +617,17 @@ def test_restore_archive_with_no_json(mocker): backup_restore(name="badbackup", force=True) -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_archive_with_bad_archive(mocker): # Break the archive os.system( - "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_4p2.tar > /home/yunohost.backup/archives/backup_wordpress_from_4p2_bad.tar" + "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_11p2.tar > /home/yunohost.backup/archives/backup_wordpress_from_11p2_bad.tar" ) - assert "backup_wordpress_from_4p2_bad" in backup_list()["archives"] + assert "backup_wordpress_from_11p2_bad" in backup_list()["archives"] with raiseYunohostError(mocker, "backup_archive_corrupted"): - backup_restore(name="backup_wordpress_from_4p2_bad", force=True) + backup_restore(name="backup_wordpress_from_11p2_bad", force=True) clean_tmp_backup_directory() From aed8ecb64594f71c0340a172b7bb8fb6ca2d82ed Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 23:47:57 +0200 Subject: [PATCH 083/436] do not skip tests from 11.2 --- src/tests/test_backuprestore.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index 4a59e75745..c0a55b65ba 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -439,7 +439,6 @@ def test_backup_using_copy_method(): # App restore # # -@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_wordpress_from_Ynh11p2(): @@ -504,7 +503,6 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") -@pytest.mark.skip @pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): From 142fad4b7898686f970d9bbae704e7da64203ed5 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 8 Sep 2023 23:50:30 +0200 Subject: [PATCH 084/436] typo --- src/tests/test_backuprestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index c0a55b65ba..ad9a1c1f2e 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -72,7 +72,7 @@ def setup_function(function): ) assert app_is_installed("backup_recommended_app") - if "with_system_archive_from11p2" in markers: + if "with_system_archive_from_11p2" in markers: add_archive_system_from_11p2() assert len(backup_list()["archives"]) == 1 From 9e87ea88df37bd7e924f5659c6ffe33ded8dbebf Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 19:30:55 +0200 Subject: [PATCH 085/436] portal-api: improve semantic for yunohost public portal stuff --- share/config_domain.toml | 2 +- src/portal.py | 20 ++++++++++---------- src/utils/configpanel.py | 2 +- src/utils/form.py | 7 ++++--- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/share/config_domain.toml b/share/config_domain.toml index 27493b4e75..1239b1fea7 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -30,7 +30,7 @@ name = "Features" type = "app" filter = "is_webapp" default = "_none" - list_portal = true + add_yunohost_portal_to_choices = true [feature.mail] diff --git a/src/portal.py b/src/portal.py index 61d0c301ad..cc6c03e4ba 100644 --- a/src/portal.py +++ b/src/portal.py @@ -76,13 +76,13 @@ def _get_portal_settings(domain: Union[str, None] = None): def portal_public(): - settings = _get_portal_settings() - settings["apps"] = {} - settings["public"] = settings.pop("default_app") == "portal_public_apps" + portal_settings = _get_portal_settings() + portal_settings["apps"] = {} + portal_settings["public"] = portal_settings.pop("default_app") == "_yunohost_portal_with_public_apps" - if settings["public"]: + if portal_settings["public"]: ssowat_conf = read_json("/etc/ssowat/conf.json") - settings["apps"] = { + portal_settings["apps"] = { perm.replace(".main", ""): { "label": infos["label"], "url": infos["uris"][0], @@ -91,14 +91,14 @@ def portal_public(): if infos["show_tile"] and infos["public"] } - if not settings["show_other_domains_apps"]: - settings["apps"] = { + if not portal_settings["show_other_domains_apps"]: + portal_settings["apps"] = { name: data - for name, data in settings["apps"].items() - if settings["domain"] in data["url"] + for name, data in portal_settings["apps"].items() + if portal_settings["domain"] in data["url"] } - return settings + return portal_settings def portal_me(): diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 30a35e4103..86dea2e7d3 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -386,7 +386,7 @@ def _get_config_panel(self): "filter", "readonly", "enabled", - "list_portal", + "add_yunohost_portal_to_choices", # "confirm", # TODO: to ask confirmation before running an action ], "defaults": {}, diff --git a/src/utils/form.py b/src/utils/form.py index c7bc25305b..f201f507b7 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -914,7 +914,7 @@ def __init__(self, question): super().__init__(question) self.filter = question.get("filter", None) - self.list_portal = question.get("list_portal", False) + self.add_yunohost_portal_to_choices = question.get("add_yunohost_portal_to_choices", False) apps = app_list(full=True)["apps"] @@ -930,8 +930,9 @@ def _app_display(app): return app["label"] + domain_path_or_id self.choices = {"_none": "---"} - if self.list_portal: - self.choices["portal_public_apps"] = "Portal" + if self.add_yunohost_portal_to_choices: + # FIXME: i18n + self.choices["_yunohost_portal_with_public_apps"] = "YunoHost's portal with public apps" self.choices.update({app["id"]: _app_display(app) for app in apps}) From a0dbf6a5b0ecf3965563b196f69f459b4db00c81 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:40:50 +0200 Subject: [PATCH 086/436] portal: improve domain setting fetch + set show_other_domains_apps to false by default ? --- share/config_domain.toml | 2 +- src/portal.py | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/share/config_domain.toml b/share/config_domain.toml index 1239b1fea7..6fc5fc50a9 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -9,7 +9,7 @@ name = "Features" [feature.portal.show_other_domains_apps] type = "boolean" - default = 1 + default = false [feature.portal.portal_title] type = "string" diff --git a/src/portal.py b/src/portal.py index cc6c03e4ba..c7a0eb68b9 100644 --- a/src/portal.py +++ b/src/portal.py @@ -59,18 +59,19 @@ def _get_portal_settings(domain: Union[str, None] = None): domain = request.get_header("host") - if Path(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml").exists(): - settings = read_yaml(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml") - else: - settings = { - "public": False, - "portal_logo": "", - "portal_theme": "system", - "portal_title": "YunoHost", - "show_other_domains_apps": 1, - } + assert domain and "/" not in domain + + settings = { + "public": False, + "portal_logo": "", + "portal_theme": "system", + "portal_title": "YunoHost", + "show_other_domains_apps": false, + "domain": domain, + } - settings["domain"] = domain + if Path(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml").exists(): + settings.update(read_yaml(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml")) return settings From b61a16421b134d7899542a94c5da3f14ca7d639c Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 20:51:45 +0200 Subject: [PATCH 087/436] portal-api: fix cookie secret initialization --- hooks/conf_regen/01-yunohost | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 4d53997a58..4f934db769 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -57,6 +57,12 @@ do_init_regen() { chmod 700 /var/cache/yunohost getent passwd ynh-portal &>/dev/null || useradd --no-create-home --shell /usr/sbin/nologin --system --user-group ynh-portal + if [ ! -e /etc/yunohost/.ssowat_cookie_secret ]; then + # NB: we need this to be exactly 32 char long, because it is later used as a key for AES256 + dd if=/dev/urandom bs=1 count=1000 2>/dev/null | tr --complement --delete 'A-Za-z0-9' | head -c 32 > /etc/yunohost/.ssowat_cookie_secret + fi + chown ynh-portal:root /etc/yunohost/.ssowat_cookie_secret + chmod 400 /etc/yunohost/.ssowat_cookie_secret cp yunohost-api.service /etc/systemd/system/yunohost-api.service cp yunohost-portal-api.service /etc/systemd/system/yunohost-portal-api.service @@ -68,6 +74,8 @@ do_init_regen() { systemctl enable yunohost-api.service --quiet systemctl start yunohost-api.service + + systemctl enable yunohost-portal-api.service systemctl start yunohost-portal-api.service From f617b97d806c264d3999a236945805dae9d985ca Mon Sep 17 00:00:00 2001 From: Alexandre Aubin <4533074+alexAubin@users.noreply.github.com> Date: Wed, 27 Sep 2023 22:08:26 +0200 Subject: [PATCH 088/436] portal/ssowat: fix conf initialization --- hooks/conf_regen/01-yunohost | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 4f934db769..edf64012e0 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -48,6 +48,9 @@ do_init_regen() { echo "{}" >'/etc/ssowat/conf.json.persistent' chmod 644 /etc/ssowat/conf.json.persistent chown root:root /etc/ssowat/conf.json.persistent + echo "{}" >'/etc/ssowat/conf.json' + chmod 644 /etc/ssowat/conf.json + chown root:root /etc/ssowat/conf.json # Empty service conf touch /etc/yunohost/services.yml From d3418479a20696f6a8ae78aecba2dc3c6dd01e88 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 22:25:44 +0200 Subject: [PATCH 089/436] fix remaining log.getActionLogger --- src/diagnosers/00-basesystem.py | 4 ++-- src/diagnosers/10-ip.py | 4 ++-- src/diagnosers/12-dnsrecords.py | 4 ++-- src/diagnosers/24-mail.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/diagnosers/00-basesystem.py b/src/diagnosers/00-basesystem.py index 336271bd17..dec8c3c2cd 100644 --- a/src/diagnosers/00-basesystem.py +++ b/src/diagnosers/00-basesystem.py @@ -19,9 +19,9 @@ import os import json import subprocess +import logging from typing import List -from moulinette.utils import log from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file, read_json, write_to_json from yunohost.diagnosis import Diagnoser @@ -31,7 +31,7 @@ system_arch, ) -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): diff --git a/src/diagnosers/10-ip.py b/src/diagnosers/10-ip.py index 4f9cd97083..3d457991e4 100644 --- a/src/diagnosers/10-ip.py +++ b/src/diagnosers/10-ip.py @@ -19,9 +19,9 @@ import re import os import random +import logging from typing import List -from moulinette.utils import log from moulinette.utils.network import download_text from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file @@ -30,7 +30,7 @@ from yunohost.utils.network import get_network_interfaces from yunohost.settings import settings_get -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): diff --git a/src/diagnosers/12-dnsrecords.py b/src/diagnosers/12-dnsrecords.py index 19becb753d..59a5e9857f 100644 --- a/src/diagnosers/12-dnsrecords.py +++ b/src/diagnosers/12-dnsrecords.py @@ -18,11 +18,11 @@ # import os import re +import logging from typing import List from datetime import datetime, timedelta from publicsuffix2 import PublicSuffixList -from moulinette.utils import log from moulinette.utils.process import check_output from yunohost.utils.dns import ( @@ -39,7 +39,7 @@ _get_relative_name_for_dns_zone, ) -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): diff --git a/src/diagnosers/24-mail.py b/src/diagnosers/24-mail.py index c7fe9d04b3..88748661c4 100644 --- a/src/diagnosers/24-mail.py +++ b/src/diagnosers/24-mail.py @@ -19,11 +19,11 @@ import os import dns.resolver import re +import logging from typing import List from subprocess import CalledProcessError -from moulinette.utils import log from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_yaml @@ -34,7 +34,7 @@ DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/dnsbl_list.yml" -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): From 127b6121d1692222f44ce48c8a48e6d30a3c996d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Wed, 27 Sep 2023 22:29:27 +0200 Subject: [PATCH 090/436] meh --- src/diagnosis.py | 1 - src/portal.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/diagnosis.py b/src/diagnosis.py index be3208b02f..90709ad238 100644 --- a/src/diagnosis.py +++ b/src/diagnosis.py @@ -24,7 +24,6 @@ from logging import getLogger from moulinette import m18n, Moulinette -from moulinette.utils import log from moulinette.utils.filesystem import ( read_json, write_to_json, diff --git a/src/portal.py b/src/portal.py index c7a0eb68b9..5b3af4f49f 100644 --- a/src/portal.py +++ b/src/portal.py @@ -66,7 +66,7 @@ def _get_portal_settings(domain: Union[str, None] = None): "portal_logo": "", "portal_theme": "system", "portal_title": "YunoHost", - "show_other_domains_apps": false, + "show_other_domains_apps": False, "domain": domain, } From 814696e9c10425c115d28be804602f06ec862338 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 29 Sep 2023 14:34:01 +0200 Subject: [PATCH 091/436] portal: redirect to $host/yunohost/admin by default (cf recent commit in ssowat) --- src/domain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/domain.py b/src/domain.py index 273fa9e6d9..a5a68ca0f9 100644 --- a/src/domain.py +++ b/src/domain.py @@ -117,6 +117,10 @@ def _get_domain_portal_dict(): out[domain] = f'{parent or domain}/yunohost/sso' + # By default, redirect to $host/yunohost/admin for domains not listed in the dict + # maybe in the future, we can allow to tweak this + out["default"] = "/yunohost/admin" + return dict(out) From 385c131d0cf472f6e94d9bef9fa0d068eaa4c697 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 29 Sep 2023 16:53:18 +0200 Subject: [PATCH 092/436] regenconf: fix dummy warning --- hooks/conf_regen/01-yunohost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index edf64012e0..adccfa17c6 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -217,7 +217,7 @@ do_post_regen() { find /etc/systemd/system/*.service -type f | xargs -r chown root:root find /etc/systemd/system/*.service -type f | xargs -r chmod 0644 - if ls -l /etc/php/*/fpm/pool.d/*.conf + if ls -l /etc/php/*/fpm/pool.d/*.conf 2>/dev/null then chown root:root /etc/php/*/fpm/pool.d/*.conf chmod 644 /etc/php/*/fpm/pool.d/*.conf From 2fc2acea51d0fa1d743634506b9e6a1bb09ebecb Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Tue, 3 Oct 2023 19:58:22 +0200 Subject: [PATCH 093/436] portalapi: misc fixes related to logging, edgecases --- src/portal.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/portal.py b/src/portal.py index 5b3af4f49f..e89d176fd2 100644 --- a/src/portal.py +++ b/src/portal.py @@ -20,10 +20,10 @@ """ from pathlib import Path from typing import Any, Union - +import logging import ldap + from moulinette.utils.filesystem import read_json, read_yaml -from moulinette.utils.log import getActionLogger from yunohost.authenticators.ldap_ynhuser import URI, USERDN, Authenticator as Auth from yunohost.user import _hash_user_password from yunohost.utils.error import YunohostError, YunohostValidationError @@ -33,7 +33,7 @@ assert_password_is_strong_enough, ) -logger = getActionLogger("portal") +logger = logging.getLogger("portal") ADMIN_ALIASES = ["root", "admin", "admins", "webmaster", "postmaster", "abuse"] @@ -77,9 +77,10 @@ def _get_portal_settings(domain: Union[str, None] = None): def portal_public(): + portal_settings = _get_portal_settings() portal_settings["apps"] = {} - portal_settings["public"] = portal_settings.pop("default_app") == "_yunohost_portal_with_public_apps" + portal_settings["public"] = portal_settings.pop("default_app", None) == "_yunohost_portal_with_public_apps" if portal_settings["public"]: ssowat_conf = read_json("/etc/ssowat/conf.json") @@ -219,6 +220,9 @@ def portal_update( ] if newpassword: + + # FIXME: this ldap stuff should be handled in utils/ldap.py imho ? + # Check that current password is valid try: con = ldap.ldapobject.ReconnectLDAPObject(URI, retry_max=0) From 0548af0c25fe6e008c36e29a4e8054f78b8c2dad Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Thu, 5 Oct 2023 18:07:58 +0200 Subject: [PATCH 094/436] ci: add git status to debug what commit exactly is used during builds ... --- .gitlab/ci/build.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/ci/build.gitlab-ci.yml b/.gitlab/ci/build.gitlab-ci.yml index 610580dacd..f1c137ad38 100644 --- a/.gitlab/ci/build.gitlab-ci.yml +++ b/.gitlab/ci/build.gitlab-ci.yml @@ -13,6 +13,8 @@ .build_script: &build_script - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" install devscripts --no-install-recommends - cd $YNH_BUILD_DIR/$PACKAGE + - git status || true + - git log -n 1 || true - VERSION=$(dpkg-parsechangelog -S Version 2>/dev/null) - VERSION_NIGHTLY="${VERSION}+$(date +%Y%m%d%H%M)" - dch --package "${PACKAGE}" --force-bad-version -v "${VERSION_NIGHTLY}" -D "unstable" --force-distribution "Daily build." From fae3b676ea760f43999c7f13b3e201af2baff31d Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Fri, 6 Oct 2023 17:06:53 +0200 Subject: [PATCH 095/436] Getting crazy about the ssowat/nginx stupid issue ... --- .gitlab/ci/install.gitlab-ci.yml | 4 ++++ .gitlab/ci/test.gitlab-ci.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.gitlab/ci/install.gitlab-ci.yml b/.gitlab/ci/install.gitlab-ci.yml index 65409c6ebb..0f5571a570 100644 --- a/.gitlab/ci/install.gitlab-ci.yml +++ b/.gitlab/ci/install.gitlab-ci.yml @@ -17,7 +17,9 @@ upgrade: image: "after-install" script: - apt-get update -o Acquire::Retries=3 + - systemctl restart nginx || journalctl -u nginx -n 50 --no-pager --no-hostname - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb + - systemctl restart nginx || journalctl -u nginx -n 50 --no-pager --no-hostname install-postinstall: @@ -25,5 +27,7 @@ install-postinstall: image: "before-install" script: - apt-get update -o Acquire::Retries=3 + - systemctl restart nginx || journalctl -u nginx -n 50 --no-pager --no-hostname - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb + - systemctl restart nginx || journalctl -u nginx -n 50 --no-pager --no-hostname - yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 2ec84c114e..f0a61c3c9f 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -1,6 +1,8 @@ .install_debs: &install_debs - apt-get update -o Acquire::Retries=3 + - systemctl restart nginx || journalctl -u nginx -n 50 --no-pager --no-hostname - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb + - systemctl restart nginx || journalctl -u nginx -n 50 --no-pager --no-hostname .test-stage: stage: test From 80362269357aea416362bef57c2852985f45fad1 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 7 Oct 2023 17:12:26 +0200 Subject: [PATCH 096/436] Typo --- src/app.py | 4 ++-- src/utils/legacy.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app.py b/src/app.py index e8317099d5..eca5286825 100644 --- a/src/app.py +++ b/src/app.py @@ -1657,10 +1657,10 @@ def app_ssowatconf(): redirected_regex.update(app_settings.get("redirected_regex", {})) from .utils.legacy import ( - translate_legacy_default_app_in_ssowant_conf_json_persistent, + translate_legacy_default_app_in_ssowat_conf_json_persistent, ) - translate_legacy_default_app_in_ssowant_conf_json_persistent() + translate_legacy_default_app_in_ssowat_conf_json_persistent() for domain in domains: default_app = domain_config_get(domain, "feature.app.default_app") diff --git a/src/utils/legacy.py b/src/utils/legacy.py index dfe8da250b..af9c08b980 100644 --- a/src/utils/legacy.py +++ b/src/utils/legacy.py @@ -114,7 +114,7 @@ def legacy_permission_label(app, permission_type): ) -def translate_legacy_default_app_in_ssowant_conf_json_persistent(): +def translate_legacy_default_app_in_ssowat_conf_json_persistent(): from yunohost.app import app_list from yunohost.domain import domain_config_set From a0ce7c2d28ebba2df27bba2e7e5064c99533f310 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Sat, 7 Oct 2023 17:40:06 +0200 Subject: [PATCH 097/436] ssowatconf: drop unused redirected_regex mechanism + we don't need the label+show_tile info in ssowat conf anymore --- src/app.py | 28 ++------------------------ src/utils/legacy.py | 49 --------------------------------------------- 2 files changed, 2 insertions(+), 75 deletions(-) diff --git a/src/app.py b/src/app.py index eca5286825..adcea56e4b 100644 --- a/src/app.py +++ b/src/app.py @@ -1619,8 +1619,6 @@ def app_ssowatconf(): permissions = { "core_skipped": { "users": [], - "label": "Core permissions - skipped", - "show_tile": False, "auth_header": False, "public": True, "uris": [domain + "/yunohost/admin" for domain in domains] @@ -1635,11 +1633,6 @@ def app_ssowatconf(): } } - # FIXME : what's the reason we do this only for the maindomain ? x_X - redirected_regex = { - main_domain + r"/yunohost[\/]?$": "https://" + main_domain + "/yunohost/sso/" - } - redirected_urls = {} apps_using_remote_user_var_in_nginx = ( check_output( @@ -1649,19 +1642,8 @@ def app_ssowatconf(): .split("\n") ) - for app in _installed_apps(): - app_settings = read_yaml(APPS_SETTING_PATH + app + "/settings.yml") or {} - - # Redirected - redirected_urls.update(app_settings.get("redirected_urls", {})) - redirected_regex.update(app_settings.get("redirected_regex", {})) - - from .utils.legacy import ( - translate_legacy_default_app_in_ssowat_conf_json_persistent, - ) - - translate_legacy_default_app_in_ssowat_conf_json_persistent() - + # FIXME : this could be handled by nginx's regen conf to further simplify ssowat's code ... + redirected_urls = {} for domain in domains: default_app = domain_config_get(domain, "feature.app.default_app") if default_app != "_none" and _is_installed(default_app): @@ -1691,10 +1673,6 @@ def app_ssowatconf(): "use_remote_user_var_in_nginx_conf": app_id in apps_using_remote_user_var_in_nginx, "users": perm_info["corresponding_users"], - "label": perm_info["label"], - "show_tile": perm_info["show_tile"] - and perm_info["url"] - and (not perm_info["url"].startswith("re:")), "auth_header": perm_info["auth_header"], "public": "visitors" in perm_info["allowed"], "uris": uris, @@ -1703,9 +1681,7 @@ def app_ssowatconf(): conf_dict = { "cookie_secret_file": "/etc/yunohost/.ssowat_cookie_secret", "cookie_name": "yunohost.portal", - "theme": settings_get("misc.portal.portal_theme"), "redirected_urls": redirected_urls, - "redirected_regex": redirected_regex, "domain_portal_urls": _get_domain_portal_dict(), "permissions": permissions, } diff --git a/src/utils/legacy.py b/src/utils/legacy.py index af9c08b980..4f11d28d74 100644 --- a/src/utils/legacy.py +++ b/src/utils/legacy.py @@ -114,55 +114,6 @@ def legacy_permission_label(app, permission_type): ) -def translate_legacy_default_app_in_ssowat_conf_json_persistent(): - from yunohost.app import app_list - from yunohost.domain import domain_config_set - - persistent_file_name = "/etc/ssowat/conf.json.persistent" - if not os.path.exists(persistent_file_name): - return - - # Ugly hack because for some reason so many people have tabs in their conf.json.persistent ... - os.system(r"sed -i 's/\t/ /g' /etc/ssowat/conf.json.persistent") - - # Ugly hack to try not to misarably fail migration - persistent = read_yaml(persistent_file_name) - - if "redirected_urls" not in persistent: - return - - redirected_urls = persistent["redirected_urls"] - - if not any( - from_url.count("/") == 1 and from_url.endswith("/") - for from_url in redirected_urls - ): - return - - apps = app_list()["apps"] - - if not any(app.get("domain_path") in redirected_urls.values() for app in apps): - return - - for from_url, dest_url in redirected_urls.copy().items(): - # Not a root domain, skip - if from_url.count("/") != 1 or not from_url.endswith("/"): - continue - for app in apps: - if app.get("domain_path") != dest_url: - continue - domain_config_set(from_url.strip("/"), "feature.app.default_app", app["id"]) - del redirected_urls[from_url] - - persistent["redirected_urls"] = redirected_urls - - write_to_json(persistent_file_name, persistent, sort_keys=True, indent=4) - - logger.warning( - "YunoHost automatically translated some legacy redirections in /etc/ssowat/conf.json.persistent to match the new default application using domain configuration" - ) - - LEGACY_PHP_VERSION_REPLACEMENTS = [ ("/etc/php5", "/etc/php/8.2"), ("/etc/php/7.0", "/etc/php/8.2"), From 089e0001c20681dc7e4c3b594e6cd4815d6c1048 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 17 Oct 2023 14:15:22 +0200 Subject: [PATCH 098/436] portal: retreive app permissions from ldap --- src/portal.py | 83 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/src/portal.py b/src/portal.py index e89d176fd2..6cd4b78168 100644 --- a/src/portal.py +++ b/src/portal.py @@ -18,16 +18,16 @@ along with this program; if not, see http://www.gnu.org/licenses """ +import logging from pathlib import Path from typing import Any, Union -import logging -import ldap -from moulinette.utils.filesystem import read_json, read_yaml +import ldap +from moulinette.utils.filesystem import read_yaml from yunohost.authenticators.ldap_ynhuser import URI, USERDN, Authenticator as Auth from yunohost.user import _hash_user_password from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.ldap import LDAPInterface +from yunohost.utils.ldap import LDAPInterface, _ldap_path_extract from yunohost.utils.password import ( assert_password_is_compatible, assert_password_is_strong_enough, @@ -51,6 +51,49 @@ def _get_user_infos( return username, auth["host"], result[0], ldap_interface +def _get_apps(username: Union[str, None] = None): + """Get public + user's authorized apps. + If `username` is not given, returns only public apps + (e.g. with `visitors` in group permissions) + """ + SYSTEM_PERMS = ("mail", "xmpp", "sftp", "ssh") + + ldap_interface = LDAPInterface("root") + permissions_infos = ldap_interface.search( + "ou=permission", + "(objectclass=permissionYnh)", + [ + "cn", + "groupPermission", + "inheritPermission", + "URL", + "label", + "showTile", + ], + ) + + apps = {} + + for perm in permissions_infos: + name = perm["cn"][0].replace(".main", "") + + if name in SYSTEM_PERMS or not perm.get("showTile", [False])[0]: + continue + + groups = [_ldap_path_extract(g, "cn") for g in perm["groupPermission"]] + users = [ + _ldap_path_extract(u, "uid") for u in perm.get("inheritPermission", []) + ] + + if username in users or "visitors" in groups: + apps[name] = { + "label": perm["label"][0], + "url": perm["URL"][0], + } + + return apps + + def _get_portal_settings(domain: Union[str, None] = None): from yunohost.domain import DOMAIN_SETTINGS_DIR @@ -80,18 +123,12 @@ def portal_public(): portal_settings = _get_portal_settings() portal_settings["apps"] = {} - portal_settings["public"] = portal_settings.pop("default_app", None) == "_yunohost_portal_with_public_apps" + portal_settings["public"] = ( + portal_settings.pop("default_app", None) == "_yunohost_portal_with_public_apps" + ) if portal_settings["public"]: - ssowat_conf = read_json("/etc/ssowat/conf.json") - portal_settings["apps"] = { - perm.replace(".main", ""): { - "label": infos["label"], - "url": infos["uris"][0], - } - for perm, infos in ssowat_conf["permissions"].items() - if infos["show_tile"] and infos["public"] - } + portal_settings["apps"] = _get_apps() if not portal_settings["show_other_domains_apps"]: portal_settings["apps"] = { @@ -111,23 +148,9 @@ def portal_me(): ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] ) - groups = [ - g.replace("cn=", "").replace(",ou=groups,dc=yunohost,dc=org", "") - for g in user["memberOf"] - ] + groups = [_ldap_path_extract(g, "cn") for g in user["memberOf"]] groups = [g for g in groups if g not in [username, "all_users"]] - - permissions = [ - p.replace("cn=", "").replace(",ou=permission,dc=yunohost,dc=org", "") - for p in user["permission"] - ] - - ssowat_conf = read_json("/etc/ssowat/conf.json") - apps = { - perm.replace(".main", ""): {"label": infos["label"], "url": infos["uris"][0]} - for perm, infos in ssowat_conf["permissions"].items() - if perm in permissions and infos["show_tile"] and username in infos["users"] - } + apps = _get_apps(username) settings = _get_portal_settings(domain=domain) if not settings["show_other_domains_apps"]: From d65cca5ab165863e4598eb5dce88b4b8d0cf3d9e Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 17 Oct 2023 14:15:59 +0200 Subject: [PATCH 099/436] portal: fix decode error --- src/authenticators/ldap_ynhuser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py index 331cf9e259..8a2b10aa0c 100644 --- a/src/authenticators/ldap_ynhuser.py +++ b/src/authenticators/ldap_ynhuser.py @@ -133,7 +133,7 @@ def set_session_cookie(self, infos): response.set_cookie( "yunohost.portal", - jwt.encode(new_infos, session_secret, algorithm="HS256").decode(), + jwt.encode(new_infos, session_secret, algorithm="HS256"), secure=True, httponly=True, path="/", From c5771253636c9155097c67c2a697bc1065f4a52c Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 17 Oct 2023 14:18:21 +0200 Subject: [PATCH 100/436] portal: temp disable 'show_other_domains_apps' settings due to missing domain info in ldap --- share/config_domain.toml | 6 +++--- src/domain.py | 2 +- src/portal.py | 25 +++++++++++++++---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/share/config_domain.toml b/share/config_domain.toml index 6fc5fc50a9..d56a46b8eb 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -7,9 +7,9 @@ name = "Features" [feature.portal] name = "Portal" - [feature.portal.show_other_domains_apps] - type = "boolean" - default = false + # [feature.portal.show_other_domains_apps] + # type = "boolean" + # default = false [feature.portal.portal_title] type = "string" diff --git a/src/domain.py b/src/domain.py index a5a68ca0f9..98420b4be0 100644 --- a/src/domain.py +++ b/src/domain.py @@ -735,7 +735,7 @@ def _apply(self): portal_options = [ "default_app", - "show_other_domains_apps", + # "show_other_domains_apps", "portal_title", "portal_logo", "portal_theme", diff --git a/src/portal.py b/src/portal.py index 6cd4b78168..9326bcc4bf 100644 --- a/src/portal.py +++ b/src/portal.py @@ -109,7 +109,7 @@ def _get_portal_settings(domain: Union[str, None] = None): "portal_logo": "", "portal_theme": "system", "portal_title": "YunoHost", - "show_other_domains_apps": False, + # "show_other_domains_apps": False, "domain": domain, } @@ -130,12 +130,13 @@ def portal_public(): if portal_settings["public"]: portal_settings["apps"] = _get_apps() - if not portal_settings["show_other_domains_apps"]: - portal_settings["apps"] = { - name: data - for name, data in portal_settings["apps"].items() - if portal_settings["domain"] in data["url"] - } + # FIXME/TODO; See: filter apps that are available on specified domain + # if not portal_settings["show_other_domains_apps"]: + # portal_settings["apps"] = { + # name: data + # for name, data in portal_settings["apps"].items() + # if portal_settings["domain"] in data["url"] + # } return portal_settings @@ -152,9 +153,13 @@ def portal_me(): groups = [g for g in groups if g not in [username, "all_users"]] apps = _get_apps(username) - settings = _get_portal_settings(domain=domain) - if not settings["show_other_domains_apps"]: - apps = {name: data for name, data in apps.items() if domain in data["url"]} + # FIXME / TODO: filter apps that are available on specified domain + # settings = _get_portal_settings(domain=domain) + # if not settings["show_other_domains_apps"]: + # apps = {name: data for name, data in apps.items() if domain in data["url"]} + # App's `domain` info is not available in LDAP data, we need another config file + # that would be readable by the `ynh-portal` user. This conf file could be generated + # in `app_ssowatconf()` result_dict = { "username": username, From 827fbe337ddf964fad6e3b74052c3b479687c619 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 18 Oct 2023 18:21:03 +0200 Subject: [PATCH 101/436] conf_regen:yunohost: setup /etc/yunohost/portal --- hooks/conf_regen/01-yunohost | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index adccfa17c6..256104a555 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -29,6 +29,11 @@ do_init_regen() { chown -R root:ssl-cert /etc/yunohost/certs chmod 750 /etc/yunohost/certs + # Portal folder + mkdir -p /etc/yunohost/portal + chmod 500 /etc/yunohost/portal + chown ynh-portal:ynh-portal /etc/yunohost/portal + # App folders mkdir -p /etc/yunohost/apps chmod 700 /etc/yunohost/apps @@ -243,6 +248,11 @@ do_post_regen() { [ ! -e "/home/$USER" ] || setfacl -m g:all_users:--- /home/$USER done + # Portal settings + mkdir -p /etc/yunohost/portal + [[ ! -e /etc/yunohost/portal ]] || (chown ynh-portal:ynh-portal /etc/yunohost/portal && chmod 500 /etc/yunohost/portal) + chmod g+s /etc/yunohost/portal + # Domain settings mkdir -p /etc/yunohost/domains @@ -267,7 +277,7 @@ do_post_regen() { systemctl restart ntp } fi - + [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload [[ ! "$regen_conf_files" =~ "login.conf.d/ynh-override.conf" ]] || { systemctl daemon-reload From 8d366e67b04f411061136085ba9c8d0699103300 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 18 Oct 2023 18:29:55 +0200 Subject: [PATCH 102/436] app_ssowatconf: generate per domain portal config with available apps --- src/app.py | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/app.py b/src/app.py index adcea56e4b..754a920a01 100644 --- a/src/app.py +++ b/src/app.py @@ -29,6 +29,7 @@ from typing import List, Tuple, Dict, Any, Iterator, Optional from packaging import version from logging import getLogger +from pathlib import Path from moulinette import Moulinette, m18n from moulinette.utils.process import run_commands, check_output @@ -36,7 +37,6 @@ read_file, read_json, read_toml, - read_yaml, write_to_file, write_to_json, cp, @@ -1606,12 +1606,16 @@ def app_ssowatconf(): """ - from yunohost.domain import domain_list, _get_maindomain, domain_config_get, _get_domain_portal_dict + from yunohost.domain import ( + domain_list, + domain_config_get, + _get_domain_portal_dict, + ) from yunohost.permission import user_permission_list - from yunohost.settings import settings_get + from yunohost.portal import PORTAL_SETTINGS_DIR - main_domain = _get_maindomain() domains = domain_list()["domains"] + portal_domains = domain_list(exclude_subdomains=True)["domains"] all_permissions = user_permission_list( full=True, ignore_system_perms=True, absolute_urls=True )["permissions"] @@ -1633,7 +1637,6 @@ def app_ssowatconf(): } } - apps_using_remote_user_var_in_nginx = ( check_output( "grep -nri '$remote_user' /etc/yunohost/apps/*/conf/*nginx*conf | awk -F/ '{print $5}' || true" @@ -1655,6 +1658,9 @@ def app_ssowatconf(): if domain + "/" != app_domain + app_path: redirected_urls[domain + "/"] = app_domain + app_path + # Will organize apps by portal domain + portal_domains_apps = {domain: {} for domain in portal_domains} + # New permission system for perm_name, perm_info in all_permissions.items(): uris = ( @@ -1678,6 +1684,23 @@ def app_ssowatconf(): "uris": uris, } + # Next: portal related + # No need to keep apps that aren't supposed to be displayed in portal + if not perm_info.get("show_tile", False): + continue + + app_domain = uris[0].split("/")[0] + # get "topest" domain + app_portal_domain = next( + domain for domain in portal_domains if domain in app_domain + ) + portal_domains_apps[app_portal_domain][app_id] = { + "label": perm_info["label"], + "users": perm_info["corresponding_users"], + "public": "visitors" in perm_info["allowed"], + "url": uris[0], + } + conf_dict = { "cookie_secret_file": "/etc/yunohost/.ssowat_cookie_secret", "cookie_name": "yunohost.portal", @@ -1688,6 +1711,22 @@ def app_ssowatconf(): write_to_json("/etc/ssowat/conf.json", conf_dict, sort_keys=True, indent=4) + # Generate a file per possible portal with available apps + for domain, apps in portal_domains_apps.items(): + portal_settings = {} + + portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{domain}.json") + if portal_settings_path.exists(): + portal_settings.update(read_json(str(portal_settings_path))) + + # Do no override anything else than "apps" since the file is shared + # with domain's config panel "portal" options + portal_settings["apps"] = apps + + write_to_json( + str(portal_settings_path), portal_settings, sort_keys=True, indent=4 + ) + logger.debug(m18n.n("ssowat_conf_generated")) From 9d21501648fc9cb6dbe71f419c95c8c9b9546b1f Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 18 Oct 2023 18:33:54 +0200 Subject: [PATCH 103/436] domain:config: update portal option saving --- src/domain.py | 37 +++++++++++++++++++++++++++++-------- src/portal.py | 1 + 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/domain.py b/src/domain.py index 98420b4be0..2a583db668 100644 --- a/src/domain.py +++ b/src/domain.py @@ -18,13 +18,21 @@ # import os import time +from pathlib import Path from typing import List, Optional from collections import OrderedDict from logging import getLogger from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError -from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm +from moulinette.utils.filesystem import ( + read_json, + read_yaml, + rm, + write_to_file, + write_to_json, + write_to_yaml, +) from yunohost.app import ( app_ssowatconf, @@ -735,16 +743,18 @@ def _apply(self): portal_options = [ "default_app", - # "show_other_domains_apps", + "show_other_domains_apps", "portal_title", "portal_logo", "portal_theme", ] if any( option in self.future_values - and self.future_values[option] != self.values[option] + and self.future_values[option] != self.values.get(option) for option in portal_options ): + from yunohost.portal import PORTAL_SETTINGS_DIR + # Portal options are also saved in a `domain.portal.yml` file # that can be read by the portal API. # FIXME remove those from the config panel saved values? @@ -775,11 +785,22 @@ def _apply(self): self.new_values["portal_logo"] = filepath # save the base64 content with mimetype to portal settings with open(filepath, "rb") as f: - portal_values["portal_logo"] = mimetype[0] + ":" + base64.b64encode(f.read()).decode("utf-8") - - # FIXME config file should be readable by non-root portal entity - write_to_yaml( - f"{DOMAIN_SETTINGS_DIR}/{self.entity}.portal.yml", portal_values + portal_values["portal_logo"] = ( + mimetype[0] + + ":" + + base64.b64encode(f.read()).decode("utf-8") + ) + + portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") + portal_settings = {"apps": {}} + + if portal_settings_path.exists(): + portal_settings.update(read_json(str(portal_settings_path))) + + # Merge settings since this config file is shared with `app_ssowatconf()` which populate the `apps` key. + portal_settings.update(portal_values) + write_to_json( + str(portal_settings_path), portal_settings, sort_keys=True, indent=4 ) super()._apply() diff --git a/src/portal.py b/src/portal.py index 9326bcc4bf..da9564d3f4 100644 --- a/src/portal.py +++ b/src/portal.py @@ -35,6 +35,7 @@ logger = logging.getLogger("portal") +PORTAL_SETTINGS_DIR = "/etc/yunohost/portal" ADMIN_ALIASES = ["root", "admin", "admins", "webmaster", "postmaster", "abuse"] From 2b5726f4a803035f821be068ec4594866f89e557 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 18 Oct 2023 18:36:07 +0200 Subject: [PATCH 104/436] portal: update settings reading from new config file --- src/portal.py | 124 +++++++++++++++++++------------------------------- 1 file changed, 47 insertions(+), 77 deletions(-) diff --git a/src/portal.py b/src/portal.py index da9564d3f4..de852f5a66 100644 --- a/src/portal.py +++ b/src/portal.py @@ -23,7 +23,7 @@ from typing import Any, Union import ldap -from moulinette.utils.filesystem import read_yaml +from moulinette.utils.filesystem import read_json from yunohost.authenticators.ldap_ynhuser import URI, USERDN, Authenticator as Auth from yunohost.user import _hash_user_password from yunohost.utils.error import YunohostError, YunohostValidationError @@ -52,51 +52,13 @@ def _get_user_infos( return username, auth["host"], result[0], ldap_interface -def _get_apps(username: Union[str, None] = None): - """Get public + user's authorized apps. - If `username` is not given, returns only public apps - (e.g. with `visitors` in group permissions) +def _get_portal_settings( + domain: Union[str, None] = None, username: Union[str, None] = None +): + """ + Returns domain's portal settings which are a combo of domain's portal config panel options + and the list of apps availables on this domain computed by `app.app_ssowatconf()`. """ - SYSTEM_PERMS = ("mail", "xmpp", "sftp", "ssh") - - ldap_interface = LDAPInterface("root") - permissions_infos = ldap_interface.search( - "ou=permission", - "(objectclass=permissionYnh)", - [ - "cn", - "groupPermission", - "inheritPermission", - "URL", - "label", - "showTile", - ], - ) - - apps = {} - - for perm in permissions_infos: - name = perm["cn"][0].replace(".main", "") - - if name in SYSTEM_PERMS or not perm.get("showTile", [False])[0]: - continue - - groups = [_ldap_path_extract(g, "cn") for g in perm["groupPermission"]] - users = [ - _ldap_path_extract(u, "uid") for u in perm.get("inheritPermission", []) - ] - - if username in users or "visitors" in groups: - apps[name] = { - "label": perm["label"][0], - "url": perm["URL"][0], - } - - return apps - - -def _get_portal_settings(domain: Union[str, None] = None): - from yunohost.domain import DOMAIN_SETTINGS_DIR if not domain: from bottle import request @@ -105,41 +67,56 @@ def _get_portal_settings(domain: Union[str, None] = None): assert domain and "/" not in domain - settings = { + settings: dict[str, Any] = { + "apps": {}, "public": False, "portal_logo": "", "portal_theme": "system", "portal_title": "YunoHost", - # "show_other_domains_apps": False, + "show_other_domains_apps": False, "domain": domain, } - if Path(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml").exists(): - settings.update(read_yaml(f"{DOMAIN_SETTINGS_DIR}/{domain}.portal.yml")) + portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{domain}.json") + + if portal_settings_path.exists(): + settings.update(read_json(str(portal_settings_path))) + # Portal may be public (no login required) + settings["public"] = ( + settings.pop("default_app", None) == "_yunohost_portal_with_public_apps" + ) + + # First clear apps since it may contains private apps + apps: dict[str, Any] = settings.pop("apps", {}) + settings["apps"] = {} + + if settings["show_other_domains_apps"]: + # Enhanced apps with all other domain's apps + import glob + + for path in glob.glob(f"{PORTAL_SETTINGS_DIR}/*.json"): + if path != str(portal_settings_path): + apps.update(read_json(path)["apps"]) + + if username: + # Add user allowed or public apps + settings["apps"] = { + name: app + for name, app in apps.items() + if username in app["users"] or app["public"] + } + elif settings["public"]: + # Add public apps (e.g. with "visitors" in group permission) + settings["apps"] = {name: app for name, app in apps.items() if app["public"]} return settings def portal_public(): - - portal_settings = _get_portal_settings() - portal_settings["apps"] = {} - portal_settings["public"] = ( - portal_settings.pop("default_app", None) == "_yunohost_portal_with_public_apps" - ) - - if portal_settings["public"]: - portal_settings["apps"] = _get_apps() - - # FIXME/TODO; See: filter apps that are available on specified domain - # if not portal_settings["show_other_domains_apps"]: - # portal_settings["apps"] = { - # name: data - # for name, data in portal_settings["apps"].items() - # if portal_settings["domain"] in data["url"] - # } - - return portal_settings + """Get public settings + If the portal is set as public, it will include the list of public apps + """ + return _get_portal_settings() def portal_me(): @@ -152,15 +129,8 @@ def portal_me(): groups = [_ldap_path_extract(g, "cn") for g in user["memberOf"]] groups = [g for g in groups if g not in [username, "all_users"]] - apps = _get_apps(username) - - # FIXME / TODO: filter apps that are available on specified domain - # settings = _get_portal_settings(domain=domain) - # if not settings["show_other_domains_apps"]: - # apps = {name: data for name, data in apps.items() if domain in data["url"]} - # App's `domain` info is not available in LDAP data, we need another config file - # that would be readable by the `ynh-portal` user. This conf file could be generated - # in `app_ssowatconf()` + # Get user allowed apps + apps = _get_portal_settings(domain, username)["apps"] result_dict = { "username": username, From 6f085ad255451e4cf6cc7330f37636b584ad5cfb Mon Sep 17 00:00:00 2001 From: axolotle Date: Thu, 19 Oct 2023 14:33:52 +0200 Subject: [PATCH 105/436] conf_regen:yunohost: repeat init portal setup in post hook --- hooks/conf_regen/01-yunohost | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 256104a555..3b810de303 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -248,10 +248,10 @@ do_post_regen() { [ ! -e "/home/$USER" ] || setfacl -m g:all_users:--- /home/$USER done - # Portal settings + # Portal folder mkdir -p /etc/yunohost/portal - [[ ! -e /etc/yunohost/portal ]] || (chown ynh-portal:ynh-portal /etc/yunohost/portal && chmod 500 /etc/yunohost/portal) - chmod g+s /etc/yunohost/portal + chmod 500 /etc/yunohost/portal + chown ynh-portal:ynh-portal /etc/yunohost/portal # Domain settings mkdir -p /etc/yunohost/domains From 163dd4d3594374401e552d56fb93bc8a01e1db7a Mon Sep 17 00:00:00 2001 From: axolotle Date: Thu, 19 Oct 2023 18:28:29 +0200 Subject: [PATCH 106/436] domain:config: remove 'portal_logo' for now --- share/config_domain.toml | 10 +++++----- src/domain.py | 33 ++------------------------------- 2 files changed, 7 insertions(+), 36 deletions(-) diff --git a/share/config_domain.toml b/share/config_domain.toml index d56a46b8eb..e67407eaa9 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -7,16 +7,16 @@ name = "Features" [feature.portal] name = "Portal" - # [feature.portal.show_other_domains_apps] - # type = "boolean" - # default = false + [feature.portal.show_other_domains_apps] + type = "boolean" + default = false [feature.portal.portal_title] type = "string" default = "YunoHost" - [feature.portal.portal_logo] - type = "file" + # [feature.portal.portal_logo] + # type = "file" [feature.portal.portal_theme] type = "select" diff --git a/src/domain.py b/src/domain.py index 2a583db668..831b6fdc27 100644 --- a/src/domain.py +++ b/src/domain.py @@ -745,12 +745,12 @@ def _apply(self): "default_app", "show_other_domains_apps", "portal_title", - "portal_logo", + # "portal_logo", "portal_theme", ] if any( option in self.future_values - and self.future_values[option] != self.values.get(option) + and self.new_values[option] != self.values.get(option) for option in portal_options ): from yunohost.portal import PORTAL_SETTINGS_DIR @@ -761,35 +761,6 @@ def _apply(self): portal_values = { option: self.future_values[option] for option in portal_options } - if portal_values["portal_logo"].startswith("/tmp/ynh_filequestion_"): - # FIXME rework this whole mess - # currently only handling API sent images, need to adapt FileOption - # to handle file extensions and file saving since "bind" is only - # done in bash helpers which are not executed in domain config - if "portal_logo[name]" in self.args or self.values["portal_logo"]: - import mimetypes - import base64 - - if "portal_logo[name]" in self.args: - # FIXME choose where to save the file - filepath = os.path.join("/tmp", self.args["portal_logo[name]"]) - # move the temp file created by FileOption with proper name and extension - os.rename(self.new_values["portal_logo"], filepath) - mimetype = mimetypes.guess_type(filepath) - else: - # image has already been saved, do not overwrite it with the empty temp file created by the FileOption - filepath = self.values["portal_logo"] - mimetype = mimetypes.guess_type(filepath) - - # save the proper path to config panel settings - self.new_values["portal_logo"] = filepath - # save the base64 content with mimetype to portal settings - with open(filepath, "rb") as f: - portal_values["portal_logo"] = ( - mimetype[0] - + ":" - + base64.b64encode(f.read()).decode("utf-8") - ) portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") portal_settings = {"apps": {}} From 5bd86808470f8a4e94c16c4100ecb3ef7dc56ec1 Mon Sep 17 00:00:00 2001 From: axolotle Date: Thu, 19 Oct 2023 18:39:15 +0200 Subject: [PATCH 107/436] domain:config: restrict portal options to topest domains --- src/domain.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/domain.py b/src/domain.py index 831b6fdc27..2a897c6256 100644 --- a/src/domain.py +++ b/src/domain.py @@ -673,6 +673,10 @@ def _get_raw_config(self): 1 if self.entity == _get_maindomain() else 0 ) + # Portal settings are only available on "topest" domains + if _get_parent_domain_of(self.entity, topest=True) is not None: + del toml["feature"]["portal"] + # Optimize wether or not to load the DNS section, # e.g. we don't want to trigger the whole _get_registary_config_section # when just getting the current value from the feature section @@ -748,7 +752,7 @@ def _apply(self): # "portal_logo", "portal_theme", ] - if any( + if _get_parent_domain_of(self.entity, topest=True) is None and any( option in self.future_values and self.new_values[option] != self.values.get(option) for option in portal_options From f5c56db10ea6e27fae15c959b4ee9c09c16fef6f Mon Sep 17 00:00:00 2001 From: axolotle Date: Thu, 13 Apr 2023 20:11:03 +0200 Subject: [PATCH 108/436] form: use pydantic BaseModel in Options and add some validators --- src/utils/form.py | 472 +++++++++++++++++++++++++--------------------- 1 file changed, 262 insertions(+), 210 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index f201f507b7..8b47be4302 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -17,6 +17,7 @@ # along with this program. If not, see . # import ast +import datetime import operator as op import os import re @@ -24,9 +25,18 @@ import tempfile import urllib.parse from enum import Enum -from typing import Any, Callable, Dict, List, Literal, Mapping, Optional, Union +from typing import Any, Callable, Dict, List, Literal, Mapping, Union from logging import getLogger +from pydantic import ( + BaseModel, + root_validator, + validator, +) +from pydantic.color import Color +from pydantic.networks import EmailStr, HttpUrl +from pydantic.types import FilePath + from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize from moulinette.utils.filesystem import read_file, write_to_file @@ -36,7 +46,6 @@ logger = getLogger("yunohost.form") -Context = dict[str, Any] # ╭───────────────────────────────────────────────────────╮ # │ ┌─╴╷ ╷╭─┐╷ │ @@ -240,27 +249,58 @@ class OptionType(str, Enum): } -class BaseOption: - def __init__( - self, - question: Dict[str, Any], - ): - self.id = question["id"] - self.type = question.get("type", OptionType.string) - self.visible = question.get("visible", True) - - self.readonly = question.get("readonly", False) - if self.readonly and self.type in FORBIDDEN_READONLY_TYPES: - # FIXME i18n - raise YunohostError( - "config_forbidden_readonly_type", - type=self.type, - id=self.id, - ) +Context = dict[str, Any] +Translation = Union[dict[str, str], str] +JSExpression = str +Values = dict[str, Any] + + +class Pattern(BaseModel): + regexp: str + error: Translation = "error_pattern" # FIXME add generic i18n key + + +class BaseOption(BaseModel): + type: OptionType + id: str + ask: Union[Translation, None] + readonly: bool = False + visible: Union[JSExpression, bool] = True + bind: Union[str, None] = None + + class Config: + arbitrary_types_allowed = True + use_enum_values = True + validate_assignment = True + + @staticmethod + def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: + # FIXME Do proper doctstring for Options + del schema["description"] + schema["additionalProperties"] = False + + @validator("ask", always=True) + def parse_or_set_default_ask( + cls, value: Union[Translation, None], values: Values + ) -> Translation: + if value is None: + return {"en": values["id"]} + if isinstance(value, str): + return {"en": value} + return value - self.ask = question.get("ask", self.id) - if not isinstance(self.ask, dict): - self.ask = {"en": self.ask} + @validator("readonly", pre=True) + def can_be_readonly(cls, value: bool, values: Values) -> bool: + forbidden_types = ("password", "app", "domain", "user", "file") + if value is True and values["type"] in forbidden_types: + raise ValueError( + m18n.n( + "config_forbidden_readonly_type", + type=values["type"], + id=values["id"], + ) + ) + return value def is_visible(self, context: Context) -> bool: if isinstance(self.visible, bool): @@ -268,7 +308,7 @@ def is_visible(self, context: Context) -> bool: return evaluate_simple_js_expression(self.visible, context=context) - def _get_prompt_message(self) -> str: + def _get_prompt_message(self, value: None) -> str: return _value_for_locale(self.ask) @@ -278,9 +318,7 @@ def _get_prompt_message(self) -> str: class BaseReadonlyOption(BaseOption): - def __init__(self, question): - super().__init__(question) - self.readonly = True + readonly: Literal[True] = True class DisplayTextOption(BaseReadonlyOption): @@ -291,38 +329,35 @@ class MarkdownOption(BaseReadonlyOption): type: Literal[OptionType.markdown] = OptionType.markdown -class AlertOption(BaseReadonlyOption): - type: Literal[OptionType.alert] = OptionType.alert - - def __init__(self, question): - super().__init__(question) - self.style = question.get("style", "info") +class State(str, Enum): + success = "success" + info = "info" + warning = "warning" + danger = "danger" - def _get_prompt_message(self) -> str: - text = _value_for_locale(self.ask) - if self.style in ["success", "info", "warning", "danger"]: - color = { - "success": "green", - "info": "cyan", - "warning": "yellow", - "danger": "red", - } - prompt = m18n.g(self.style) if self.style != "danger" else m18n.n("danger") - return colorize(prompt, color[self.style]) + f" {text}" - else: - return text +class AlertOption(BaseReadonlyOption): + type: Literal[OptionType.alert] = OptionType.alert + style: State = State.info + icon: Union[str, None] = None + + def _get_prompt_message(self, value: None) -> str: + colors = { + State.success: "green", + State.info: "cyan", + State.warning: "yellow", + State.danger: "red", + } + message = m18n.g(self.style) if self.style != State.danger else m18n.n("danger") + return f"{colorize(message, colors[self.style])} {_value_for_locale(self.ask)}" class ButtonOption(BaseReadonlyOption): type: Literal[OptionType.button] = OptionType.button - enabled = True - - def __init__(self, question): - super().__init__(question) - self.help = question.get("help") - self.style = question.get("style", "success") - self.enabled = question.get("enabled", True) + help: Union[Translation, None] = None + style: State = State.success + icon: Union[str, None] = None + enabled: Union[JSExpression, bool] = True def is_enabled(self, context: Context) -> bool: if isinstance(self.enabled, bool): @@ -337,16 +372,21 @@ def is_enabled(self, context: Context) -> bool: class BaseInputOption(BaseOption): - hide_user_input_in_prompt = False - pattern: Optional[Dict] = None - - def __init__(self, question: Dict[str, Any]): - super().__init__(question) - self.default = question.get("default", None) - self.optional = question.get("optional", False) - self.pattern = question.get("pattern", self.pattern) - self.help = question.get("help") - self.redact = question.get("redact", False) + help: Union[Translation, None] = None + example: Union[str, None] = None + placeholder: Union[str, None] = None + redact: bool = False + optional: bool = False # FIXME keep required as default? + default: Any = None + + @validator("default", pre=True) + def check_empty_default(value: Any) -> Any: + if value == "": + return None + return value + + # FIXME remove + def old__init__(self, question: Dict[str, Any]): # .current_value is the currently stored value self.current_value = question.get("current_value") # .value is the "proposed" value which we got from the user @@ -354,10 +394,6 @@ def __init__(self, question: Dict[str, Any]): # Use to return several values in case answer is in mutipart self.values: Dict[str, Any] = {} - # Empty value is parsed as empty string - if self.default == "": - self.default = None - @staticmethod def humanize(value, option={}): return str(value) @@ -368,12 +404,12 @@ def normalize(value, option={}): value = value.strip() return value - def _get_prompt_message(self) -> str: - message = super()._get_prompt_message() + def _get_prompt_message(self, value: Any) -> str: + message = super()._get_prompt_message(value) if self.readonly: message = colorize(message, "purple") - return f"{message} {self.humanize(self.current_value)}" + return f"{message} {self.humanize(value, self)}" return message @@ -418,7 +454,8 @@ def _value_post_validator(self): class BaseStringOption(BaseInputOption): - default_value = "" + default: Union[str, None] + pattern: Union[Pattern, None] = None class StringOption(BaseStringOption): @@ -429,27 +466,23 @@ class TextOption(BaseStringOption): type: Literal[OptionType.text] = OptionType.text +FORBIDDEN_PASSWORD_CHARS = r"{}" + + class PasswordOption(BaseInputOption): type: Literal[OptionType.password] = OptionType.password - hide_user_input_in_prompt = True - default_value = "" - forbidden_chars = "{}" - - def __init__(self, question): - super().__init__(question) - self.redact = True - if self.default is not None: - raise YunohostValidationError( - "app_argument_password_no_default", name=self.id - ) + example: Literal[None] = None + default: Literal[None] = None + redact: Literal[True] = True + _forbidden_chars: str = FORBIDDEN_PASSWORD_CHARS def _value_pre_validator(self): super()._value_pre_validator() if self.value not in [None, ""]: - if any(char in self.value for char in self.forbidden_chars): + if any(char in self.value for char in self._forbidden_chars): raise YunohostValidationError( - "pattern_password_app", forbidden_chars=self.forbidden_chars + "pattern_password_app", forbidden_chars=self._forbidden_chars ) # If it's an optional argument the value should be empty or strong enough @@ -458,26 +491,25 @@ def _value_pre_validator(self): assert_password_is_strong_enough("user", self.value) -class ColorOption(BaseStringOption): +class ColorOption(BaseInputOption): type: Literal[OptionType.color] = OptionType.color - pattern = { - "regexp": r"^#[ABCDEFabcdef\d]{3,6}$", - "error": "config_validate_color", # i18n: config_validate_color - } + default: Union[str, None] + # pattern = { + # "regexp": r"^#[ABCDEFabcdef\d]{3,6}$", + # "error": "config_validate_color", # i18n: config_validate_color + # } # ─ NUMERIC ─────────────────────────────────────────────── class NumberOption(BaseInputOption): + # `number` and `range` are exactly the same, but `range` does render as a slider in web-admin type: Literal[OptionType.number, OptionType.range] = OptionType.number - default_value = None - - def __init__(self, question): - super().__init__(question) - self.min = question.get("min", None) - self.max = question.get("max", None) - self.step = question.get("step", None) + default: Union[int, None] + min: Union[int, None] = None + max: Union[int, None] = None + step: Union[int, None] = None @staticmethod def normalize(value, option={}): @@ -493,7 +525,7 @@ def normalize(value, option={}): if value in [None, ""]: return None - option = option.__dict__ if isinstance(option, BaseOption) else option + option = option.dict() if isinstance(option, BaseOption) else option raise YunohostValidationError( "app_argument_invalid", name=option.get("id"), @@ -525,20 +557,15 @@ def _value_pre_validator(self): class BooleanOption(BaseInputOption): type: Literal[OptionType.boolean] = OptionType.boolean - default_value = 0 - yes_answers = ["1", "yes", "y", "true", "t", "on"] - no_answers = ["0", "no", "n", "false", "f", "off"] - - def __init__(self, question): - super().__init__(question) - self.yes = question.get("yes", 1) - self.no = question.get("no", 0) - if self.default is None: - self.default = self.no + yes: Any = 1 + no: Any = 0 + default: Union[bool, int, str, None] = 0 + _yes_answers: set[str] = {"1", "yes", "y", "true", "t", "on"} + _no_answers: set[str] = {"0", "no", "n", "false", "f", "off"} @staticmethod def humanize(value, option={}): - option = option.__dict__ if isinstance(option, BaseOption) else option + option = option.dict() if isinstance(option, BaseOption) else option yes = option.get("yes", 1) no = option.get("no", 0) @@ -561,7 +588,7 @@ def humanize(value, option={}): @staticmethod def normalize(value, option={}): - option = option.__dict__ if isinstance(option, BaseOption) else option + option = option.dict() if isinstance(option, BaseOption) else option if isinstance(value, str): value = value.strip() @@ -569,8 +596,8 @@ def normalize(value, option={}): technical_yes = option.get("yes", 1) technical_no = option.get("no", 0) - no_answers = BooleanOption.no_answers - yes_answers = BooleanOption.yes_answers + no_answers = BooleanOption._no_answers + yes_answers = BooleanOption._yes_answers assert ( str(technical_yes).lower() not in no_answers @@ -579,8 +606,8 @@ def normalize(value, option={}): str(technical_no).lower() not in yes_answers ), f"'no' value can't be in {yes_answers}" - no_answers += [str(technical_no).lower()] - yes_answers += [str(technical_yes).lower()] + no_answers.add(str(technical_no).lower()) + yes_answers.add(str(technical_yes).lower()) strvalue = str(value).lower() @@ -602,8 +629,8 @@ def normalize(value, option={}): def get(self, key, default=None): return getattr(self, key, default) - def _get_prompt_message(self): - message = super()._get_prompt_message() + def _get_prompt_message(self, value: Union[bool, None]) -> str: + message = super()._get_prompt_message(value) if not self.readonly: message += " [yes | no]" @@ -614,12 +641,13 @@ def _get_prompt_message(self): # ─ TIME ────────────────────────────────────────────────── -class DateOption(BaseStringOption): +class DateOption(BaseInputOption): type: Literal[OptionType.date] = OptionType.date - pattern = { - "regexp": r"^\d{4}-\d\d-\d\d$", - "error": "config_validate_date", # i18n: config_validate_date - } + default: Union[str, None] + # pattern = { + # "regexp": r"^\d{4}-\d\d-\d\d$", + # "error": "config_validate_date", # i18n: config_validate_date + # } def _value_pre_validator(self): from datetime import datetime @@ -633,32 +661,34 @@ def _value_pre_validator(self): raise YunohostValidationError("config_validate_date") -class TimeOption(BaseStringOption): +class TimeOption(BaseInputOption): type: Literal[OptionType.time] = OptionType.time - pattern = { - "regexp": r"^(?:\d|[01]\d|2[0-3]):[0-5]\d$", - "error": "config_validate_time", # i18n: config_validate_time - } + default: Union[str, int, None] + # pattern = { + # "regexp": r"^(?:\d|[01]\d|2[0-3]):[0-5]\d$", + # "error": "config_validate_time", # i18n: config_validate_time + # } # ─ LOCATIONS ───────────────────────────────────────────── -class EmailOption(BaseStringOption): +class EmailOption(BaseInputOption): type: Literal[OptionType.email] = OptionType.email - pattern = { - "regexp": r"^.+@.+", - "error": "config_validate_email", # i18n: config_validate_email - } + default: Union[EmailStr, None] + # pattern = { + # "regexp": r"^.+@.+", + # "error": "config_validate_email", # i18n: config_validate_email + # } class WebPathOption(BaseInputOption): type: Literal[OptionType.path] = OptionType.path - default_value = "" + default: Union[str, None] @staticmethod def normalize(value, option={}): - option = option.__dict__ if isinstance(option, BaseOption) else option + option = option.dict() if isinstance(option, BaseOption) else option if not isinstance(value, str): raise YunohostValidationError( @@ -685,10 +715,11 @@ def normalize(value, option={}): class URLOption(BaseStringOption): type: Literal[OptionType.url] = OptionType.url - pattern = { - "regexp": r"^https?://.*$", - "error": "config_validate_url", # i18n: config_validate_url - } + default: Union[str, None] + # pattern = { + # "regexp": r"^https?://.*$", + # "error": "config_validate_url", # i18n: config_validate_url + # } # ─ FILE ────────────────────────────────────────────────── @@ -696,16 +727,16 @@ class URLOption(BaseStringOption): class FileOption(BaseInputOption): type: Literal[OptionType.file] = OptionType.file - upload_dirs: List[str] = [] - - def __init__(self, question): - super().__init__(question) - self.accept = question.get("accept", "") + # `FilePath` for CLI (path must exists and must be a file) + # `bytes` for API (a base64 encoded file actually) + accept: Union[str, None] = "" # currently only used by the web-admin + default: Union[str, None] + _upload_dirs: set[str] = set() @classmethod def clean_upload_dirs(cls): # Delete files uploaded from API - for upload_dir in cls.upload_dirs: + for upload_dir in cls._upload_dirs: if os.path.exists(upload_dir): shutil.rmtree(upload_dir) @@ -738,7 +769,7 @@ def _value_post_validator(self): upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_") _, file_path = tempfile.mkstemp(dir=upload_dir) - FileOption.upload_dirs += [upload_dir] + FileOption._upload_dirs.add(upload_dir) logger.debug(f"Saving file {self.id} for file question into {file_path}") @@ -760,26 +791,30 @@ def is_file_path(s): # ─ CHOICES ─────────────────────────────────────────────── +ChoosableOptions = Literal[ + OptionType.string, + OptionType.color, + OptionType.number, + OptionType.date, + OptionType.time, + OptionType.email, + OptionType.path, + OptionType.url, +] + + class BaseChoicesOption(BaseInputOption): - def __init__( - self, - question: Dict[str, Any], - ): - super().__init__(question) - # Don't restrict choices if there's none specified - self.choices = question.get("choices", None) + # FIXME probably forbid choices to be None? + choices: Union[dict[str, Any], list[Any], None] - def _get_prompt_message(self) -> str: - message = super()._get_prompt_message() + def _get_prompt_message(self, value: Any) -> str: + message = super()._get_prompt_message(value) if self.readonly: - message = message - choice = self.current_value - - if isinstance(self.choices, dict) and choice is not None: - choice = self.choices[choice] + if isinstance(self.choices, dict) and value is not None: + value = self.choices[value] - return f"{colorize(message, 'purple')} {choice}" + return f"{colorize(message, 'purple')} {value}" if self.choices: # Prevent displaying a shitload of choices @@ -789,17 +824,15 @@ def _get_prompt_message(self) -> str: if isinstance(self.choices, dict) else self.choices ) - choices_to_display = choices[:20] + splitted_choices = choices[:20] remaining_choices = len(choices[20:]) if remaining_choices > 0: - choices_to_display += [ + splitted_choices += [ m18n.n("other_available_options", n=remaining_choices) ] - choices_to_display = " | ".join( - str(choice) for choice in choices_to_display - ) + choices_to_display = " | ".join(str(choice) for choice in splitted_choices) return f"{message} [{choices_to_display}]" @@ -821,12 +854,15 @@ def _value_pre_validator(self): class SelectOption(BaseChoicesOption): type: Literal[OptionType.select] = OptionType.select - default_value = "" + choices: Union[dict[str, Any], list[Any]] + default: Union[str, None] class TagsOption(BaseChoicesOption): type: Literal[OptionType.tags] = OptionType.tags - default_value = "" + choices: Union[list[str], None] = None + pattern: Union[Pattern, None] = None + default: Union[str, list[str], None] @staticmethod def humanize(value, option={}): @@ -879,19 +915,23 @@ def _value_post_validator(self): class DomainOption(BaseChoicesOption): type: Literal[OptionType.domain] = OptionType.domain + choices: Union[dict[str, str], None] - def __init__(self, question): - from yunohost.domain import domain_list, _get_maindomain + @root_validator() + def inject_domains_choices_and_default(cls, values: Values) -> Values: + # TODO remove calls to resources in validators (pydantic V2 should adress this) + from yunohost.domain import domain_list - super().__init__(question) + data = domain_list() + values["choices"] = { + domain: domain + " ★" if domain == data["main"] else domain + for domain in data["domains"] + } - if self.default is None: - self.default = _get_maindomain() + if values["default"] is None: + values["default"] = data["main"] - self.choices = { - domain: domain + " ★" if domain == self.default else domain - for domain in domain_list()["domains"] - } + return values @staticmethod def normalize(value, option={}): @@ -908,87 +948,99 @@ def normalize(value, option={}): class AppOption(BaseChoicesOption): type: Literal[OptionType.app] = OptionType.app + choices: Union[dict[str, str], None] + add_yunohost_portal_to_choices: bool = False + filter: Union[str, None] = None - def __init__(self, question): + @root_validator() + def inject_apps_choices(cls, values: Values) -> Values: from yunohost.app import app_list - super().__init__(question) - self.filter = question.get("filter", None) - self.add_yunohost_portal_to_choices = question.get("add_yunohost_portal_to_choices", False) - apps = app_list(full=True)["apps"] - if self.filter: + if values.get("filter", None): apps = [ app for app in apps - if evaluate_simple_js_expression(self.filter, context=app) + if evaluate_simple_js_expression(values["filter"], context=app) ] + values["choices"] = {"_none": "---"} - def _app_display(app): - domain_path_or_id = f" ({app.get('domain_path', app['id'])})" - return app["label"] + domain_path_or_id + if values.get("add_yunohost_portal_to_choices", False): + values["choices"]["_yunohost_portal_with_public_apps"] = "YunoHost's portal with public apps" - self.choices = {"_none": "---"} - if self.add_yunohost_portal_to_choices: - # FIXME: i18n - self.choices["_yunohost_portal_with_public_apps"] = "YunoHost's portal with public apps" - self.choices.update({app["id"]: _app_display(app) for app in apps}) + values["choices"].update( + { + app["id"]: f"{app['label']} ({app.get('domain_path', app['id'])})" + for app in apps + } + ) + + return values class UserOption(BaseChoicesOption): type: Literal[OptionType.user] = OptionType.user + choices: Union[dict[str, str], None] - def __init__(self, question): - from yunohost.user import user_list, user_info + @root_validator() + def inject_users_choices_and_default(cls, values: dict[str, Any]) -> dict[str, Any]: from yunohost.domain import _get_maindomain + from yunohost.user import user_info, user_list - super().__init__(question) - - self.choices = { + values["choices"] = { username: f"{infos['fullname']} ({infos['mail']})" for username, infos in user_list()["users"].items() } - if not self.choices: + # FIXME keep this to test if any user, do not raise error if no admin? + if not values["choices"]: raise YunohostValidationError( "app_argument_invalid", - name=self.id, + name=values["id"], error="You should create a YunoHost user first.", ) - if self.default is None: + if values["default"] is None: # FIXME: this code is obsolete with the new admins group # Should be replaced by something like "any first user we find in the admin group" root_mail = "root@%s" % _get_maindomain() - for user in self.choices.keys(): + for user in values["choices"].keys(): if root_mail in user_info(user).get("mail-aliases", []): - self.default = user + values["default"] = user break + return values + class GroupOption(BaseChoicesOption): type: Literal[OptionType.group] = OptionType.group + choices: Union[dict[str, str], None] - def __init__(self, question): + @root_validator() + def inject_groups_choices_and_default(cls, values: Values) -> Values: from yunohost.user import user_group_list - super().__init__(question) - - self.choices = list( - user_group_list(short=True, include_primary_groups=False)["groups"] - ) + groups = user_group_list(short=True, include_primary_groups=False)["groups"] - def _human_readable_group(g): + def _human_readable_group(groupname): # i18n: visitors # i18n: all_users # i18n: admins - return m18n.n(g) if g in ["visitors", "all_users", "admins"] else g + return ( + m18n.n(groupname) + if groupname in ["visitors", "all_users", "admins"] + else groupname + ) + + values["choices"] = { + groupname: _human_readable_group(groupname) for groupname in groups + } - self.choices = {g: _human_readable_group(g) for g in self.choices} + if values["default"] is None: + values["default"] = "all_users" - if self.default is None: - self.default = "all_users" + return values OPTIONS = { @@ -997,7 +1049,7 @@ def _human_readable_group(g): OptionType.alert: AlertOption, OptionType.button: ButtonOption, OptionType.string: StringOption, - OptionType.text: StringOption, + OptionType.text: TextOption, OptionType.password: PasswordOption, OptionType.color: ColorOption, OptionType.number: NumberOption, From 89ae5e654de8104a0c2fe85b5582078c755c13d3 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 15:06:53 +0200 Subject: [PATCH 109/436] form: update asking flow, separate form and options --- src/tests/test_questions.py | 32 ++-- src/utils/form.py | 297 +++++++++++++++++++++++++----------- 2 files changed, 225 insertions(+), 104 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index a695e834d9..9eceedcaf1 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -11,11 +11,11 @@ from _pytest.mark.structures import ParameterSet - from moulinette import Moulinette from yunohost import app, domain, user from yunohost.utils.form import ( OPTIONS, + FORBIDDEN_PASSWORD_CHARS, ask_questions_and_parse_answers, BaseChoicesOption, BaseInputOption, @@ -378,8 +378,8 @@ def _fill_or_prompt_one_option(raw_option, intake): options = {id_: raw_option} answers = {id_: intake} if intake is not None else {} - option = ask_questions_and_parse_answers(options, answers)[0] - return (option, option.value if isinstance(option, BaseInputOption) else None) + options, form = ask_questions_and_parse_answers(options, answers) + return (options[0], form[id_] if isinstance(options[0], BaseInputOption) else None) def _test_value_is_expected_output(value, expected_output): @@ -551,7 +551,7 @@ def test_scenarios(self, intake, expected_output, raw_option, data): ask_questions_and_parse_answers({_id: raw_option}, answers) else: with patch.object(sys, "stdout", new_callable=StringIO) as stdout: - options = ask_questions_and_parse_answers( + options, form = ask_questions_and_parse_answers( {_id: raw_option}, answers ) assert stdout.getvalue() == f"{options[0].ask['en']}\n" @@ -604,7 +604,7 @@ def test_scenarios(self, intake, expected_output, raw_option, data): ) else: with patch.object(sys, "stdout", new_callable=StringIO) as stdout: - options = ask_questions_and_parse_answers( + options, form = ask_questions_and_parse_answers( {"display_text_id": raw_option}, answers ) ask = options[0].ask["en"] @@ -1925,9 +1925,7 @@ def patch_query_string(file_repr): "&fake_id=fake_value" ) - def _assert_correct_values(options, raw_options): - form = {option.id: option.value for option in options} - + def _assert_correct_values(options, form, raw_options): for k, v in results.items(): if k == "file_id": assert os.path.exists(form["file_id"]) and os.path.isfile( @@ -1943,24 +1941,24 @@ def _assert_correct_values(options, raw_options): with patch_interface("api"), patch_file_api(file_content1) as b64content: with patch_query_string(b64content.decode("utf-8")) as query_string: - options = ask_questions_and_parse_answers(raw_options, query_string) - _assert_correct_values(options, raw_options) + options, form = ask_questions_and_parse_answers(raw_options, query_string) + _assert_correct_values(options, form, raw_options) with patch_interface("cli"), patch_file_cli(file_content1) as filepath: with patch_query_string(filepath) as query_string: - options = ask_questions_and_parse_answers(raw_options, query_string) - _assert_correct_values(options, raw_options) + options, form = ask_questions_and_parse_answers(raw_options, query_string) + _assert_correct_values(options, form, raw_options) def test_question_string_default_type(): questions = {"some_string": {}} answers = {"some_string": "some_value"} - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.id == "some_string" - assert out.type == "string" - assert out.value == "some_value" + options, form = ask_questions_and_parse_answers(questions, answers) + option = options[0] + assert option.id == "some_string" + assert option.type == "string" + assert form[option.id] == "some_value" def test_option_default_type_with_choices_is_select(): diff --git a/src/utils/form.py b/src/utils/form.py index 8b47be4302..8a2a34f4be 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -25,17 +25,30 @@ import tempfile import urllib.parse from enum import Enum -from typing import Any, Callable, Dict, List, Literal, Mapping, Union from logging import getLogger +from typing import ( + Annotated, + Any, + Callable, + List, + Literal, + Mapping, + Type, + Union, + cast, +) from pydantic import ( BaseModel, + Extra, + ValidationError, + create_model, root_validator, validator, ) from pydantic.color import Color +from pydantic.fields import Field from pydantic.networks import EmailStr, HttpUrl -from pydantic.types import FilePath from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize @@ -385,15 +398,6 @@ def check_empty_default(value: Any) -> Any: return None return value - # FIXME remove - def old__init__(self, question: Dict[str, Any]): - # .current_value is the currently stored value - self.current_value = question.get("current_value") - # .value is the "proposed" value which we got from the user - self.value = question.get("value") - # Use to return several values in case answer is in mutipart - self.values: Dict[str, Any] = {} - @staticmethod def humanize(value, option={}): return str(value) @@ -650,8 +654,6 @@ class DateOption(BaseInputOption): # } def _value_pre_validator(self): - from datetime import datetime - super()._value_pre_validator() if self.value not in [None, ""]: @@ -1069,6 +1071,120 @@ def _human_readable_group(groupname): OptionType.group: GroupOption, } +AnyOption = Union[ + DisplayTextOption, + MarkdownOption, + AlertOption, + ButtonOption, + StringOption, + TextOption, + PasswordOption, + ColorOption, + NumberOption, + BooleanOption, + DateOption, + TimeOption, + EmailOption, + WebPathOption, + URLOption, + FileOption, + SelectOption, + TagsOption, + DomainOption, + AppOption, + UserOption, + GroupOption, +] + + +# ╭───────────────────────────────────────────────────────╮ +# │ ┌─╴╭─╮┌─╮╭╮╮ │ +# │ ├─╴│ │├┬╯│││ │ +# │ ╵ ╰─╯╵ ╰╵╵╵ │ +# ╰───────────────────────────────────────────────────────╯ + + +class OptionsModel(BaseModel): + # Pydantic will match option types to their models class based on the "type" attribute + options: list[Annotated[AnyOption, Field(discriminator="type")]] + + @staticmethod + def options_dict_to_list(options: dict[str, Any], defaults: dict[str, Any] = {}): + return [ + option + | { + "id": id_, + "type": option.get("type", "string"), + } + for id_, option in options.items() + ] + + def __init__(self, **kwargs) -> None: + super().__init__(options=self.options_dict_to_list(kwargs)) + + +class FormModel(BaseModel): + """ + Base form on which dynamic forms are built upon Options. + """ + + class Config: + validate_assignment = True + extra = Extra.ignore + + def __getitem__(self, name: str): + # FIXME + # if a FormModel's required field is not instancied with a value, it is + # not available as an attr and therefor triggers an `AttributeError` + # Also since `BaseReadonlyOption`s do not end up in form, + # `form[AlertOption.id]` would also triggers an error + # For convinience in those 2 cases, we return `None` + if not hasattr(self, name): + # Return None to trigger a validation error instead for required fields + return None + + return getattr(self, name) + + def __setitem__(self, name: str, value: Any): + setattr(self, name, value) + + def get(self, attr: str, default: Any = None) -> Any: + try: + return getattr(self, attr) + except AttributeError: + return default + + +def build_form(options: list[AnyOption], name: str = "DynamicForm") -> Type[FormModel]: + """ + Returns a dynamic pydantic model class that can be used as a form. + Parsing/validation occurs at instanciation and assignements. + To avoid validation at instanciation, use `my_form.construct(**values)` + """ + options_as_fields: Any = {} + validators: dict[str, Any] = {} + + for option in options: + if not isinstance(option, BaseInputOption): + continue # filter out non input options + + options_as_fields[option.id] = option._as_dynamic_model_field() + + for step in ("pre", "post"): + validators[f"{option.id}_{step}_validator"] = validator( + option.id, allow_reuse=True, pre=step == "pre" + )(getattr(option, f"_value_{step}_validator")) + + return cast( + Type[FormModel], + create_model( + name, + __base__=FormModel, + __validators__=validators, + **options_as_fields, + ), + ) + def hydrate_option_type(raw_option: dict[str, Any]) -> dict[str, Any]: type_ = raw_option.get( @@ -1097,20 +1213,16 @@ def hydrate_option_type(raw_option: dict[str, Any]) -> dict[str, Any]: def prompt_or_validate_form( - raw_options: dict[str, Any], + options: list[AnyOption], + form: FormModel, prefilled_answers: dict[str, Any] = {}, context: Context = {}, hooks: Hooks = {}, -) -> list[BaseOption]: - options = [] +) -> FormModel: answers = {**prefilled_answers} + values = {} - for id_, raw_option in raw_options.items(): - raw_option["id"] = id_ - raw_option["value"] = answers.get(id_) - raw_option = hydrate_option_type(raw_option) - option = OPTIONS[raw_option["type"]](raw_option) - + for option in options: interactive = Moulinette.interface.type == "cli" and os.isatty(1) if isinstance(option, ButtonOption): @@ -1123,89 +1235,88 @@ def prompt_or_validate_form( help=_value_for_locale(option.help), ) - # FIXME not sure why we do not append Buttons to returned options - options.append(option) - if not option.is_visible(context): if isinstance(option, BaseInputOption): # FIXME There could be several use case if the question is not displayed: # - we doesn't want to give a specific value # - we want to keep the previous value # - we want the default value - option.value = context[option.id] = None + context[option.id] = form[option.id] = None continue - message = option._get_prompt_message() - - if option.readonly: - if interactive: - Moulinette.display(message) + # if we try to get a `BaseReadonlyOption` value, which doesn't exists in the form, + # we get `None` + value = form[option.id] + if isinstance(option, BaseReadonlyOption) or option.readonly: if isinstance(option, BaseInputOption): - option.value = context[option.id] = option.current_value + # FIXME normalized needed, form[option.id] should already be normalized + # only update the context with the value + context[option.id] = form[option.id] + + # FIXME here we could error out + if option.id in prefilled_answers: + logger.warning( + f"'{option.id}' is readonly, value '{prefilled_answers[option.id]}' is then ignored." + ) - continue + if interactive: + Moulinette.display(option._get_prompt_message(value)) - if isinstance(option, BaseInputOption): - for i in range(5): - if interactive and option.value is None: - prefill = "" - choices = ( - option.choices if isinstance(option, BaseChoicesOption) else [] - ) + continue - if option.current_value is not None: - prefill = option.humanize(option.current_value, option) - elif option.default is not None: - prefill = option.humanize(option.default, option) - - option.value = Moulinette.prompt( - message=message, - is_password=isinstance(option, PasswordOption), - confirm=False, - prefill=prefill, - is_multiline=(option.type == "text"), - autocomplete=choices, - help=_value_for_locale(option.help), - ) + for i in range(5): + if option.id in prefilled_answers: + value = prefilled_answers[option.id] + elif interactive: + value = option.humanize(value, option) + choices = ( + option.choices if isinstance(option, BaseChoicesOption) else [] + ) + value = Moulinette.prompt( + message=option._get_prompt_message(value), + is_password=isinstance(option, PasswordOption), + confirm=False, + prefill=value, + is_multiline=isinstance(option, TextOption), + autocomplete=choices, + help=_value_for_locale(option.help), + ) - # Apply default value - class_default = getattr(option, "default_value", None) - if option.value in [None, ""] and ( - option.default is not None or class_default is not None - ): - option.value = ( - class_default if option.default is None else option.default - ) + # Apply default value if none + if value is None or value == "" and option.default is not None: + value = option.default - try: - # Normalize and validate - option.value = option.normalize(option.value, option) - option._value_pre_validator() - except YunohostValidationError as e: - # If in interactive cli, re-ask the current question - if i < 4 and interactive: - logger.error(str(e)) - option.value = None - continue + try: + # Normalize and validate + values[option.id] = form[option.id] = option.normalize(value, option) + except (ValidationError, YunohostValidationError) as e: + # If in interactive cli, re-ask the current question + if i < 4 and interactive: + logger.error(str(e)) + value = None + continue - # Otherwise raise the ValidationError - raise + if isinstance(e, ValidationError): + error = "\n".join([err["msg"] for err in e.errors()]) + raise YunohostValidationError(error, raw_msg=True) - break + # Otherwise raise the ValidationError + raise e - option.value = option.values[option.id] = option._value_post_validator() + break - # Search for post actions in hooks - post_hook = f"post_ask__{option.id}" - if post_hook in hooks: - option.values.update(hooks[post_hook](option)) + # Search for post actions in hooks + post_hook = f"post_ask__{option.id}" + if post_hook in hooks: + values.update(hooks[post_hook](option)) + # FIXME reapply new values to form to validate it - answers.update(option.values) - context.update(option.values) + answers.update(values) + context.update(values) - return options + return form def ask_questions_and_parse_answers( @@ -1213,7 +1324,7 @@ def ask_questions_and_parse_answers( prefilled_answers: Union[str, Mapping[str, Any]] = {}, current_values: Mapping[str, Any] = {}, hooks: Hooks = {}, -) -> list[BaseOption]: +) -> tuple[list[AnyOption], FormModel]: """Parse arguments store in either manifest.json or actions.json or from a config panel against the user answers when they are present. @@ -1241,9 +1352,21 @@ def ask_questions_and_parse_answers( context = {**current_values, **answers} - return prompt_or_validate_form( - raw_options, prefilled_answers=answers, context=context, hooks=hooks + # Validate/parse the options attributes + try: + model = OptionsModel(**raw_options) + except ValidationError as e: + error = "\n".join([err["msg"] for err in e.errors()]) + # FIXME use YunohostError instead since it is not really a user mistake? + raise YunohostValidationError(error, raw_msg=True) + + # Build the form from those questions and instantiate it without + # parsing/validation (construct) since it may contains required questions. + form = build_form(model.options).construct() + form = prompt_or_validate_form( + model.options, form, prefilled_answers=answers, context=context, hooks=hooks ) + return (model.options, form) def hydrate_questions_with_choices(raw_questions: List) -> List: @@ -1251,7 +1374,7 @@ def hydrate_questions_with_choices(raw_questions: List) -> List: for raw_question in raw_questions: raw_question = hydrate_option_type(raw_question) - question = OPTIONS[raw_question["type"]](raw_question) + question = OPTIONS[raw_question["type"]](**raw_question) if isinstance(question, BaseChoicesOption) and question.choices: raw_question["choices"] = question.choices raw_question["default"] = question.default From 39437748111fc850003ccdb305f71876e49c2f6e Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 15:23:05 +0200 Subject: [PATCH 110/436] form: add dynamic annotation getters --- src/utils/form.py | 161 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 21 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index 8a2a34f4be..17b0a9432a 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -27,6 +27,8 @@ from enum import Enum from logging import getLogger from typing import ( + TYPE_CHECKING, + cast, Annotated, Any, Callable, @@ -35,7 +37,6 @@ Mapping, Type, Union, - cast, ) from pydantic import ( @@ -49,6 +50,7 @@ from pydantic.color import Color from pydantic.fields import Field from pydantic.networks import EmailStr, HttpUrl +from pydantic.types import constr from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize @@ -57,6 +59,9 @@ from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.i18n import _value_for_locale +if TYPE_CHECKING: + from pydantic.fields import FieldInfo + logger = getLogger("yunohost.form") @@ -391,6 +396,7 @@ class BaseInputOption(BaseOption): redact: bool = False optional: bool = False # FIXME keep required as default? default: Any = None + _annotation = Any @validator("default", pre=True) def check_empty_default(value: Any) -> Any: @@ -408,6 +414,57 @@ def normalize(value, option={}): value = value.strip() return value + @property + def _dynamic_annotation(self) -> Any: + """ + Returns the expected type of an Option's value. + This may be dynamic based on constraints. + """ + return self._annotation + + def _get_field_attrs(self) -> dict[str, Any]: + """ + Returns attributes to build a `pydantic.Field`. + This may contains non `Field` attrs that will end up in `Field.extra`. + Those extra can be used as constraints in custom validators and ends up + in the JSON Schema. + """ + # TODO + # - help + # - placeholder + attrs: dict[str, Any] = { + "redact": self.redact, # extra + "none_as_empty_str": self._none_as_empty_str, + } + + if self.readonly: + attrs["allow_mutation"] = False + + if self.example: + attrs["examples"] = [self.example] + + if self.default is not None: + attrs["default"] = self.default + else: + attrs["default"] = ... if not self.optional else None + + return attrs + + def _as_dynamic_model_field(self) -> tuple[Any, "FieldInfo"]: + """ + Return a tuple of a type and a Field instance to be injected in a + custom form declaration. + """ + attrs = self._get_field_attrs() + anno = ( + self._dynamic_annotation + if not self.optional + else Union[self._dynamic_annotation, None] + ) + field = Field(default=attrs.pop("default", None), **attrs) + + return (anno, field) + def _get_prompt_message(self, value: Any) -> str: message = super()._get_prompt_message(value) @@ -460,6 +517,22 @@ def _value_post_validator(self): class BaseStringOption(BaseInputOption): default: Union[str, None] pattern: Union[Pattern, None] = None + _annotation = str + + @property + def _dynamic_annotation(self) -> Type[str]: + if self.pattern: + return constr(regex=self.pattern.regexp) + + return self._annotation + + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + + if self.pattern: + attrs["regex_error"] = self.pattern.error # extra + + return attrs class StringOption(BaseStringOption): @@ -478,8 +551,16 @@ class PasswordOption(BaseInputOption): example: Literal[None] = None default: Literal[None] = None redact: Literal[True] = True + _annotation = str _forbidden_chars: str = FORBIDDEN_PASSWORD_CHARS + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + + attrs["forbidden_chars"] = self._forbidden_chars # extra + + return attrs + def _value_pre_validator(self): super()._value_pre_validator() @@ -498,10 +579,7 @@ def _value_pre_validator(self): class ColorOption(BaseInputOption): type: Literal[OptionType.color] = OptionType.color default: Union[str, None] - # pattern = { - # "regexp": r"^#[ABCDEFabcdef\d]{3,6}$", - # "error": "config_validate_color", # i18n: config_validate_color - # } + _annotation = Color # ─ NUMERIC ─────────────────────────────────────────────── @@ -514,6 +592,7 @@ class NumberOption(BaseInputOption): min: Union[int, None] = None max: Union[int, None] = None step: Union[int, None] = None + _annotation = int @staticmethod def normalize(value, option={}): @@ -536,6 +615,14 @@ def normalize(value, option={}): error=m18n.n("invalid_number"), ) + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + attrs["ge"] = self.min + attrs["le"] = self.max + attrs["step"] = self.step # extra + + return attrs + def _value_pre_validator(self): super()._value_pre_validator() if self.value in [None, ""]: @@ -564,6 +651,7 @@ class BooleanOption(BaseInputOption): yes: Any = 1 no: Any = 0 default: Union[bool, int, str, None] = 0 + _annotation = Union[bool, int, str] _yes_answers: set[str] = {"1", "yes", "y", "true", "t", "on"} _no_answers: set[str] = {"0", "no", "n", "false", "f", "off"} @@ -633,6 +721,14 @@ def normalize(value, option={}): def get(self, key, default=None): return getattr(self, key, default) + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + attrs["parse"] = { # extra + True: self.yes, + False: self.no, + } + return attrs + def _get_prompt_message(self, value: Union[bool, None]) -> str: message = super()._get_prompt_message(value) @@ -648,10 +744,7 @@ def _get_prompt_message(self, value: Union[bool, None]) -> str: class DateOption(BaseInputOption): type: Literal[OptionType.date] = OptionType.date default: Union[str, None] - # pattern = { - # "regexp": r"^\d{4}-\d\d-\d\d$", - # "error": "config_validate_date", # i18n: config_validate_date - # } + _annotation = datetime.date def _value_pre_validator(self): super()._value_pre_validator() @@ -666,10 +759,7 @@ def _value_pre_validator(self): class TimeOption(BaseInputOption): type: Literal[OptionType.time] = OptionType.time default: Union[str, int, None] - # pattern = { - # "regexp": r"^(?:\d|[01]\d|2[0-3]):[0-5]\d$", - # "error": "config_validate_time", # i18n: config_validate_time - # } + _annotation = datetime.time # ─ LOCATIONS ───────────────────────────────────────────── @@ -678,15 +768,13 @@ class TimeOption(BaseInputOption): class EmailOption(BaseInputOption): type: Literal[OptionType.email] = OptionType.email default: Union[EmailStr, None] - # pattern = { - # "regexp": r"^.+@.+", - # "error": "config_validate_email", # i18n: config_validate_email - # } + _annotation = EmailStr class WebPathOption(BaseInputOption): type: Literal[OptionType.path] = OptionType.path default: Union[str, None] + _annotation = str @staticmethod def normalize(value, option={}): @@ -718,10 +806,7 @@ def normalize(value, option={}): class URLOption(BaseStringOption): type: Literal[OptionType.url] = OptionType.url default: Union[str, None] - # pattern = { - # "regexp": r"^https?://.*$", - # "error": "config_validate_url", # i18n: config_validate_url - # } + _annotation = HttpUrl # ─ FILE ────────────────────────────────────────────────── @@ -733,6 +818,7 @@ class FileOption(BaseInputOption): # `bytes` for API (a base64 encoded file actually) accept: Union[str, None] = "" # currently only used by the web-admin default: Union[str, None] + _annotation = str # TODO could be Path at some point _upload_dirs: set[str] = set() @classmethod @@ -809,6 +895,17 @@ class BaseChoicesOption(BaseInputOption): # FIXME probably forbid choices to be None? choices: Union[dict[str, Any], list[Any], None] + @property + def _dynamic_annotation(self) -> Union[object, Type[str]]: + if self.choices is not None: + choices = ( + self.choices if isinstance(self.choices, list) else self.choices.keys() + ) + # FIXME in case of dict, try to parse keys with `item_type` (at least number) + return Literal[tuple(choices)] + + return self._annotation + def _get_prompt_message(self, value: Any) -> str: message = super()._get_prompt_message(value) @@ -858,6 +955,7 @@ class SelectOption(BaseChoicesOption): type: Literal[OptionType.select] = OptionType.select choices: Union[dict[str, Any], list[Any]] default: Union[str, None] + _annotation = str class TagsOption(BaseChoicesOption): @@ -865,6 +963,7 @@ class TagsOption(BaseChoicesOption): choices: Union[list[str], None] = None pattern: Union[Pattern, None] = None default: Union[str, list[str], None] + _annotation = str @staticmethod def humanize(value, option={}): @@ -880,6 +979,26 @@ def normalize(value, option={}): value = value.strip() return value + @property + def _dynamic_annotation(self): + # TODO use Literal when serialization is seperated from validation + # if self.choices is not None: + # return Literal[tuple(self.choices)] + + # Repeat pattern stuff since we can't call the bare class `_dynamic_annotation` prop without instantiating it + if self.pattern: + return constr(regex=self.pattern.regexp) + + return self._annotation + + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + + if self.choices: + attrs["choices"] = self.choices # extra + + return attrs + def _value_pre_validator(self): values = self.value if isinstance(values, str): From ec5da99a79512b0864b549ce916d8cd805d33c4b Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 15:33:34 +0200 Subject: [PATCH 111/436] form: rework pre + post options validators --- src/utils/form.py | 279 ++++++++++++++++++++++++++-------------------- 1 file changed, 156 insertions(+), 123 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index 17b0a9432a..cbe64b499f 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -60,7 +60,7 @@ from yunohost.utils.i18n import _value_for_locale if TYPE_CHECKING: - from pydantic.fields import FieldInfo + from pydantic.fields import ModelField, FieldInfo logger = getLogger("yunohost.form") @@ -397,6 +397,7 @@ class BaseInputOption(BaseOption): optional: bool = False # FIXME keep required as default? default: Any = None _annotation = Any + _none_as_empty_str: bool = True @validator("default", pre=True) def check_empty_default(value: Any) -> Any: @@ -405,7 +406,9 @@ def check_empty_default(value: Any) -> Any: return value @staticmethod - def humanize(value, option={}): + def humanize(value: Any, option={}) -> str: + if value is None: + return "" return str(value) @staticmethod @@ -474,31 +477,30 @@ def _get_prompt_message(self, value: Any) -> str: return message - def _value_pre_validator(self): - if self.value in [None, ""] and not self.optional: - raise YunohostValidationError("app_argument_required", name=self.id) + @classmethod + def _value_pre_validator(cls, value: Any, field: "ModelField") -> Any: + if value == "": + return None - # we have an answer, do some post checks - if self.value not in [None, ""]: - if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): - raise YunohostValidationError( - self.pattern["error"], - name=self.id, - value=self.value, - ) + return value + + @classmethod + def _value_post_validator(cls, value: Any, field: "ModelField") -> Any: + extras = field.field_info.extra - def _value_post_validator(self): - if not self.redact: - return self.value + if value is None and extras["none_as_empty_str"]: + value = "" + + if not extras.get("redact"): + return value # Tell the operation_logger to redact all password-type / secret args # Also redact the % escaped version of the password that might appear in # the 'args' section of metadata (relevant for password with non-alphanumeric char) data_to_redact = [] - if self.value and isinstance(self.value, str): - data_to_redact.append(self.value) - if self.current_value and isinstance(self.current_value, str): - data_to_redact.append(self.current_value) + if value and isinstance(value, str): + data_to_redact.append(value) + data_to_redact += [ urllib.parse.quote(data) for data in data_to_redact @@ -508,7 +510,7 @@ def _value_post_validator(self): for operation_logger in OperationLogger._instances: operation_logger.data_to_redact.extend(data_to_redact) - return self.value + return value # ─ STRINGS ─────────────────────────────────────────────── @@ -561,19 +563,25 @@ def _get_field_attrs(self) -> dict[str, Any]: return attrs - def _value_pre_validator(self): - super()._value_pre_validator() - - if self.value not in [None, ""]: - if any(char in self.value for char in self._forbidden_chars): + @classmethod + def _value_pre_validator( + cls, value: Union[str, None], field: "ModelField" + ) -> Union[str, None]: + value = super()._value_pre_validator(value, field) + + if value is not None and value != "": + forbidden_chars: str = field.field_info.extra["forbidden_chars"] + if any(char in value for char in forbidden_chars): raise YunohostValidationError( - "pattern_password_app", forbidden_chars=self._forbidden_chars + "pattern_password_app", forbidden_chars=forbidden_chars ) # If it's an optional argument the value should be empty or strong enough from yunohost.utils.password import assert_password_is_strong_enough - assert_password_is_strong_enough("user", self.value) + assert_password_is_strong_enough("user", value) + + return value class ColorOption(BaseInputOption): @@ -581,6 +589,29 @@ class ColorOption(BaseInputOption): default: Union[str, None] _annotation = Color + @staticmethod + def humanize(value: Union[Color, str, None], option={}) -> str: + if isinstance(value, Color): + value.as_named(fallback=True) + + return super(ColorOption, ColorOption).humanize(value, option) + + @staticmethod + def normalize(value: Union[Color, str, None], option={}) -> str: + if isinstance(value, Color): + return value.as_hex() + + return super(ColorOption, ColorOption).normalize(value, option) + + @classmethod + def _value_post_validator( + cls, value: Union[Color, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, Color): + return value.as_hex() + + return super()._value_post_validator(value, field) + # ─ NUMERIC ─────────────────────────────────────────────── @@ -623,24 +654,16 @@ def _get_field_attrs(self) -> dict[str, Any]: return attrs - def _value_pre_validator(self): - super()._value_pre_validator() - if self.value in [None, ""]: - return + @classmethod + def _value_pre_validator( + cls, value: Union[int, None], field: "ModelField" + ) -> Union[int, None]: + value = super()._value_pre_validator(value, field) - if self.min is not None and int(self.value) < self.min: - raise YunohostValidationError( - "app_argument_invalid", - name=self.id, - error=m18n.n("invalid_number_min", min=self.min), - ) + if value is None: + return None - if self.max is not None and int(self.value) > self.max: - raise YunohostValidationError( - "app_argument_invalid", - name=self.id, - error=m18n.n("invalid_number_max", max=self.max), - ) + return value # ─ BOOLEAN ─────────────────────────────────────────────── @@ -654,6 +677,7 @@ class BooleanOption(BaseInputOption): _annotation = Union[bool, int, str] _yes_answers: set[str] = {"1", "yes", "y", "true", "t", "on"} _no_answers: set[str] = {"0", "no", "n", "false", "f", "off"} + _none_as_empty_str = False @staticmethod def humanize(value, option={}): @@ -737,6 +761,15 @@ def _get_prompt_message(self, value: Union[bool, None]) -> str: return message + @classmethod + def _value_post_validator( + cls, value: Union[bool, None], field: "ModelField" + ) -> Any: + if isinstance(value, bool): + return field.field_info.extra["parse"][value] + + return super()._value_post_validator(value, field) + # ─ TIME ────────────────────────────────────────────────── @@ -746,14 +779,14 @@ class DateOption(BaseInputOption): default: Union[str, None] _annotation = datetime.date - def _value_pre_validator(self): - super()._value_pre_validator() + @classmethod + def _value_post_validator( + cls, value: Union[datetime.date, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, datetime.date): + return value.isoformat() - if self.value not in [None, ""]: - try: - datetime.strptime(self.value, "%Y-%m-%d") - except ValueError: - raise YunohostValidationError("config_validate_date") + return super()._value_post_validator(value, field) class TimeOption(BaseInputOption): @@ -761,6 +794,16 @@ class TimeOption(BaseInputOption): default: Union[str, int, None] _annotation = datetime.time + @classmethod + def _value_post_validator( + cls, value: Union[datetime.date, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, datetime.time): + # FIXME could use `value.isoformat()` to get `%H:%M:%S` + return value.strftime("%H:%M") + + return super()._value_post_validator(value, field) + # ─ LOCATIONS ───────────────────────────────────────────── @@ -780,6 +823,9 @@ class WebPathOption(BaseInputOption): def normalize(value, option={}): option = option.dict() if isinstance(option, BaseOption) else option + if value is None: + value = "" + if not isinstance(value, str): raise YunohostValidationError( "app_argument_invalid", @@ -828,52 +874,40 @@ def clean_upload_dirs(cls): if os.path.exists(upload_dir): shutil.rmtree(upload_dir) - def _value_pre_validator(self): - if self.value is None: - self.value = self.current_value - - super()._value_pre_validator() - - # Validation should have already failed if required - if self.value in [None, ""]: - return self.value - - if Moulinette.interface.type != "api": - if not os.path.exists(str(self.value)) or not os.path.isfile( - str(self.value) - ): - raise YunohostValidationError( - "app_argument_invalid", - name=self.id, - error=m18n.n("file_does_not_exist", path=str(self.value)), - ) - - def _value_post_validator(self): + @classmethod + def _value_post_validator(cls, value: Any, field: "ModelField") -> Any: from base64 import b64decode - if not self.value: + if not value: return "" + def is_file_path(s): + return ( + isinstance(s, str) + and s.startswith("/") + and os.path.exists(s) + and os.path.isfile(s) + ) + + file_exists = is_file_path(value) + if Moulinette.interface.type != "api" and not file_exists: + # FIXME error + raise YunohostValidationError("File doesn't exists", raw_msg=True) + elif file_exists: + content = read_file(str(value), file_mode="rb") + else: + content = b64decode(value) + upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_") _, file_path = tempfile.mkstemp(dir=upload_dir) FileOption._upload_dirs.add(upload_dir) - logger.debug(f"Saving file {self.id} for file question into {file_path}") - - def is_file_path(s): - return isinstance(s, str) and s.startswith("/") and os.path.exists(s) - - if Moulinette.interface.type != "api" or is_file_path(self.value): - content = read_file(str(self.value), file_mode="rb") - else: - content = b64decode(self.value) + logger.debug(f"Saving file {field.name} for file question into {file_path}") write_to_file(file_path, content, file_mode="wb") - self.value = file_path - - return self.value + return file_path # ─ CHOICES ─────────────────────────────────────────────── @@ -895,6 +929,13 @@ class BaseChoicesOption(BaseInputOption): # FIXME probably forbid choices to be None? choices: Union[dict[str, Any], list[Any], None] + @validator("choices", pre=True) + def parse_comalist_choices(value: Any) -> Union[dict[str, Any], list[Any], None]: + if isinstance(value, str): + values = [value.strip() for value in value.split(",")] + return [value for value in values if value] + return value + @property def _dynamic_annotation(self) -> Union[object, Type[str]]: if self.choices is not None: @@ -937,19 +978,6 @@ def _get_prompt_message(self, value: Any) -> str: return message - def _value_pre_validator(self): - super()._value_pre_validator() - - # we have an answer, do some post checks - if self.value not in [None, ""]: - if self.choices and self.value not in self.choices: - raise YunohostValidationError( - "app_argument_choice_invalid", - name=self.id, - value=self.value, - choices=", ".join(str(choice) for choice in self.choices), - ) - class SelectOption(BaseChoicesOption): type: Literal[OptionType.select] = OptionType.select @@ -969,6 +997,8 @@ class TagsOption(BaseChoicesOption): def humanize(value, option={}): if isinstance(value, list): return ",".join(str(v) for v in value) + if not value: + return "" return value @staticmethod @@ -976,7 +1006,9 @@ def normalize(value, option={}): if isinstance(value, list): return ",".join(str(v) for v in value) if isinstance(value, str): - value = value.strip() + value = value.strip().strip(",") + if value is None or value == "": + return "" return value @property @@ -999,36 +1031,37 @@ def _get_field_attrs(self) -> dict[str, Any]: return attrs - def _value_pre_validator(self): - values = self.value - if isinstance(values, str): - values = values.split(",") - elif values is None: - values = [] + @classmethod + def _value_pre_validator( + cls, value: Union[list, str, None], field: "ModelField" + ) -> Union[str, None]: + if value is None or value == "": + return None - if not isinstance(values, list): - if self.choices: - raise YunohostValidationError( - "app_argument_choice_invalid", - name=self.id, - value=self.value, - choices=", ".join(str(choice) for choice in self.choices), - ) + if not isinstance(value, (list, str, type(None))): raise YunohostValidationError( "app_argument_invalid", - name=self.id, - error=f"'{str(self.value)}' is not a list", + name=field.name, + error=f"'{str(value)}' is not a list", ) - for value in values: - self.value = value - super()._value_pre_validator() - self.value = values + if isinstance(value, str): + value = [v.strip() for v in value.split(",")] + value = [v for v in value if v] - def _value_post_validator(self): - if isinstance(self.value, list): - self.value = ",".join(self.value) - return super()._value_post_validator() + if isinstance(value, list): + choices = field.field_info.extra.get("choices") + if choices: + if not all(v in choices for v in value): + raise YunohostValidationError( + "app_argument_choice_invalid", + name=field.name, + value=value, + choices=", ".join(str(choice) for choice in choices), + ) + + return ",".join(str(v) for v in value) + return value # ─ ENTITIES ────────────────────────────────────────────── From c428ba616a3f2d63a22257560345429f0af29729 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 15:37:56 +0200 Subject: [PATCH 112/436] test:options: update tests results to pydantic parsing --- src/tests/test_questions.py | 374 +++++++++++++++++++----------------- 1 file changed, 197 insertions(+), 177 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 9eceedcaf1..1f8667adfb 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -20,7 +20,6 @@ BaseChoicesOption, BaseInputOption, BaseReadonlyOption, - PasswordOption, DomainOption, WebPathOption, BooleanOption, @@ -436,6 +435,10 @@ def get_raw_option(cls, raw_option={}, **kwargs): @classmethod def _test_basic_attrs(self): raw_option = self.get_raw_option(optional=True) + + if raw_option["type"] == "select": + raw_option["choices"] = ["one"] + id_ = raw_option["id"] option, value = _fill_or_prompt_one_option(raw_option, None) @@ -481,6 +484,7 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): base_raw_option = prefill_data["raw_option"] prefill = prefill_data["prefill"] + # FIXME could patch prompt with prefill if we switch to "do not apply default if value is None|''" with patch_prompt("") as prompt: raw_option = self.get_raw_option( raw_option=base_raw_option, @@ -583,9 +587,7 @@ class TestAlert(TestDisplayText): (None, None, {"ask": "Some text\na new line"}), (None, None, {"ask": {"en": "Some text\na new line", "fr": "Un peu de texte\nune nouvelle ligne"}}), *[(None, None, {"ask": "question", "style": style}) for style in ("success", "info", "warning", "danger")], - *xpass(scenarios=[ - (None, None, {"ask": "question", "style": "nimp"}), - ], reason="Should fail, wrong style"), + (None, FAIL, {"ask": "question", "style": "nimp"}), ] # fmt: on @@ -643,11 +645,15 @@ class TestString(BaseTest): scenarios = [ *nones(None, "", output=""), # basic typed values - *unchanged(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), # FIXME should output as str? + (False, "False"), + (True, "True"), + (0, "0"), + (1, "1"), + (-1, "-1"), + (1337, "1337"), + (13.37, "13.37"), + *all_fails([], ["one"], {}), *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", raw_option={"optional": True}), - *xpass(scenarios=[ - ([], []), - ], reason="Should fail"), # test strip ("value", "value"), ("value\n", "value"), @@ -660,7 +666,7 @@ class TestString(BaseTest): (" ##value \n \tvalue\n ", "##value \n \tvalue"), ], reason=r"should fail or without `\n`?"), # readonly - ("overwrite", "expected value", {"readonly": True, "current_value": "expected value"}), + ("overwrite", "expected value", {"readonly": True, "default": "expected value"}), # FIXME do we want to fail instead? ] # fmt: on @@ -680,11 +686,15 @@ class TestText(BaseTest): scenarios = [ *nones(None, "", output=""), # basic typed values - *unchanged(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), # FIXME should fail or output as str? + (False, "False"), + (True, "True"), + (0, "0"), + (1, "1"), + (-1, "-1"), + (1337, "1337"), + (13.37, "13.37"), + *all_fails([], ["one"], {}), *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", raw_option={"optional": True}), - *xpass(scenarios=[ - ([], []) - ], reason="Should fail"), ("value", "value"), ("value\n value", "value\n value"), # test no strip @@ -697,7 +707,7 @@ class TestText(BaseTest): (r" ##value \n \tvalue\n ", r"##value \n \tvalue\n"), ], reason="Should not be stripped"), # readonly - ("overwrite", "expected value", {"readonly": True, "current_value": "expected value"}), + ("overwrite", "expected value", {"readonly": True, "default": "expected value"}), ] # fmt: on @@ -715,7 +725,7 @@ class TestPassword(BaseTest): } # fmt: off scenarios = [ - *all_fails(False, True, 0, 1, -1, 1337, 13.37, raw_option={"optional": True}, error=TypeError), # FIXME those fails with TypeError + *all_fails(False, True, 0, 1, -1, 1337, 13.37, raw_option={"optional": True}), *all_fails([], ["one"], {}, raw_option={"optional": True}, error=AttributeError), # FIXME those fails with AttributeError *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), @@ -729,9 +739,9 @@ class TestPassword(BaseTest): ], reason="Should output exactly the same"), ("s3cr3t!!", "s3cr3t!!"), ("secret", FAIL), - *[("supersecret" + char, FAIL) for char in PasswordOption.forbidden_chars], # FIXME maybe add ` \n` to the list? + *[("supersecret" + char, FAIL) for char in FORBIDDEN_PASSWORD_CHARS], # FIXME maybe add ` \n` to the list? # readonly - ("s3cr3t!!", YunohostError, {"readonly": True, "current_value": "isforbidden"}), # readonly is forbidden + ("s3cr3t!!", FAIL, {"readonly": True, "current_value": "isforbidden"}), # readonly is forbidden ] # fmt: on @@ -744,35 +754,31 @@ class TestPassword(BaseTest): class TestColor(BaseTest): raw_option = {"type": "color", "id": "color_id"} prefill = { - "raw_option": {"default": "#ff0000"}, - "prefill": "#ff0000", - # "intake": "#ff00ff", + "raw_option": {"default": "red"}, + "prefill": "red", } # fmt: off scenarios = [ *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), - *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), # custom valid - ("#000000", "#000000"), + (" #fe1 ", "#fe1"), + ("#000000", "#000"), ("#000", "#000"), - ("#fe100", "#fe100"), - (" #fe100 ", "#fe100"), - ("#ABCDEF", "#ABCDEF"), + ("#ABCDEF", "#abcdef"), + ('1337', "#1337"), # rgba=(17, 51, 51, 0.47) + ("000000", "#000"), + ("#feaf", "#fea"), # `#feaf` is `#fea` with alpha at `f|100%` -> equivalent to `#fea` + # named + ("red", "#f00"), + ("yellow", "#ff0"), # custom fail - *xpass(scenarios=[ - ("#feaf", "#feaf"), - ], reason="Should fail; not a legal color value"), - ("000000", FAIL), ("#12", FAIL), ("#gggggg", FAIL), ("#01010101af", FAIL), - *xfail(scenarios=[ - ("red", "#ff0000"), - ("yellow", "#ffff00"), - ], reason="Should work with pydantic"), # readonly - ("#ffff00", "#fe100", {"readonly": True, "current_value": "#fe100"}), + ("#ffff00", "#000", {"readonly": True, "default": "#000"}), ] # fmt: on @@ -796,10 +802,8 @@ class TestNumber(BaseTest): *nones(None, "", output=None), *unchanged(0, 1, -1, 1337), - *xpass(scenarios=[(False, False)], reason="should fail or output as `0`"), - *xpass(scenarios=[(True, True)], reason="should fail or output as `1`"), - *all_as("0", 0, output=0), - *all_as("1", 1, output=1), + *all_as(False, "0", 0, output=0), # FIXME should `False` fail instead? + *all_as(True, "1", 1, output=1), # FIXME should `True` fail instead? *all_as("1337", 1337, output=1337), *xfail(scenarios=[ ("-1", -1) @@ -814,7 +818,7 @@ class TestNumber(BaseTest): (-10, -10, {"default": 10}), (-10, -10, {"default": 10, "optional": True}), # readonly - (1337, 10000, {"readonly": True, "current_value": 10000}), + (1337, 10000, {"readonly": True, "default": "10000"}), ] # fmt: on # FIXME should `step` be some kind of "multiple of"? @@ -839,14 +843,20 @@ class TestBoolean(BaseTest): *all_fails("none", "None"), # FIXME should output as `0` (default) like other none values when required? *all_as(None, "", output=0, raw_option={"optional": True}), # FIXME should output as `None`? *all_as("none", "None", output=None, raw_option={"optional": True}), - # FIXME even if default is explicity `None|""`, it ends up with class_default `0` - *all_as(None, "", output=0, raw_option={"default": None}), # FIXME this should fail, default is `None` - *all_as(None, "", output=0, raw_option={"optional": True, "default": None}), # FIXME even if default is explicity None, it ends up with class_default - *all_as(None, "", output=0, raw_option={"default": ""}), # FIXME this should fail, default is `""` - *all_as(None, "", output=0, raw_option={"optional": True, "default": ""}), # FIXME even if default is explicity None, it ends up with class_default - # With "none" behavior is ok - *all_fails(None, "", raw_option={"default": "none"}), - *all_as(None, "", output=None, raw_option={"optional": True, "default": "none"}), + { + "raw_options": [ + {"default": None}, + {"default": ""}, + {"default": "none"}, + {"default": "None"} + ], + "scenarios": [ + # All none values fails if default is overriden + *all_fails(None, "", "none", "None"), + # All none values ends up as None if default is overriden + *all_as(None, "", "none", "None", output=None, raw_option={"optional": True}), + ] + }, # Unhandled types should fail *all_fails(1337, "1337", "string", [], "[]", ",", "one,two"), *all_fails(1337, "1337", "string", [], "[]", ",", "one,two", {"optional": True}), @@ -879,7 +889,7 @@ class TestBoolean(BaseTest): "scenarios": all_fails("", "y", "n", error=AssertionError), }, # readonly - (1, 0, {"readonly": True, "current_value": 0}), + (1, 0, {"readonly": True, "default": 0}), ] @@ -896,8 +906,12 @@ class TestDate(BaseTest): } # fmt: off scenarios = [ - *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), - *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + # Those passes since False|True are parsed as 0|1 then int|float are considered a timestamp in seconds which ends up as default Unix date + *all_as(False, True, 0, 1, 1337, 13.37, "0", "1", "1337", "13.37", output="1970-01-01"), + # Those are negative one second timestamp ending up as Unix date - 1 sec (so day change) + *all_as(-1, "-1", output="1969-12-31"), + *all_fails([], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), # custom valid ("2070-12-31", "2070-12-31"), @@ -906,18 +920,16 @@ class TestDate(BaseTest): ("2025-06-15T13:45:30", "2025-06-15"), ("2025-06-15 13:45:30", "2025-06-15") ], reason="iso date repr should be valid and extra data striped"), - *xfail(scenarios=[ - (1749938400, "2025-06-15"), - (1749938400.0, "2025-06-15"), - ("1749938400", "2025-06-15"), - ("1749938400.0", "2025-06-15"), - ], reason="timestamp could be an accepted value"), + (1749938400, "2025-06-14"), + (1749938400.0, "2025-06-14"), + ("1749938400", "2025-06-14"), + ("1749938400.0", "2025-06-14"), # custom invalid ("29-12-2070", FAIL), ("12-01-10", FAIL), ("2022-02-29", FAIL), # readonly - ("2070-12-31", "2024-02-29", {"readonly": True, "current_value": "2024-02-29"}), + ("2070-12-31", "2024-02-29", {"readonly": True, "default": "2024-02-29"}), ] # fmt: on @@ -935,22 +947,26 @@ class TestTime(BaseTest): } # fmt: off scenarios = [ - *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), - *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + # Those passes since False|True are parsed as 0|1 then int|float are considered a timestamp in seconds but we don't take seconds into account so -> 00:00 + *all_as(False, True, 0, 1, 13.37, "0", "1", "13.37", output="00:00"), + # 1337 seconds == 22 minutes + *all_as(1337, "1337", output="00:22"), + # Negative timestamp fails + *all_fails(-1, "-1", error=OverflowError), # FIXME should handle that as a validation error + # *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), # custom valid *unchanged("00:00", "08:00", "12:19", "20:59", "23:59"), - ("3:00", "3:00"), # FIXME should fail or output as `"03:00"`? - *xfail(scenarios=[ - ("22:35:05", "22:35"), - ("22:35:03.514", "22:35"), - ], reason="time as iso format could be valid"), + ("3:00", "03:00"), + ("23:1", "23:01"), + ("22:35:05", "22:35"), + ("22:35:03.514", "22:35"), # custom invalid ("24:00", FAIL), - ("23:1", FAIL), ("23:005", FAIL), # readonly - ("00:00", "08:00", {"readonly": True, "current_value": "08:00"}), + ("00:00", "08:00", {"readonly": True, "default": "08:00"}), ] # fmt: on @@ -973,72 +989,75 @@ class TestEmail(BaseTest): *nones(None, "", output=""), ("\n Abc@example.tld ", "Abc@example.tld"), + *xfail(scenarios=[("admin@ynh.local", "admin@ynh.local")], reason="Should this pass?"), # readonly - ("Abc@example.tld", "admin@ynh.local", {"readonly": True, "current_value": "admin@ynh.local"}), + ("Abc@example.tld", "admin@ynh.org", {"readonly": True, "default": "admin@ynh.org"}), # Next examples are from https://github.com/JoshData/python-email-validator/blob/main/tests/test_syntax.py # valid email values - ("Abc@example.tld", "Abc@example.tld"), - ("Abc.123@test-example.com", "Abc.123@test-example.com"), - ("user+mailbox/department=shipping@example.tld", "user+mailbox/department=shipping@example.tld"), - ("伊昭傑@郵件.商務", "伊昭傑@郵件.商務"), - ("राम@मोहन.ईन्फो", "राम@मोहन.ईन्फो"), - ("юзер@екзампл.ком", "юзер@екзампл.ком"), - ("θσερ@εχαμπλε.ψομ", "θσερ@εχαμπλε.ψομ"), - ("葉士豪@臺網中心.tw", "葉士豪@臺網中心.tw"), - ("jeff@臺網中心.tw", "jeff@臺網中心.tw"), - ("葉士豪@臺網中心.台灣", "葉士豪@臺網中心.台灣"), - ("jeff葉@臺網中心.tw", "jeff葉@臺網中心.tw"), - ("ñoñó@example.tld", "ñoñó@example.tld"), - ("甲斐黒川日本@example.tld", "甲斐黒川日本@example.tld"), - ("чебурашкаящик-с-апельсинами.рф@example.tld", "чебурашкаящик-с-апельсинами.рф@example.tld"), - ("उदाहरण.परीक्ष@domain.with.idn.tld", "उदाहरण.परीक्ष@domain.with.idn.tld"), - ("ιωάννης@εεττ.gr", "ιωάννης@εεττ.gr"), + *unchanged( + "Abc@example.tld", + "Abc.123@test-example.com", + "user+mailbox/department=shipping@example.tld", + "伊昭傑@郵件.商務", + "राम@मोहन.ईन्फो", + "юзер@екзампл.ком", + "θσερ@εχαμπλε.ψομ", + "葉士豪@臺網中心.tw", + "jeff@臺網中心.tw", + "葉士豪@臺網中心.台灣", + "jeff葉@臺網中心.tw", + "ñoñó@example.tld", + "甲斐黒川日本@example.tld", + "чебурашкаящик-с-апельсинами.рф@example.tld", + "उदाहरण.परीक्ष@domain.with.idn.tld", + "ιωάννης@εεττ.gr", + ), # invalid email (Hiding because our current regex is very permissive) - # ("my@localhost", FAIL), - # ("my@.leadingdot.com", FAIL), - # ("my@．leadingfwdot.com", FAIL), - # ("my@twodots..com", FAIL), - # ("my@twofwdots．．.com", FAIL), - # ("my@trailingdot.com.", FAIL), - # ("my@trailingfwdot.com．", FAIL), - # ("me@-leadingdash", FAIL), - # ("me@－leadingdashfw", FAIL), - # ("me@trailingdash-", FAIL), - # ("me@trailingdashfw－", FAIL), - # ("my@baddash.-.com", FAIL), - # ("my@baddash.-a.com", FAIL), - # ("my@baddash.b-.com", FAIL), - # ("my@baddashfw.－.com", FAIL), - # ("my@baddashfw.－a.com", FAIL), - # ("my@baddashfw.b－.com", FAIL), - # ("my@example.com\n", FAIL), - # ("my@example\n.com", FAIL), - # ("me@x!", FAIL), - # ("me@x ", FAIL), - # (".leadingdot@domain.com", FAIL), - # ("twodots..here@domain.com", FAIL), - # ("trailingdot.@domain.email", FAIL), - # ("me@⒈wouldbeinvalid.com", FAIL), - ("@example.com", FAIL), - # ("\nmy@example.com", FAIL), - ("m\ny@example.com", FAIL), - ("my\n@example.com", FAIL), - # ("11111111112222222222333333333344444444445555555555666666666677777@example.com", FAIL), - # ("111111111122222222223333333333444444444455555555556666666666777777@example.com", FAIL), - # ("me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444444444455555555556.com", FAIL), - # ("me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", FAIL), - # ("me@中1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", FAIL), - # ("my.long.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333333344444.info", FAIL), - # ("my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333.info", FAIL), - # ("my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", FAIL), - # ("my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444.info", FAIL), - # ("my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", FAIL), - # ("me@bad-tld-1", FAIL), - # ("me@bad.tld-2", FAIL), - # ("me@xn--0.tld", FAIL), - # ("me@yy--0.tld", FAIL), - # ("me@yy－－0.tld", FAIL), + *all_fails( + "my@localhost", + "my@.leadingdot.com", + "my@．leadingfwdot.com", + "my@twodots..com", + "my@twofwdots．．.com", + "my@trailingdot.com.", + "my@trailingfwdot.com．", + "me@-leadingdash", + "me@－leadingdashfw", + "me@trailingdash-", + "me@trailingdashfw－", + "my@baddash.-.com", + "my@baddash.-a.com", + "my@baddash.b-.com", + "my@baddashfw.－.com", + "my@baddashfw.－a.com", + "my@baddashfw.b－.com", + "my@example\n.com", + "me@x!", + "me@x ", + ".leadingdot@domain.com", + "twodots..here@domain.com", + "trailingdot.@domain.email", + "me@⒈wouldbeinvalid.com", + "@example.com", + "m\ny@example.com", + "my\n@example.com", + "11111111112222222222333333333344444444445555555555666666666677777@example.com", + "111111111122222222223333333333444444444455555555556666666666777777@example.com", + "me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444444444455555555556.com", + "me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", + "me@中1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", + "my.long.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333333344444.info", + "my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333.info", + "my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", + "my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444.info", + "my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", + "me@bad-tld-1", + "me@bad.tld-2", + "me@xn--0.tld", + "me@yy--0.tld", + "me@yy－－0.tld", + ) ] # fmt: on @@ -1087,7 +1106,7 @@ class TestWebPath(BaseTest): ("https://example.com/folder", "/https://example.com/folder") ], reason="Should fail or scheme+domain removed"), # readonly - ("/overwrite", "/value", {"readonly": True, "current_value": "/value"}), + ("/overwrite", "/value", {"readonly": True, "default": "/value"}), # FIXME should path have forbidden_chars? ] # fmt: on @@ -1111,21 +1130,17 @@ class TestUrl(BaseTest): *nones(None, "", output=""), ("http://some.org/folder/file.txt", "http://some.org/folder/file.txt"), + (' https://www.example.com \n', 'https://www.example.com'), # readonly - ("https://overwrite.org", "https://example.org", {"readonly": True, "current_value": "https://example.org"}), + ("https://overwrite.org", "https://example.org", {"readonly": True, "default": "https://example.org"}), # rest is taken from https://github.com/pydantic/pydantic/blob/main/tests/test_networks.py # valid *unchanged( # Those are valid but not sure how they will output with pydantic 'http://example.org', - 'http://test', - 'http://localhost', 'https://example.org/whatever/next/', 'https://example.org', - 'http://localhost', - 'http://localhost/', - 'http://localhost:8000', - 'http://localhost:8000/', + 'https://foo_bar.example.com/', 'http://example.co.jp', 'http://www.example.com/a%C2%B1b', @@ -1149,29 +1164,31 @@ class TestUrl(BaseTest): 'http://twitter.com/@handle/', 'http://11.11.11.11.example.com/action', 'http://abc.11.11.11.11.example.com/action', - 'http://example#', - 'http://example/#', - 'http://example/#fragment', - 'http://example/?#', 'http://example.org/path#', 'http://example.org/path#fragment', 'http://example.org/path?query#', 'http://example.org/path?query#fragment', + 'https://foo_bar.example.com/', + 'https://exam_ple.com/', + 'HTTP://EXAMPLE.ORG', + 'https://example.org', + 'https://example.org?a=1&b=2', + 'https://example.org#a=3;b=3', + 'https://example.xn--p1ai', + 'https://example.xn--vermgensberatung-pwb', + 'https://example.xn--zfr164b', ), - # Pydantic default parsing add a final `/` - ('https://foo_bar.example.com/', 'https://foo_bar.example.com/'), - ('https://exam_ple.com/', 'https://exam_ple.com/'), *xfail(scenarios=[ - (' https://www.example.com \n', 'https://www.example.com/'), - ('HTTP://EXAMPLE.ORG', 'http://example.org/'), - ('https://example.org', 'https://example.org/'), - ('https://example.org?a=1&b=2', 'https://example.org/?a=1&b=2'), - ('https://example.org#a=3;b=3', 'https://example.org/#a=3;b=3'), - ('https://example.xn--p1ai', 'https://example.xn--p1ai/'), - ('https://example.xn--vermgensberatung-pwb', 'https://example.xn--vermgensberatung-pwb/'), - ('https://example.xn--zfr164b', 'https://example.xn--zfr164b/'), - ], reason="pydantic default behavior would append a final `/`"), - + ('http://test', 'http://test'), + ('http://localhost', 'http://localhost'), + ('http://localhost/', 'http://localhost/'), + ('http://localhost:8000', 'http://localhost:8000'), + ('http://localhost:8000/', 'http://localhost:8000/'), + ('http://example#', 'http://example#'), + ('http://example/#', 'http://example/#'), + ('http://example/#fragment', 'http://example/#fragment'), + ('http://example/?#', 'http://example/?#'), + ], reason="Should this be valid?"), # invalid *all_fails( 'ftp://example.com/', @@ -1182,15 +1199,13 @@ class TestUrl(BaseTest): "/", "+http://example.com/", "ht*tp://example.com/", + "http:///", + "http://??", + "https://example.org more", + "http://2001:db8::ff00:42:8329", + "http://[192.168.1.1]:8329", + "http://example.com:99999", ), - *xpass(scenarios=[ - ("http:///", "http:///"), - ("http://??", "http://??"), - ("https://example.org more", "https://example.org more"), - ("http://2001:db8::ff00:42:8329", "http://2001:db8::ff00:42:8329"), - ("http://[192.168.1.1]:8329", "http://[192.168.1.1]:8329"), - ("http://example.com:99999", "http://example.com:99999"), - ], reason="Should fail"), ] # fmt: on @@ -1361,7 +1376,6 @@ class TestSelect(BaseTest): # [-1, 0, 1] "raw_options": [ {"choices": [-1, 0, 1, 10]}, - {"choices": {-1: "verbose -one", 0: "verbose zero", 1: "verbose one", 10: "verbose ten"}}, ], "scenarios": [ *nones(None, "", output=""), @@ -1375,6 +1389,18 @@ class TestSelect(BaseTest): *all_fails("100", 100), ] }, + { + "raw_options": [ + {"choices": {-1: "verbose -one", 0: "verbose zero", 1: "verbose one", 10: "verbose ten"}}, + {"choices": {"-1": "verbose -one", "0": "verbose zero", "1": "verbose one", "10": "verbose ten"}}, + ], + "scenarios": [ + *nones(None, "", output=""), + *all_fails(-1, 0, 1, 10), # Should pass? converted to str? + *unchanged("-1", "0", "1", "10"), + *all_fails("100", 100), + ] + }, # [True, False, None] *unchanged(True, False, raw_option={"choices": [True, False, None]}), # FIXME we should probably forbid None in choices (None, FAIL, {"choices": [True, False, None]}), @@ -1402,7 +1428,7 @@ class TestSelect(BaseTest): ] }, # readonly - ("one", "two", {"readonly": True, "choices": ["one", "two"], "current_value": "two"}), + ("one", "two", {"readonly": True, "choices": ["one", "two"], "default": "two"}), ] # fmt: on @@ -1411,7 +1437,7 @@ class TestSelect(BaseTest): # │ TAGS │ # ╰───────────────────────────────────────────────────────╯ - +# [], ["one"], {} class TestTags(BaseTest): raw_option = {"type": "tags", "id": "tags_id"} prefill = { @@ -1420,12 +1446,7 @@ class TestTags(BaseTest): } # fmt: off scenarios = [ - *nones(None, [], "", output=""), - # FIXME `","` could be considered a none value which kinda already is since it fail when required - (",", FAIL), - *xpass(scenarios=[ - (",", ",", {"optional": True}) - ], reason="Should output as `''`? ie: None"), + *nones(None, [], "", ",", output=""), { "raw_options": [ {}, @@ -1450,7 +1471,7 @@ class TestTags(BaseTest): *all_fails(*([t] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), *all_fails(*([str(t)] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), # readonly - ("one", "one,two", {"readonly": True, "choices": ["one", "two"], "current_value": "one,two"}), + ("one", "one,two", {"readonly": True, "choices": ["one", "two"], "default": "one,two"}), ] # fmt: on @@ -1566,8 +1587,7 @@ class TestApp(BaseTest): ], "scenarios": [ # FIXME there are currently 3 different nones (`None`, `""` and `_none`), choose one? - *nones(None, output=None), # FIXME Should return chosen none? - *nones("", output=""), # FIXME Should return chosen none? + *nones(None, "", output=""), # FIXME Should return chosen none? *xpass(scenarios=[ ("_none", "_none"), ("_none", "_none", {"default": "_none"}), @@ -1590,7 +1610,7 @@ class TestApp(BaseTest): (installed_webapp["id"], installed_webapp["id"], {"filter": "is_webapp"}), (installed_webapp["id"], FAIL, {"filter": "is_webapp == false"}), (installed_webapp["id"], FAIL, {"filter": "id != 'my_webapp'"}), - (None, None, {"filter": "id == 'fake_app'", "optional": True}), + (None, "", {"filter": "id == 'fake_app'", "optional": True}), ] }, { @@ -1800,7 +1820,7 @@ class TestGroup(BaseTest): ("", "custom_group", {"default": "custom_group"}), ], reason="Should throw 'default must be in (None, 'all_users', 'visitors', 'admins')"), # readonly - ("admins", YunohostError, {"readonly": True}), # readonly is forbidden + ("admins", "all_users", {"readonly": True}), # readonly is forbidden (default is "all_users") ] }, ] @@ -1880,12 +1900,12 @@ def test_options_query_string(): "string_id": "string", "text_id": "text\ntext", "password_id": "sUpRSCRT", - "color_id": "#ffff00", + "color_id": "#ff0", "number_id": 10, "boolean_id": 1, "date_id": "2030-03-06", "time_id": "20:55", - "email_id": "coucou@ynh.local", + "email_id": "coucou@ynh.org", "path_id": "/ynh-dev", "url_id": "https://yunohost.org", "file_id": file_content1, @@ -1908,7 +1928,7 @@ def patch_query_string(file_repr): "&boolean_id=y" "&date_id=2030-03-06" "&time_id=20:55" - "&email_id=coucou@ynh.local" + "&email_id=coucou@ynh.org" "&path_id=ynh-dev/" "&url_id=https://yunohost.org" f"&file_id={file_repr}" From 3ff6e6ed96c91e9ae94821f54f9939811edb930e Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 16:20:52 +0200 Subject: [PATCH 113/436] app: update app_install --- src/app.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/src/app.py b/src/app.py index 754a920a01..1a2e80442d 100644 --- a/src/app.py +++ b/src/app.py @@ -1098,13 +1098,9 @@ def app_install( app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) # Retrieve arguments list for install script - raw_questions = manifest["install"] - questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args) - args = { - question.id: question.value - for question in questions - if not question.readonly and question.value is not None - } + raw_options = manifest["install"] + options, form = ask_questions_and_parse_answers(raw_options, prefilled_answers=args) + args = form.dict(exclude_none=True) # Validate domain / path availability for webapps # (ideally this should be handled by the resource system for manifest v >= 2 @@ -1141,15 +1137,15 @@ def app_install( "current_revision": manifest.get("remote", {}).get("revision", "?"), } - # If packaging_format v2+, save all install questions as settings + # If packaging_format v2+, save all install options as settings if packaging_format >= 2: - for question in questions: + for option in options: # Except user-provider passwords # ... which we need to reinject later in the env_dict - if question.type == "password": + if option.type == "password": continue - app_settings[question.id] = question.value + app_settings[option.id] = form[option.id] _set_app_settings(app_instance_name, app_settings) @@ -1202,23 +1198,23 @@ def app_install( app_instance_name, args=args, workdir=extracted_app_folder, action="install" ) - # If packaging_format v2+, save all install questions as settings + # If packaging_format v2+, save all install options as settings if packaging_format >= 2: - for question in questions: + for option in options: # Reinject user-provider passwords which are not in the app settings # (cf a few line before) - if question.type == "password": - env_dict[question.id] = question.value + if option.type == "password": + env_dict[option.id] = form[option.id] # We want to hav the env_dict in the log ... but not password values env_dict_for_logging = env_dict.copy() - for question in questions: - # Or should it be more generally question.redact ? - if question.type == "password": - if f"YNH_APP_ARG_{question.id.upper()}" in env_dict_for_logging: - del env_dict_for_logging[f"YNH_APP_ARG_{question.id.upper()}"] - if question.id in env_dict_for_logging: - del env_dict_for_logging[question.id] + for option in options: + # Or should it be more generally option.redact ? + if option.type == "password": + if f"YNH_APP_ARG_{option.id.upper()}" in env_dict_for_logging: + del env_dict_for_logging[f"YNH_APP_ARG_{option.id.upper()}"] + if option.id in env_dict_for_logging: + del env_dict_for_logging[option.id] operation_logger.extra.update({"env": env_dict_for_logging}) From 582b1ed311e5feb0823a2742a1f8b58c64a2165d Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 19:51:16 +0200 Subject: [PATCH 114/436] form: add translating method --- src/tests/test_questions.py | 10 ++++----- src/utils/form.py | 41 +++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 1f8667adfb..99ab9f156b 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -447,7 +447,7 @@ def _test_basic_attrs(self): assert isinstance(option, OPTIONS[raw_option["type"]]) assert option.type == raw_option["type"] assert option.id == id_ - assert option.ask == {"en": id_} + assert option.ask == id_ assert option.readonly is (True if is_special_readonly_option else False) assert option.visible is True # assert option.bind is None @@ -493,7 +493,7 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): ) option, value = _fill_or_prompt_one_option(raw_option, None) - expected_message = option.ask["en"] + expected_message = option.ask choices = [] if isinstance(option, BaseChoicesOption): @@ -510,7 +510,7 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): prefill=prefill, is_multiline=option.type == "text", autocomplete=choices, - help=option.help["en"], + help=option.help, ) def test_scenarios(self, intake, expected_output, raw_option, data): @@ -558,7 +558,7 @@ def test_scenarios(self, intake, expected_output, raw_option, data): options, form = ask_questions_and_parse_answers( {_id: raw_option}, answers ) - assert stdout.getvalue() == f"{options[0].ask['en']}\n" + assert stdout.getvalue() == f"{options[0].ask}\n" # ╭───────────────────────────────────────────────────────╮ @@ -609,7 +609,7 @@ def test_scenarios(self, intake, expected_output, raw_option, data): options, form = ask_questions_and_parse_answers( {"display_text_id": raw_option}, answers ) - ask = options[0].ask["en"] + ask = options[0].ask if style in colors: color = colors[style] title = style.title() + (":" if style != "success" else "!") diff --git a/src/utils/form.py b/src/utils/form.py index cbe64b499f..71954ee2b3 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -297,15 +297,6 @@ def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: del schema["description"] schema["additionalProperties"] = False - @validator("ask", always=True) - def parse_or_set_default_ask( - cls, value: Union[Translation, None], values: Values - ) -> Translation: - if value is None: - return {"en": values["id"]} - if isinstance(value, str): - return {"en": value} - return value @validator("readonly", pre=True) def can_be_readonly(cls, value: bool, values: Values) -> bool: @@ -327,7 +318,9 @@ def is_visible(self, context: Context) -> bool: return evaluate_simple_js_expression(self.visible, context=context) def _get_prompt_message(self, value: None) -> str: - return _value_for_locale(self.ask) + # force type to str + # `OptionsModel.translate_options()` should have been called before calling this method + return cast(str, self.ask) # ╭───────────────────────────────────────────────────────╮ @@ -367,7 +360,7 @@ def _get_prompt_message(self, value: None) -> str: State.danger: "red", } message = m18n.g(self.style) if self.style != State.danger else m18n.n("danger") - return f"{colorize(message, colors[self.style])} {_value_for_locale(self.ask)}" + return f"{colorize(message, colors[self.style])} {self.ask}" class ButtonOption(BaseReadonlyOption): @@ -624,6 +617,7 @@ class NumberOption(BaseInputOption): max: Union[int, None] = None step: Union[int, None] = None _annotation = int + _none_as_empty_str = False @staticmethod def normalize(value, option={}): @@ -1274,6 +1268,26 @@ def options_dict_to_list(options: dict[str, Any], defaults: dict[str, Any] = {}) def __init__(self, **kwargs) -> None: super().__init__(options=self.options_dict_to_list(kwargs)) + def translate_options(self, i18n_key: Union[str, None] = None): + """ + Mutate in place translatable attributes of options to their translations + """ + for option in self.options: + for key in ("ask", "help"): + if not hasattr(option, key): + continue + + value = getattr(option, key) + if value: + setattr(option, key, _value_for_locale(value)) + elif key == "ask" and m18n.key_exists(f"{i18n_key}_{option.id}"): + setattr(option, key, m18n.n(f"{i18n_key}_{option.id}")) + elif key == "help" and m18n.key_exists(f"{i18n_key}_{option.id}_help"): + setattr(option, key, m18n.n(f"{i18n_key}_{option.id}_help")) + elif key == "ask": + # FIXME warn? + option.ask = option.id + class FormModel(BaseModel): """ @@ -1384,7 +1398,7 @@ def prompt_or_validate_form( raise YunohostValidationError( "config_action_disabled", action=option.id, - help=_value_for_locale(option.help), + help=option.help, ) if not option.is_visible(context): @@ -1433,7 +1447,7 @@ def prompt_or_validate_form( prefill=value, is_multiline=isinstance(option, TextOption), autocomplete=choices, - help=_value_for_locale(option.help), + help=option.help, ) # Apply default value if none @@ -1512,6 +1526,7 @@ def ask_questions_and_parse_answers( # FIXME use YunohostError instead since it is not really a user mistake? raise YunohostValidationError(error, raw_msg=True) + model.translate_options() # Build the form from those questions and instantiate it without # parsing/validation (construct) since it may contains required questions. form = build_form(model.options).construct() From a574855a039b0ef39cd459690e0ac62c31ae7cda Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 20:01:31 +0200 Subject: [PATCH 115/436] form: fix forbidden readonly type --- src/tests/test_questions.py | 10 +++++----- src/utils/form.py | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 99ab9f156b..959f2c8b7a 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -741,7 +741,7 @@ class TestPassword(BaseTest): ("secret", FAIL), *[("supersecret" + char, FAIL) for char in FORBIDDEN_PASSWORD_CHARS], # FIXME maybe add ` \n` to the list? # readonly - ("s3cr3t!!", FAIL, {"readonly": True, "current_value": "isforbidden"}), # readonly is forbidden + ("s3cr3t!!", FAIL, {"readonly": True}), # readonly is forbidden ] # fmt: on @@ -1519,7 +1519,7 @@ class TestDomain(BaseTest): ("doesnt_exist.pouet", FAIL, {}), ("fake.com", FAIL, {"choices": ["fake.com"]}), # readonly - (domains1[0], YunohostError, {"readonly": True}), # readonly is forbidden + (domains1[0], FAIL, {"readonly": True}), # readonly is forbidden ] }, { @@ -1619,7 +1619,7 @@ class TestApp(BaseTest): (installed_non_webapp["id"], installed_non_webapp["id"]), (installed_non_webapp["id"], FAIL, {"filter": "is_webapp"}), # readonly - (installed_non_webapp["id"], YunohostError, {"readonly": True}), # readonly is forbidden + (installed_non_webapp["id"], FAIL, {"readonly": True}), # readonly is forbidden ] }, ] @@ -1736,7 +1736,7 @@ class TestUser(BaseTest): ("", regular_username, {"default": regular_username}) ], reason="Should throw 'no default allowed'"), # readonly - (admin_username, YunohostError, {"readonly": True}), # readonly is forbidden + (admin_username, FAIL, {"readonly": True}), # readonly is forbidden ] }, ] @@ -1820,7 +1820,7 @@ class TestGroup(BaseTest): ("", "custom_group", {"default": "custom_group"}), ], reason="Should throw 'default must be in (None, 'all_users', 'visitors', 'admins')"), # readonly - ("admins", "all_users", {"readonly": True}), # readonly is forbidden (default is "all_users") + ("admins", FAIL, {"readonly": True}), # readonly is forbidden ] }, ] diff --git a/src/utils/form.py b/src/utils/form.py index 71954ee2b3..6246a91e5a 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -300,8 +300,7 @@ def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: @validator("readonly", pre=True) def can_be_readonly(cls, value: bool, values: Values) -> bool: - forbidden_types = ("password", "app", "domain", "user", "file") - if value is True and values["type"] in forbidden_types: + if value is True and values["type"] in FORBIDDEN_READONLY_TYPES: raise ValueError( m18n.n( "config_forbidden_readonly_type", From 774b11cbbeffcc2480b146f8557143876d992509 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 20:02:02 +0200 Subject: [PATCH 116/436] form: add legacy "name" attr --- src/utils/form.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/utils/form.py b/src/utils/form.py index 6246a91e5a..7e009c5b59 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -285,6 +285,7 @@ class BaseOption(BaseModel): readonly: bool = False visible: Union[JSExpression, bool] = True bind: Union[str, None] = None + name: Union[str, None] = None # LEGACY (replaced by `id`) class Config: arbitrary_types_allowed = True @@ -297,6 +298,12 @@ def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: del schema["description"] schema["additionalProperties"] = False + # FIXME Legacy, is `name` still needed? + @validator("name", pre=True, always=True) + def apply_legacy_name(cls, value: Union[str, None], values: Values) -> str: + if value is None: + return values["id"] + return value @validator("readonly", pre=True) def can_be_readonly(cls, value: bool, values: Values) -> bool: From bec34b92b065f964b88be37eef7c79bafa4598fc Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 20:03:35 +0200 Subject: [PATCH 117/436] form: add reserved "id" validator --- src/utils/configpanel.py | 25 ------------------------- src/utils/form.py | 27 ++++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 86dea2e7d3..53cd4b9c8b 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -455,31 +455,6 @@ def _build_internal_config_panel(raw_infos, level): "config_unknown_filter_key", filter_key=self.filter_key ) - # List forbidden keywords from helpers and sections toml (to avoid conflict) - forbidden_keywords = [ - "old", - "app", - "changed", - "file_hash", - "binds", - "types", - "formats", - "getter", - "setter", - "short_setting", - "type", - "bind", - "nothing_changed", - "changes_validated", - "result", - "max_progression", - ] - forbidden_keywords += format_description["sections"] - - for _, _, option in self._iterate(): - if option["id"] in forbidden_keywords: - raise YunohostError("config_forbidden_keyword", keyword=option["id"]) - return self.config def _get_default_values(self): diff --git a/src/utils/form.py b/src/utils/form.py index 7e009c5b59..6c14bcdf06 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -265,7 +265,26 @@ class OptionType(str, Enum): OptionType.user, OptionType.group, } - +FORBIDDEN_KEYWORDS = { + "old", + "app", + "changed", + "file_hash", + "binds", + "types", + "formats", + "getter", + "setter", + "short_setting", + "type", + "bind", + "nothing_changed", + "changes_validated", + "result", + "max_progression", + "properties", + "defaults", +} Context = dict[str, Any] Translation = Union[dict[str, str], str] @@ -298,6 +317,12 @@ def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: del schema["description"] schema["additionalProperties"] = False + @validator("id", pre=True) + def check_id_is_not_forbidden(cls, value: str) -> str: + if value in FORBIDDEN_KEYWORDS: + raise ValueError(m18n.n("config_forbidden_keyword", keyword=value)) + return value + # FIXME Legacy, is `name` still needed? @validator("name", pre=True, always=True) def apply_legacy_name(cls, value: Union[str, None], values: Values) -> str: From 564a66de2fab6b29b6d4457ea5e5b9084fdfbb0c Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 17 Apr 2023 20:08:52 +0200 Subject: [PATCH 118/436] configpanel: add config panel models --- src/utils/configpanel.py | 192 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 2 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 53cd4b9c8b..bf441798cf 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -21,8 +21,10 @@ import re import urllib.parse from collections import OrderedDict -from typing import Union from logging import getLogger +from typing import TYPE_CHECKING, Any, Literal, Sequence, Type, Union + +from pydantic import BaseModel, Extra, validator from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize @@ -30,20 +32,206 @@ from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.form import ( OPTIONS, - BaseChoicesOption, BaseInputOption, BaseOption, + BaseReadonlyOption, FileOption, + OptionsModel, OptionType, + Translation, ask_questions_and_parse_answers, evaluate_simple_js_expression, ) from yunohost.utils.i18n import _value_for_locale +if TYPE_CHECKING: + from pydantic.fields import ModelField + logger = getLogger("yunohost.configpanel") + + +# ╭───────────────────────────────────────────────────────╮ +# │ ╭╮╮╭─╮┌─╮┌─╴╷ ╭─╴ │ +# │ ││││ ││ │├─╴│ ╰─╮ │ +# │ ╵╵╵╰─╯└─╯╰─╴╰─╴╶─╯ │ +# ╰───────────────────────────────────────────────────────╯ + CONFIG_PANEL_VERSION_SUPPORTED = 1.0 +class ContainerModel(BaseModel): + id: str + name: Union[Translation, None] = None + services: list[str] = [] + help: Union[Translation, None] = None + + def translate(self, i18n_key: Union[str, None] = None): + """ + Translate `ask` and `name` attributes of panels and section. + This is in place mutation. + """ + + for key in ("help", "name"): + value = getattr(self, key) + if value: + setattr(self, key, _value_for_locale(value)) + elif key == "help" and m18n.key_exists(f"{i18n_key}_{self.id}_help"): + setattr(self, key, m18n.n(f"{i18n_key}_{self.id}_help")) + + +class SectionModel(ContainerModel, OptionsModel): + visible: Union[bool, str] = True + optional: bool = True + + # Don't forget to pass arguments to super init + def __init__( + self, + id: str, + name: Union[Translation, None] = None, + services: list[str] = [], + help: Union[Translation, None] = None, + visible: Union[bool, str] = True, + **kwargs, + ) -> None: + options = self.options_dict_to_list(kwargs, defaults={"optional": True}) + + ContainerModel.__init__( + self, + id=id, + name=name, + services=services, + help=help, + visible=visible, + options=options, + ) + + @property + def is_action_section(self): + return any([option.type is OptionType.button for option in self.options]) + + def is_visible(self, context: dict[str, Any]): + if isinstance(self.visible, bool): + return self.visible + + return evaluate_simple_js_expression(self.visible, context=context) + + def translate(self, i18n_key: Union[str, None] = None): + """ + Call to `Container`'s `translate` for self translation + + Call to `OptionsContainer`'s `translate_options` for options translation + """ + super().translate(i18n_key) + self.translate_options(i18n_key) + + +class PanelModel(ContainerModel): + # FIXME what to do with `actions? + actions: dict[str, Translation] = {"apply": {"en": "Apply"}} + sections: list[SectionModel] + + class Config: + extra = Extra.allow + + # Don't forget to pass arguments to super init + def __init__( + self, + id: str, + name: Union[Translation, None] = None, + services: list[str] = [], + help: Union[Translation, None] = None, + **kwargs, + ) -> None: + sections = [data | {"id": name} for name, data in kwargs.items()] + super().__init__( + id=id, name=name, services=services, help=help, sections=sections + ) + + def translate(self, i18n_key: Union[str, None] = None): + """ + Recursivly mutate translatable attributes to their translation + """ + super().translate(i18n_key) + + for section in self.sections: + section.translate(i18n_key) + + +class ConfigPanelModel(BaseModel): + version: float = CONFIG_PANEL_VERSION_SUPPORTED + i18n: Union[str, None] = None + panels: list[PanelModel] + + class Config: + arbitrary_types_allowed = True + extra = Extra.allow + + # Don't forget to pass arguments to super init + def __init__( + self, + version: float, + i18n: Union[str, None] = None, + **kwargs, + ) -> None: + panels = [data | {"id": name} for name, data in kwargs.items()] + super().__init__(version=version, i18n=i18n, panels=panels) + + @property + def sections(self): + """Convinient prop to iter on all sections""" + for panel in self.panels: + for section in panel.sections: + yield section + + @property + def options(self): + """Convinient prop to iter on all options""" + for section in self.sections: + for option in section.options: + yield option + + + def iter_children( + self, + trigger: list[Literal["panel", "section", "option", "action"]] = ["option"], + ): + for panel in self.panels: + if "panel" in trigger: + yield (panel, None, None) + for section in panel.sections: + if "section" in trigger: + yield (panel, section, None) + if "action" in trigger: + for option in section.options: + if option.type is OptionType.button: + yield (panel, section, option) + if "option" in trigger: + for option in section.options: + yield (panel, section, option) + + def translate(self): + """ + Recursivly mutate translatable attributes to their translation + """ + for panel in self.panels: + panel.translate(self.i18n) + + @validator("version", always=True) + def check_version(cls, value, field: "ModelField"): + if value < CONFIG_PANEL_VERSION_SUPPORTED: + raise ValueError( + f"Config panels version '{value}' are no longer supported." + ) + + return value + + +# ╭───────────────────────────────────────────────────────╮ +# │ ╭─╴╭─╮╭╮╷┌─╴╶┬╴╭─╮ ╶┬╴╭╮╮┌─╮╷ │ +# │ │ │ ││││├─╴ │ │╶╮ │ │││├─╯│ │ +# │ ╰─╴╰─╯╵╰╯╵ ╶┴╴╰─╯ ╶┴╴╵╵╵╵ ╰─╴ │ +# ╰───────────────────────────────────────────────────────╯ + + class ConfigPanel: entity_type = "config" save_path_tpl: Union[str, None] = None From 02948ad49c909f3d8eaed9bf3fdde0aada5bf11a Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 01:26:41 +0200 Subject: [PATCH 119/436] config: rework config+settings getter methods --- src/app.py | 10 +- src/dns.py | 10 +- src/domain.py | 59 ++++--- src/settings.py | 26 ++-- src/tests/test_dns.py | 6 +- src/utils/configpanel.py | 327 +++++++++++++-------------------------- src/utils/form.py | 4 +- 7 files changed, 166 insertions(+), 276 deletions(-) diff --git a/src/app.py b/src/app.py index 1a2e80442d..ffcd1ecc32 100644 --- a/src/app.py +++ b/src/app.py @@ -26,7 +26,8 @@ import subprocess import tempfile import copy -from typing import List, Tuple, Dict, Any, Iterator, Optional +from collections import OrderedDict +from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Iterator, Optional from packaging import version from logging import getLogger from pathlib import Path @@ -71,6 +72,9 @@ APPS_CATALOG_LOGOS, ) +if TYPE_CHECKING: + from yunohost.utils.configpanel import ConfigPanelModel, RawSettings + logger = getLogger("yunohost.app") APPS_SETTING_PATH = "/etc/yunohost/apps/" @@ -1802,8 +1806,8 @@ def _run_action(self, action): env = {key: str(value) for key, value in self.new_values.items()} self._call_config_script(action, env=env) - def _get_raw_settings(self): - self.values = self._call_config_script("show") + def _get_raw_settings(self, config: "ConfigPanelModel") -> "RawSettings": + return self._call_config_script("show") def _apply(self): env = {key: str(value) for key, value in self.new_values.items()} diff --git a/src/dns.py b/src/dns.py index 9a081e228a..07ff2fb215 100644 --- a/src/dns.py +++ b/src/dns.py @@ -528,7 +528,7 @@ def _get_registrar_config_section(domain): parent_domain=parent_domain, parent_domain_link=parent_domain_link, ), - "value": "parent_domain", + "default": "parent_domain", } ) return OrderedDict(registrar_infos) @@ -541,7 +541,7 @@ def _get_registrar_config_section(domain): "type": "alert", "style": "success", "ask": m18n.n("domain_dns_registrar_yunohost"), - "value": "yunohost", + "default": "yunohost", } ) return OrderedDict(registrar_infos) @@ -551,7 +551,7 @@ def _get_registrar_config_section(domain): "type": "alert", "style": "info", "ask": m18n.n("domain_dns_conf_special_use_tld"), - "value": None, + "default": None, } ) @@ -563,7 +563,7 @@ def _get_registrar_config_section(domain): "type": "alert", "style": "warning", "ask": m18n.n("domain_dns_registrar_not_supported"), - "value": None, + "default": None, } ) else: @@ -572,7 +572,7 @@ def _get_registrar_config_section(domain): "type": "alert", "style": "info", "ask": m18n.n("domain_dns_registrar_supported", registrar=registrar), - "value": registrar, + "default": registrar, } ) diff --git a/src/domain.py b/src/domain.py index 2a897c6256..a796e0142b 100644 --- a/src/domain.py +++ b/src/domain.py @@ -19,7 +19,7 @@ import os import time from pathlib import Path -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from collections import OrderedDict from logging import getLogger @@ -47,6 +47,9 @@ from yunohost.utils.dns import is_yunohost_dyndns_domain from yunohost.log import is_unit_operation +if TYPE_CHECKING: + from yunohost.utils.configpanel import RawConfig + logger = getLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" @@ -666,10 +669,14 @@ def get(self, key="", mode="classic"): return result - def _get_raw_config(self): - toml = super()._get_raw_config() + def _get_raw_config(self) -> "RawConfig": + # TODO add mechanism to share some settings with other domains on the same zone + raw_config = super()._get_raw_config() + + any_filter = all(self.filter_key) + panel_id, section_id, option_id = self.filter_key - toml["feature"]["xmpp"]["xmpp"]["default"] = ( + raw_config["feature"]["xmpp"]["xmpp"]["default"] = ( 1 if self.entity == _get_maindomain() else 0 ) @@ -680,55 +687,43 @@ def _get_raw_config(self): # Optimize wether or not to load the DNS section, # e.g. we don't want to trigger the whole _get_registary_config_section # when just getting the current value from the feature section - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if not filter_key or filter_key[0] == "dns": + if not any_filter or panel_id == "dns": from yunohost.dns import _get_registrar_config_section - toml["dns"]["registrar"] = _get_registrar_config_section(self.entity) - - # FIXME: Ugly hack to save the registar id/value and reinject it in _get_raw_settings ... - self.registar_id = toml["dns"]["registrar"]["registrar"]["value"] - del toml["dns"]["registrar"]["registrar"]["value"] + raw_config["dns"]["registrar"] = _get_registrar_config_section(self.entity) # Cert stuff - if not filter_key or filter_key[0] == "cert": + if not any_filter or panel_id == "cert": from yunohost.certificate import certificate_status status = certificate_status([self.entity], full=True)["certificates"][ self.entity ] - toml["cert"]["cert"]["cert_summary"]["style"] = status["style"] + raw_config["cert"]["cert"]["cert_summary"]["style"] = status["style"] # i18n: domain_config_cert_summary_expired # i18n: domain_config_cert_summary_selfsigned # i18n: domain_config_cert_summary_abouttoexpire # i18n: domain_config_cert_summary_ok # i18n: domain_config_cert_summary_letsencrypt - toml["cert"]["cert"]["cert_summary"]["ask"] = m18n.n( + raw_config["cert"]["cert"]["cert_summary"]["ask"] = m18n.n( f"domain_config_cert_summary_{status['summary']}" ) - # FIXME: Ugly hack to save the cert status and reinject it in _get_raw_settings ... - self.cert_status = status + for option_id, status_key in [ + ("cert_validity", "validity"), + ("cert_issuer", "CA_type"), + ("acme_eligible", "ACME_eligible"), + # FIXME not sure why "summary" was injected in settings values + # ("summary", "summary") + ]: + raw_config["cert"]["cert"][option_id]["default"] = status[status_key] - return toml + # Other specific strings used in config panels + # i18n: domain_config_cert_renew_help - def _get_raw_settings(self): - # TODO add mechanism to share some settings with other domains on the same zone - super()._get_raw_settings() - - # FIXME: Ugly hack to save the registar id/value and reinject it in _get_raw_settings ... - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if not filter_key or filter_key[0] == "dns": - self.values["registrar"] = self.registar_id - - # FIXME: Ugly hack to save the cert status and reinject it in _get_raw_settings ... - if not filter_key or filter_key[0] == "cert": - self.values["cert_validity"] = self.cert_status["validity"] - self.values["cert_issuer"] = self.cert_status["CA_type"] - self.values["acme_eligible"] = self.cert_status["ACME_eligible"] - self.values["summary"] = self.cert_status["summary"] + return raw_config def _apply(self): if ( diff --git a/src/settings.py b/src/settings.py index e2f34bda9a..f3340e8e9d 100644 --- a/src/settings.py +++ b/src/settings.py @@ -19,6 +19,7 @@ import os import subprocess from logging import getLogger +from typing import TYPE_CHECKING from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError @@ -29,6 +30,9 @@ from yunohost.log import is_unit_operation from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings +if TYPE_CHECKING: + from yunohost.utils.configpanel import ConfigPanelModel, RawConfig, RawSettings + logger = getLogger("yunohost.settings") SETTINGS_PATH = "/etc/yunohost/settings.yml" @@ -180,8 +184,8 @@ def reset(self, key="", operation_logger=None): logger.success(m18n.n("global_settings_reset_success")) operation_logger.success() - def _get_raw_config(self): - toml = super()._get_raw_config() + def _get_raw_config(self) -> "RawConfig": + raw_config = super()._get_raw_config() # Dynamic choice list for portal themes THEMEDIR = "/usr/share/ssowat/portal/assets/themes/" @@ -189,28 +193,30 @@ def _get_raw_config(self): themes = [d for d in os.listdir(THEMEDIR) if os.path.isdir(THEMEDIR + d)] except Exception: themes = ["unsplash", "vapor", "light", "default", "clouds"] - toml["misc"]["portal"]["portal_theme"]["choices"] = themes + raw_config["misc"]["portal"]["portal_theme"]["choices"] = themes - return toml + return raw_config - def _get_raw_settings(self): - super()._get_raw_settings() + def _get_raw_settings(self, config: "ConfigPanelModel") -> "RawSettings": + raw_settings = super()._get_raw_settings(config) # Specific logic for those settings who are "virtual" settings # and only meant to have a custom setter mapped to tools_rootpw - self.values["root_password"] = "" - self.values["root_password_confirm"] = "" + raw_settings["root_password"] = "" + raw_settings["root_password_confirm"] = "" # Specific logic for virtual setting "passwordless_sudo" try: from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() - self.values["passwordless_sudo"] = "!authenticate" in ldap.search( + raw_settings["passwordless_sudo"] = "!authenticate" in ldap.search( "ou=sudo", "cn=admins", ["sudoOption"] )[0].get("sudoOption", []) except Exception: - self.values["passwordless_sudo"] = False + raw_settings["passwordless_sudo"] = False + + return raw_settings def _apply(self): root_password = self.new_values.pop("root_password", None) diff --git a/src/tests/test_dns.py b/src/tests/test_dns.py index e896d9c9f7..744e3e7899 100644 --- a/src/tests/test_dns.py +++ b/src/tests/test_dns.py @@ -49,19 +49,19 @@ def test_registrar_list_integrity(): def test_magic_guess_registrar_weird_domain(): - assert _get_registrar_config_section("yolo.tld")["registrar"]["value"] is None + assert _get_registrar_config_section("yolo.tld")["registrar"]["default"] is None def test_magic_guess_registrar_ovh(): assert ( - _get_registrar_config_section("yolo.yunohost.org")["registrar"]["value"] + _get_registrar_config_section("yolo.yunohost.org")["registrar"]["default"] == "ovh" ) def test_magic_guess_registrar_yunodyndns(): assert ( - _get_registrar_config_section("yolo.nohost.me")["registrar"]["value"] + _get_registrar_config_section("yolo.nohost.me")["registrar"]["default"] == "yunohost" ) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index bf441798cf..e113d007b9 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -31,15 +31,16 @@ from moulinette.utils.filesystem import mkdir, read_toml, read_yaml, write_to_yaml from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.form import ( - OPTIONS, BaseInputOption, BaseOption, BaseReadonlyOption, FileOption, + FormModel, OptionsModel, OptionType, Translation, ask_questions_and_parse_answers, + build_form, evaluate_simple_js_expression, ) from yunohost.utils.i18n import _value_for_locale @@ -93,7 +94,7 @@ def __init__( visible: Union[bool, str] = True, **kwargs, ) -> None: - options = self.options_dict_to_list(kwargs, defaults={"optional": True}) + options = self.options_dict_to_list(kwargs, optional=True) ContainerModel.__init__( self, @@ -231,12 +232,33 @@ def check_version(cls, value, field: "ModelField"): # │ ╰─╴╰─╯╵╰╯╵ ╶┴╴╰─╯ ╶┴╴╵╵╵╵ ╰─╴ │ # ╰───────────────────────────────────────────────────────╯ +if TYPE_CHECKING: + FilterKey = Sequence[Union[str, None]] + RawConfig = OrderedDict[str, Any] + RawSettings = dict[str, Any] + + +def parse_filter_key(key: Union[str, None] = None) -> "FilterKey": + if key and key.count(".") > 2: + raise YunohostError( + f"The filter key {key} has too many sub-levels, the max is 3.", + raw_msg=True, + ) + + if not key: + return (None, None, None) + keys = key.split(".") + return tuple(keys[i] if len(keys) > i else None for i in range(3)) + class ConfigPanel: entity_type = "config" save_path_tpl: Union[str, None] = None config_path_tpl = "/usr/share/yunohost/config_{entity_type}.toml" save_mode = "full" + filter_key: "FilterKey" = (None, None, None) + config: Union[ConfigPanelModel, None] = None + form: Union[FormModel, None] = None @classmethod def list(cls): @@ -265,9 +287,6 @@ def __init__(self, entity, config_path=None, save_path=None, creation=False): self.save_path = save_path if not save_path and self.save_path_tpl: self.save_path = self.save_path_tpl.format(entity=entity) - self.config = {} - self.values = {} - self.new_values = {} if ( self.save_path @@ -501,215 +520,103 @@ def run_action(self, action=None, args=None, args_file=None, operation_logger=No logger.success(f"Action {action_id} successful") operation_logger.success() - def _get_raw_config(self): + def _get_raw_config(self) -> "RawConfig": + if not os.path.exists(self.config_path): + raise YunohostValidationError("config_no_panel") + return read_toml(self.config_path) - def _get_config_panel(self): - # Split filter_key - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if len(filter_key) > 3: - raise YunohostError( - f"The filter key {filter_key} has too many sub-levels, the max is 3.", - raw_msg=True, + def _get_raw_settings(self, config: ConfigPanelModel) -> "RawSettings": + if not self.save_path or not os.path.exists(self.save_path): + raise YunohostValidationError("config_no_settings") + + return read_yaml(self.save_path) + + def _get_partial_raw_config(self) -> "RawConfig": + def filter_keys( + data: "RawConfig", + key: str, + model: Union[Type[ConfigPanelModel], Type[PanelModel], Type[SectionModel]], + ) -> "RawConfig": + # filter in keys defined in model, filter out panels/sections/options that aren't `key` + return OrderedDict( + {k: v for k, v in data.items() if k in model.__fields__ or k == key} ) - if not os.path.exists(self.config_path): - logger.debug(f"Config panel {self.config_path} doesn't exists") - return None + raw_config = self._get_raw_config() - toml_config_panel = self._get_raw_config() + panel_id, section_id, option_id = self.filter_key + if panel_id: + raw_config = filter_keys(raw_config, panel_id, ConfigPanelModel) - # Check TOML config panel is in a supported version - if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED: - logger.error( - f"Config panels version {toml_config_panel['version']} are not supported" - ) - return None - - # Transform toml format into internal format - format_description = { - "root": { - "properties": ["version", "i18n"], - "defaults": {"version": 1.0}, - }, - "panels": { - "properties": ["name", "services", "actions", "help"], - "defaults": { - "services": [], - "actions": {"apply": {"en": "Apply"}}, - }, - }, - "sections": { - "properties": ["name", "services", "optional", "help", "visible"], - "defaults": { - "name": "", - "services": [], - "optional": True, - "is_action_section": False, - }, - }, - "options": { - "properties": [ - "ask", - "type", - "bind", - "help", - "example", - "default", - "style", - "icon", - "placeholder", - "visible", - "optional", - "choices", - "yes", - "no", - "pattern", - "limit", - "min", - "max", - "step", - "accept", - "redact", - "filter", - "readonly", - "enabled", - "add_yunohost_portal_to_choices", - # "confirm", # TODO: to ask confirmation before running an action - ], - "defaults": {}, - }, - } + if section_id: + raw_config[panel_id] = filter_keys( + raw_config[panel_id], section_id, PanelModel + ) - def _build_internal_config_panel(raw_infos, level): - """Convert TOML in internal format ('full' mode used by webadmin) - Here are some properties of 1.0 config panel in toml: - - node properties and node children are mixed, - - text are in english only - - some properties have default values - This function detects all children nodes and put them in a list - """ - - defaults = format_description[level]["defaults"] - properties = format_description[level]["properties"] - - # Start building the ouput (merging the raw infos + defaults) - out = {key: raw_infos.get(key, value) for key, value in defaults.items()} - - # Now fill the sublevels (+ apply filter_key) - i = list(format_description).index(level) - sublevel = list(format_description)[i + 1] if level != "options" else None - search_key = filter_key[i] if len(filter_key) > i else False - - for key, value in raw_infos.items(): - # Key/value are a child node - if ( - isinstance(value, OrderedDict) - and key not in properties - and sublevel - ): - # We exclude all nodes not referenced by the filter_key - if search_key and key != search_key: - continue - subnode = _build_internal_config_panel(value, sublevel) - subnode["id"] = key - if level == "root": - subnode.setdefault("name", {"en": key.capitalize()}) - elif level == "sections": - subnode["name"] = key # legacy - subnode.setdefault("optional", raw_infos.get("optional", True)) - # If this section contains at least one button, it becomes an "action" section - if subnode.get("type") == OptionType.button: - out["is_action_section"] = True - out.setdefault(sublevel, []).append(subnode) - # Key/value are a property - else: - if key not in properties: - logger.warning(f"Unknown key '{key}' found in config panel") - # Todo search all i18n keys - out[key] = ( - value - if key not in ["ask", "help", "name"] or isinstance(value, dict) - else {"en": value} + if option_id: + raw_config[panel_id][section_id] = filter_keys( + raw_config[panel_id][section_id], option_id, SectionModel ) - return out - self.config = _build_internal_config_panel(toml_config_panel, "root") + return raw_config + + def _get_partial_raw_settings_and_mutate_config( + self, config: ConfigPanelModel + ) -> tuple[ConfigPanelModel, "RawSettings"]: + raw_settings = self._get_raw_settings(config) + values = {} + + for _, section, option in config.iter_children(): + value = data = raw_settings.get(option.id, getattr(option, "default", None)) + + if isinstance(data, dict): + # Settings data if gathered from bash "ynh_app_config_show" + # may be a custom getter that returns a dict with `value` or `current_value` + # and other attributes meant to override those of the option. + + if "value" in data: + value = data.pop("value") + + # Allow to use value instead of current_value in app config script. + # e.g. apps may write `echo 'value: "foobar"'` in the config file (which is more intuitive that `echo 'current_value: "foobar"'` + # For example hotspot used it... + # See https://github.com/YunoHost/yunohost/pull/1546 + # FIXME do we still need the `current_value`? + if "current_value" in data: + value = data.pop("current_value") + + # Mutate other possible option attributes + for k, v in data.items(): + setattr(option, k, v) + + if isinstance(option, BaseInputOption): # or option.bind == "null": + values[option.id] = value + + return (config, values) + + def _get_config_panel( + self, prevalidate: bool = False + ) -> tuple[ConfigPanelModel, FormModel]: + raw_config = self._get_partial_raw_config() + config = ConfigPanelModel(**raw_config) + config, raw_settings = self._get_partial_raw_settings_and_mutate_config(config) + config.translate() + Settings = build_form(config.options) + settings = ( + Settings(**raw_settings) + if prevalidate + else Settings.construct(**raw_settings) + ) try: - self.config["panels"][0]["sections"][0]["options"][0] + config.panels[0].sections[0].options[0] except (KeyError, IndexError): raise YunohostValidationError( "config_unknown_filter_key", filter_key=self.filter_key ) - return self.config - - def _get_default_values(self): - return { - option["id"]: option["default"] - for _, _, option in self._iterate() - if "default" in option - } - - def _get_raw_settings(self): - """ - Retrieve entries in YAML file - And set default values if needed - """ - - # Inject defaults if needed (using the magic .update() ;)) - self.values = self._get_default_values() - - # Retrieve entries in the YAML - if os.path.exists(self.save_path) and os.path.isfile(self.save_path): - self.values.update(read_yaml(self.save_path) or {}) - - def _hydrate(self): - # Hydrating config panel with current value - for _, section, option in self._iterate(): - if option["id"] not in self.values: - allowed_empty_types = { - OptionType.alert, - OptionType.display_text, - OptionType.markdown, - OptionType.file, - OptionType.button, - } - - if section["is_action_section"] and option.get("default") is not None: - self.values[option["id"]] = option["default"] - elif ( - option["type"] in allowed_empty_types - or option.get("bind") == "null" - ): - continue - else: - raise YunohostError( - f"Config panel question '{option['id']}' should be initialized with a value during install or upgrade.", - raw_msg=True, - ) - value = self.values[option["id"]] - - # Allow to use value instead of current_value in app config script. - # e.g. apps may write `echo 'value: "foobar"'` in the config file (which is more intuitive that `echo 'current_value: "foobar"'` - # For example hotspot used it... - # See https://github.com/YunoHost/yunohost/pull/1546 - if ( - isinstance(value, dict) - and "value" in value - and "current_value" not in value - ): - value["current_value"] = value["value"] - - # In general, the value is just a simple value. - # Sometimes it could be a dict used to overwrite the option itself - value = value if isinstance(value, dict) else {"current_value": value} - option.update(value) - - self.values[option["id"]] = value.get("current_value") - - return self.values + return (config, settings) def _ask(self, action=None): logger.debug("Ask unanswered question and prevalidate data") @@ -781,19 +688,6 @@ def display_header(message): } ) - @property - def future_values(self): - return {**self.values, **self.new_values} - - def __getattr__(self, name): - if "new_values" in self.__dict__ and name in self.new_values: - return self.new_values[name] - - if "values" in self.__dict__ and name in self.values: - return self.values[name] - - return self.__dict__[name] - def _parse_pre_answered(self, args, value, args_file): args = urllib.parse.parse_qs(args or "", keep_blank_values=True) self.args = {key: ",".join(value_) for key, value_ in args.items()} @@ -836,14 +730,3 @@ def _reload_services(self): if hasattr(self, "entity"): service = service.replace("__APP__", self.entity) service_reload_or_restart(service) - - def _iterate(self, trigger=["option"]): - for panel in self.config.get("panels", []): - if "panel" in trigger: - yield (panel, None, panel) - for section in panel.get("sections", []): - if "section" in trigger: - yield (panel, section, section) - if "option" in trigger: - for option in section.get("options", []): - yield (panel, section, option) diff --git a/src/utils/form.py b/src/utils/form.py index 6c14bcdf06..9ca0393d00 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1286,12 +1286,14 @@ class OptionsModel(BaseModel): options: list[Annotated[AnyOption, Field(discriminator="type")]] @staticmethod - def options_dict_to_list(options: dict[str, Any], defaults: dict[str, Any] = {}): + def options_dict_to_list(options: dict[str, Any], optional: bool = False): return [ option | { "id": id_, "type": option.get("type", "string"), + # ConfigPanel options needs to be set as optional by default + "optional": option.get("optional", optional) } for id_, option in options.items() ] From 80dbd6dac46cbe2848a063758f29f3b369417529 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 15:25:40 +0200 Subject: [PATCH 120/436] form: rework entities validators to avoid multiple calls to them --- src/utils/form.py | 90 +++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index 9ca0393d00..8ed83393eb 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -44,7 +44,6 @@ Extra, ValidationError, create_model, - root_validator, validator, ) from pydantic.color import Color @@ -324,7 +323,7 @@ def check_id_is_not_forbidden(cls, value: str) -> str: return value # FIXME Legacy, is `name` still needed? - @validator("name", pre=True, always=True) + @validator("name") def apply_legacy_name(cls, value: Union[str, None], values: Values) -> str: if value is None: return values["id"] @@ -1096,21 +1095,30 @@ class DomainOption(BaseChoicesOption): type: Literal[OptionType.domain] = OptionType.domain choices: Union[dict[str, str], None] - @root_validator() - def inject_domains_choices_and_default(cls, values: Values) -> Values: + @validator("choices", pre=True, always=True) + def inject_domains_choices( + cls, value: Union[dict[str, str], None], values: Values + ) -> dict[str, str]: # TODO remove calls to resources in validators (pydantic V2 should adress this) from yunohost.domain import domain_list data = domain_list() - values["choices"] = { + return { domain: domain + " ★" if domain == data["main"] else domain for domain in data["domains"] } - if values["default"] is None: - values["default"] = data["main"] + @validator("default") + def inject_default( + cls, value: Union[str, None], values: Values + ) -> Union[str, None]: + # TODO remove calls to resources in validators (pydantic V2 should adress this) + from yunohost.domain import _get_maindomain - return values + if value is None: + return _get_maindomain() + + return value @staticmethod def normalize(value, option={}): @@ -1131,8 +1139,11 @@ class AppOption(BaseChoicesOption): add_yunohost_portal_to_choices: bool = False filter: Union[str, None] = None - @root_validator() - def inject_apps_choices(cls, values: Values) -> Values: + @validator("choices", pre=True, always=True) + def inject_apps_choices( + cls, value: Union[dict[str, str], None], values: Values + ) -> dict[str, str]: + # TODO remove calls to resources in validators (pydantic V2 should adress this) from yunohost.app import app_list apps = app_list(full=True)["apps"] @@ -1143,61 +1154,77 @@ def inject_apps_choices(cls, values: Values) -> Values: for app in apps if evaluate_simple_js_expression(values["filter"], context=app) ] - values["choices"] = {"_none": "---"} + + value = {"_none": "---"} if values.get("add_yunohost_portal_to_choices", False): - values["choices"]["_yunohost_portal_with_public_apps"] = "YunoHost's portal with public apps" + value["_yunohost_portal_with_public_apps"] = "YunoHost's portal with public apps" - values["choices"].update( + value.update( { app["id"]: f"{app['label']} ({app.get('domain_path', app['id'])})" for app in apps } ) - return values + return value class UserOption(BaseChoicesOption): type: Literal[OptionType.user] = OptionType.user choices: Union[dict[str, str], None] - @root_validator() - def inject_users_choices_and_default(cls, values: dict[str, Any]) -> dict[str, Any]: - from yunohost.domain import _get_maindomain - from yunohost.user import user_info, user_list + @validator("choices", pre=True, always=True) + def inject_users_choices( + cls, value: Union[dict[str, str], None], values: Values + ) -> dict[str, str]: + # TODO remove calls to resources in validators (pydantic V2 should adress this) + from yunohost.user import user_list - values["choices"] = { + value = { username: f"{infos['fullname']} ({infos['mail']})" for username, infos in user_list()["users"].items() } # FIXME keep this to test if any user, do not raise error if no admin? - if not values["choices"]: + if not value: raise YunohostValidationError( "app_argument_invalid", name=values["id"], error="You should create a YunoHost user first.", ) - if values["default"] is None: + return value + + @validator("default") + def inject_default( + cls, value: Union[str, None], values: Values + ) -> Union[str, None]: + # TODO remove calls to resources in validators (pydantic V2 should adress this) + from yunohost.domain import _get_maindomain + from yunohost.user import user_info + + if value is None: # FIXME: this code is obsolete with the new admins group # Should be replaced by something like "any first user we find in the admin group" root_mail = "root@%s" % _get_maindomain() for user in values["choices"].keys(): if root_mail in user_info(user).get("mail-aliases", []): - values["default"] = user - break + return user - return values + return value class GroupOption(BaseChoicesOption): type: Literal[OptionType.group] = OptionType.group choices: Union[dict[str, str], None] + default: Union[Literal["visitors", "all_users", "admins"], None] = "all_users" - @root_validator() - def inject_groups_choices_and_default(cls, values: Values) -> Values: + @validator("choices", pre=True, always=True) + def inject_groups_choices( + cls, value: Union[dict[str, str], None], values: Values + ) -> dict[str, str]: + # TODO remove calls to resources in validators (pydantic V2 should adress this) from yunohost.user import user_group_list groups = user_group_list(short=True, include_primary_groups=False)["groups"] @@ -1212,14 +1239,7 @@ def _human_readable_group(groupname): else groupname ) - values["choices"] = { - groupname: _human_readable_group(groupname) for groupname in groups - } - - if values["default"] is None: - values["default"] = "all_users" - - return values + return {groupname: _human_readable_group(groupname) for groupname in groups} OPTIONS = { @@ -1293,7 +1313,7 @@ def options_dict_to_list(options: dict[str, Any], optional: bool = False): "id": id_, "type": option.get("type", "string"), # ConfigPanel options needs to be set as optional by default - "optional": option.get("optional", optional) + "optional": option.get("optional", optional), } for id_, option in options.items() ] From a92e22b6534e22e4ce95e50b4312f44953e10713 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 16:11:14 +0200 Subject: [PATCH 121/436] config: rework get method --- src/domain.py | 19 ++----- src/settings.py | 21 ++++---- src/utils/configpanel.py | 112 ++++++++++++++++++--------------------- 3 files changed, 64 insertions(+), 88 deletions(-) diff --git a/src/domain.py b/src/domain.py index a796e0142b..6d57a0f10d 100644 --- a/src/domain.py +++ b/src/domain.py @@ -652,22 +652,9 @@ class DomainConfigPanel(ConfigPanel): save_path_tpl = f"{DOMAIN_SETTINGS_DIR}/{{entity}}.yml" save_mode = "diff" - def get(self, key="", mode="classic"): - result = super().get(key=key, mode=mode) - - if mode == "full": - for panel, section, option in self._iterate(): - # This injects: - # i18n: domain_config_cert_renew_help - # i18n: domain_config_default_app_help - # i18n: domain_config_xmpp_help - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) - return self.config - - return result + # i18n: domain_config_cert_renew_help + # i18n: domain_config_default_app_help + # i18n: domain_config_xmpp_help def _get_raw_config(self) -> "RawConfig": # TODO add mechanism to share some settings with other domains on the same zone diff --git a/src/settings.py b/src/settings.py index f3340e8e9d..e66195802b 100644 --- a/src/settings.py +++ b/src/settings.py @@ -19,7 +19,7 @@ import os import subprocess from logging import getLogger -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Union from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError @@ -31,7 +31,12 @@ from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings if TYPE_CHECKING: - from yunohost.utils.configpanel import ConfigPanelModel, RawConfig, RawSettings + from yunohost.utils.configpanel import ( + ConfigPanelGetMode, + ConfigPanelModel, + RawConfig, + RawSettings, + ) logger = getLogger("yunohost.settings") @@ -129,17 +134,11 @@ class SettingsConfigPanel(ConfigPanel): def __init__(self, config_path=None, save_path=None, creation=False): super().__init__("settings") - def get(self, key="", mode="classic"): + def get( + self, key: Union[str, None] = None, mode: "ConfigPanelGetMode" = "classic" + ) -> Any: result = super().get(key=key, mode=mode) - if mode == "full": - for panel, section, option in self._iterate(): - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) - return self.config - # Dirty hack to let settings_get() to work from a python script if isinstance(result, str) and result in ["True", "False"]: result = bool(result == "True") diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index e113d007b9..e39d5c91c6 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -31,6 +31,7 @@ from moulinette.utils.filesystem import mkdir, read_toml, read_yaml, write_to_yaml from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.form import ( + AnyOption, BaseInputOption, BaseOption, BaseReadonlyOption, @@ -190,6 +191,13 @@ def options(self): for option in section.options: yield option + def get_option(self, option_id) -> Union[AnyOption, None]: + for option in self.options: + if option.id == option_id: + return option + # FIXME raise error? + return None + def iter_children( self, @@ -236,6 +244,7 @@ def check_version(cls, value, field: "ModelField"): FilterKey = Sequence[Union[str, None]] RawConfig = OrderedDict[str, Any] RawSettings = dict[str, Any] + ConfigPanelGetMode = Literal["classic", "full", "export"] def parse_filter_key(key: Union[str, None] = None) -> "FilterKey": @@ -310,78 +319,59 @@ def __init__(self, entity, config_path=None, save_path=None, creation=False): and re.match("^(validate|post_ask)__", func) } - def get(self, key="", mode="classic"): - self.filter_key = key or "" - - # Read config panel toml - self._get_config_panel() - - if not self.config: - raise YunohostValidationError("config_no_panel") + def get( + self, key: Union[str, None] = None, mode: "ConfigPanelGetMode" = "classic" + ) -> Any: + self.filter_key = parse_filter_key(key) + self.config, self.form = self._get_config_panel(prevalidate=False) - # Read or get values and hydrate the config - self._get_raw_settings() - self._hydrate() + panel_id, section_id, option_id = self.filter_key # In 'classic' mode, we display the current value if key refer to an option - if self.filter_key.count(".") == 2 and mode == "classic": - option = self.filter_key.split(".")[-1] - value = self.values.get(option, None) + if option_id and mode == "classic": + option = self.config.get_option(option_id) - option_type = None - for _, _, option_ in self._iterate(): - if option_["id"] == option: - option_type = OPTIONS[option_["type"]] - break + if option is None: + # FIXME i18n + raise YunohostValidationError( + f"Couldn't find any option with id {option_id}" + ) - return option_type.normalize(value) if option_type else value + if isinstance(option, BaseReadonlyOption): + return None + + return self.form[option_id] # Format result in 'classic' or 'export' mode + self.config.translate() logger.debug(f"Formating result in '{mode}' mode") - result = {} - for panel, section, option in self._iterate(): - if section["is_action_section"] and mode != "full": - continue - - key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == "export": - result[option["id"]] = option.get("current_value") - continue + result = OrderedDict() + for panel in self.config.panels: + for section in panel.sections: + if section.is_action_section and mode != "full": + continue - ask = None - if "ask" in option: - ask = _value_for_locale(option["ask"]) - elif "i18n" in self.config: - ask = m18n.n(self.config["i18n"] + "_" + option["id"]) - - if mode == "full": - option["ask"] = ask - question_class = OPTIONS[option.get("type", OptionType.string)] - # FIXME : maybe other properties should be taken from the question, not just choices ?. - if issubclass(question_class, BaseChoicesOption): - option["choices"] = question_class(option).choices - if issubclass(question_class, BaseInputOption): - option["default"] = question_class(option).default - option["pattern"] = question_class(option).pattern - else: - result[key] = {"ask": ask} - if "current_value" in option: - question_class = OPTIONS[option.get("type", OptionType.string)] - if hasattr(question_class, "humanize"): - result[key]["value"] = question_class.humanize( - option["current_value"], option - ) - else: - result[key]["value"] = option["current_value"] - - # FIXME: semantics, technically here this is not about a prompt... - if getattr(question_class, "hide_user_input_in_prompt", None): - result[key][ - "value" - ] = "**************" # Prevent displaying password in `config get` + for option in section.options: + if mode == "export": + if isinstance(option, BaseInputOption): + result[option.id] = self.form[option.id] + continue + + if mode == "classic": + key = f"{panel.id}.{section.id}.{option.id}" + result[key] = {"ask": option.ask} + + if isinstance(option, BaseInputOption): + result[key]["value"] = option.humanize( + self.form[option.id], option + ) + if option.type is OptionType.password: + result[key][ + "value" + ] = "**************" # Prevent displaying password in `config get` if mode == "full": - return self.config + return self.config.dict(exclude_none=True) else: return result From dbaea019fe1945fe8e126d51c00fcc019eeca1c2 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 16:54:54 +0200 Subject: [PATCH 122/436] form+config: replace _parse_pre_answered method with generic function --- src/utils/configpanel.py | 19 ++++++------------- src/utils/form.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index e39d5c91c6..1f240a1057 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -43,6 +43,7 @@ ask_questions_and_parse_answers, build_form, evaluate_simple_js_expression, + parse_prefilled_values, ) from yunohost.utils.i18n import _value_for_locale @@ -397,7 +398,10 @@ def set( # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") - self._parse_pre_answered(args, value, args_file) + if option_id and value is not None: + self.args = {option_id: value} + else: + self.args = parse_prefilled_values(args, value, args_file) # Read or get values and hydrate the config self._get_raw_settings() @@ -468,7 +472,7 @@ def run_action(self, action=None, args=None, args_file=None, operation_logger=No # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") - self._parse_pre_answered(args, None, args_file) + self.args = parse_prefilled_values(args, args_file) # Read or get values and hydrate the config self._get_raw_settings() @@ -678,17 +682,6 @@ def display_header(message): } ) - def _parse_pre_answered(self, args, value, args_file): - args = urllib.parse.parse_qs(args or "", keep_blank_values=True) - self.args = {key: ",".join(value_) for key, value_ in args.items()} - - if args_file: - # Import YAML / JSON file but keep --args values - self.args = {**read_yaml(args_file), **self.args} - - if value is not None: - self.args = {self.filter_key.split(".")[-1]: value} - def _apply(self): logger.info("Saving the new configuration...") dir_path = os.path.dirname(os.path.realpath(self.save_path)) diff --git a/src/utils/form.py b/src/utils/form.py index 8ed83393eb..7a97259b9c 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -53,7 +53,7 @@ from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize -from moulinette.utils.filesystem import read_file, write_to_file +from moulinette.utils.filesystem import read_file, read_yaml, write_to_file from yunohost.log import OperationLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.i18n import _value_for_locale @@ -1431,6 +1431,31 @@ def hydrate_option_type(raw_option: dict[str, Any]) -> dict[str, Any]: Hooks = dict[str, Callable[[BaseInputOption], Any]] +def parse_prefilled_values( + args: Union[str, None] = None, + args_file: Union[str, None] = None, + method: Literal["parse_qs", "parse_qsl"] = "parse_qs", +) -> dict[str, Any]: + """ + Retrieve form values from yaml file or query string. + """ + values: Values = {} + if args_file: + # Import YAML / JSON file + values |= read_yaml(args_file) + if args: + # FIXME See `ask_questions_and_parse_answers` + parsed = getattr(urllib.parse, method)(args, keep_blank_values=True) + if isinstance(parsed, dict): # parse_qs + # FIXME could do the following to get a list directly? + # k: None if not len(v) else (v if len(v) > 1 else v[0]) + values |= {k: ",".join(v) for k, v in parsed.items()} + else: + values |= dict(parsed) + + return values + + def prompt_or_validate_form( options: list[AnyOption], form: FormModel, @@ -1561,9 +1586,7 @@ def ask_questions_and_parse_answers( # whereas parse.qs return list of values (which is useful for tags, etc) # For now, let's not migrate this piece of code to parse_qs # Because Aleks believes some bits of the app CI rely on overriding values (e.g. foo=foo&...&foo=bar) - answers = dict( - urllib.parse.parse_qsl(prefilled_answers or "", keep_blank_values=True) - ) + answers = parse_prefilled_values(prefilled_answers, method="parse_qsl") elif isinstance(prefilled_answers, Mapping): answers = {**prefilled_answers} else: From f1038de56d0289d770317109d770bf5144129703 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 17:51:21 +0200 Subject: [PATCH 123/436] form: fix entities validators order for filter and apply the right default --- src/tests/test_questions.py | 11 +---------- src/utils/form.py | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 959f2c8b7a..387d5c0f9f 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -1816,9 +1816,7 @@ class TestGroup(BaseTest): "scenarios": [ ("custom_group", "custom_group"), *all_as("", None, output="visitors", raw_option={"default": "visitors"}), - *xpass(scenarios=[ - ("", "custom_group", {"default": "custom_group"}), - ], reason="Should throw 'default must be in (None, 'all_users', 'visitors', 'admins')"), + ("", FAIL, {"default": "custom_group"}), # Not allowed to set a default which is not a default group # readonly ("admins", FAIL, {"readonly": True}), # readonly is forbidden ] @@ -1837,13 +1835,6 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): "prefill": "admins", } ) - # FIXME This should fail, not allowed to set a default which is not a default group - super().test_options_prompted_with_ask_help( - prefill_data={ - "raw_option": {"default": "custom_group"}, - "prefill": "custom_group", - } - ) def test_scenarios(self, intake, expected_output, raw_option, data): with patch_groups(**data): diff --git a/src/utils/form.py b/src/utils/form.py index 7a97259b9c..7098692a46 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -951,6 +951,7 @@ def is_file_path(s): class BaseChoicesOption(BaseInputOption): # FIXME probably forbid choices to be None? + filter: Union[JSExpression, None] = None # filter before choices choices: Union[dict[str, Any], list[Any], None] @validator("choices", pre=True) @@ -1093,6 +1094,7 @@ def _value_pre_validator( class DomainOption(BaseChoicesOption): type: Literal[OptionType.domain] = OptionType.domain + filter: Literal[None] = None choices: Union[dict[str, str], None] @validator("choices", pre=True, always=True) @@ -1108,17 +1110,14 @@ def inject_domains_choices( for domain in data["domains"] } - @validator("default") + @validator("default", pre=True, always=True) def inject_default( cls, value: Union[str, None], values: Values ) -> Union[str, None]: # TODO remove calls to resources in validators (pydantic V2 should adress this) from yunohost.domain import _get_maindomain - if value is None: - return _get_maindomain() - - return value + return _get_maindomain() @staticmethod def normalize(value, option={}): @@ -1135,9 +1134,9 @@ def normalize(value, option={}): class AppOption(BaseChoicesOption): type: Literal[OptionType.app] = OptionType.app - choices: Union[dict[str, str], None] + filter: Union[JSExpression, None] = None add_yunohost_portal_to_choices: bool = False - filter: Union[str, None] = None + choices: Union[dict[str, str], None] @validator("choices", pre=True, always=True) def inject_apps_choices( @@ -1172,6 +1171,7 @@ def inject_apps_choices( class UserOption(BaseChoicesOption): type: Literal[OptionType.user] = OptionType.user + filter: Literal[None] = None choices: Union[dict[str, str], None] @validator("choices", pre=True, always=True) @@ -1196,19 +1196,19 @@ def inject_users_choices( return value - @validator("default") + @validator("default", pre=True, always=True) def inject_default( cls, value: Union[str, None], values: Values ) -> Union[str, None]: # TODO remove calls to resources in validators (pydantic V2 should adress this) from yunohost.domain import _get_maindomain - from yunohost.user import user_info + from yunohost.user import user_list, user_info if value is None: # FIXME: this code is obsolete with the new admins group # Should be replaced by something like "any first user we find in the admin group" root_mail = "root@%s" % _get_maindomain() - for user in values["choices"].keys(): + for user in user_list()["users"].keys(): if root_mail in user_info(user).get("mail-aliases", []): return user @@ -1217,6 +1217,7 @@ def inject_default( class GroupOption(BaseChoicesOption): type: Literal[OptionType.group] = OptionType.group + filter: Literal[None] = None choices: Union[dict[str, str], None] default: Union[Literal["visitors", "all_users", "admins"], None] = "all_users" @@ -1241,6 +1242,13 @@ def _human_readable_group(groupname): return {groupname: _human_readable_group(groupname) for groupname in groups} + @validator("default", pre=True, always=True) + def inject_default(cls, value: Union[str, None], values: Values) -> str: + # FIXME do we really want to default to something all the time? + if value is None: + return "all_users" + return value + OPTIONS = { OptionType.display_text: DisplayTextOption, From 2c35dcbb2498f2227282a10a8d8eac6da06f82e6 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 20:04:20 +0200 Subject: [PATCH 124/436] configpanel: update _reload_services --- src/utils/configpanel.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 1f240a1057..22ea5b3b89 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -199,6 +199,17 @@ def get_option(self, option_id) -> Union[AnyOption, None]: # FIXME raise error? return None + @property + def services(self) -> list[str]: + services = set() + for panel in self.panels: + services |= set(panel.services) + for section in panel.sections: + services |= set(section.services) + + services_ = list(services) + services_.sort(key="nginx".__eq__) + return services_ def iter_children( self, @@ -701,12 +712,8 @@ def _apply(self): def _reload_services(self): from yunohost.service import service_reload_or_restart - services_to_reload = set() - for panel, section, obj in self._iterate(["panel", "section", "option"]): - services_to_reload |= set(obj.get("services", [])) + services_to_reload = self.config.services - services_to_reload = list(services_to_reload) - services_to_reload.sort(key="nginx".__eq__) if services_to_reload: logger.info("Reloading services...") for service in services_to_reload: From 7a60703ef58b17de4e1cde9e7861fa4e50298e6e Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 20:13:01 +0200 Subject: [PATCH 125/436] configpanel: update _ask --- src/utils/configpanel.py | 97 +++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 22ea5b3b89..22318e5e0c 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -19,7 +19,6 @@ import glob import os import re -import urllib.parse from collections import OrderedDict from logging import getLogger from typing import TYPE_CHECKING, Any, Literal, Sequence, Type, Union @@ -36,19 +35,19 @@ BaseOption, BaseReadonlyOption, FileOption, - FormModel, OptionsModel, OptionType, Translation, - ask_questions_and_parse_answers, build_form, evaluate_simple_js_expression, parse_prefilled_values, + prompt_or_validate_form, ) from yunohost.utils.i18n import _value_for_locale if TYPE_CHECKING: from pydantic.fields import ModelField + from yunohost.utils.form import FormModel, Hooks logger = getLogger("yunohost.configpanel") @@ -279,7 +278,8 @@ class ConfigPanel: save_mode = "full" filter_key: "FilterKey" = (None, None, None) config: Union[ConfigPanelModel, None] = None - form: Union[FormModel, None] = None + form: Union["FormModel", None] = None + hooks: "Hooks" = {} @classmethod def list(cls): @@ -602,7 +602,7 @@ def _get_partial_raw_settings_and_mutate_config( def _get_config_panel( self, prevalidate: bool = False - ) -> tuple[ConfigPanelModel, FormModel]: + ) -> tuple[ConfigPanelModel, "FormModel"]: raw_config = self._get_partial_raw_config() config = ConfigPanelModel(**raw_config) config, raw_settings = self._get_partial_raw_settings_and_mutate_config(config) @@ -623,58 +623,62 @@ def _get_config_panel( return (config, settings) - def _ask(self, action=None): + def ask( + self, + config: ConfigPanelModel, + settings: "FormModel", + prefilled_answers: dict[str, Any] = {}, + action_id: Union[str, None] = None, + hooks: "Hooks" = {}, + ) -> "FormModel": + # FIXME could be turned into a staticmethod logger.debug("Ask unanswered question and prevalidate data") - if "i18n" in self.config: - for panel, section, option in self._iterate(): - if "ask" not in option: - option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"]) - # auto add i18n help text if present in locales - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) + interactive = Moulinette.interface.type == "cli" and os.isatty(1) - def display_header(message): - """CLI panel/section header display""" - if Moulinette.interface.type == "cli" and self.filter_key.count(".") < 2: - Moulinette.display(colorize(message, "purple")) - - for panel, section, obj in self._iterate(["panel", "section"]): - if ( - section - and section.get("visible") - and not evaluate_simple_js_expression( - section["visible"], context=self.future_values + if interactive: + config.translate() + + for panel in config.panels: + if interactive: + Moulinette.display( + colorize(f"\n{'='*40}\n>>>> {panel.name}\n{'='*40}", "purple") ) - ): - continue - # Ugly hack to skip action section ... except when when explicitly running actions - if not action: - if section and section["is_action_section"]: + # A section or option may only evaluate its conditions (`visible` + # and `enabled`) with its panel's local context that is built + # prompt after prompt. + # That means that a condition can only reference options of its + # own panel and options that are previously defined. + context: dict[str, Any] = {} + + for section in panel.sections: + if ( + action_id is None and section.is_action_section + ) or not section.is_visible(context): + # FIXME useless? + Moulinette.display("Skipping section '{panel.id}.{section.id}'…") continue - if panel == obj: - name = _value_for_locale(panel["name"]) - display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") - else: - name = _value_for_locale(section["name"]) - if name: - display_header(f"\n# {name}") - elif section: + if interactive and section.name: + Moulinette.display(colorize(f"\n# {section.name}", "purple")) + # filter action section options in case of multiple buttons - section["options"] = [ + options = [ option - for option in section["options"] - if option.get("type", OptionType.string) != OptionType.button - or option["id"] == action + for option in section.options + if option.type is not OptionType.button or option.id == action_id ] - if panel == obj: - continue + settings = prompt_or_validate_form( + options, + settings, + prefilled_answers=prefilled_answers, + context=context, + hooks=hooks, + ) +<<<<<<< HEAD # Check and ask unanswered questions prefilled_answers = self.args.copy() prefilled_answers.update(self.new_values) @@ -692,6 +696,9 @@ def display_header(message): if not question.readonly and question.value is not None } ) +======= + return settings +>>>>>>> be777b928 (configpanel: update _ask) def _apply(self): logger.info("Saving the new configuration...") From 5f9ea5831391908eeace58c3f496796c69351bd5 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 20:15:25 +0200 Subject: [PATCH 126/436] configpanel: update _apply --- src/app.py | 14 +++++++-- src/domain.py | 68 +++++++++++++++++----------------------- src/settings.py | 42 ++++++++++++++----------- src/utils/configpanel.py | 47 +++++++++++---------------- 4 files changed, 82 insertions(+), 89 deletions(-) diff --git a/src/app.py b/src/app.py index ffcd1ecc32..08e2225790 100644 --- a/src/app.py +++ b/src/app.py @@ -27,7 +27,7 @@ import tempfile import copy from collections import OrderedDict -from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Iterator, Optional +from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Iterator, Optional, Union from packaging import version from logging import getLogger from pathlib import Path @@ -73,7 +73,10 @@ ) if TYPE_CHECKING: + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + from yunohost.utils.configpanel import ConfigPanelModel, RawSettings + from yunohost.utils.form import FormModel logger = getLogger("yunohost.app") @@ -1809,8 +1812,13 @@ def _run_action(self, action): def _get_raw_settings(self, config: "ConfigPanelModel") -> "RawSettings": return self._call_config_script("show") - def _apply(self): - env = {key: str(value) for key, value in self.new_values.items()} + def _apply( + self, + form: "FormModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ): + env = {key: str(value) for key, value in form.dict().items()} return_content = self._call_config_script("apply", env=env) # If the script returned validation error diff --git a/src/domain.py b/src/domain.py index 6d57a0f10d..f0531e624a 100644 --- a/src/domain.py +++ b/src/domain.py @@ -19,7 +19,7 @@ import os import time from pathlib import Path -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional, Union from collections import OrderedDict from logging import getLogger @@ -48,7 +48,10 @@ from yunohost.log import is_unit_operation if TYPE_CHECKING: + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + from yunohost.utils.configpanel import RawConfig + from yunohost.utils.form import FormModel logger = getLogger("yunohost.domain") @@ -669,7 +672,7 @@ def _get_raw_config(self) -> "RawConfig": # Portal settings are only available on "topest" domains if _get_parent_domain_of(self.entity, topest=True) is not None: - del toml["feature"]["portal"] + del raw_config["feature"]["portal"] # Optimize wether or not to load the DNS section, # e.g. we don't want to trigger the whole _get_registary_config_section @@ -712,17 +715,23 @@ def _get_raw_config(self) -> "RawConfig": return raw_config - def _apply(self): - if ( - "default_app" in self.future_values - and self.future_values["default_app"] != self.values["default_app"] - ): + def _apply( + self, + form: "FormModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ): + next_settings = { + k: v for k, v in form.dict().items() if previous_settings.get(k) != v + } + + if "default_app" in next_settings: from yunohost.app import app_ssowatconf, app_map if "/" in app_map(raw=True).get(self.entity, {}): raise YunohostValidationError( "app_make_default_location_already_used", - app=self.future_values["default_app"], + app=next_settings["default_app"], domain=self.entity, other_app=app_map(raw=True)[self.entity]["/"]["id"], ) @@ -735,8 +744,7 @@ def _apply(self): "portal_theme", ] if _get_parent_domain_of(self.entity, topest=True) is None and any( - option in self.future_values - and self.new_values[option] != self.values.get(option) + option in next_settings for option in portal_options ): from yunohost.portal import PORTAL_SETTINGS_DIR @@ -744,9 +752,8 @@ def _apply(self): # Portal options are also saved in a `domain.portal.yml` file # that can be read by the portal API. # FIXME remove those from the config panel saved values? - portal_values = { - option: self.future_values[option] for option in portal_options - } + + portal_values = form.dict(include=portal_options) portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") portal_settings = {"apps": {}} @@ -760,38 +767,21 @@ def _apply(self): str(portal_settings_path), portal_settings, sort_keys=True, indent=4 ) - super()._apply() + super()._apply(form, previous_settings) # Reload ssowat if default app changed - if ( - "default_app" in self.future_values - and self.future_values["default_app"] != self.values["default_app"] - ): + if "default_app" in next_settings: app_ssowatconf() - stuff_to_regen_conf = [] - if ( - "xmpp" in self.future_values - and self.future_values["xmpp"] != self.values["xmpp"] - ): - stuff_to_regen_conf.append("nginx") - stuff_to_regen_conf.append("metronome") - - if ( - "mail_in" in self.future_values - and self.future_values["mail_in"] != self.values["mail_in"] - ) or ( - "mail_out" in self.future_values - and self.future_values["mail_out"] != self.values["mail_out"] - ): - if "nginx" not in stuff_to_regen_conf: - stuff_to_regen_conf.append("nginx") - stuff_to_regen_conf.append("postfix") - stuff_to_regen_conf.append("dovecot") - stuff_to_regen_conf.append("rspamd") + stuff_to_regen_conf = set() + if "xmpp" in next_settings: + stuff_to_regen_conf.update({"nginx", "metronome"}) + + if "mail_in" in next_settings or "mail_out" in next_settings: + stuff_to_regen_conf.update({"nginx", "postfix", "dovecot", "rspamd"}) if stuff_to_regen_conf: - regen_conf(names=stuff_to_regen_conf) + regen_conf(names=list(stuff_to_regen_conf)) def domain_action_run(domain, action, args=None): diff --git a/src/settings.py b/src/settings.py index e66195802b..6a05217dc4 100644 --- a/src/settings.py +++ b/src/settings.py @@ -31,12 +31,15 @@ from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings if TYPE_CHECKING: + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + from yunohost.utils.configpanel import ( ConfigPanelGetMode, ConfigPanelModel, RawConfig, RawSettings, ) + from yunohost.utils.form import FormModel logger = getLogger("yunohost.settings") @@ -217,19 +220,15 @@ def _get_raw_settings(self, config: "ConfigPanelModel") -> "RawSettings": return raw_settings - def _apply(self): - root_password = self.new_values.pop("root_password", None) - root_password_confirm = self.new_values.pop("root_password_confirm", None) - passwordless_sudo = self.new_values.pop("passwordless_sudo", None) - - self.values = { - k: v for k, v in self.values.items() if k not in self.virtual_settings - } - self.new_values = { - k: v for k, v in self.new_values.items() if k not in self.virtual_settings - } - - assert all(v not in self.future_values for v in self.virtual_settings) + def _apply( + self, + form: "FormModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ): + root_password = form.get("root_password", None) + root_password_confirm = form.get("root_password_confirm", None) + passwordless_sudo = form.get("passwordless_sudo", None) if root_password and root_password.strip(): if root_password != root_password_confirm: @@ -248,15 +247,20 @@ def _apply(self): {"sudoOption": ["!authenticate"] if passwordless_sudo else []}, ) - super()._apply() - - settings = { - k: v for k, v in self.future_values.items() if self.values.get(k) != v + # First save settings except virtual + default ones + super()._apply(form, previous_settings, exclude=self.virtual_settings) + next_settings = { + k: v + for k, v in form.dict(exclude=self.virtual_settings).items() + if previous_settings.get(k) != v } - for setting_name, value in settings.items(): + + for setting_name, value in next_settings.items(): try: + # FIXME not sure to understand why we need the previous value if + # updated_settings has already been filtered trigger_post_change_hook( - setting_name, self.values.get(setting_name), value + setting_name, previous_settings.get(setting_name), value ) except Exception as e: logger.error(f"Post-change hook for setting failed : {e}") diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 22318e5e0c..56b0584e36 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -47,6 +47,8 @@ if TYPE_CHECKING: from pydantic.fields import ModelField + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + from yunohost.utils.form import FormModel, Hooks logger = getLogger("yunohost.configpanel") @@ -678,43 +680,32 @@ def ask( hooks=hooks, ) -<<<<<<< HEAD - # Check and ask unanswered questions - prefilled_answers = self.args.copy() - prefilled_answers.update(self.new_values) - - questions = ask_questions_and_parse_answers( - {question["id"]: question for question in section["options"]}, - prefilled_answers=prefilled_answers, - current_values=self.values, - hooks=self.hooks, - ) - self.new_values.update( - { - question.id: question.value - for question in questions - if not question.readonly and question.value is not None - } - ) -======= return settings ->>>>>>> be777b928 (configpanel: update _ask) - def _apply(self): + def _apply( + self, + form: "FormModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> dict[str, Any]: + """ + Save settings in yaml file. + If `save_mode` is `"diff"` (which is the default), only values that are + different from their default value will be saved. + """ logger.info("Saving the new configuration...") + dir_path = os.path.dirname(os.path.realpath(self.save_path)) if not os.path.exists(dir_path): mkdir(dir_path, mode=0o700) - values_to_save = self.future_values - if self.save_mode == "diff": - defaults = self._get_default_values() - values_to_save = { - k: v for k, v in values_to_save.items() if defaults.get(k) != v - } + exclude_defaults = self.save_mode == "diff" + settings = form.dict(exclude_defaults=exclude_defaults, exclude=exclude) # type: ignore # Save the settings to the .yaml file - write_to_yaml(self.save_path, values_to_save) + write_to_yaml(self.save_path, settings) + + return settings def _reload_services(self): from yunohost.service import service_reload_or_restart From 6b3691ce534df73c98479e4eba3387024cca8000 Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 21:04:05 +0200 Subject: [PATCH 127/436] configpanel: update set --- src/settings.py | 2 +- src/utils/configpanel.py | 49 ++++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/settings.py b/src/settings.py index 6a05217dc4..0d0a5406bc 100644 --- a/src/settings.py +++ b/src/settings.py @@ -132,7 +132,7 @@ class SettingsConfigPanel(ConfigPanel): entity_type = "global" save_path_tpl = SETTINGS_PATH save_mode = "diff" - virtual_settings = ["root_password", "root_password_confirm", "passwordless_sudo"] + virtual_settings = {"root_password", "root_password_confirm", "passwordless_sudo"} def __init__(self, config_path=None, save_path=None, creation=False): super().__init__("settings") diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 56b0584e36..20a4219255 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -50,6 +50,7 @@ from pydantic.typing import AbstractSetIntStr, MappingIntStrAny from yunohost.utils.form import FormModel, Hooks + from yunohost.log import OperationLogger logger = getLogger("yunohost.configpanel") @@ -390,15 +391,15 @@ def get( return result def set( - self, key=None, value=None, args=None, args_file=None, operation_logger=None + self, + key: Union[str, None] = None, + value: Any = None, + args: Union[str, None] = None, + args_file: Union[str, None] = None, + operation_logger: Union["OperationLogger", None] = None, ): - self.filter_key = key or "" - - # Read config panel toml - self._get_config_panel() - - if not self.config: - raise YunohostValidationError("config_no_panel") + self.filter_key = parse_filter_key(key) + panel_id, section_id, option_id = self.filter_key if (args is not None or args_file is not None) and value is not None: raise YunohostValidationError( @@ -406,27 +407,35 @@ def set( raw_msg=True, ) - if self.filter_key.count(".") != 2 and value is not None: + if not option_id and value is not None: raise YunohostValidationError("config_cant_set_value_on_section") # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") if option_id and value is not None: - self.args = {option_id: value} + prefilled_answers = {option_id: value} else: - self.args = parse_prefilled_values(args, value, args_file) + prefilled_answers = parse_prefilled_values(args, args_file) - # Read or get values and hydrate the config - self._get_raw_settings() - self._hydrate() - BaseOption.operation_logger = operation_logger - self._ask() + self.config, self.form = self._get_config_panel() + # FIXME find a better way to exclude previous settings + previous_settings = self.form.dict() + + # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) + # BaseOption.operation_logger = operation_logger + + self.form = self._ask( + self.config, + self.form, + prefilled_answers=prefilled_answers, + hooks=self.hooks, + ) if operation_logger: operation_logger.start() try: - self._apply() + self._apply(self.form, previous_settings) except YunohostError: raise # Script got manually interrupted ... @@ -452,7 +461,9 @@ def set( self._reload_services() logger.success("Config updated as expected") - operation_logger.success() + + if operation_logger: + operation_logger.success() def list_actions(self): actions = {} @@ -625,7 +636,7 @@ def _get_config_panel( return (config, settings) - def ask( + def _ask( self, config: ConfigPanelModel, settings: "FormModel", From 15c827908f917ab57e61b303d4af5e36a1a9d5ce Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 21:05:25 +0200 Subject: [PATCH 128/436] configpanel: update run_action --- src/app.py | 11 ++++---- src/utils/configpanel.py | 59 +++++++++++++++++++++++++--------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/app.py b/src/app.py index 08e2225790..970a66fb2e 100644 --- a/src/app.py +++ b/src/app.py @@ -46,10 +46,11 @@ chmod, ) -from yunohost.utils.configpanel import ConfigPanel, ask_questions_and_parse_answers +from yunohost.utils.configpanel import ConfigPanel from yunohost.utils.form import ( DomainOption, WebPathOption, + ask_questions_and_parse_answers, hydrate_questions_with_choices, ) from yunohost.utils.i18n import _value_for_locale @@ -1805,10 +1806,6 @@ class AppConfigPanel(ConfigPanel): save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml") config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.toml") - def _run_action(self, action): - env = {key: str(value) for key, value in self.new_values.items()} - self._call_config_script(action, env=env) - def _get_raw_settings(self, config: "ConfigPanelModel") -> "RawSettings": return self._call_config_script("show") @@ -1832,6 +1829,10 @@ def _apply( error=message, ) + def _run_action(self, form: "FormModel", action_id: str): + env = {key: str(value) for key, value in form.dict().items()} + self._call_config_script(action_id, env=env) + def _call_config_script(self, action, env=None): from yunohost.hook import hook_exec diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 20a4219255..eb244aa082 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -479,30 +479,41 @@ def list_actions(self): return actions - def run_action(self, action=None, args=None, args_file=None, operation_logger=None): + def run_action( + self, + key: Union[str, None] = None, + args: Union[str, None] = None, + args_file: Union[str, None] = None, + operation_logger: Union["OperationLogger", None] = None, + ): # # FIXME : this stuff looks a lot like set() ... # + panel_id, section_id, action_id = parse_filter_key(key) + # since an action may require some options from its section, + # remove the action_id from the filter + self.filter_key = (panel_id, section_id, None) - self.filter_key = ".".join(action.split(".")[:2]) - action_id = action.split(".")[2] - - # Read config panel toml - self._get_config_panel() + self.config, self.form = self._get_config_panel() # FIXME: should also check that there's indeed a key called action - if not self.config: - raise YunohostValidationError(f"No action named {action}", raw_msg=True) + if not action_id or not self.config.get_option(action_id): + raise YunohostValidationError(f"No action named {action_id}", raw_msg=True) # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") - self.args = parse_prefilled_values(args, args_file) + prefilled_answers = parse_prefilled_values(args, args_file) + + self.form = self._ask( + self.config, + self.form, + prefilled_answers=prefilled_answers, + action_id=action_id, + hooks=self.hooks, + ) - # Read or get values and hydrate the config - self._get_raw_settings() - self._hydrate() - BaseOption.operation_logger = operation_logger - self._ask(action=action_id) + # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) + # BaseOption.operation_logger = operation_logger # FIXME: here, we could want to check constrains on # the action's visibility / requirements wrt to the answer to questions ... @@ -511,21 +522,21 @@ def run_action(self, action=None, args=None, args_file=None, operation_logger=No operation_logger.start() try: - self._run_action(action_id) + self._run_action(self.form, action_id) except YunohostError: raise # Script got manually interrupted ... # N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): error = m18n.n("operation_interrupted") - logger.error(m18n.n("config_action_failed", action=action, error=error)) + logger.error(m18n.n("config_action_failed", action=key, error=error)) raise # Something wrong happened in Yunohost's code (most probably hook_exec) except Exception: import traceback error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("config_action_failed", action=action, error=error)) + logger.error(m18n.n("config_action_failed", action=key, error=error)) raise finally: # Delete files uploaded from API @@ -536,7 +547,9 @@ def run_action(self, action=None, args=None, args_file=None, operation_logger=No # FIXME: i18n logger.success(f"Action {action_id} successful") - operation_logger.success() + + if operation_logger: + operation_logger.success() def _get_raw_config(self) -> "RawConfig": if not os.path.exists(self.config_path): @@ -648,12 +661,13 @@ def _ask( logger.debug("Ask unanswered question and prevalidate data") interactive = Moulinette.interface.type == "cli" and os.isatty(1) + verbose = action_id is None or len(list(config.options)) > 1 if interactive: config.translate() for panel in config.panels: - if interactive: + if interactive and verbose: Moulinette.display( colorize(f"\n{'='*40}\n>>>> {panel.name}\n{'='*40}", "purple") ) @@ -669,11 +683,9 @@ def _ask( if ( action_id is None and section.is_action_section ) or not section.is_visible(context): - # FIXME useless? - Moulinette.display("Skipping section '{panel.id}.{section.id}'…") continue - if interactive and section.name: + if interactive and verbose and section.name: Moulinette.display(colorize(f"\n# {section.name}", "purple")) # filter action section options in case of multiple buttons @@ -718,6 +730,9 @@ def _apply( return settings + def _run_action(self, form: "FormModel", action_id: str): + raise NotImplementedError() + def _reload_services(self): from yunohost.service import service_reload_or_restart From 25ccbd5f78e25cc7a75128ae7f0e2fcc8f8f74fe Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 18 Apr 2023 22:00:58 +0200 Subject: [PATCH 129/436] configpanel: quickly update list_actions --- src/utils/configpanel.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index eb244aa082..96bf910b04 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -470,12 +470,12 @@ def list_actions(self): # FIXME : meh, loading the entire config panel is again going to cause # stupid issues for domain (e.g loading registrar stuff when willing to just list available actions ...) - self.filter_key = "" - self._get_config_panel() - for panel, section, option in self._iterate(): - if option["type"] == OptionType.button: - key = f"{panel['id']}.{section['id']}.{option['id']}" - actions[key] = _value_for_locale(option["ask"]) + self.config, self.form = self._get_config_panel() + + for panel, section, option in self.config.iter_children(): + if option.type == OptionType.button: + key = f"{panel.id}.{section.id}.{option.id}" + actions[key] = _value_for_locale(option.ask) return actions From f087a6b967d434c32517822bd1b8741c62547798 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 14:34:43 +0200 Subject: [PATCH 130/436] config: normalize get option value --- src/utils/configpanel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 96bf910b04..4e1e0e8ce4 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -355,7 +355,7 @@ def get( if isinstance(option, BaseReadonlyOption): return None - return self.form[option_id] + return option.normalize(self.form[option_id], option) # Format result in 'classic' or 'export' mode self.config.translate() From 73b795be1d192329936a052787ec67178dc81ddd Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 14:37:49 +0200 Subject: [PATCH 131/436] config: readd value has been initialized + change test removed value from settings since boolean has a native default value --- src/tests/test_app_config.py | 4 ++-- src/utils/configpanel.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/tests/test_app_config.py b/src/tests/test_app_config.py index 4a74cbc0de..24abdc5dc1 100644 --- a/src/tests/test_app_config.py +++ b/src/tests/test_app_config.py @@ -125,9 +125,9 @@ def test_app_config_get_nonexistentstuff(config_app): with pytest.raises(YunohostValidationError): app_config_get(config_app, "main.components.nonexistent") - app_setting(config_app, "boolean", delete=True) + app_setting(config_app, "number", delete=True) with pytest.raises(YunohostError): - app_config_get(config_app, "main.components.boolean") + app_config_get(config_app, "main.components.number") def test_app_config_regular_setting(config_app): diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 4e1e0e8ce4..eb9455a805 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -601,6 +601,17 @@ def _get_partial_raw_settings_and_mutate_config( for _, section, option in config.iter_children(): value = data = raw_settings.get(option.id, getattr(option, "default", None)) + if isinstance(option, BaseInputOption) and option.id not in raw_settings: + if option.default is not None: + value = option.default + elif option.type is OptionType.file or option.bind == "null": + continue + else: + raise YunohostError( + f"Config panel question '{option.id}' should be initialized with a value during install or upgrade.", + raw_msg=True, + ) + if isinstance(data, dict): # Settings data if gathered from bash "ynh_app_config_show" # may be a custom getter that returns a dict with `value` or `current_value` From 98d3b4ffc8d7bfed4eea3c147a4da68a2f80dd68 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 15:05:16 +0200 Subject: [PATCH 132/436] form: rework context/values/hooks in prompt_or_validate_form --- src/utils/form.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index 7098692a46..1496f445b1 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1471,9 +1471,6 @@ def prompt_or_validate_form( context: Context = {}, hooks: Hooks = {}, ) -> FormModel: - answers = {**prefilled_answers} - values = {} - for option in options: interactive = Moulinette.interface.type == "cli" and os.isatty(1) @@ -1505,7 +1502,7 @@ def prompt_or_validate_form( if isinstance(option, BaseInputOption): # FIXME normalized needed, form[option.id] should already be normalized # only update the context with the value - context[option.id] = form[option.id] + context[option.id] = option.normalize(form[option.id]) # FIXME here we could error out if option.id in prefilled_answers: @@ -1542,7 +1539,8 @@ def prompt_or_validate_form( try: # Normalize and validate - values[option.id] = form[option.id] = option.normalize(value, option) + form[option.id] = option.normalize(value, option) + context[option.id] = form[option.id] except (ValidationError, YunohostValidationError) as e: # If in interactive cli, re-ask the current question if i < 4 and interactive: @@ -1562,11 +1560,13 @@ def prompt_or_validate_form( # Search for post actions in hooks post_hook = f"post_ask__{option.id}" if post_hook in hooks: - values.update(hooks[post_hook](option)) - # FIXME reapply new values to form to validate it - - answers.update(values) - context.update(values) + # Hooks looks like they can return multiple values, validate those + values = hooks[post_hook](option) + for option_id, value in values.items(): + option = next(opt for opt in options if option.id == option_id) + if option and isinstance(option, BaseInputOption): + form[option.id] = option.normalize(value, option) + context[option.id] = form[option.id] return form From 98ec5448f2aba6e8b51c5720b5f80fe600cf51ab Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 15:07:34 +0200 Subject: [PATCH 133/436] form: cli retries as variable to be patched in tests --- src/tests/test_questions.py | 8 ++++++++ src/utils/form.py | 5 ++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 387d5c0f9f..6aca55e1a3 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -26,6 +26,7 @@ FileOption, evaluate_simple_js_expression, ) +from yunohost.utils import form from yunohost.utils.error import YunohostError, YunohostValidationError @@ -94,6 +95,12 @@ def patch_with_tty(): yield +@pytest.fixture +def patch_cli_retries(): + with patch.object(form, "MAX_RETRIES", 0): + yield + + # ╭───────────────────────────────────────────────────────╮ # │ ╭─╴╭─╴┌─╴╭╮╷╭─┐┌─╮╶┬╴╭─╮╭─╴ │ # │ ╰─╮│ ├─╴│││├─┤├┬╯ │ │ │╰─╮ │ @@ -405,6 +412,7 @@ def _test_intake_may_fail(raw_option, intake, expected_output): _test_intake(raw_option, intake, expected_output) +@pytest.mark.usefixtures("patch_cli_retries") # To avoid chain error logging class BaseTest: raw_option: dict[str, Any] = {} prefill: dict[Literal["raw_option", "prefill", "intake"], Any] diff --git a/src/utils/form.py b/src/utils/form.py index 1496f445b1..bf50d93a40 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1464,6 +1464,9 @@ def parse_prefilled_values( return values +MAX_RETRIES = 4 + + def prompt_or_validate_form( options: list[AnyOption], form: FormModel, @@ -1543,7 +1546,7 @@ def prompt_or_validate_form( context[option.id] = form[option.id] except (ValidationError, YunohostValidationError) as e: # If in interactive cli, re-ask the current question - if i < 4 and interactive: + if i < MAX_RETRIES and interactive: logger.error(str(e)) value = None continue From b45515049dfee9af447a4244ba653b86cb1225fb Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 17:26:46 +0200 Subject: [PATCH 134/436] config: fix wrong diff logic on settings apply --- src/utils/configpanel.py | 31 ++++++++++++++++++++++++++++--- src/utils/form.py | 2 +- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index eb9455a805..b48046ca39 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -282,6 +282,7 @@ class ConfigPanel: filter_key: "FilterKey" = (None, None, None) config: Union[ConfigPanelModel, None] = None form: Union["FormModel", None] = None + raw_settings: "RawSettings" = {} hooks: "Hooks" = {} @classmethod @@ -596,6 +597,8 @@ def _get_partial_raw_settings_and_mutate_config( self, config: ConfigPanelModel ) -> tuple[ConfigPanelModel, "RawSettings"]: raw_settings = self._get_raw_settings(config) + # Save `raw_settings` for diff at `_apply` + self.raw_settings = raw_settings values = {} for _, section, option in config.iter_children(): @@ -734,12 +737,34 @@ def _apply( mkdir(dir_path, mode=0o700) exclude_defaults = self.save_mode == "diff" - settings = form.dict(exclude_defaults=exclude_defaults, exclude=exclude) # type: ignore + # get settings keys filtered by filter_key + partial_settings_keys = form.__fields__.keys() + # get filtered settings + partial_settings = form.dict(exclude_defaults=exclude_defaults, exclude=exclude) # type: ignore + # get previous settings that we will updated with new settings + current_settings = self.raw_settings.copy() + + if exclude: + current_settings = { + key: value + for key, value in current_settings.items() + if key not in exclude + } + + for key in partial_settings_keys: + if ( + exclude_defaults + and key not in partial_settings + and key in current_settings + ): + del current_settings[key] + elif key in partial_settings: + current_settings[key] = partial_settings[key] # Save the settings to the .yaml file - write_to_yaml(self.save_path, settings) + write_to_yaml(self.save_path, current_settings) - return settings + return current_settings def _run_action(self, form: "FormModel", action_id: str): raise NotImplementedError() diff --git a/src/utils/form.py b/src/utils/form.py index bf50d93a40..d8ff4b8c70 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1493,7 +1493,7 @@ def prompt_or_validate_form( # - we doesn't want to give a specific value # - we want to keep the previous value # - we want the default value - context[option.id] = form[option.id] = None + context[option.id] = None continue From 54cc23c90cc87d853cfac9e355efc8891e9bda07 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 17:46:55 +0200 Subject: [PATCH 135/436] config: update SettingsConfigPanel.reset --- src/settings.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/settings.py b/src/settings.py index 0d0a5406bc..5f645e3dcc 100644 --- a/src/settings.py +++ b/src/settings.py @@ -23,7 +23,7 @@ from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.configpanel import ConfigPanel +from yunohost.utils.configpanel import ConfigPanel, parse_filter_key from yunohost.utils.form import BaseOption from yunohost.regenconf import regen_conf from yunohost.firewall import firewall_reload @@ -31,6 +31,8 @@ from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings if TYPE_CHECKING: + from yunohost.log import OperationLogger + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny from yunohost.utils.configpanel import ( @@ -148,25 +150,26 @@ def get( return result - def reset(self, key="", operation_logger=None): - self.filter_key = key + def reset(self, key: Union[str, None] = None, operation_logger: Union["OperationLogger", None] = None,): + self.filter_key = parse_filter_key(key) # Read config panel toml - self._get_config_panel() + self.config, self.form = self._get_config_panel(prevalidate=True) - if not self.config: - raise YunohostValidationError("config_no_panel") + # FIXME find a better way to exclude previous settings + previous_settings = self.form.dict() - # Replace all values with default values - self.values = self._get_default_values() + for option in self.config.options: + if not option.readonly and (option.optional or option.default not in {None, ""}): + self.form[option.id] = option.normalize(option.default, option) - BaseOption.operation_logger = operation_logger + # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) + # BaseOption.operation_logger = operation_logger if operation_logger: operation_logger.start() - try: - self._apply() + self._apply(self.form, previous_settings) except YunohostError: raise # Script got manually interrupted ... @@ -184,7 +187,9 @@ def reset(self, key="", operation_logger=None): raise logger.success(m18n.n("global_settings_reset_success")) - operation_logger.success() + + if operation_logger: + operation_logger.success() def _get_raw_config(self) -> "RawConfig": raw_config = super()._get_raw_config() From 37b4eb956da2afe63e1c8bc4480d4fa4654fc7ca Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 18:52:59 +0200 Subject: [PATCH 136/436] typing: add missing type + misc typing fixes --- src/app.py | 13 ++++++---- src/domain.py | 2 +- src/log.py | 2 +- src/settings.py | 17 ++++++++++---- src/utils/configpanel.py | 51 ++++++++++++++++++++-------------------- src/utils/form.py | 46 +++++++++++++++++++----------------- 6 files changed, 72 insertions(+), 59 deletions(-) diff --git a/src/app.py b/src/app.py index 970a66fb2e..0514066c9c 100644 --- a/src/app.py +++ b/src/app.py @@ -1814,26 +1814,29 @@ def _apply( form: "FormModel", previous_settings: dict[str, Any], exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, - ): + ) -> None: env = {key: str(value) for key, value in form.dict().items()} return_content = self._call_config_script("apply", env=env) # If the script returned validation error # raise a ValidationError exception using # the first key - if return_content: - for key, message in return_content.get("validation_errors").items(): + errors = return_content.get("validation_errors") + if errors: + for key, message in errors.items(): raise YunohostValidationError( "app_argument_invalid", name=key, error=message, ) - def _run_action(self, form: "FormModel", action_id: str): + def _run_action(self, form: "FormModel", action_id: str) -> None: env = {key: str(value) for key, value in form.dict().items()} self._call_config_script(action_id, env=env) - def _call_config_script(self, action, env=None): + def _call_config_script( + self, action: str, env: Union[dict[str, Any], None] = None + ) -> dict[str, Any]: from yunohost.hook import hook_exec if env is None: diff --git a/src/domain.py b/src/domain.py index f0531e624a..892220a681 100644 --- a/src/domain.py +++ b/src/domain.py @@ -720,7 +720,7 @@ def _apply( form: "FormModel", previous_settings: dict[str, Any], exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, - ): + ) -> None: next_settings = { k: v for k, v in form.dict().items() if previous_settings.get(k) != v } diff --git a/src/log.py b/src/log.py index 13683d8ef7..5a72411d80 100644 --- a/src/log.py +++ b/src/log.py @@ -469,7 +469,7 @@ class OperationLogger: This class record logs and metadata like context or start time/end time. """ - _instances: List[object] = [] + _instances: List["OperationLogger"] = [] def __init__(self, operation, related_to=None, **kwargs): # TODO add a way to not save password on app installation diff --git a/src/settings.py b/src/settings.py index 5f645e3dcc..f70f9df618 100644 --- a/src/settings.py +++ b/src/settings.py @@ -136,7 +136,7 @@ class SettingsConfigPanel(ConfigPanel): save_mode = "diff" virtual_settings = {"root_password", "root_password_confirm", "passwordless_sudo"} - def __init__(self, config_path=None, save_path=None, creation=False): + def __init__(self, config_path=None, save_path=None, creation=False) -> None: super().__init__("settings") def get( @@ -150,7 +150,11 @@ def get( return result - def reset(self, key: Union[str, None] = None, operation_logger: Union["OperationLogger", None] = None,): + def reset( + self, + key: Union[str, None] = None, + operation_logger: Union["OperationLogger", None] = None, + ) -> None: self.filter_key = parse_filter_key(key) # Read config panel toml @@ -160,8 +164,11 @@ def reset(self, key: Union[str, None] = None, operation_logger: Union["Operation previous_settings = self.form.dict() for option in self.config.options: - if not option.readonly and (option.optional or option.default not in {None, ""}): - self.form[option.id] = option.normalize(option.default, option) + if not option.readonly and ( + option.optional or option.default not in {None, ""} + ): + # FIXME Mypy complains about option.default not being a valid type for normalize but this should be ok + self.form[option.id] = option.normalize(option.default, option) # type: ignore # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) # BaseOption.operation_logger = operation_logger @@ -230,7 +237,7 @@ def _apply( form: "FormModel", previous_settings: dict[str, Any], exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, - ): + ) -> None: root_password = form.get("root_password", None) root_password_confirm = form.get("root_password_confirm", None) passwordless_sudo = form.get("passwordless_sudo", None) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index b48046ca39..b23df6dddf 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -21,7 +21,7 @@ import re from collections import OrderedDict from logging import getLogger -from typing import TYPE_CHECKING, Any, Literal, Sequence, Type, Union +from typing import TYPE_CHECKING, Any, Iterator, Literal, Sequence, Type, Union from pydantic import BaseModel, Extra, validator @@ -32,7 +32,6 @@ from yunohost.utils.form import ( AnyOption, BaseInputOption, - BaseOption, BaseReadonlyOption, FileOption, OptionsModel, @@ -70,7 +69,7 @@ class ContainerModel(BaseModel): services: list[str] = [] help: Union[Translation, None] = None - def translate(self, i18n_key: Union[str, None] = None): + def translate(self, i18n_key: Union[str, None] = None) -> None: """ Translate `ask` and `name` attributes of panels and section. This is in place mutation. @@ -111,16 +110,16 @@ def __init__( ) @property - def is_action_section(self): + def is_action_section(self) -> bool: return any([option.type is OptionType.button for option in self.options]) - def is_visible(self, context: dict[str, Any]): + def is_visible(self, context: dict[str, Any]) -> bool: if isinstance(self.visible, bool): return self.visible return evaluate_simple_js_expression(self.visible, context=context) - def translate(self, i18n_key: Union[str, None] = None): + def translate(self, i18n_key: Union[str, None] = None) -> None: """ Call to `Container`'s `translate` for self translation + Call to `OptionsContainer`'s `translate_options` for options translation @@ -151,7 +150,7 @@ def __init__( id=id, name=name, services=services, help=help, sections=sections ) - def translate(self, i18n_key: Union[str, None] = None): + def translate(self, i18n_key: Union[str, None] = None) -> None: """ Recursivly mutate translatable attributes to their translation """ @@ -181,14 +180,14 @@ def __init__( super().__init__(version=version, i18n=i18n, panels=panels) @property - def sections(self): + def sections(self) -> Iterator[SectionModel]: """Convinient prop to iter on all sections""" for panel in self.panels: for section in panel.sections: yield section @property - def options(self): + def options(self) -> Iterator[AnyOption]: """Convinient prop to iter on all options""" for section in self.sections: for option in section.options: @@ -231,7 +230,7 @@ def iter_children( for option in section.options: yield (panel, section, option) - def translate(self): + def translate(self) -> None: """ Recursivly mutate translatable attributes to their translation """ @@ -239,7 +238,7 @@ def translate(self): panel.translate(self.i18n) @validator("version", always=True) - def check_version(cls, value, field: "ModelField"): + def check_version(cls, value: float, field: "ModelField") -> float: if value < CONFIG_PANEL_VERSION_SUPPORTED: raise ValueError( f"Config panels version '{value}' are no longer supported." @@ -302,7 +301,9 @@ def list(cls): entities = [] return entities - def __init__(self, entity, config_path=None, save_path=None, creation=False): + def __init__( + self, entity, config_path=None, save_path=None, creation=False + ) -> None: self.entity = entity self.config_path = config_path if not config_path: @@ -350,7 +351,7 @@ def get( if option is None: # FIXME i18n raise YunohostValidationError( - f"Couldn't find any option with id {option_id}" + f"Couldn't find any option with id {option_id}", raw_msg=True ) if isinstance(option, BaseReadonlyOption): @@ -398,7 +399,7 @@ def set( args: Union[str, None] = None, args_file: Union[str, None] = None, operation_logger: Union["OperationLogger", None] = None, - ): + ) -> None: self.filter_key = parse_filter_key(key) panel_id, section_id, option_id = self.filter_key @@ -466,7 +467,7 @@ def set( if operation_logger: operation_logger.success() - def list_actions(self): + def list_actions(self) -> dict[str, str]: actions = {} # FIXME : meh, loading the entire config panel is again going to cause @@ -486,7 +487,7 @@ def run_action( args: Union[str, None] = None, args_file: Union[str, None] = None, operation_logger: Union["OperationLogger", None] = None, - ): + ) -> None: # # FIXME : this stuff looks a lot like set() ... # @@ -666,7 +667,7 @@ def _get_config_panel( def _ask( self, config: ConfigPanelModel, - settings: "FormModel", + form: "FormModel", prefilled_answers: dict[str, Any] = {}, action_id: Union[str, None] = None, hooks: "Hooks" = {}, @@ -709,22 +710,22 @@ def _ask( if option.type is not OptionType.button or option.id == action_id ] - settings = prompt_or_validate_form( + form = prompt_or_validate_form( options, - settings, + form, prefilled_answers=prefilled_answers, context=context, hooks=hooks, ) - return settings + return form def _apply( self, form: "FormModel", previous_settings: dict[str, Any], exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, - ) -> dict[str, Any]: + ) -> None: """ Save settings in yaml file. If `save_mode` is `"diff"` (which is the default), only values that are @@ -764,15 +765,13 @@ def _apply( # Save the settings to the .yaml file write_to_yaml(self.save_path, current_settings) - return current_settings - - def _run_action(self, form: "FormModel", action_id: str): + def _run_action(self, form: "FormModel", action_id: str) -> None: raise NotImplementedError() - def _reload_services(self): + def _reload_services(self) -> None: from yunohost.service import service_reload_or_restart - services_to_reload = self.config.services + services_to_reload = self.config.services if self.config else [] if services_to_reload: logger.info("Reloading services...") diff --git a/src/utils/form.py b/src/utils/form.py index d8ff4b8c70..4d62b0a294 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -32,7 +32,7 @@ Annotated, Any, Callable, - List, + Iterable, Literal, Mapping, Type, @@ -207,7 +207,7 @@ def js_to_python(expr): return py_expr -def evaluate_simple_js_expression(expr, context={}): +def evaluate_simple_js_expression(expr: str, context: dict[str, Any] = {}) -> bool: if not expr.strip(): return False node = ast.parse(js_to_python(expr), mode="eval").body @@ -650,7 +650,7 @@ class NumberOption(BaseInputOption): _none_as_empty_str = False @staticmethod - def normalize(value, option={}): + def normalize(value, option={}) -> Union[int, None]: if isinstance(value, int): return value @@ -704,7 +704,7 @@ class BooleanOption(BaseInputOption): _none_as_empty_str = False @staticmethod - def humanize(value, option={}): + def humanize(value, option={}) -> str: option = option.dict() if isinstance(option, BaseOption) else option yes = option.get("yes", 1) @@ -727,7 +727,7 @@ def humanize(value, option={}): ) @staticmethod - def normalize(value, option={}): + def normalize(value, option={}) -> Any: option = option.dict() if isinstance(option, BaseOption) else option if isinstance(value, str): @@ -844,7 +844,7 @@ class WebPathOption(BaseInputOption): _annotation = str @staticmethod - def normalize(value, option={}): + def normalize(value, option={}) -> str: option = option.dict() if isinstance(option, BaseOption) else option if value is None: @@ -892,14 +892,14 @@ class FileOption(BaseInputOption): _upload_dirs: set[str] = set() @classmethod - def clean_upload_dirs(cls): + def clean_upload_dirs(cls) -> None: # Delete files uploaded from API for upload_dir in cls._upload_dirs: if os.path.exists(upload_dir): shutil.rmtree(upload_dir) @classmethod - def _value_post_validator(cls, value: Any, field: "ModelField") -> Any: + def _value_post_validator(cls, value: Any, field: "ModelField") -> str: from base64 import b64decode if not value: @@ -967,7 +967,6 @@ def _dynamic_annotation(self) -> Union[object, Type[str]]: choices = ( self.choices if isinstance(self.choices, list) else self.choices.keys() ) - # FIXME in case of dict, try to parse keys with `item_type` (at least number) return Literal[tuple(choices)] return self._annotation @@ -1006,6 +1005,7 @@ def _get_prompt_message(self, value: Any) -> str: class SelectOption(BaseChoicesOption): type: Literal[OptionType.select] = OptionType.select + filter: Literal[None] = None choices: Union[dict[str, Any], list[Any]] default: Union[str, None] _annotation = str @@ -1013,13 +1013,14 @@ class SelectOption(BaseChoicesOption): class TagsOption(BaseChoicesOption): type: Literal[OptionType.tags] = OptionType.tags + filter: Literal[None] = None choices: Union[list[str], None] = None pattern: Union[Pattern, None] = None default: Union[str, list[str], None] _annotation = str @staticmethod - def humanize(value, option={}): + def humanize(value, option={}) -> str: if isinstance(value, list): return ",".join(str(v) for v in value) if not value: @@ -1027,7 +1028,7 @@ def humanize(value, option={}): return value @staticmethod - def normalize(value, option={}): + def normalize(value, option={}) -> str: if isinstance(value, list): return ",".join(str(v) for v in value) if isinstance(value, str): @@ -1037,7 +1038,7 @@ def normalize(value, option={}): return value @property - def _dynamic_annotation(self): + def _dynamic_annotation(self) -> Type[str]: # TODO use Literal when serialization is seperated from validation # if self.choices is not None: # return Literal[tuple(self.choices)] @@ -1120,7 +1121,7 @@ def inject_default( return _get_maindomain() @staticmethod - def normalize(value, option={}): + def normalize(value, option={}) -> str: if value.startswith("https://"): value = value[len("https://") :] elif value.startswith("http://"): @@ -1314,7 +1315,9 @@ class OptionsModel(BaseModel): options: list[Annotated[AnyOption, Field(discriminator="type")]] @staticmethod - def options_dict_to_list(options: dict[str, Any], optional: bool = False): + def options_dict_to_list( + options: dict[str, Any], optional: bool = False + ) -> list[dict[str, Any]]: return [ option | { @@ -1329,7 +1332,7 @@ def options_dict_to_list(options: dict[str, Any], optional: bool = False): def __init__(self, **kwargs) -> None: super().__init__(options=self.options_dict_to_list(kwargs)) - def translate_options(self, i18n_key: Union[str, None] = None): + def translate_options(self, i18n_key: Union[str, None] = None) -> None: """ Mutate in place translatable attributes of options to their translations """ @@ -1359,7 +1362,7 @@ class Config: validate_assignment = True extra = Extra.ignore - def __getitem__(self, name: str): + def __getitem__(self, name: str) -> Any: # FIXME # if a FormModel's required field is not instancied with a value, it is # not available as an attr and therefor triggers an `AttributeError` @@ -1372,7 +1375,7 @@ def __getitem__(self, name: str): return getattr(self, name) - def __setitem__(self, name: str, value: Any): + def __setitem__(self, name: str, value: Any) -> None: setattr(self, name, value) def get(self, attr: str, default: Any = None) -> Any: @@ -1382,7 +1385,9 @@ def get(self, attr: str, default: Any = None) -> Any: return default -def build_form(options: list[AnyOption], name: str = "DynamicForm") -> Type[FormModel]: +def build_form( + options: Iterable[AnyOption], name: str = "DynamicForm" +) -> Type[FormModel]: """ Returns a dynamic pydantic model class that can be used as a form. Parsing/validation occurs at instanciation and assignements. @@ -1468,7 +1473,7 @@ def parse_prefilled_values( def prompt_or_validate_form( - options: list[AnyOption], + options: Iterable[AnyOption], form: FormModel, prefilled_answers: dict[str, Any] = {}, context: Context = {}, @@ -1503,7 +1508,6 @@ def prompt_or_validate_form( if isinstance(option, BaseReadonlyOption) or option.readonly: if isinstance(option, BaseInputOption): - # FIXME normalized needed, form[option.id] should already be normalized # only update the context with the value context[option.id] = option.normalize(form[option.id]) @@ -1623,7 +1627,7 @@ def ask_questions_and_parse_answers( return (model.options, form) -def hydrate_questions_with_choices(raw_questions: List) -> List: +def hydrate_questions_with_choices(raw_questions: list[dict[str, Any]]) -> list[dict[str, Any]]: out = [] for raw_question in raw_questions: From 2a28e289adcd0224ecd4856314c2ae75b2135b7c Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 19:58:14 +0200 Subject: [PATCH 137/436] form: rework 'hydrate_questions...' with a new 'parse_raw_options' that parse and validate options --- src/app.py | 5 ++--- src/utils/form.py | 50 +++++++++++++++++++++++++++++------------------ 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/app.py b/src/app.py index 0514066c9c..8ab683d81a 100644 --- a/src/app.py +++ b/src/app.py @@ -51,7 +51,7 @@ DomainOption, WebPathOption, ask_questions_and_parse_answers, - hydrate_questions_with_choices, + parse_raw_options, ) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError @@ -963,8 +963,7 @@ def app_upgrade( def app_manifest(app, with_screenshot=False): manifest, extracted_app_folder = _extract_app(app) - raw_questions = manifest.get("install", {}).values() - manifest["install"] = hydrate_questions_with_choices(raw_questions) + manifest["install"] = parse_raw_options(manifest.get("install", {}), serialize=True) # Add a base64 image to be displayed in web-admin if with_screenshot and Moulinette.interface.type == "api": diff --git a/src/utils/form.py b/src/utils/form.py index 4d62b0a294..dce4b94c83 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -29,6 +29,7 @@ from typing import ( TYPE_CHECKING, cast, + overload, Annotated, Any, Callable, @@ -1609,6 +1610,33 @@ def ask_questions_and_parse_answers( context = {**current_values, **answers} + model_options = parse_raw_options(raw_options, serialize=False) + # Build the form from those questions and instantiate it without + # parsing/validation (construct) since it may contains required questions. + form = build_form(model_options).construct() + form = prompt_or_validate_form( + model_options, form, prefilled_answers=answers, context=context, hooks=hooks + ) + return (model_options, form) + + +@overload +def parse_raw_options( + raw_options: dict[str, Any], serialize: Literal[True] +) -> list[dict[str, Any]]: + ... + + +@overload +def parse_raw_options( + raw_options: dict[str, Any], serialize: Literal[False] = False +) -> list[AnyOption]: + ... + + +def parse_raw_options( + raw_options: dict[str, Any], serialize: bool = False +) -> Union[list[dict[str, Any]], list[AnyOption]]: # Validate/parse the options attributes try: model = OptionsModel(**raw_options) @@ -1618,24 +1646,8 @@ def ask_questions_and_parse_answers( raise YunohostValidationError(error, raw_msg=True) model.translate_options() - # Build the form from those questions and instantiate it without - # parsing/validation (construct) since it may contains required questions. - form = build_form(model.options).construct() - form = prompt_or_validate_form( - model.options, form, prefilled_answers=answers, context=context, hooks=hooks - ) - return (model.options, form) - - -def hydrate_questions_with_choices(raw_questions: list[dict[str, Any]]) -> list[dict[str, Any]]: - out = [] - for raw_question in raw_questions: - raw_question = hydrate_option_type(raw_question) - question = OPTIONS[raw_question["type"]](**raw_question) - if isinstance(question, BaseChoicesOption) and question.choices: - raw_question["choices"] = question.choices - raw_question["default"] = question.default - out.append(raw_question) + if serialize: + return model.dict()["options"] - return out + return model.options From 6bef4b1e0e9a856ec4dfb026d704addedfb13dc6 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 20:00:02 +0200 Subject: [PATCH 138/436] app: remove call of 'domain_config_get' to avoid infinite recursion --- src/app.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/app.py b/src/app.py index 8ab683d81a..b3e34221d1 100644 --- a/src/app.py +++ b/src/app.py @@ -130,7 +130,6 @@ def app_info(app, full=False, upgradable=False): Get info for a specific app """ from yunohost.permission import user_permission_list - from yunohost.domain import domain_config_get _assert_is_installed(app) @@ -229,9 +228,7 @@ def app_info(app, full=False, upgradable=False): ret["is_webapp"] = "domain" in settings and "path" in settings if ret["is_webapp"]: - ret["is_default"] = ( - domain_config_get(settings["domain"], "feature.app.default_app") == app - ) + ret["is_default"] = settings.get("default_app", "_none") ret["supports_change_url"] = os.path.exists( os.path.join(setting_path, "scripts", "change_url") From b778aaf780ff5deb708bd2624547b4fc739a3102 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 19 Apr 2023 20:01:18 +0200 Subject: [PATCH 139/436] form: remove ChoosableOptions for now --- src/utils/form.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index dce4b94c83..81bd10ba83 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -938,18 +938,6 @@ def is_file_path(s): # ─ CHOICES ─────────────────────────────────────────────── -ChoosableOptions = Literal[ - OptionType.string, - OptionType.color, - OptionType.number, - OptionType.date, - OptionType.time, - OptionType.email, - OptionType.path, - OptionType.url, -] - - class BaseChoicesOption(BaseInputOption): # FIXME probably forbid choices to be None? filter: Union[JSExpression, None] = None # filter before choices From bd9bf29a88670efc495680137125f49a01c6ddbe Mon Sep 17 00:00:00 2001 From: axolotle Date: Thu, 20 Apr 2023 15:48:51 +0200 Subject: [PATCH 140/436] debian: add python3-pydantic + python3-email-validator dependencies --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 9f156ddea0..70a780af75 100644 --- a/debian/control +++ b/debian/control @@ -16,7 +16,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-toml, python3-packaging, python3-publicsuffix2 , python3-ldap, python3-zeroconf (>= 0.47), python3-lexicon, , python3-cryptography, python3-jwt - , python-is-python3 + , python-is-python3, python3-pydantic, python3-email-validator , nginx, nginx-extras (>=1.22) , apt, apt-transport-https, apt-utils, dirmngr , openssh-server, iptables, fail2ban, bind9-dnsutils From fccb291d7885c92a9e26a34867a715e9b9ef6045 Mon Sep 17 00:00:00 2001 From: axolotle Date: Fri, 21 Apr 2023 22:05:49 +0200 Subject: [PATCH 141/436] form: readd `pattern` to `path` --- src/utils/form.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index 81bd10ba83..76328f1136 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -396,6 +396,7 @@ def _get_prompt_message(self, value: None) -> str: class ButtonOption(BaseReadonlyOption): type: Literal[OptionType.button] = OptionType.button + bind: Literal["null"] = "null" help: Union[Translation, None] = None style: State = State.success icon: Union[str, None] = None @@ -839,10 +840,8 @@ class EmailOption(BaseInputOption): _annotation = EmailStr -class WebPathOption(BaseInputOption): +class WebPathOption(BaseStringOption): type: Literal[OptionType.path] = OptionType.path - default: Union[str, None] - _annotation = str @staticmethod def normalize(value, option={}) -> str: @@ -876,7 +875,6 @@ def normalize(value, option={}) -> str: class URLOption(BaseStringOption): type: Literal[OptionType.url] = OptionType.url - default: Union[str, None] _annotation = HttpUrl From 48f882ecd314ba5cb9849dc55372a8ebc074834b Mon Sep 17 00:00:00 2001 From: axolotle Date: Sat, 22 Apr 2023 18:47:05 +0200 Subject: [PATCH 142/436] form+configpanel: reflect Section `optional` value to all its Options --- src/utils/configpanel.py | 3 ++- src/utils/form.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index b23df6dddf..55dd077871 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -95,9 +95,10 @@ def __init__( services: list[str] = [], help: Union[Translation, None] = None, visible: Union[bool, str] = True, + optional: bool = True, **kwargs, ) -> None: - options = self.options_dict_to_list(kwargs, optional=True) + options = self.options_dict_to_list(kwargs, optional=optional) ContainerModel.__init__( self, diff --git a/src/utils/form.py b/src/utils/form.py index 76328f1136..bc21c309a0 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1308,9 +1308,8 @@ def options_dict_to_list( return [ option | { - "id": id_, + "id": option.get("id", id_), "type": option.get("type", "string"), - # ConfigPanel options needs to be set as optional by default "optional": option.get("optional", optional), } for id_, option in options.items() From d370cb0b241695d9d1dd6cf6e0c2482ac3f66648 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 24 Apr 2023 15:09:44 +0200 Subject: [PATCH 143/436] configpanel: add `value` in options dict for config get --full --- src/utils/configpanel.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 55dd077871..776577d3ee 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -363,7 +363,20 @@ def get( # Format result in 'classic' or 'export' mode self.config.translate() logger.debug(f"Formating result in '{mode}' mode") + + if mode == "full": + result = self.config.dict(exclude_none=True) + + for panel in result["panels"]: + for section in panel["sections"]: + for opt in section["options"]: + instance = self.config.get_option(opt["id"]) + if isinstance(instance, BaseInputOption): + opt["value"] = instance.normalize(self.form[opt["id"]], instance) + return result + result = OrderedDict() + for panel in self.config.panels: for section in panel.sections: if section.is_action_section and mode != "full": @@ -388,10 +401,7 @@ def get( "value" ] = "**************" # Prevent displaying password in `config get` - if mode == "full": - return self.config.dict(exclude_none=True) - else: - return result + return result def set( self, From 51d302bf180bf6f8fbc3cbf16494147f2a7f2b7d Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 24 Apr 2023 15:10:27 +0200 Subject: [PATCH 144/436] configpanel: `is_action_section` as attr --- src/utils/configpanel.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 776577d3ee..756981b63a 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -86,6 +86,7 @@ def translate(self, i18n_key: Union[str, None] = None) -> None: class SectionModel(ContainerModel, OptionsModel): visible: Union[bool, str] = True optional: bool = True + is_action_section: bool = False # Don't forget to pass arguments to super init def __init__( @@ -99,7 +100,7 @@ def __init__( **kwargs, ) -> None: options = self.options_dict_to_list(kwargs, optional=optional) - + is_action_section = any([option["type"] == OptionType.button for option in options]) ContainerModel.__init__( self, id=id, @@ -108,12 +109,9 @@ def __init__( help=help, visible=visible, options=options, + is_action_section=is_action_section, ) - @property - def is_action_section(self) -> bool: - return any([option.type is OptionType.button for option in self.options]) - def is_visible(self, context: dict[str, Any]) -> bool: if isinstance(self.visible, bool): return self.visible From 2f4c88ec55b85fbe813470d0881896331a30551e Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 24 Apr 2023 15:12:54 +0200 Subject: [PATCH 145/436] form: parse pydantic error in logging --- src/utils/form.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index bc21c309a0..36e816110d 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1537,7 +1537,11 @@ def prompt_or_validate_form( except (ValidationError, YunohostValidationError) as e: # If in interactive cli, re-ask the current question if i < MAX_RETRIES and interactive: - logger.error(str(e)) + logger.error( + "\n".join([err["msg"] for err in e.errors()]) + if isinstance(e, ValidationError) + else str(e) + ) value = None continue @@ -1627,8 +1631,7 @@ def parse_raw_options( model = OptionsModel(**raw_options) except ValidationError as e: error = "\n".join([err["msg"] for err in e.errors()]) - # FIXME use YunohostError instead since it is not really a user mistake? - raise YunohostValidationError(error, raw_msg=True) + raise YunohostError(error, raw_msg=True) model.translate_options() From 6953a8bf15202856c25c9114f3491dabe9b8f5ea Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 24 Apr 2023 15:22:10 +0200 Subject: [PATCH 146/436] configpanel: quick fix option typing --- src/utils/configpanel.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 756981b63a..a602266da6 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -21,7 +21,7 @@ import re from collections import OrderedDict from logging import getLogger -from typing import TYPE_CHECKING, Any, Iterator, Literal, Sequence, Type, Union +from typing import TYPE_CHECKING, Any, Iterator, Literal, Sequence, Type, Union, cast from pydantic import BaseModel, Extra, validator @@ -100,7 +100,9 @@ def __init__( **kwargs, ) -> None: options = self.options_dict_to_list(kwargs, optional=optional) - is_action_section = any([option["type"] == OptionType.button for option in options]) + is_action_section = any( + [option["type"] == OptionType.button for option in options] + ) ContainerModel.__init__( self, id=id, @@ -370,7 +372,9 @@ def get( for opt in section["options"]: instance = self.config.get_option(opt["id"]) if isinstance(instance, BaseInputOption): - opt["value"] = instance.normalize(self.form[opt["id"]], instance) + opt["value"] = instance.normalize( + self.form[opt["id"]], instance + ) return result result = OrderedDict() @@ -381,6 +385,9 @@ def get( continue for option in section.options: + # FIXME not sure why option resolves as possibly `None` + option = cast(AnyOption, option) + if mode == "export": if isinstance(option, BaseInputOption): result[option.id] = self.form[option.id] From ef860ee6eebb8d64aaad2707dd06363755183eed Mon Sep 17 00:00:00 2001 From: axolotle Date: Fri, 28 Apr 2023 16:11:38 +0200 Subject: [PATCH 147/436] form: default type to "select" if choices in option --- src/utils/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/form.py b/src/utils/form.py index 36e816110d..396419f3e8 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1309,7 +1309,7 @@ def options_dict_to_list( option | { "id": option.get("id", id_), - "type": option.get("type", "string"), + "type": option.get("type", "select" if "choices" in option else "string"), "optional": option.get("optional", optional), } for id_, option in options.items() From 3f417bb9b32a45be1e8dd48c8ee806cc8c0cdde9 Mon Sep 17 00:00:00 2001 From: axolotle Date: Fri, 28 Apr 2023 17:23:02 +0200 Subject: [PATCH 148/436] tests: update error instance in tests to YunohostError for packaging errors --- src/tests/test_questions.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 6aca55e1a3..8f8e701e9c 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -595,7 +595,7 @@ class TestAlert(TestDisplayText): (None, None, {"ask": "Some text\na new line"}), (None, None, {"ask": {"en": "Some text\na new line", "fr": "Un peu de texte\nune nouvelle ligne"}}), *[(None, None, {"ask": "question", "style": style}) for style in ("success", "info", "warning", "danger")], - (None, FAIL, {"ask": "question", "style": "nimp"}), + (None, YunohostError, {"ask": "question", "style": "nimp"}), ] # fmt: on @@ -737,7 +737,7 @@ class TestPassword(BaseTest): *all_fails([], ["one"], {}, raw_option={"optional": True}, error=AttributeError), # FIXME those fails with AttributeError *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), - ("s3cr3t!!", FAIL, {"default": "SUPAs3cr3t!!"}), # default is forbidden + ("s3cr3t!!", YunohostError, {"default": "SUPAs3cr3t!!"}), # default is forbidden *xpass(scenarios=[ ("s3cr3t!!", "s3cr3t!!", {"example": "SUPAs3cr3t!!"}), # example is forbidden ], reason="Should fail; example is forbidden"), @@ -749,7 +749,7 @@ class TestPassword(BaseTest): ("secret", FAIL), *[("supersecret" + char, FAIL) for char in FORBIDDEN_PASSWORD_CHARS], # FIXME maybe add ` \n` to the list? # readonly - ("s3cr3t!!", FAIL, {"readonly": True}), # readonly is forbidden + ("s3cr3t!!", YunohostError, {"readonly": True}), # readonly is forbidden ] # fmt: on @@ -1474,10 +1474,10 @@ class TestTags(BaseTest): # basic types (not in a list) should fail *all_fails(True, False, -1, 0, 1, 1337, 13.37, {}), # Mixed choices should fail - ([False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}], FAIL, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), - ("False,True,-1,0,1,1337,13.37,[],['one'],{}", FAIL, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), - *all_fails(*([t] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), - *all_fails(*([str(t)] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + ([False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}], YunohostError, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + ("False,True,-1,0,1,1337,13.37,[],['one'],{}", YunohostError, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + *all_fails(*([t] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}, error=YunohostError), + *all_fails(*([str(t)] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}, error=YunohostError), # readonly ("one", "one,two", {"readonly": True, "choices": ["one", "two"], "default": "one,two"}), ] @@ -1527,7 +1527,7 @@ class TestDomain(BaseTest): ("doesnt_exist.pouet", FAIL, {}), ("fake.com", FAIL, {"choices": ["fake.com"]}), # readonly - (domains1[0], FAIL, {"readonly": True}), # readonly is forbidden + (domains1[0], YunohostError, {"readonly": True}), # readonly is forbidden ] }, { @@ -1627,7 +1627,7 @@ class TestApp(BaseTest): (installed_non_webapp["id"], installed_non_webapp["id"]), (installed_non_webapp["id"], FAIL, {"filter": "is_webapp"}), # readonly - (installed_non_webapp["id"], FAIL, {"readonly": True}), # readonly is forbidden + (installed_non_webapp["id"], YunohostError, {"readonly": True}), # readonly is forbidden ] }, ] @@ -1744,7 +1744,7 @@ class TestUser(BaseTest): ("", regular_username, {"default": regular_username}) ], reason="Should throw 'no default allowed'"), # readonly - (admin_username, FAIL, {"readonly": True}), # readonly is forbidden + (admin_username, YunohostError, {"readonly": True}), # readonly is forbidden ] }, ] @@ -1824,9 +1824,9 @@ class TestGroup(BaseTest): "scenarios": [ ("custom_group", "custom_group"), *all_as("", None, output="visitors", raw_option={"default": "visitors"}), - ("", FAIL, {"default": "custom_group"}), # Not allowed to set a default which is not a default group + ("", YunohostError, {"default": "custom_group"}), # Not allowed to set a default which is not a default group # readonly - ("admins", FAIL, {"readonly": True}), # readonly is forbidden + ("admins", YunohostError, {"readonly": True}), # readonly is forbidden ] }, ] From 3a5d353c4b60770b0ed5ddaaa1a9caa85e66895d Mon Sep 17 00:00:00 2001 From: axolotle Date: Fri, 28 Apr 2023 17:24:11 +0200 Subject: [PATCH 149/436] form: force option type to 'select' if there's 'choices' --- src/tests/test_questions.py | 4 ++-- src/utils/form.py | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index 8f8e701e9c..fbbf757c9b 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -1989,10 +1989,10 @@ def test_option_default_type_with_choices_is_select(): } answers = {"some_choices": "a", "some_legacy": "a"} - options = ask_questions_and_parse_answers(questions, answers) + options, form = ask_questions_and_parse_answers(questions, answers) for option in options: assert option.type == "select" - assert option.value == "a" + assert form[option.id] == "a" @pytest.mark.skip # we should do something with this example diff --git a/src/utils/form.py b/src/utils/form.py index 396419f3e8..c586467fca 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1305,15 +1305,25 @@ class OptionsModel(BaseModel): def options_dict_to_list( options: dict[str, Any], optional: bool = False ) -> list[dict[str, Any]]: - return [ - option - | { - "id": option.get("id", id_), - "type": option.get("type", "select" if "choices" in option else "string"), - "optional": option.get("optional", optional), + options_list = [] + + for id_, data in options.items(): + option = data | { + "id": data.get("id", id_), + "type": data.get("type", OptionType.select if "choices" in data else OptionType.string), + "optional": data.get("optional", optional), } - for id_, option in options.items() - ] + + # LEGACY (`choices` in option `string` used to be valid) + if "choices" in option and option["type"] == OptionType.string: + logger.warning( + f"Packagers: option {id_} has 'choices' but has type 'string', use 'select' instead to remove this warning." + ) + option["type"] = OptionType.select + + options_list.append(option) + + return options_list def __init__(self, **kwargs) -> None: super().__init__(options=self.options_dict_to_list(kwargs)) From 3cae07970e913f04a249b3e930b9693a08aebaa9 Mon Sep 17 00:00:00 2001 From: axolotle Date: Sun, 30 Apr 2023 17:29:28 +0200 Subject: [PATCH 150/436] form: remove no longer used hydrate_option_type method --- src/utils/form.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/utils/form.py b/src/utils/form.py index c586467fca..07be55312e 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -1310,7 +1310,10 @@ def options_dict_to_list( for id_, data in options.items(): option = data | { "id": data.get("id", id_), - "type": data.get("type", OptionType.select if "choices" in data else OptionType.string), + "type": data.get( + "type", + OptionType.select if "choices" in data else OptionType.string, + ), "optional": data.get("optional", optional), } @@ -1414,22 +1417,6 @@ def build_form( ) -def hydrate_option_type(raw_option: dict[str, Any]) -> dict[str, Any]: - type_ = raw_option.get( - "type", OptionType.select if "choices" in raw_option else OptionType.string - ) - # LEGACY (`choices` in option `string` used to be valid) - if "choices" in raw_option and type_ == OptionType.string: - logger.warning( - f"Packagers: option {raw_option['id']} has 'choices' but has type 'string', use 'select' instead to remove this warning." - ) - type_ = OptionType.select - - raw_option["type"] = type_ - - return raw_option - - # ╭───────────────────────────────────────────────────────╮ # │ ╷ ╷╶┬╴╶┬╴╷ ╭─╴ │ # │ │ │ │ │ │ ╰─╮ │ From 66cb855c0ca0f484e62db4087bac8b802af0b0bc Mon Sep 17 00:00:00 2001 From: axolotle Date: Sun, 22 Oct 2023 17:47:43 +0200 Subject: [PATCH 151/436] domain: type fix --- src/domain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domain.py b/src/domain.py index 892220a681..7ac8a50cb5 100644 --- a/src/domain.py +++ b/src/domain.py @@ -753,10 +753,10 @@ def _apply( # that can be read by the portal API. # FIXME remove those from the config panel saved values? - portal_values = form.dict(include=portal_options) + portal_values = form.dict(include=set(portal_options)) portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") - portal_settings = {"apps": {}} + portal_settings: dict[str, Any] = {"apps": {}} if portal_settings_path.exists(): portal_settings.update(read_json(str(portal_settings_path))) From e7b43c763c28af767fef4b7e0acd97df9fb9059a Mon Sep 17 00:00:00 2001 From: axolotle Date: Sun, 22 Oct 2023 17:49:08 +0200 Subject: [PATCH 152/436] configpanel: do not raise error if no settings file --- src/utils/configpanel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index a602266da6..d79b8a80a9 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -577,7 +577,8 @@ def _get_raw_config(self) -> "RawConfig": def _get_raw_settings(self, config: ConfigPanelModel) -> "RawSettings": if not self.save_path or not os.path.exists(self.save_path): - raise YunohostValidationError("config_no_settings") + return {} + # raise YunohostValidationError("config_no_settings") return read_yaml(self.save_path) From 3a31984e3c2c193fc3ab5b2b27acbabd8e50d6db Mon Sep 17 00:00:00 2001 From: axolotle Date: Sun, 22 Oct 2023 17:51:04 +0200 Subject: [PATCH 153/436] configpanel: allow other ConfigPanels to have no settings defined --- src/app.py | 1 + src/utils/configpanel.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index b3e34221d1..21219b2ce6 100644 --- a/src/app.py +++ b/src/app.py @@ -1801,6 +1801,7 @@ class AppConfigPanel(ConfigPanel): entity_type = "app" save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml") config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.toml") + settings_must_be_defined: bool = True def _get_raw_settings(self, config: "ConfigPanelModel") -> "RawSettings": return self._call_config_script("show") diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index d79b8a80a9..325f6579dd 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -279,6 +279,7 @@ class ConfigPanel: save_path_tpl: Union[str, None] = None config_path_tpl = "/usr/share/yunohost/config_{entity_type}.toml" save_mode = "full" + settings_must_be_defined: bool = False filter_key: "FilterKey" = (None, None, None) config: Union[ConfigPanelModel, None] = None form: Union["FormModel", None] = None @@ -627,7 +628,7 @@ def _get_partial_raw_settings_and_mutate_config( value = option.default elif option.type is OptionType.file or option.bind == "null": continue - else: + elif self.settings_must_be_defined: raise YunohostError( f"Config panel question '{option.id}' should be initialized with a value during install or upgrade.", raw_msg=True, From 9134515604782cbde14a08745c2da393a5880958 Mon Sep 17 00:00:00 2001 From: axolotle Date: Sun, 22 Oct 2023 17:53:50 +0200 Subject: [PATCH 154/436] domain:config: make 'registrar' info a frozen input since an alert has no value --- src/dns.py | 88 +++++++++++++++++++++++------------------------------- 1 file changed, 38 insertions(+), 50 deletions(-) diff --git a/src/dns.py b/src/dns.py index 07ff2fb215..fc4b26a758 100644 --- a/src/dns.py +++ b/src/dns.py @@ -502,11 +502,26 @@ def _get_relative_name_for_dns_zone(domain, base_dns_zone): def _get_registrar_config_section(domain): from lexicon.providers.auto import _relevant_provider_for_domain - registrar_infos = { - "name": m18n.n( - "registrar_infos" - ), # This is meant to name the config panel section, for proper display in the webadmin - } + registrar_infos = OrderedDict( + { + "name": m18n.n( + "registrar_infos" + ), # This is meant to name the config panel section, for proper display in the webadmin + "registrar": OrderedDict( + { + "readonly": True, + "visible": False, + "default": None, + } + ), + "infos": OrderedDict( + { + "type": "alert", + "style": "info", + } + ), + } + ) dns_zone = _get_dns_zone_for_domain(domain) @@ -519,61 +534,34 @@ def _get_registrar_config_section(domain): else: parent_domain_link = parent_domain - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "info", - "ask": m18n.n( - "domain_dns_registrar_managed_in_parent_domain", - parent_domain=parent_domain, - parent_domain_link=parent_domain_link, - ), - "default": "parent_domain", - } + registrar_infos["registrar"]["default"] = "parent_domain" + registrar_infos["infos"]["ask"] = m18n.n( + "domain_dns_registrar_managed_in_parent_domain", + parent_domain=parent_domain, + parent_domain_link=parent_domain_link, ) - return OrderedDict(registrar_infos) + return registrar_infos # TODO big project, integrate yunohost's dynette as a registrar-like provider # TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron... if is_yunohost_dyndns_domain(dns_zone): - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "success", - "ask": m18n.n("domain_dns_registrar_yunohost"), - "default": "yunohost", - } - ) - return OrderedDict(registrar_infos) + registrar_infos["registrar"]["default"] = "yunohost" + registrar_infos["infos"]["style"] = "success" + registrar_infos["infos"]["ask"] = m18n.n("domain_dns_registrar_yunohost") + + return registrar_infos elif is_special_use_tld(dns_zone): - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "info", - "ask": m18n.n("domain_dns_conf_special_use_tld"), - "default": None, - } - ) + registrar_infos["infos"]["ask"] = m18n.n("domain_dns_conf_special_use_tld") try: registrar = _relevant_provider_for_domain(dns_zone)[0] except ValueError: - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "warning", - "ask": m18n.n("domain_dns_registrar_not_supported"), - "default": None, - } - ) + registrar_infos["registrar"]["default"] = None + registrar_infos["infos"]["ask"] = m18n.n("domain_dns_registrar_not_supported") else: - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "info", - "ask": m18n.n("domain_dns_registrar_supported", registrar=registrar), - "default": registrar, - } + registrar_infos["registrar"]["default"] = registrar + registrar_infos["infos"]["ask"] = m18n.n( + "domain_dns_registrar_supported", registrar=registrar ) TESTED_REGISTRARS = ["ovh", "gandi"] @@ -601,7 +589,7 @@ def _get_registrar_config_section(domain): infos["optional"] = infos.get("optional", "False") registrar_infos.update(registrar_credentials) - return OrderedDict(registrar_infos) + return registrar_infos def _get_registar_settings(domain): From 1221fd1458a2cd8a2d9249f2ee7c38c3466cb923 Mon Sep 17 00:00:00 2001 From: axolotle Date: Fri, 21 Apr 2023 22:15:34 +0200 Subject: [PATCH 155/436] doc:options: add documentation and generator for configpanel/manifest options --- doc/generate_options_doc.py | 365 +++++++++++++++++++++++++++ src/utils/form.py | 486 ++++++++++++++++++++++++++++++++++++ 2 files changed, 851 insertions(+) create mode 100644 doc/generate_options_doc.py diff --git a/doc/generate_options_doc.py b/doc/generate_options_doc.py new file mode 100644 index 0000000000..fc60783708 --- /dev/null +++ b/doc/generate_options_doc.py @@ -0,0 +1,365 @@ +import ast +import datetime +import subprocess + +version = open("../debian/changelog").readlines()[0].split()[1].strip("()") +today = datetime.datetime.now().strftime("%d/%m/%Y") + + +def get_current_commit(): + p = subprocess.Popen( + "git rev-parse --verify HEAD", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + stdout, stderr = p.communicate() + + current_commit = stdout.strip().decode("utf-8") + return current_commit + + +current_commit = get_current_commit() + + +print( + f"""--- +title: Options +template: docs +taxonomy: + category: docs +routes: + default: '/packaging_apps_options' +--- + +# Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{current_commit}/doc/generate_options_doc.py) on {today} (YunoHost version {version}) + +# Options + +Options are fields declaration that renders as form items in the web-admin and prompts in cli. +They are used in app manifests to declare the before installation form and in config panels. + +## Glossary + +You may encounter some named types which are used for simplicity. + +- `Translation`: a translated property + - used for properties: `ask`, `help` and `Pattern.error` + - a `dict` with locales as keys and translations as values: + ```toml + ask.en = "The text in english" + ask.fr = "Le texte en français" + ``` + It is not currently possible for translators to translate those string in weblate. + - a single `str` for a single english default string + ```toml + help = "The text in english" + ``` +- `JSExpression`: a `str` JS expression to be evaluated to `true` or `false`: + - used for properties: `visible` and `enabled` + - operators availables: `==`, `!=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%`, `match()` + - [examples available in the advanced section](#advanced-use-cases) +- `Binding`: bind a value to a file/property/variable/getter/setter/validator + - save the value in `settings.yaml` when not defined (`None`) + - nothing at all with `"null"` + - a custom getter/setter/validator with `"null"` + a function starting with `get__`, `set__`, `validate__` in `scripts/config` + - a variable/property in a file with `:__FINALPATH__/my_file.php` + - a whole file with `__FINALPATH__/my_file.php` + - [examples available in the advanced section](#bind) +- `Pattern`: a `dict` with a regex to match the value against and an error message + ```toml + pattern.regexp = "^[A-F]\d\d$" + pattern.error = "Provide a room like F12: one uppercase and 2 numbers" + # or with translated error + pattern.error.en = "Provide a room like F12: one uppercase and 2 numbers" + pattern.error.fr = "Entrez un numéro de salle comme F12: une lettre majuscule et deux chiffres." + ``` + +""" +) + + +fname = "../src/utils/form.py" +content = open(fname).read() + +# NB: This magic is because we want to be able to run this script outside of a YunoHost context, +# in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... +tree = ast.parse(content) + +OptionClasses = [ + c + for c in tree.body + if isinstance(c, ast.ClassDef) and c.name.endswith("Option") +] + +OptionDocString = {} + +for c in OptionClasses: + if not isinstance(c.body[0], ast.Expr): + continue + option_type = None + + if c.name in {"BaseOption", "BaseInputOption"}: + option_type = c.name + elif c.body[1].target.id == "type": + option_type = c.body[1].value.attr + + docstring = ast.get_docstring(c) + if docstring: + if "##### Properties" not in docstring: + docstring += """ +##### Properties + +- [common properties](#common-option-properties) + """ + OptionDocString[option_type] = docstring + +for option_type, doc in OptionDocString.items(): + print("") + if option_type == "BaseOption": + print("## Common Option properties") + elif option_type == "BaseInputOption": + print("## Common Inputs properties") + elif option_type == "display_text": + print("----------------") + print("## Readonly Options") + print(f"### Option `{option_type}`") + elif option_type == "string": + print("----------------") + print("## Input Options") + print(f"### Option `{option_type}`") + else: + print(f"### Option `{option_type}`") + print("") + print(doc) + print("") + +print( + """ +---------------- + +## Advanced use cases + +### `visible` & `enabled` expression evaluation + +Sometimes we may want to conditionaly display a message or prompt for a value, for this we have the `visible` prop. +And we may want to allow a user to trigger an action only if some condition are met, for this we have the `enabled` prop. + +Expressions are evaluated against a context containing previous values of the current section's options. This quite limited current design exists because on the web-admin or on the CLI we cannot guarantee that a value will be present in the form if the user queried only a single panel/section/option. +In the case of an action, the user will be shown or asked for each of the options of the section in which the button is present. + +The expression has to be written in javascript (this has been designed for the web-admin first and is converted to python on the fly on the cli). + +Available operators are: `==`, `!=`, `>`, `>=`, `<`, `<=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%` and `match()`. + +##### Examples + +```toml +# simple "my_option_id" is thruthy/falsy +visible = "my_option_id" +visible = "!my_option_id" +# misc +visible = "my_value >= 10" +visible = "-(my_value + 1) < 0" +visible = "!!my_value || my_other_value" +``` +For a more complete set of examples, [check the tests at the end of the file](https://github.com/YunoHost/yunohost/blob/dev/src/tests/test_questions.py). + +##### match() + +For more complex evaluation we can use regex matching. + +```toml +[my_string] +default = "Lorem ipsum dolor et si qua met!" + +[my_boolean] +type = "boolean" +visible = "my_string && match(my_string, '^Lorem [ia]psumE?')" +``` + +Match the content of a file. + +```toml +[my_file] +type = "file" +accept = ".txt" +bind = "/etc/random/lorem.txt" + +[my_boolean] +type = "boolean" +visible = "my_file && match(my_file, '^Lorem [ia]psumE?')" +``` + +with a file with content like: +```txt +Lorem ipsum dolor et si qua met! +``` + + +### `bind` + +Config panels only + +`bind` allows us to alter the generic behavior of option's values which is: get from and set in `settings.yml`. + +We can: +- alter the source the value comes from with getters. +- alter the destination with setters +- parse/validate the value before destination with validators + +##### Getters + +Define an option's custom getter in a bash script `script/config`. +It has to be named after an option's id prepended by `get__`. + +To display a custom alert message for example. We setup a base option in `config_panel.toml`. + +```toml +[panel.section.alert] +type = "alert" +# bind to "null" to inform there's something in `scripts/config` +bind = "null" +# `ask` & `style` will be injected by a custom getter +``` + +Then add a custom getter that output yaml, every properties defined here will override any property previously declared. + +```bash +get__alert() { + if [ "$whatever" ]; then + cat << EOF +style: success +ask: Your VPN is running :) +EOF + else + cat << EOF +style: danger +ask: Your VPN is down +EOF + fi +} +``` + +Or to inject a custom value: + +```toml +[panel.section.my_hidden_value] +type = "number" +bind = "null" +# option will act as an hidden variable that can be used in context evaluation +# (ie: `visible` or `enabled`) +readonly = true +visible = false +# `default` injected by a custom getter +``` + +```bash +get__my_hidden_value() { + if [ "$whatever" ]; then + # if only a value is needed + echo "10" + else + # or if we need to override some other props + # (use `default` or `value` to inject the value) + cat << EOF +ask: Here's a number +visible: true +default: 0 +EOF + fi +} +``` + +##### Setters + +Define an option's custom setter in a bash script `script/config`. +It has to be named after an option's id prepended by `set__`. + +```toml +[panel.section.my_value] +type = "string" +bind = "null" +ask = "gimme complex string" +``` + +```bash +set__my_value() { + if [ -n "$my_value" ]; then + # split the string into multiple elements or idk + fi + # To save the value or modified value as a setting: + ynh_app_setting_set --app=$app --key=my_value --value="$my_value" +} +``` + +##### Validators + +Define an option's custom validator in a bash script `script/config`. +It has to be named after an option's id prepended by `validate__`. + +Validators allows us to return custom error messages depending on the value. + +```toml +[panel.section.my_value] +type = "string" +bind = "null" +ask = "Gimme a long string" +default = "too short" +``` + +```bash +validate__my_value() { + if [[ "${#my_value}" -lt 12 ]]; then echo 'Too short!'; fi +} +``` + +##### Actions + +Define an option's action in a bash script `script/config`. +It has to be named after a `button`'s id prepended by `run__`. + +```toml +[panel.section.my_action] +type = "button" +# no need to set `bind` to "null" it is its hard default +ask = "Run action" +``` + +```bash +run__my_action() { + ynh_print_info "Running 'my_action'..." +} +``` + +A more advanced example could look like: + +```toml +[panel.my_action_section] +name = "Action section" + [panel.my_action_section.my_repo] + type = "url" + bind = "null" # value will not be saved as a setting + ask = "gimme a repo link" + + [panel.my_action_section.my_repo_name] + type = "string" + bind = "null" # value will not be saved as a setting + ask = "gimme a custom folder name" + + [panel.my_action_section.my_action] + type = "button" + ask = "Clone the repo" + # enabled the button only if the above values is defined + enabled = "my_repo && my_repo_name" +``` + +```bash +run__my_action() { + ynh_print_info "Cloning '$my_repo'..." + cd /tmp + git clone "$my_repo" "$my_repo_name" +} +``` +""" +) diff --git a/src/utils/form.py b/src/utils/form.py index 07be55312e..f030523d64 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -298,6 +298,76 @@ class Pattern(BaseModel): class BaseOption(BaseModel): + """ + ##### Examples + + ```toml + [my_option_id] + type = "string" + # ask as `str` + ask = "The text in english" + # ask as `dict` + ask.en = "The text in english" + ask.fr = "Le texte en français" + # advanced props + visible = "my_other_option_id != 'success'" + readonly = true + # much advanced: config panel only? + bind = "null" + ``` + + ##### Properties + + - `type`: + - readonly types: + - [`display_text`](#option-display_text) + - [`markdown`](#option-markdown) + - [`alert`](#option-alert) + - [`button`](#option-button) + - inputs types: + - [`string`](#option-string) + - [`text`](#option-text) + - [`password`](#option-password) + - [`color`](#option-color) + - [`number`](#option-number) + - [`range`](#option-range) + - [`boolean`](#option-boolean) + - [`date`](#option-date) + - [`time`](#option-time) + - [`email`](#option-email) + - [`path`](#option-path) + - [`url`](#option-url) + - [`file`](#option-file) + - [`select`](#option-select) + - [`tags`](#option-tags) + - [`domain`](#option-domain) + - [`app`](#option-app) + - [`user`](#option-user) + - [`group`](#option-group) + - `ask`: `Translation` (default to the option's `id` if not defined): + - text to display as the option's label for inputs or text to display for readonly options + - `visible` (optional): `bool` or `JSExpression` (default: `true`) + - define if the option is diplayed/asked + - if `false` and used alongside `readonly = true`, you get a context only value that can still be used in `JSExpression`s + - `readonly` (optional): `bool` (default: `false`, forced to `true` for readonly types): + - If `true` for input types: forbid mutation of its value + - `bind` (optional): `Binding` (default: `None`): + - (config panels only!) allow to choose where an option's is gathered/stored: + - if not specified, the value will be gathered/stored in the `settings.yml` + - if `"null"`: + - the value will not be stored at all (can still be used in context evaluations) + - if in `scripts/config` there's a function named: + - `get__my_option_id`: the value will be gathered from this custom getter + - `set__my_option_id`: the value will be passed to this custom setter where you can do whatever you want with the value + - `validate__my_option_id`: the value will be passed to this custom validator before any custom setter + - if `bind` is a file path: + - if the path starts with `:`, the value be saved as its id's variable/property counterpart + - this only works for first level variables/properties and simple types (no array) + - else the value will be stored as the whole content of the file + - you can use `__FINALPATH__` in your path to point to dynamic install paths + - FIXME are other global variables accessible? + """ + type: OptionType id: str ask: Union[Translation, None] @@ -364,10 +434,34 @@ class BaseReadonlyOption(BaseOption): class DisplayTextOption(BaseReadonlyOption): + """ + Display simple multi-line content. + + ##### Examples + + ```toml + [my_option_id] + type = "display_text" + ask = "Simple text rendered as is." + ``` + """ + type: Literal[OptionType.display_text] = OptionType.display_text class MarkdownOption(BaseReadonlyOption): + """ + Display markdown multi-line content. + Markdown is currently only rendered in the web-admin + + ##### Examples + ```toml + [my_option_id] + type = "display_text" + ask = "Text **rendered** in markdown." + ``` + """ + type: Literal[OptionType.markdown] = OptionType.markdown @@ -379,6 +473,27 @@ class State(str, Enum): class AlertOption(BaseReadonlyOption): + """ + Alerts displays a important message with a level of severity. + You can use markdown in `ask` but will only be rendered in the web-admin. + + ##### Examples + + ```toml + [my_option_id] + type = "alert" + ask = "The configuration seems to be manually modified..." + style = "warning" + icon = "warning" + ``` + ##### Properties + + - [common properties](#common-option-properties) + - `style`: any of `"success|info|warning|danger"` (default: `"info"`) + - `icon` (optional): any icon name from [Fork Awesome](https://forkaweso.me/Fork-Awesome/icons/) + - Currently only displayed in the web-admin + """ + type: Literal[OptionType.alert] = OptionType.alert style: State = State.info icon: Union[str, None] = None @@ -395,6 +510,45 @@ def _get_prompt_message(self, value: None) -> str: class ButtonOption(BaseReadonlyOption): + """ + Triggers actions. + Available only in config panels. + Renders as a `button` in the web-admin and can be called with `yunohost [app|domain|settings] action run ` in CLI. + + Every options defined in an action section (a config panel section with at least one `button`) is guaranted to be shown/asked to the user and available in `scripts/config`'s scope. + [check examples in advanced use cases](#actions). + + ##### Examples + + ```toml + [my_action_id] + type = "button" + ask = "Break the system" + style = "danger" + icon = "bug" + # enabled only if another option's value (a `boolean` for example) is positive + enabled = "aknowledged" + ``` + + To be able to trigger an action we have to add a bash function starting with `run__` in your `scripts/config` + + ```bash + run__my_action_id() { + ynh_print_info "Running 'my_action_id' action" + } + ``` + + ##### Properties + + - [common properties](#common-option-properties) + - `bind`: forced to `"null"` + - `style`: any of `"success|info|warning|danger"` (default: `"success"`) + - `enabled`: `Binding` or `bool` (default: `true`) + - when used with `Binding` you can enable/disable the button depending on context + - `icon` (optional): any icon name from [Fork Awesome](https://forkaweso.me/Fork-Awesome/icons/) + - Currently only displayed in the web-admin + """ + type: Literal[OptionType.button] = OptionType.button bind: Literal["null"] = "null" help: Union[Translation, None] = None @@ -415,6 +569,35 @@ def is_enabled(self, context: Context) -> bool: class BaseInputOption(BaseOption): + """ + Rest of the option types available are considered `inputs`. + + ##### Examples + + ```toml + [my_option_id] + type = "string" + # …any common props… + + optional = false + redact = False + default = "some default string" + help = "You can enter almost anything!" + example = "an example string" + placeholder = "write something…" + ``` + + ##### Properties + + - [common properties](#common-option-properties) + - `optional`: `bool` (default: `false`, but `true` in config panels) + - `redact`: `bool` (default: `false`), to redact the value in the logs when the value contain private information + - `default`: depends on `type`, the default value to assign to the option + - in case of readonly values, you can use this `default` to assign a value (or return a dynamic `default` from a custom getter) + - `help` (optional): `Translation`, to display a short help message in cli and web-admin + - `example` (optional): `str`, to display an example value in web-admin only + - `placeholder` (optional): `str`, shown in the web-admin fields only + """ + help: Union[Translation, None] = None example: Union[str, None] = None placeholder: Union[str, None] = None @@ -563,10 +746,44 @@ def _get_field_attrs(self) -> dict[str, Any]: class StringOption(BaseStringOption): + """ + Ask for a simple string. + + ##### Examples + ```toml + [my_option_id] + type = "string" + default = "E10" + pattern.regexp = "^[A-F]\d\d$" + pattern.error = "Provide a room like F12 : one uppercase and 2 numbers" + ``` + + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.string] = OptionType.string class TextOption(BaseStringOption): + """ + Ask for a multiline string. + Renders as a `textarea` in the web-admin and by opening a text editor on the CLI. + + ##### Examples + ```toml + [my_option_id] + type = "text" + default = "multi\\nline\\ncontent" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.text] = OptionType.text @@ -574,6 +791,22 @@ class TextOption(BaseStringOption): class PasswordOption(BaseInputOption): + """ + Ask for a password. + The password is tested as a regular user password (at least 8 chars) + + ##### Examples + ```toml + [my_option_id] + type = "password" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: forced to `""` + - `redact`: forced to `true` + - `example`: forbidden + """ + type: Literal[OptionType.password] = OptionType.password example: Literal[None] = None default: Literal[None] = None @@ -610,6 +843,21 @@ def _value_pre_validator( class ColorOption(BaseInputOption): + """ + Ask for a color represented as a hex value (with possibly an alpha channel). + Renders as color picker in the web-admin and as a prompt that accept named color like `yellow` in CLI. + + ##### Examples + ```toml + [my_option_id] + type = "color" + default = "#ff0" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.color] = OptionType.color default: Union[str, None] _annotation = Color @@ -642,6 +890,26 @@ def _value_post_validator( class NumberOption(BaseInputOption): + """ + Ask for a number (an integer). + + ##### Examples + ```toml + [my_option_id] + type = "number" + default = 100 + min = 50 + max = 200 + step = 5 + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `type`: `number` or `range` (input or slider in the web-admin) + - `min` (optional): minimal int value inclusive + - `max` (optional): maximal int value inclusive + - `step` (optional): currently only used in the webadmin as the `` step jump + """ + # `number` and `range` are exactly the same, but `range` does render as a slider in web-admin type: Literal[OptionType.number, OptionType.range] = OptionType.number default: Union[int, None] @@ -696,6 +964,27 @@ def _value_pre_validator( class BooleanOption(BaseInputOption): + """ + Ask for a boolean. + Renders as a switch in the web-admin and a yes/no prompt in CLI. + + ##### Examples + ```toml + [my_option_id] + type = "boolean" + default = 1 + yes = "agree" + no = "disagree" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `0` + - `yes` (optional): (default: `1`) define as what the thruthy value should output + - can be `true`, `True`, `"yes"`, etc. + - `no` (optional): (default: `0`) define as what the thruthy value should output + - can be `0`, `"false"`, `"n"`, etc. + """ + type: Literal[OptionType.boolean] = OptionType.boolean yes: Any = 1 no: Any = 0 @@ -801,6 +1090,23 @@ def _value_post_validator( class DateOption(BaseInputOption): + """ + Ask for a date in the form `"2025-06-14"`. + Renders as a date-picker in the web-admin and a regular prompt in CLI. + + Can also take a timestamp as value that will output as an ISO date string. + + ##### Examples + ```toml + [my_option_id] + type = "date" + default = "2070-12-31" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.date] = OptionType.date default: Union[str, None] _annotation = datetime.date @@ -816,6 +1122,21 @@ def _value_post_validator( class TimeOption(BaseInputOption): + """ + Ask for an hour in the form `"22:35"`. + Renders as a date-picker in the web-admin and a regular prompt in CLI. + + ##### Examples + ```toml + [my_option_id] + type = "time" + default = "12:26" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.time] = OptionType.time default: Union[str, int, None] _annotation = datetime.time @@ -835,12 +1156,41 @@ def _value_post_validator( class EmailOption(BaseInputOption): + """ + Ask for an email. Validation made with [python-email-validator](https://github.com/JoshData/python-email-validator) + + ##### Examples + ```toml + [my_option_id] + type = "email" + default = "Abc.123@test-example.com" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.email] = OptionType.email default: Union[EmailStr, None] _annotation = EmailStr class WebPathOption(BaseStringOption): + """ + Ask for an web path (the part of an url after the domain). Used by default in app install to define from where the app will be accessible. + + ##### Examples + ```toml + [my_option_id] + type = "path" + default = "/" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.path] = OptionType.path @staticmethod @@ -874,6 +1224,21 @@ def normalize(value, option={}) -> str: class URLOption(BaseStringOption): + """ + Ask for any url. + + ##### Examples + ```toml + [my_option_id] + type = "url" + default = "https://example.xn--zfr164b/@handle/" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.url] = OptionType.url _annotation = HttpUrl @@ -882,6 +1247,25 @@ class URLOption(BaseStringOption): class FileOption(BaseInputOption): + """ + Ask for file. + Renders a file prompt in the web-admin and ask for a path in CLI. + + ##### Examples + ```toml + [my_option_id] + type = "file" + accept = ".json" + # bind the file to a location to save the file there + bind = "/tmp/my_file.json" + ``` + ##### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `accept`: a comma separated list of extension to accept like `".conf, .ini` + - /!\ currently only work on the web-admin + """ + type: Literal[OptionType.file] = OptionType.file # `FilePath` for CLI (path must exists and must be a file) # `bytes` for API (a base64 encoded file actually) @@ -991,6 +1375,24 @@ def _get_prompt_message(self, value: Any) -> str: class SelectOption(BaseChoicesOption): + """ + Ask for value from a limited set of values. + Renders as a regular `` in the web-admin and as a regular prompt in CLI with autocompletion of `choices`. - ##### Examples + #### Examples ```toml [section.my_option_id] type = "select" @@ -1398,7 +1395,7 @@ class SelectOption(BaseChoicesOption): choices = "one,two,three" default = "two" ``` - ##### Properties + #### Properties - [common inputs properties](#common-inputs-properties) - `default`: `""`, obviously the default has to be empty or an available `choices` item. - `choices`: a (coma separated) list of values @@ -1418,7 +1415,7 @@ class TagsOption(BaseChoicesOption): This output as a coma separated list of strings `"one,two,three"` - ##### Examples + #### Examples ```toml [section.my_option_id] type = "tags" @@ -1430,7 +1427,7 @@ class TagsOption(BaseChoicesOption): # choices = "one,two,three" default = "two,three" ``` - ##### Properties + #### Properties - [common inputs properties](#common-inputs-properties) - `default`: `""`, obviously the default has to be empty or an available `choices` item. - `pattern` (optional): `Pattern`, a regex to match all the values against @@ -1523,12 +1520,12 @@ class DomainOption(BaseChoicesOption): Ask for a user domain. Renders as a select in the web-admin and as a regular prompt in CLI with autocompletion of registered domains. - ##### Examples + #### Examples ```toml [section.my_option_id] type = "domain" ``` - ##### Properties + #### Properties - [common inputs properties](#common-inputs-properties) - `default`: forced to the instance main domain """ @@ -1577,13 +1574,13 @@ class AppOption(BaseChoicesOption): Ask for a user app. Renders as a select in the web-admin and as a regular prompt in CLI with autocompletion of installed apps. - ##### Examples + #### Examples ```toml [section.my_option_id] type = "app" filter = "is_webapp" ``` - ##### Properties + #### Properties - [common inputs properties](#common-inputs-properties) - `default`: `""` - `filter` (optional): `JSExpression` with what `yunohost app info --full` returns as context (only first level keys) @@ -1630,12 +1627,12 @@ class UserOption(BaseChoicesOption): Ask for a user. Renders as a select in the web-admin and as a regular prompt in CLI with autocompletion of available usernames. - ##### Examples + #### Examples ```toml [section.my_option_id] type = "user" ``` - ##### Properties + #### Properties - [common inputs properties](#common-inputs-properties) - `default`: the first admin user found """ @@ -1690,13 +1687,13 @@ class GroupOption(BaseChoicesOption): Ask for a group. Renders as a select in the web-admin and as a regular prompt in CLI with autocompletion of available groups. - ##### Examples + #### Examples ```toml [section.my_option_id] type = "group" default = "visitors" ``` - ##### Properties + #### Properties - [common inputs properties](#common-inputs-properties) - `default`: `"all_users"`, `"visitors"` or `"admins"` (default: `"all_users"`) """ From 02619e8284bfa068202a5aa941448bd70cc92ecb Mon Sep 17 00:00:00 2001 From: axolotle Date: Tue, 24 Oct 2023 15:05:26 +0200 Subject: [PATCH 161/436] doc:config fix missing aleks additions --- doc/generate_configpanel_doc.py | 64 +++++------ doc/generate_options_doc.py | 185 ++++++++++++++++---------------- src/utils/configpanel.py | 22 ++-- 3 files changed, 136 insertions(+), 135 deletions(-) diff --git a/doc/generate_configpanel_doc.py b/doc/generate_configpanel_doc.py index e29a80dbc7..1eb7b5ebbb 100644 --- a/doc/generate_configpanel_doc.py +++ b/doc/generate_configpanel_doc.py @@ -88,41 +88,41 @@ def get_current_commit(): """ ## Full example -We supposed we have an upstream app with this simple config.yml file: +Let's imagine that the upstream app is configured using this simple `config.yml` file stored in the app's install directory (typically `/var/www/$app/config.yml`): ```yaml -title: 'My dummy apps' +title: 'My dummy app' theme: 'white' max_rate: 10 max_age: 365 ``` -We could for example create a simple configuration panel for it like this one, by following the syntax `\[PANEL.SECTION.QUESTION\]`: +We could for example create a simple configuration panel for it like this one, by following the syntax `[PANEL.SECTION.QUESTION]`: ```toml version = "1.0" [main] -[main.main] -[main.main.title] -ask.en = "Title" -type = "string" -bind = ":__INSTALL_DIR__/config.yml" - -[main.main.theme] -ask.en = "Theme" -type = "select" -choices = ["white", "dark"] -bind = ":__INSTALL_DIR__/config.yml" - -[main.limits] -[main.limits.max_rate] -ask.en = "Maximum display rate" -type = "number" -bind = ":__INSTALL_DIR__/config.yml" - -[main.limits.max_age] -ask.en = "Duration of a dummy" -type = "number" -bind = ":__INSTALL_DIR__/config.yml" + [main.main] + [main.main.title] + ask.en = "Title" + type = "string" + bind = ":__INSTALL_DIR__/config.yml" + + [main.main.theme] + ask.en = "Theme" + type = "select" + choices = ["white", "dark"] + bind = ":__INSTALL_DIR__/config.yml" + + [main.limits] + [main.limits.max_rate] + ask.en = "Maximum display rate" + type = "number" + bind = ":__INSTALL_DIR__/config.yml" + + [main.limits.max_age] + ask.en = "Duration of a dummy" + type = "number" + bind = ":__INSTALL_DIR__/config.yml" ``` Here we have created one `main` panel, containing the `main` and `limits` sections, containing questions according to params name of our `config.yml` file. Thanks to the `bind` properties, all those questions are bind to their values in the `config.yml` file. @@ -147,13 +147,13 @@ def get_current_commit(): } ``` -List of main configuration helpers -* ynh_app_config_get -* ynh_app_config_show -* ynh_app_config_validate -* ynh_app_config_apply -* ynh_app_config_run +List of main configuration helpers: + * `ynh_app_config_get` + * `ynh_app_config_show` + * `ynh_app_config_validate` + * `ynh_app_config_apply` + * `ynh_app_config_run` -More info on this could be found by reading [vpnclient_ynh config script](https://github.com/YunoHost-Apps/vpnclient_ynh/blob/master/scripts/config) +More info on this can be found by reading [vpnclient_ynh config script](https://github.com/YunoHost-Apps/vpnclient_ynh/blob/master/scripts/config) """ ) diff --git a/doc/generate_options_doc.py b/doc/generate_options_doc.py index 88f6deb204..ea7febe6d1 100644 --- a/doc/generate_options_doc.py +++ b/doc/generate_options_doc.py @@ -132,62 +132,70 @@ def get_current_commit(): """ ---------------- -## Read and write values: the `bind` property +## Reading and writing values ! Config panels only You can read and write values with 2 mechanisms: the `bind` property in the `config_panel.toml` and for complex use cases the getter/setter in a `config` script. -`bind` allows us to alter the default behavior of applying option's values, which is: get from and set in the app `settings.yml`. +If you did not define a specific getter/setter (see below), and no `bind` argument was defined, YunoHost will read/write the value from/to the app's `/etc/yunohost/$app/settings.yml` file. -We can: +With `bind`, we can: - alter the source the value comes from with binds to file or custom getters. -- alter the destination with binds to file or settings. +- alter the destination with binds to file or custom setters. - parse/validate the value before destination with validators -! IMPORTANT: with the exception of `bind = "null"` options, options ids should almost **always** correspond to an app setting initialized / reused during install/upgrade. +! IMPORTANT: with the exception of `bind = "null"` options, options ids should almost **always** correspond to an app setting initialized/reused during install/upgrade. Not doing so may result in inconsistencies between the config panel mechanism and the use of ynh_add_config -### Read / write into a var of a configuration file +### Read / write into a var of an actual configuration file Settings usually correspond to key/values in actual app configurations. Hence, a more useful mode is to have `bind = ":FILENAME"` with a colon `:` before. In that case, YunoHost will automagically find a line with `KEY=VALUE` in `FILENAME` (with the adequate separator between `KEY` and `VALUE`). YunoHost will then use this value for the read/get operation. During write/set operations, YunoHost will overwrite the value in **both** FILENAME and in the app's settings.yml -Configuration file format supported: `yaml`, `toml`, `json`, `ini`, `env`, `php`, `python`. +Configuration file format supported: `YAML`, `TOML`, `JSON`, `INI`, `PHP`, `.env`-like, `.py`. The feature probably works with others formats, but should be tested carefully. -Note that this feature only works with relatively simple cases such as `KEY: VALUE`, but won't properly work with complex data structures like multiline array/lists or dictionnaries. -It also doesn't work with XML format, custom config function call, php define(), … -If you need to save complex/multiline content in a configuration variable, you should do it via a specific getter/setter. - ```toml -[panel.section.config_value] +[main.main.theme] # Do not use `file` for this since we only want to insert/save a value type = "string" -bind = ":__FINALPATH__/config.ini" -default = "" +bind = ":__INSTALL_DIR__/config.yml" ``` +In which case, YunoHost will look for something like a key/value, with the key being `theme`. -By default, `bind = ":FILENAME"` will use the option id as `KEY` but the option id may sometime not be the exact same `KEY` name in the configuration file. -For example, [In pepettes app](https://github.com/YunoHost-Apps/pepettes_ynh/blob/5cc2d3ffd6529cc7356ff93af92dbb6785c3ab9a/conf/settings.py##L11), the python variable is `name` and not `project_name`. In that case, the key name can be specified before the colon `:`. - +If the question id in the config panel (here, `theme`) differs from the key in the actual conf file (let's say it's not `theme` but `css_theme`), then you can write: ```toml -[panel.section.project_name] -bind = "name:__FINALPATH__/config.ini" +[main.main.theme] +type = "string" +bind = "css_theme:__FINALPATH__/config.yml" +``` +!!!! Note: This mechanism is quasi language agnostic and will use regexes to find something that looks like a key=value or common variants. However, it does assume that the key and value are stored on the same line. It doesn't support multiline text or file in a variable with this method. If you need to save multiline content in a configuration variable, you should create a custom getter/setter (see below). + +Nested syntax is also supported, which may be useful for example to remove ambiguities about stuff looking like: +```json +{ + "foo": { + "max": 123 + }, + "bar": { + "max": 456 + } +} ``` -Sometimes, you want to read and save a value in a variable name that appears several time in the configuration file (for example variables called `max`). The `bind` property allows you to change the value on the variable following a regex in a the file: +which we can `bind` to using: ```toml -bind = "importExportRateLimiting>max:__INSTALL_DIR__/conf.json" +bind = "foo>max:__INSTALL_DIR__/conf.json" ``` ### Read / write an entire file -You can bind a `text` or directly a `file` to a specific file by using `bind = "FILEPATH`. +Useful when using a question `file` or `text` for which you want to save the raw content directly as a file on the system. ```toml [panel.section.config_file] @@ -207,17 +215,14 @@ def get_current_commit(): Sometimes the `bind` mechanism is not enough: * the config file format is not supported (e.g. xml, csv) * the data is not contained in a config file (e.g. database, directory, web resources...) - * the data should be writen but not read (e.g. password) - * the data should be read but not writen (e.g. status information) + * the data should be written but not read (e.g. password) + * the data should be read but not written (e.g. fetching status information) * we want to change other things than the value (e.g. the choices list of a select) * the question answer contains several values to dispatch in several places * and so on -For all of those use cases, there are the specific getter or setter mechanism for an option! - -To create specific getter/setter, you first need to create a `config` script inside the `scripts` directory +You can create specific getter/setters functions inside the `scripts/config` of your app to customize how the information is read/written. -`scripts/config` ```bash #!/bin/bash source /usr/share/yunohost/helpers @@ -232,54 +237,65 @@ def get_current_commit(): ### Getters -Define an option's custom getter in a bash script `script/config`. -It has to be named after an option's `id` prepended by `get__`. +A question's getter is the function used to read the current value/state. Custom getters are defined using bash functions called `getter__QUESTION_SHORT_KEY()` which returns data through stdout. -The function should returns one of these two formats: - * a raw format, in this case the return is binded directly to the value of the question - * a yaml format, in this case you can rewrite several properties of your option (like the `style` of an `alert`, the list of `choices` of a `select`, etc.) +Stdout can generated using one of those formats: + 1) either a raw format, in which case the return is binded directly to the value of the question + 2) or a yaml format, in this case you dynamically provide properties for your question (for example the `style` of an `alert`, the list of available `choices` of a `select`, etc.) -[details summary="Basic example : Get the login inside the first line of a file " class="helper-card-subtitle text-muted"] -scripts/config -```bash -get__login_user() { - if [ -s /etc/openvpn/keys/credentials ] - then - echo "$(sed -n 1p /etc/openvpn/keys/credentials)" - else - echo "" - fi -} -``` +[details summary="Basic example with raw stdout: get the timezone on the system" class="helper-card-subtitle text-muted"] + +`config_panel.toml` -config_panel.toml ```toml -[main.auth.login_user] -ask = "Username" +[main.main.timezone] +ask = "Timezone" type = "string" ``` -[/details] -[details summary="Advanced example 1 : Display a list of available plugins" class="helper-card-subtitle text-muted"] -scripts/config +`scripts/config` + ```bash -get__plugins() { - echo "choices: [$(ls $install_dir/plugins/ | tr '\n' ',')]" +get__timezone() { + echo "$(cat /etc/timezone)" } ``` +[/details] -config_panel.toml +[details summary="Basic example with yaml-formated stdout : Display a list of available plugins" class="helper-card-subtitle text-muted"] + +`config_panel.toml` ```toml [main.plugins.plugins] ask = "Plugin to activate" type = "tags" choices = [] ``` + +`scripts/config` + +```bash +get__plugins() { + echo "choices: [$(ls $install_dir/plugins/ | tr '\n' ',')]" +} +``` + [/details] -[details summary="Example 2 : Display the status of a VPN" class="helper-card-subtitle text-muted"] -scripts/config +[details summary="Advanced example with yaml-formated stdout : Display the status of a VPN" class="helper-card-subtitle text-muted"] + +`config_panel.toml` + +```toml +[main.cube.status] +ask = "Custom getter alert" +type = "alert" +style = "info" +bind = "null" # no behaviour on +``` + +`scripts/config` ```bash get__status() { if [ -f "/sys/class/net/tun0/operstate" ] && [ "$(cat /sys/class/net/tun0/operstate)" == "up" ] @@ -298,50 +314,38 @@ def get_current_commit(): fi } ``` - -config_panel.toml -```toml -[main.cube.status] -ask = "Custom getter alert" -type = "alert" -style = "info" -bind = "null" # no behaviour on -``` [/details] ### Setters -Define an option's custom setter in a bash script `script/config`. -It has to be named after an option's id prepended by `set__`. +A question's setter is the function used to set new value/state. Custom setters are defined using bash functions called `setter__QUESTION_SHORT_KEY()`. In the context of the setter function, variables named with the various quetion's short keys are avaible ... for example the user-specified date for question `[main.main.theme]` is available as `$theme`. -This function could access new values defined by the users by using bash variable with the same name as the short key of a question. +When doing non-trivial operations to set a value, you may want to use `ynh_print_info` to inform the admin about what's going on. -You probably should use `ynh_print_info` in order to display info for user about change that has been made to help them to understand a bit what's going. +[details summary="Basic example : Set the system timezone" class="helper-card-subtitle text-muted"] -[details summary="Basic example : Set the login into the first line of a file " class="helper-card-subtitle text-muted"] -scripts/config -```bash -set__login_user() { - if [ -z "${login_user}" ] - then - echo "${login_user}" > /etc/openvpn/keys/credentials - ynh_print_info "The user login has been registered in /etc/openvpn/keys/credentials" - fi -} -``` +`config_panel.toml` -config_panel.toml ```toml -[main.auth.login_user] -ask = "Username" +[main.main.timezone] +ask = "Timezone" type = "string" ``` + +`scripts/config` + +```bash +set__timezone() { + echo "$timezone" > /etc/timezone + ynh_print_info "The timezone has been changed to $timezone" +} +``` [/details] -#### Validation +### Validation You will often need to validate data answered by the user before to save it somewhere. @@ -353,9 +357,9 @@ def get_current_commit(): You can also restrict several types with a choices list. ```toml -choices.option1 = "Plop1" -choices.option2 = "Plop2" -choices.option3 = "Plop3" +choices.foo = "Foo (some explanation)" +choices.bar = "Bar (moar explanation)" +choices.loremipsum = "Lorem Ipsum Dolor Sit Amet" ``` Some other type specific argument exist like @@ -366,15 +370,12 @@ def get_current_commit(): | `boolean` | `yes` `no` | -If you need more control over validation, you can use custom validators. -Define an option's custom validator in a bash script `script/config`. -It has to be named after an option's id prepended by `validate__`. - +Finally, if you need specific or multi variable validation, you can use custom validators function. Validators allows us to return custom error messages depending on the value. ```bash validate__login_user() { - if [[ "${#login_user}" -lt 4 ]]; then echo 'Too short user login'; fi + if [[ "${#login_user}" -lt 4 ]]; then echo 'User login is too short, should be at least 4 chars'; fi } ``` diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 4f333cc5a2..47c97a8084 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -217,30 +217,30 @@ def translate(self, i18n_key: Union[str, None] = None) -> None: class ConfigPanelModel(BaseModel): """ - Configuration panels allows instances adminitrators to manage some parameters or runs some actions for which the app's upstream doesn't provide any configuration panels itself. It's a good way to reduce manual change on config files and avoid conflicts on it. + Configuration panels allows admins to manage parameters or runs actions for which the upstream's app doesn't provide any appropriate UI itself. It's a good way to reduce manual change on config files and avoid conflicts on it. - Those panels could also be used as interface generator to extend quickly capabilities of YunoHost (e.g. VPN Client, Hotspost, Borg, etc.). + Those panels can also be used to quickly create interfaces that extend the capabilities of YunoHost (e.g. VPN Client, Hotspost, Borg, etc.). From a packager perspective, this `config_panel.toml` is coupled to the `scripts/config` script, which may be used to define custom getters/setters/validations/actions. However, most use cases should be covered automagically by the core, thus it may not be necessary to define a scripts/config at all! - ! IMPORTANT: Please: Keep in mind the YunoHost spirit, and try to build your panels in such a way as to expose only really useful parameters, and if there are many of them, to relegate those corresponding to rarer use cases to "Advanced" sub-sections. + ! Please: Keep in mind the YunoHost spirit, and try to build your panels in such a way as to expose only really useful, "high-level" parameters, and if there are many of them, to relegate those corresponding to rarer use cases to "Advanced" sub-sections. Keep it simple, focus on common needs, don't expect the admins to have 3 PhDs in computer science. - ### How does `config_panel.toml` work - Basically, configuration panels for apps uses at least a `config_panel.toml` at the root of your package. For advanced usecases, this TOML file could also be paired with a `scripts/config` to define custom getters/setters/validators/actions. However, most use cases should be covered automagically by the core, thus it may not be necessary to define a `scripts/config` at all! + ### `config_panel.toml`'s principle and general format + To create configuration panels for apps, you should at least create a `config_panel.toml` at the root of the package. For more complex cases, this TOML file can be paired with a `config` script inside the scripts directory of your package, which will handle specific controller logic. - The `config_panel.toml` file describes one or several panels, containing some sections, containing some options generally binded to a params in a configuration file. + The `config_panel.toml` describes one or several panels, containing sections, each containing questions generally binded to a params in the app's actual configuration files. ### Options short keys have to be unique - For performance reasons, questions short keys should be unique in all the `config_panel.toml` file, not just inside its panel or its section. - - So you can't have + For performance reasons, questions short keys have to be unique in all the `config_panel.toml` file, not just inside its panel or its section. Hence it's not possible to have: ```toml [manual.vpn.server_ip] [advanced.dns.server_ip] ``` - Indeed the real variable name is server_ip and here you have a conflict. + In which two questions have "real variable name" `is server_ip` and therefore conflict with each other. + + ! Some short keys are forbidden cause it can interfer with config scripts (`old`, `file_hash`, `types`, `binds`, `formats`, `changed`) and you probably should avoid to use common settings name to avoid to bind your question to this settings (e.g. `id`, `install_time`, `mysql_pwd`, `path`, `domain`, `port`, `db_name`, `current_revision`, `admin`) - ### Options + ### Supported questions types and properties [Learn more about Options](/dev/forms) in their dedicated doc page as those are also used in app install forms and core config panels. From c4c79c61fe30491388a2cda4fe5627de81428263 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 25 Oct 2023 15:06:10 +0200 Subject: [PATCH 162/436] configpanel: forbid extra props on BaseOption + accordingly fix tests --- src/tests/test_questions.py | 4 ++++ src/utils/form.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index fbbf757c9b..afd73c8617 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -16,6 +16,7 @@ from yunohost.utils.form import ( OPTIONS, FORBIDDEN_PASSWORD_CHARS, + READONLY_TYPES, ask_questions_and_parse_answers, BaseChoicesOption, BaseInputOption, @@ -444,6 +445,9 @@ def get_raw_option(cls, raw_option={}, **kwargs): def _test_basic_attrs(self): raw_option = self.get_raw_option(optional=True) + if raw_option["type"] in READONLY_TYPES: + del raw_option["optional"] + if raw_option["type"] == "select": raw_option["choices"] = ["one"] diff --git a/src/utils/form.py b/src/utils/form.py index 07be55312e..70afeb8f36 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -258,6 +258,12 @@ class OptionType(str, Enum): group = "group" +READONLY_TYPES = { + OptionType.display_text, + OptionType.markdown, + OptionType.alert, + OptionType.button, +} FORBIDDEN_READONLY_TYPES = { OptionType.password, OptionType.app, @@ -310,6 +316,7 @@ class Config: arbitrary_types_allowed = True use_enum_values = True validate_assignment = True + extra = Extra.forbid @staticmethod def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: @@ -1314,9 +1321,11 @@ def options_dict_to_list( "type", OptionType.select if "choices" in data else OptionType.string, ), - "optional": data.get("optional", optional), } + if option["type"] not in READONLY_TYPES: + option["optional"] = option.get("optional", optional) + # LEGACY (`choices` in option `string` used to be valid) if "choices" in option and option["type"] == OptionType.string: logger.warning( From 3faa5742674074562f168183560a5daf5ebc0968 Mon Sep 17 00:00:00 2001 From: axolotle Date: Wed, 25 Oct 2023 15:07:31 +0200 Subject: [PATCH 163/436] configpanel: add proper schema definition --- doc/generate_json_schema.py | 4 ++++ src/utils/configpanel.py | 35 +++++++++++++++++++++++++++++++++++ src/utils/form.py | 12 ++++++++---- 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 doc/generate_json_schema.py diff --git a/doc/generate_json_schema.py b/doc/generate_json_schema.py new file mode 100644 index 0000000000..1abf88915f --- /dev/null +++ b/doc/generate_json_schema.py @@ -0,0 +1,4 @@ +from yunohost.utils.configpanel import ConfigPanelModel + + +print(ConfigPanelModel.schema_json(indent=2)) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 325f6579dd..e3ceeff88d 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -88,6 +88,14 @@ class SectionModel(ContainerModel, OptionsModel): optional: bool = True is_action_section: bool = False + class Config: + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + del schema["properties"]["id"] + options = schema["properties"].pop("options") + del schema["required"] + schema["additionalProperties"] = options["items"] + # Don't forget to pass arguments to super init def __init__( self, @@ -137,6 +145,13 @@ class PanelModel(ContainerModel): class Config: extra = Extra.allow + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + del schema["properties"]["id"] + del schema["properties"]["sections"] + del schema["required"] + schema["additionalProperties"] = {"$ref": "#/definitions/SectionModel"} + # Don't forget to pass arguments to super init def __init__( self, @@ -170,6 +185,26 @@ class Config: arbitrary_types_allowed = True extra = Extra.allow + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + """Update the schema to the expected input + In actual TOML definition, schema is like: + ```toml + [panel_1] + [panel_1.section_1] + [panel_1.section_1.option_1] + ``` + Which is equivalent to `{"panel_1": {"section_1": {"option_1": {}}}}` + so `section_id` (and `option_id`) are additional property of `panel_id`, + which is convinient to write but not ideal to iterate. + In ConfigPanelModel we gather additional properties of panels, sections + and options as lists so that structure looks like: + `{"panels`: [{"id": "panel_1", "sections": [{"id": "section_1", "options": [{"id": "option_1"}]}]}] + """ + del schema["properties"]["panels"] + del schema["required"] + schema["additionalProperties"] = {"$ref": "#/definitions/PanelModel"} + # Don't forget to pass arguments to super init def __init__( self, diff --git a/src/utils/form.py b/src/utils/form.py index 70afeb8f36..bd373badbf 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -319,10 +319,14 @@ class Config: extra = Extra.forbid @staticmethod - def schema_extra(schema: dict[str, Any], model: Type["BaseOption"]) -> None: - # FIXME Do proper doctstring for Options - del schema["description"] - schema["additionalProperties"] = False + def schema_extra(schema: dict[str, Any]) -> None: + del schema["properties"]["id"] + del schema["properties"]["name"] + schema["required"] = [ + required for required in schema.get("required", []) if required != "id" + ] + if not schema["required"]: + del schema["required"] @validator("id", pre=True) def check_id_is_not_forbidden(cls, value: str) -> str: From 7f954af6b6e5781ee7e7db4853ac2dcc25e2baad Mon Sep 17 00:00:00 2001 From: Kay0u Date: Mon, 2 Oct 2023 04:10:58 +0200 Subject: [PATCH 164/436] fix an error in dump_script_log_extract_for_debugging --- src/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/log.py b/src/log.py index 13683d8ef7..e429bbd4a1 100644 --- a/src/log.py +++ b/src/log.py @@ -745,7 +745,7 @@ def dump_script_log_extract_for_debugging(self): # 2019-10-19 16:10:27,611: DEBUG - + mysql -u piwigo --password=********** -B piwigo # And we just want the part starting by "DEBUG - " lines = [line for line in lines if ":" in line.strip()] - lines = [line.strip().split(": ", 1)[1] for line in lines] + lines = [line.strip().split(": ", 1)[-1] for line in lines] # And we ignore boring/irrelevant lines # Annnnnnd we also ignore lines matching [number] + such as # 72971 DEBUG 29739 + ynh_exit_properly From 418df4c05f82636b40da14768f3400cc3d363fb5 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 30 Oct 2023 15:11:42 +0100 Subject: [PATCH 165/436] debian: move yunohost-portal to 'Recommends' ... mainly to bypass issue on the CI, but also because it sounds legit ... not 100% about this ? --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 9f156ddea0..6a72998dc7 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Package: yunohost Essential: yes Architecture: all Depends: ${python3:Depends}, ${misc:Depends} - , moulinette (>= 11.1), ssowat (>= 11.1), yunohost-portal (>= 11.1) + , moulinette (>= 11.1), ssowat (>= 11.1) , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 @@ -30,7 +30,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , acl , git, curl, wget, cron, unzip, jq, bc, at, procps , lsb-release, haveged, fake-hwclock, equivs, lsof, whois -Recommends: yunohost-admin +Recommends: yunohost-admin, yunohost-portal (>= 11.1) , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog , unattended-upgrades From e4182bb362776bbdab221990b299c1e28b93d3c9 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 30 Oct 2023 15:12:15 +0100 Subject: [PATCH 166/436] debian: require moulinette, ssowat, yunohost-portal to be >= 12.0 --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 6a72998dc7..a9f7f4cc66 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Package: yunohost Essential: yes Architecture: all Depends: ${python3:Depends}, ${misc:Depends} - , moulinette (>= 11.1), ssowat (>= 11.1) + , moulinette (>= 12.0), ssowat (>= 12.0) , python3-psutil, python3-requests, python3-dnspython, python3-openssl , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 @@ -30,7 +30,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , acl , git, curl, wget, cron, unzip, jq, bc, at, procps , lsb-release, haveged, fake-hwclock, equivs, lsof, whois -Recommends: yunohost-admin, yunohost-portal (>= 11.1) +Recommends: yunohost-admin, yunohost-portal (>= 12.0) , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog , unattended-upgrades From 9423168aaf836b376783692941e6e9bcf5b2f042 Mon Sep 17 00:00:00 2001 From: axolotle Date: Mon, 30 Oct 2023 15:17:01 +0100 Subject: [PATCH 167/436] configpanels: fix app `is_default` + dns alert style --- src/app.py | 6 ++++-- src/dns.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index 21219b2ce6..52e1ebede8 100644 --- a/src/app.py +++ b/src/app.py @@ -26,7 +26,6 @@ import subprocess import tempfile import copy -from collections import OrderedDict from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Iterator, Optional, Union from packaging import version from logging import getLogger @@ -129,6 +128,7 @@ def app_info(app, full=False, upgradable=False): """ Get info for a specific app """ + from yunohost.domain import domain_config_get from yunohost.permission import user_permission_list _assert_is_installed(app) @@ -228,7 +228,9 @@ def app_info(app, full=False, upgradable=False): ret["is_webapp"] = "domain" in settings and "path" in settings if ret["is_webapp"]: - ret["is_default"] = settings.get("default_app", "_none") + ret["is_default"] = ( + domain_config_get(settings["domain"], "feature.app.default_app") == app + ) ret["supports_change_url"] = os.path.exists( os.path.join(setting_path, "scripts", "change_url") diff --git a/src/dns.py b/src/dns.py index fc4b26a758..126f81dbf6 100644 --- a/src/dns.py +++ b/src/dns.py @@ -558,6 +558,7 @@ def _get_registrar_config_section(domain): except ValueError: registrar_infos["registrar"]["default"] = None registrar_infos["infos"]["ask"] = m18n.n("domain_dns_registrar_not_supported") + registrar_infos["infos"]["style"] = "warning" else: registrar_infos["registrar"]["default"] = registrar registrar_infos["infos"]["ask"] = m18n.n( From 8aee337d0f18f77876e539a3a2b96e8d21f39546 Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 30 Oct 2023 17:04:17 +0100 Subject: [PATCH 168/436] regenconf/portal: fix attempt to chown before the user is created --- hooks/conf_regen/01-yunohost | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 3b810de303..74331edc29 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -29,11 +29,6 @@ do_init_regen() { chown -R root:ssl-cert /etc/yunohost/certs chmod 750 /etc/yunohost/certs - # Portal folder - mkdir -p /etc/yunohost/portal - chmod 500 /etc/yunohost/portal - chown ynh-portal:ynh-portal /etc/yunohost/portal - # App folders mkdir -p /etc/yunohost/apps chmod 700 /etc/yunohost/apps @@ -72,6 +67,12 @@ do_init_regen() { chown ynh-portal:root /etc/yunohost/.ssowat_cookie_secret chmod 400 /etc/yunohost/.ssowat_cookie_secret + # Portal folder + mkdir -p /etc/yunohost/portal + chmod 500 /etc/yunohost/portal + chown ynh-portal:ynh-portal /etc/yunohost/portal + + # YunoHost services cp yunohost-api.service /etc/systemd/system/yunohost-api.service cp yunohost-portal-api.service /etc/systemd/system/yunohost-portal-api.service cp yunohost-firewall.service /etc/systemd/system/yunohost-firewall.service From f02538cef05744c4103fcf1af31d0114e839351e Mon Sep 17 00:00:00 2001 From: Alexandre Aubin Date: Mon, 30 Oct 2023 18:39:31 +0100 Subject: [PATCH 169/436] doc: iterate on configpanel/form documentation --- ...enerate_configpanel_and_formoptions_doc.py | 171 ++++++ doc/generate_configpanel_doc.py | 159 ------ doc/generate_options_doc.py | 486 ------------------ src/utils/configpanel.py | 57 +- src/utils/form.py | 83 ++- 5 files changed, 211 insertions(+), 745 deletions(-) create mode 100644 doc/generate_configpanel_and_formoptions_doc.py delete mode 100644 doc/generate_configpanel_doc.py delete mode 100644 doc/generate_options_doc.py diff --git a/doc/generate_configpanel_and_formoptions_doc.py b/doc/generate_configpanel_and_formoptions_doc.py new file mode 100644 index 0000000000..061ebf77cf --- /dev/null +++ b/doc/generate_configpanel_and_formoptions_doc.py @@ -0,0 +1,171 @@ +import sys +import ast +import datetime +import subprocess + +version = open("../debian/changelog").readlines()[0].split()[1].strip("()") +today = datetime.datetime.now().strftime("%d/%m/%Y") + + +def get_current_commit(): + p = subprocess.Popen( + "git rev-parse --verify HEAD", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + stdout, stderr = p.communicate() + + current_commit = stdout.strip().decode("utf-8") + return current_commit + + +current_commit = get_current_commit() + +def print_config_panel_docs(): + + fname = "../src/utils/configpanel.py" + content = open(fname).read() + + # NB: This magic is because we want to be able to run this script outside of a YunoHost context, + # in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... + tree = ast.parse(content) + + ConfigPanelClasses = reversed( + [ + c + for c in tree.body + if isinstance(c, ast.ClassDef) + and c.name in {"SectionModel", "PanelModel", "ConfigPanelModel"} + ] + ) + + + print("## Configuration panel structure") + + for c in ConfigPanelClasses: + doc = ast.get_docstring(c) + print("") + print(f"### {c.name.replace('Model', '')}") + print("") + print(doc) + print("") + print("---") + + +def print_form_doc(): + + fname = "../src/utils/form.py" + content = open(fname).read() + + # NB: This magic is because we want to be able to run this script outside of a YunoHost context, + # in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... + tree = ast.parse(content) + + OptionClasses = [ + c for c in tree.body if isinstance(c, ast.ClassDef) and c.name.endswith("Option") + ] + + OptionDocString = {} + + print("## List of all option types") + + for c in OptionClasses: + if not isinstance(c.body[0], ast.Expr): + continue + option_type = None + + if c.name in {"BaseOption", "BaseInputOption"}: + option_type = c.name + elif c.body[1].target.id == "type": + option_type = c.body[1].value.attr + + generaltype = c.bases[0].id.replace("Option", "").replace("Base", "").lower() if c.bases else None + + docstring = ast.get_docstring(c) + if docstring: + if "#### Properties" not in docstring: + docstring += """ +#### Properties + +- [common properties](#common-properties)""" + OptionDocString[option_type] = {"doc": docstring, "generaltype": generaltype} + + # Dirty hack to have "BaseOption" as first and "BaseInputOption" as 2nd in list + + base = OptionDocString.pop("BaseOption") + baseinput = OptionDocString.pop("BaseInputOption") + OptionDocString2 = { + "BaseOption": base, + "BaseInputOption": baseinput, + } + OptionDocString2.update(OptionDocString) + + for option_type, infos in OptionDocString2.items(): + if option_type == "display_text": + # display_text is kind of legacy x_x + continue + print("") + if option_type == "BaseOption": + print("### Common properties") + elif option_type == "BaseInputOption": + print("### Common inputs properties") + else: + print(f"### `{option_type}`" + (f" ({infos['generaltype']})" if infos["generaltype"] else "")) + print("") + print(infos["doc"]) + print("") + print("---") + +print( + f"""--- +title: Technical details for config panel structure and form option types +template: docs +taxonomy: + category: docs +routes: + default: '/dev/forms' +--- + +Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{current_commit}/doc/generate_options_doc.py) on {today} (YunoHost version {version}) + +## Glossary + +You may encounter some named types which are used for simplicity. + +- `Translation`: a translated property + - used for properties: `ask`, `help` and `Pattern.error` + - a `dict` with locales as keys and translations as values: + ```toml + ask.en = "The text in english" + ask.fr = "Le texte en français" + ``` + It is not currently possible for translators to translate those string in weblate. + - a single `str` for a single english default string + ```toml + help = "The text in english" + ``` +- `JSExpression`: a `str` JS expression to be evaluated to `true` or `false`: + - used for properties: `visible` and `enabled` + - operators availables: `==`, `!=`, `>`, `>=`, `<`, `<=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%` and `match()` +- `Binding`: bind a value to a file/property/variable/getter/setter/validator + - save the value in `settings.yaml` when not defined + - nothing at all with `"null"` + - a custom getter/setter/validator with `"null"` + a function starting with `get__`, `set__`, `validate__` in `scripts/config` + - a variable/property in a file with `:__FINALPATH__/my_file.php` + - a whole file with `__FINALPATH__/my_file.php` +- `Pattern`: a `dict` with a regex to match the value against and an error message + ```toml + pattern.regexp = '^[A-F]\d\d$' + pattern.error = "Provide a room number such as F12: one uppercase and 2 numbers" + # or with translated error + pattern.error.en = "Provide a room number such as F12: one uppercase and 2 numbers" + pattern.error.fr = "Entrez un numéro de salle comme F12: une lettre majuscule et deux chiffres." + ``` + - IMPORTANT: your `pattern.regexp` should be between simple quote, not double. + +""" +) + +print_config_panel_docs() +print_form_doc() diff --git a/doc/generate_configpanel_doc.py b/doc/generate_configpanel_doc.py deleted file mode 100644 index 1eb7b5ebbb..0000000000 --- a/doc/generate_configpanel_doc.py +++ /dev/null @@ -1,159 +0,0 @@ -import ast -import datetime -import subprocess - -version = open("../debian/changelog").readlines()[0].split()[1].strip("()") -today = datetime.datetime.now().strftime("%d/%m/%Y") - - -def get_current_commit(): - p = subprocess.Popen( - "git rev-parse --verify HEAD", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - stdout, stderr = p.communicate() - - current_commit = stdout.strip().decode("utf-8") - return current_commit - - -current_commit = get_current_commit() - - -print( - f"""--- -title: Config Panels -template: docs -taxonomy: - category: docs -routes: - default: '/packaging_config_panels' ---- - -Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{current_commit}/doc/generate_configpanel_doc.py) on {today} (YunoHost version {version}) - -## Glossary - -You may encounter some named types which are used for simplicity. - -- `Translation`: a translated property - - used for properties: `ask`, `help` and `Pattern.error` - - a `dict` with locales as keys and translations as values: - ```toml - ask.en = "The text in english" - ask.fr = "Le texte en français" - ``` - It is not currently possible for translators to translate those string in weblate. - - a single `str` for a single english default string - ```toml - help = "The text in english" - ``` -- `JSExpression`: a `str` JS expression to be evaluated to `true` or `false`: - - used for properties: `visible` and `enabled` - - operators availables: `==`, `!=`, `>`, `>=`, `<`, `<=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%` and `match()` - - [examples available in the advanced section of Options](/packaging_apps_options#advanced-use-cases) -""" -) - - -fname = "../src/utils/configpanel.py" -content = open(fname).read() - -# NB: This magic is because we want to be able to run this script outside of a YunoHost context, -# in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... -tree = ast.parse(content) - -OptionClasses = reversed( - [ - c - for c in tree.body - if isinstance(c, ast.ClassDef) - and c.name in {"SectionModel", "PanelModel", "ConfigPanelModel"} - ] -) - -for c in OptionClasses: - doc = ast.get_docstring(c) - print("") - print("----------------") - print(f"## {c.name.replace('Model', '')}") - print("") - print(doc) - print("") - - -print( - """ -## Full example - -Let's imagine that the upstream app is configured using this simple `config.yml` file stored in the app's install directory (typically `/var/www/$app/config.yml`): -```yaml -title: 'My dummy app' -theme: 'white' -max_rate: 10 -max_age: 365 -``` - -We could for example create a simple configuration panel for it like this one, by following the syntax `[PANEL.SECTION.QUESTION]`: -```toml -version = "1.0" -[main] - - [main.main] - [main.main.title] - ask.en = "Title" - type = "string" - bind = ":__INSTALL_DIR__/config.yml" - - [main.main.theme] - ask.en = "Theme" - type = "select" - choices = ["white", "dark"] - bind = ":__INSTALL_DIR__/config.yml" - - [main.limits] - [main.limits.max_rate] - ask.en = "Maximum display rate" - type = "number" - bind = ":__INSTALL_DIR__/config.yml" - - [main.limits.max_age] - ask.en = "Duration of a dummy" - type = "number" - bind = ":__INSTALL_DIR__/config.yml" -``` - -Here we have created one `main` panel, containing the `main` and `limits` sections, containing questions according to params name of our `config.yml` file. Thanks to the `bind` properties, all those questions are bind to their values in the `config.yml` file. - -## Overwrite config panel mechanism - -All main configuration helpers are overwritable, example: - -```bash -ynh_app_config_apply() { - - # Stop vpn client - touch /tmp/.ynh-vpnclient-stopped - systemctl stop ynh-vpnclient - - _ynh_app_config_apply - - # Start vpn client - systemctl start ynh-vpnclient - rm -f /tmp/.ynh-vpnclient-stopped - -} -``` - -List of main configuration helpers: - * `ynh_app_config_get` - * `ynh_app_config_show` - * `ynh_app_config_validate` - * `ynh_app_config_apply` - * `ynh_app_config_run` - -More info on this can be found by reading [vpnclient_ynh config script](https://github.com/YunoHost-Apps/vpnclient_ynh/blob/master/scripts/config) -""" -) diff --git a/doc/generate_options_doc.py b/doc/generate_options_doc.py deleted file mode 100644 index ea7febe6d1..0000000000 --- a/doc/generate_options_doc.py +++ /dev/null @@ -1,486 +0,0 @@ -import ast -import datetime -import subprocess - -version = open("../debian/changelog").readlines()[0].split()[1].strip("()") -today = datetime.datetime.now().strftime("%d/%m/%Y") - - -def get_current_commit(): - p = subprocess.Popen( - "git rev-parse --verify HEAD", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - stdout, stderr = p.communicate() - - current_commit = stdout.strip().decode("utf-8") - return current_commit - - -current_commit = get_current_commit() - - -print( - f"""--- -title: Options -template: docs -taxonomy: - category: docs -routes: - default: '/dev/forms' ---- - -Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{current_commit}/doc/generate_options_doc.py) on {today} (YunoHost version {version}) - -## Glossary - -You may encounter some named types which are used for simplicity. - -- `Translation`: a translated property - - used for properties: `ask`, `help` and `Pattern.error` - - a `dict` with locales as keys and translations as values: - ```toml - ask.en = "The text in english" - ask.fr = "Le texte en français" - ``` - It is not currently possible for translators to translate those string in weblate. - - a single `str` for a single english default string - ```toml - help = "The text in english" - ``` -- `JSExpression`: a `str` JS expression to be evaluated to `true` or `false`: - - used for properties: `visible` and `enabled` - - operators availables: `==`, `!=`, `>`, `>=`, `<`, `<=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%` and `match()` - - [examples available in the advanced section](#advanced-use-cases) -- `Binding`: bind a value to a file/property/variable/getter/setter/validator - - save the value in `settings.yaml` when not defined - - nothing at all with `"null"` - - a custom getter/setter/validator with `"null"` + a function starting with `get__`, `set__`, `validate__` in `scripts/config` - - a variable/property in a file with `:__FINALPATH__/my_file.php` - - a whole file with `__FINALPATH__/my_file.php` - - [examples available in the advanced section](#bind) -- `Pattern`: a `dict` with a regex to match the value against and an error message - ```toml - pattern.regexp = '^[A-F]\d\d$' - pattern.error = "Provide a room like F12: one uppercase and 2 numbers" - # or with translated error - pattern.error.en = "Provide a room like F12: one uppercase and 2 numbers" - pattern.error.fr = "Entrez un numéro de salle comme F12: une lettre majuscule et deux chiffres." - ``` - - IMPORTANT: your `pattern.regexp` should be between simple quote, not double. - -""" -) - - -fname = "../src/utils/form.py" -content = open(fname).read() - -# NB: This magic is because we want to be able to run this script outside of a YunoHost context, -# in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... -tree = ast.parse(content) - -OptionClasses = [ - c for c in tree.body if isinstance(c, ast.ClassDef) and c.name.endswith("Option") -] - -OptionDocString = {} - -for c in OptionClasses: - if not isinstance(c.body[0], ast.Expr): - continue - option_type = None - - if c.name in {"BaseOption", "BaseInputOption"}: - option_type = c.name - elif c.body[1].target.id == "type": - option_type = c.body[1].value.attr - - docstring = ast.get_docstring(c) - if docstring: - if "##### Properties" not in docstring: - docstring += """ -##### Properties - -- [common properties](#common-option-properties) - """ - OptionDocString[option_type] = docstring - -for option_type, doc in OptionDocString.items(): - print("") - if option_type == "BaseOption": - print("## Common Option properties") - elif option_type == "BaseInputOption": - print("## Common Inputs properties") - elif option_type == "display_text": - print("----------------") - print("## Readonly Options") - print(f"### Option `{option_type}`") - elif option_type == "string": - print("----------------") - print("## Input Options") - print(f"### Option `{option_type}`") - else: - print(f"### Option `{option_type}`") - print("") - print(doc) - print("") - -print( - """ ----------------- - -## Reading and writing values - -! Config panels only - -You can read and write values with 2 mechanisms: the `bind` property in the `config_panel.toml` and for complex use cases the getter/setter in a `config` script. - -If you did not define a specific getter/setter (see below), and no `bind` argument was defined, YunoHost will read/write the value from/to the app's `/etc/yunohost/$app/settings.yml` file. - -With `bind`, we can: -- alter the source the value comes from with binds to file or custom getters. -- alter the destination with binds to file or custom setters. -- parse/validate the value before destination with validators - -! IMPORTANT: with the exception of `bind = "null"` options, options ids should almost **always** correspond to an app setting initialized/reused during install/upgrade. -Not doing so may result in inconsistencies between the config panel mechanism and the use of ynh_add_config - - -### Read / write into a var of an actual configuration file - -Settings usually correspond to key/values in actual app configurations. Hence, a more useful mode is to have `bind = ":FILENAME"` with a colon `:` before. In that case, YunoHost will automagically find a line with `KEY=VALUE` in `FILENAME` (with the adequate separator between `KEY` and `VALUE`). - -YunoHost will then use this value for the read/get operation. During write/set operations, YunoHost will overwrite the value in **both** FILENAME and in the app's settings.yml - -Configuration file format supported: `YAML`, `TOML`, `JSON`, `INI`, `PHP`, `.env`-like, `.py`. -The feature probably works with others formats, but should be tested carefully. - -```toml -[main.main.theme] -# Do not use `file` for this since we only want to insert/save a value -type = "string" -bind = ":__INSTALL_DIR__/config.yml" -``` -In which case, YunoHost will look for something like a key/value, with the key being `theme`. - -If the question id in the config panel (here, `theme`) differs from the key in the actual conf file (let's say it's not `theme` but `css_theme`), then you can write: -```toml -[main.main.theme] -type = "string" -bind = "css_theme:__FINALPATH__/config.yml" -``` - -!!!! Note: This mechanism is quasi language agnostic and will use regexes to find something that looks like a key=value or common variants. However, it does assume that the key and value are stored on the same line. It doesn't support multiline text or file in a variable with this method. If you need to save multiline content in a configuration variable, you should create a custom getter/setter (see below). - -Nested syntax is also supported, which may be useful for example to remove ambiguities about stuff looking like: -```json -{ - "foo": { - "max": 123 - }, - "bar": { - "max": 456 - } -} -``` - -which we can `bind` to using: - -```toml -bind = "foo>max:__INSTALL_DIR__/conf.json" -``` - -### Read / write an entire file - -Useful when using a question `file` or `text` for which you want to save the raw content directly as a file on the system. - -```toml -[panel.section.config_file] -type = "file" -bind = "__FINALPATH__/config.ini" -``` - -```toml -[panel.section.config_content] -type = "text" -bind = "__FINALPATH__/config.ini" -default = "key: 'value'" -``` - -## Advanced use cases - -Sometimes the `bind` mechanism is not enough: - * the config file format is not supported (e.g. xml, csv) - * the data is not contained in a config file (e.g. database, directory, web resources...) - * the data should be written but not read (e.g. password) - * the data should be read but not written (e.g. fetching status information) - * we want to change other things than the value (e.g. the choices list of a select) - * the question answer contains several values to dispatch in several places - * and so on - -You can create specific getter/setters functions inside the `scripts/config` of your app to customize how the information is read/written. - -```bash -#!/bin/bash -source /usr/share/yunohost/helpers - -ynh_abort_if_errors - -# Put your getter, setter, validator or action here - -# Keep this last line -ynh_app_config_run $1 -``` - -### Getters - -A question's getter is the function used to read the current value/state. Custom getters are defined using bash functions called `getter__QUESTION_SHORT_KEY()` which returns data through stdout. - -Stdout can generated using one of those formats: - 1) either a raw format, in which case the return is binded directly to the value of the question - 2) or a yaml format, in this case you dynamically provide properties for your question (for example the `style` of an `alert`, the list of available `choices` of a `select`, etc.) - - -[details summary="Basic example with raw stdout: get the timezone on the system" class="helper-card-subtitle text-muted"] - -`config_panel.toml` - -```toml -[main.main.timezone] -ask = "Timezone" -type = "string" -``` - -`scripts/config` - -```bash -get__timezone() { - echo "$(cat /etc/timezone)" -} -``` -[/details] - -[details summary="Basic example with yaml-formated stdout : Display a list of available plugins" class="helper-card-subtitle text-muted"] - -`config_panel.toml` -```toml -[main.plugins.plugins] -ask = "Plugin to activate" -type = "tags" -choices = [] -``` - -`scripts/config` - -```bash -get__plugins() { - echo "choices: [$(ls $install_dir/plugins/ | tr '\n' ',')]" -} -``` - -[/details] - -[details summary="Advanced example with yaml-formated stdout : Display the status of a VPN" class="helper-card-subtitle text-muted"] - -`config_panel.toml` - -```toml -[main.cube.status] -ask = "Custom getter alert" -type = "alert" -style = "info" -bind = "null" # no behaviour on -``` - -`scripts/config` -```bash -get__status() { - if [ -f "/sys/class/net/tun0/operstate" ] && [ "$(cat /sys/class/net/tun0/operstate)" == "up" ] - then - cat << EOF -style: success -ask: - en: Your VPN is running :) -EOF - else - cat << EOF -style: danger -ask: - en: Your VPN is down -EOF - fi -} -``` -[/details] - - -### Setters - -A question's setter is the function used to set new value/state. Custom setters are defined using bash functions called `setter__QUESTION_SHORT_KEY()`. In the context of the setter function, variables named with the various quetion's short keys are avaible ... for example the user-specified date for question `[main.main.theme]` is available as `$theme`. - -When doing non-trivial operations to set a value, you may want to use `ynh_print_info` to inform the admin about what's going on. - - -[details summary="Basic example : Set the system timezone" class="helper-card-subtitle text-muted"] - -`config_panel.toml` - -```toml -[main.main.timezone] -ask = "Timezone" -type = "string" -``` - -`scripts/config` - -```bash -set__timezone() { - echo "$timezone" > /etc/timezone - ynh_print_info "The timezone has been changed to $timezone" -} -``` -[/details] - - -### Validation - -You will often need to validate data answered by the user before to save it somewhere. - -Validation can be made with regex through `pattern` argument -```toml -pattern.regexp = '^.+@.+$' -pattern.error = 'An email is required for this field' -``` - -You can also restrict several types with a choices list. -```toml -choices.foo = "Foo (some explanation)" -choices.bar = "Bar (moar explanation)" -choices.loremipsum = "Lorem Ipsum Dolor Sit Amet" -``` - -Some other type specific argument exist like -| type | validation arguments | -| ----- | --------------------------- | -| `number`, `range` | `min`, `max`, `step` | -| `file` | `accept` | -| `boolean` | `yes` `no` | - - -Finally, if you need specific or multi variable validation, you can use custom validators function. -Validators allows us to return custom error messages depending on the value. - -```bash -validate__login_user() { - if [[ "${#login_user}" -lt 4 ]]; then echo 'User login is too short, should be at least 4 chars'; fi -} -``` - -### Actions - -Define an option's action in a bash script `script/config`. -It has to be named after a `button`'s id prepended by `run__`. - -```toml -[panel.section.my_action] -type = "button" -# no need to set `bind` to "null" it is its hard default -ask = "Run action" -``` - -```bash -run__my_action() { - ynh_print_info "Running 'my_action'..." -} -``` - -A more advanced example could look like: - -```toml -[panel.my_action_section] -name = "Action section" - [panel.my_action_section.my_repo] - type = "url" - bind = "null" # value will not be saved as a setting - ask = "gimme a repo link" - - [panel.my_action_section.my_repo_name] - type = "string" - bind = "null" # value will not be saved as a setting - ask = "gimme a custom folder name" - - [panel.my_action_section.my_action] - type = "button" - ask = "Clone the repo" - # enabled the button only if the above values is defined - enabled = "my_repo && my_repo_name" -``` - -```bash -run__my_action() { - ynh_print_info "Cloning '$my_repo'..." - cd /tmp - git clone "$my_repo" "$my_repo_name" -} -``` - -### `visible` & `enabled` expression evaluation - -Sometimes we may want to conditionaly display a message or prompt for a value, for this we have the `visible` prop. -And we may want to allow a user to trigger an action only if some condition are met, for this we have the `enabled` prop. - -Expressions are evaluated against a context containing previous values of the current section's options. This quite limited current design exists because on the web-admin or on the CLI we cannot guarantee that a value will be present in the form if the user queried only a single panel/section/option. -In the case of an action, the user will be shown or asked for each of the options of the section in which the button is present. - -The expression has to be written in javascript (this has been designed for the web-admin first and is converted to python on the fly on the cli). - -Available operators are: `==`, `!=`, `>`, `>=`, `<`, `<=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%` and `match()`. - -#### Examples - -```toml -# simple "my_option_id" is thruthy/falsy -visible = "my_option_id" -visible = "!my_option_id" -# misc -visible = "my_value >= 10" -visible = "-(my_value + 1) < 0" -visible = "!!my_value || my_other_value" -``` -For a more complete set of examples, [check the tests at the end of the file](https://github.com/YunoHost/yunohost/blob/dev/src/tests/test_questions.py). - -#### match() - -For more complex evaluation we can use regex matching. - -```toml -[my_string] -default = "Lorem ipsum dolor et si qua met!" - -[my_boolean] -type = "boolean" -visible = "my_string && match(my_string, '^Lorem [ia]psumE?')" -``` - -Match the content of a file. - -```toml -[my_file] -type = "file" -accept = ".txt" -bind = "/etc/random/lorem.txt" - -[my_boolean] -type = "boolean" -visible = "my_file && match(my_file, '^Lorem [ia]psumE?')" -``` - -with a file with content like: -```txt -Lorem ipsum dolor et si qua met! -``` -""" -) diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index 47c97a8084..bfad41280b 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -85,13 +85,13 @@ def translate(self, i18n_key: Union[str, None] = None) -> None: class SectionModel(ContainerModel, OptionsModel): """ - Group options. Sections are `dict`s defined inside a Panel and require a unique id (in the below example, the id is `customization` prepended by the panel's id `main`). Keep in mind that this combined id will be used in CLI to refer to the section, so choose something short and meaningfull. Also make sure to not make a typo in the panel id, which would implicitly create an other entire panel. + Sections are, basically, options grouped together. Sections are `dict`s defined inside a Panel and require a unique id (in the below example, the id is `customization` prepended by the panel's id `main`). Keep in mind that this combined id will be used in CLI to refer to the section, so choose something short and meaningfull. Also make sure to not make a typo in the panel id, which would implicitly create an other entire panel. If at least one `button` is present it then become an action section. Options in action sections are not considered settings and therefor are not saved, they are more like parameters that exists only during the execution of an action. FIXME i'm not sure we have this in code. - ### Examples + #### Examples ```toml [main] @@ -106,7 +106,7 @@ class SectionModel(ContainerModel, OptionsModel): # …refer to Options doc ``` - ### Properties + #### Properties - `name` (optional): `Translation` or `str`, displayed as the section's title if any - `help`: `Translation` or `str`, text to display before the first option - `services` (optional): `list` of services names to `reload-or-restart` when any option's value contained in the section changes @@ -163,9 +163,9 @@ def translate(self, i18n_key: Union[str, None] = None) -> None: class PanelModel(ContainerModel): """ - Group sections. Panels are `dict`s defined inside a ConfigPanel file and require a unique id (in the below example, the id is `main`). Keep in mind that this id will be used in CLI to refer to the panel, so choose something short and meaningfull. + Panels are, basically, sections grouped together. Panels are `dict`s defined inside a ConfigPanel file and require a unique id (in the below example, the id is `main`). Keep in mind that this id will be used in CLI to refer to the panel, so choose something short and meaningfull. - ### Examples + #### Examples ```toml [main] name.en = "Main configuration" @@ -176,7 +176,7 @@ class PanelModel(ContainerModel): [main.customization] # …refer to Sections doc ``` - ### Properties + #### Properties - `name`: `Translation` or `str`, displayed as the panel title - `help` (optional): `Translation` or `str`, text to display before the first section - `services` (optional): `list` of services names to `reload-or-restart` when any option's value contained in the panel changes @@ -217,58 +217,23 @@ def translate(self, i18n_key: Union[str, None] = None) -> None: class ConfigPanelModel(BaseModel): """ - Configuration panels allows admins to manage parameters or runs actions for which the upstream's app doesn't provide any appropriate UI itself. It's a good way to reduce manual change on config files and avoid conflicts on it. + This is the 'root' level of the config panel toml file - Those panels can also be used to quickly create interfaces that extend the capabilities of YunoHost (e.g. VPN Client, Hotspost, Borg, etc.). + #### Examples - From a packager perspective, this `config_panel.toml` is coupled to the `scripts/config` script, which may be used to define custom getters/setters/validations/actions. However, most use cases should be covered automagically by the core, thus it may not be necessary to define a scripts/config at all! - - ! Please: Keep in mind the YunoHost spirit, and try to build your panels in such a way as to expose only really useful, "high-level" parameters, and if there are many of them, to relegate those corresponding to rarer use cases to "Advanced" sub-sections. Keep it simple, focus on common needs, don't expect the admins to have 3 PhDs in computer science. - - ### `config_panel.toml`'s principle and general format - To create configuration panels for apps, you should at least create a `config_panel.toml` at the root of the package. For more complex cases, this TOML file can be paired with a `config` script inside the scripts directory of your package, which will handle specific controller logic. - - The `config_panel.toml` describes one or several panels, containing sections, each containing questions generally binded to a params in the app's actual configuration files. - - ### Options short keys have to be unique - For performance reasons, questions short keys have to be unique in all the `config_panel.toml` file, not just inside its panel or its section. Hence it's not possible to have: - ```toml - [manual.vpn.server_ip] - [advanced.dns.server_ip] - ``` - In which two questions have "real variable name" `is server_ip` and therefore conflict with each other. - - ! Some short keys are forbidden cause it can interfer with config scripts (`old`, `file_hash`, `types`, `binds`, `formats`, `changed`) and you probably should avoid to use common settings name to avoid to bind your question to this settings (e.g. `id`, `install_time`, `mysql_pwd`, `path`, `domain`, `port`, `db_name`, `current_revision`, `admin`) - - ### Supported questions types and properties - - [Learn more about Options](/dev/forms) in their dedicated doc page as those are also used in app install forms and core config panels. - - ### YunoHost community examples - - [Check the basic example at the end of this doc](#basic-example) - - [Check the example_ynh app toml](https://github.com/YunoHost/example_ynh/blob/master/config_panel.toml.example) and the [basic `scripts/config` example](https://github.com/YunoHost/example_ynh/blob/master/scripts/config) - - [Check config panels of other apps](https://grep.app/search?q=version&filter[repo.pattern][0]=YunoHost-Apps&filter[lang][0]=TOML) - - [Check `scripts/config` of other apps](https://grep.app/search?q=ynh_app_config_apply&filter[repo.pattern][0]=YunoHost-Apps&filter[lang][0]=Shell) - - ### Examples ```toml version = 1.0 [config] # …refer to Panels doc ``` - ### Properties + + #### Properties + - `version`: `float` (default: `1.0`), version that the config panel supports in terms of features. - `i18n` (optional): `str`, an i18n property that let you internationalize options text. - However this feature is only available in core configuration panel (like `yunohost domain config`), prefer the use `Translation` in `name`, `help`, etc. - #### Version - Here a small reminder to associate config panel version with YunoHost version. - - | Config | YNH | Config panel small change log | - | ------ | --- | ------------------------------------------------------- | - | 0.1 | 3.x | 0.1 config script not compatible with YNH >= 4.3 | - | 1.0 | 4.3.x | The new config panel system with 'bind' property | """ version: float = CONFIG_PANEL_VERSION_SUPPORTED diff --git a/src/utils/form.py b/src/utils/form.py index 28c5c4d24b..1197bae016 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -306,7 +306,7 @@ class BaseOption(BaseModel): ! IMPORTANT: as for Panels and Sections you have to choose an id, but this one should be unique in all this document, even if the question is in an other panel. - #### Examples + #### Example ```toml [section.my_option_id] @@ -325,32 +325,7 @@ class BaseOption(BaseModel): #### Properties - - `type`: - - readonly types: - - [`display_text`](#option-display_text) - - [`markdown`](#option-markdown) - - [`alert`](#option-alert) - - [`button`](#option-button) - - inputs types: - - [`string`](#option-string) - - [`text`](#option-text) - - [`password`](#option-password) - - [`color`](#option-color) - - [`number`](#option-number) - - [`range`](#option-range) - - [`boolean`](#option-boolean) - - [`date`](#option-date) - - [`time`](#option-time) - - [`email`](#option-email) - - [`path`](#option-path) - - [`url`](#option-url) - - [`file`](#option-file) - - [`select`](#option-select) - - [`tags`](#option-tags) - - [`domain`](#option-domain) - - [`app`](#option-app) - - [`user`](#option-user) - - [`group`](#option-group) + - `type`: the actual type of the option, such as 'markdown', 'password', 'number', 'email', ... - `ask`: `Translation` (default to the option's `id` if not defined): - text to display as the option's label for inputs or text to display for readonly options - in config panels, questions are displayed on the left side and therefore have not much space to be rendered. Therefore, it is better to use a short question, and use the `help` property to provide additional details if necessary. @@ -445,7 +420,7 @@ class DisplayTextOption(BaseReadonlyOption): """ Display simple multi-line content. - #### Examples + #### Example ```toml [section.my_option_id] @@ -462,7 +437,7 @@ class MarkdownOption(BaseReadonlyOption): Display markdown multi-line content. Markdown is currently only rendered in the web-admin - #### Examples + #### Example ```toml [section.my_option_id] type = "display_text" @@ -485,7 +460,7 @@ class AlertOption(BaseReadonlyOption): Alerts displays a important message with a level of severity. You can use markdown in `ask` but will only be rendered in the web-admin. - #### Examples + #### Example ```toml [section.my_option_id] @@ -496,7 +471,7 @@ class AlertOption(BaseReadonlyOption): ``` #### Properties - - [common properties](#common-option-properties) + - [common properties](#common-properties) - `style`: any of `"success|info|warning|danger"` (default: `"info"`) - `icon` (optional): any icon name from [Fork Awesome](https://forkaweso.me/Fork-Awesome/icons/) - Currently only displayed in the web-admin @@ -526,7 +501,7 @@ class ButtonOption(BaseReadonlyOption): Every options defined in an action section (a config panel section with at least one `button`) is guaranted to be shown/asked to the user and available in `scripts/config`'s scope. [check examples in advanced use cases](#actions). - #### Examples + #### Example ```toml [section.my_option_id] @@ -548,7 +523,7 @@ class ButtonOption(BaseReadonlyOption): #### Properties - - [common properties](#common-option-properties) + - [common properties](#common-properties) - `bind`: forced to `"null"` - `style`: any of `"success|info|warning|danger"` (default: `"success"`) - `enabled`: `JSExpression` or `bool` (default: `true`) @@ -580,14 +555,14 @@ class BaseInputOption(BaseOption): """ Rest of the option types available are considered `inputs`. - #### Examples + #### Example ```toml [section.my_option_id] type = "string" # …any common props… + optional = false - redact = False + redact = false default = "some default string" help = "You can enter almost anything!" example = "an example string" @@ -596,7 +571,7 @@ class BaseInputOption(BaseOption): #### Properties - - [common properties](#common-option-properties) + - [common properties](#common-properties) - `optional`: `bool` (default: `false`, but `true` in config panels) - `redact`: `bool` (default: `false`), to redact the value in the logs when the value contain private information - `default`: depends on `type`, the default value to assign to the option @@ -757,7 +732,7 @@ class StringOption(BaseStringOption): """ Ask for a simple string. - #### Examples + #### Example ```toml [section.my_option_id] type = "string" @@ -780,7 +755,7 @@ class TextOption(BaseStringOption): Ask for a multiline string. Renders as a `textarea` in the web-admin and by opening a text editor on the CLI. - #### Examples + #### Example ```toml [section.my_option_id] type = "text" @@ -803,7 +778,7 @@ class PasswordOption(BaseInputOption): Ask for a password. The password is tested as a regular user password (at least 8 chars) - #### Examples + #### Example ```toml [section.my_option_id] type = "password" @@ -855,7 +830,7 @@ class ColorOption(BaseInputOption): Ask for a color represented as a hex value (with possibly an alpha channel). Renders as color picker in the web-admin and as a prompt that accept named color like `yellow` in CLI. - #### Examples + #### Example ```toml [section.my_option_id] type = "color" @@ -901,7 +876,7 @@ class NumberOption(BaseInputOption): """ Ask for a number (an integer). - #### Examples + #### Example ```toml [section.my_option_id] type = "number" @@ -976,7 +951,7 @@ class BooleanOption(BaseInputOption): Ask for a boolean. Renders as a switch in the web-admin and a yes/no prompt in CLI. - #### Examples + #### Example ```toml [section.my_option_id] type = "boolean" @@ -1104,7 +1079,7 @@ class DateOption(BaseInputOption): Can also take a timestamp as value that will output as an ISO date string. - #### Examples + #### Example ```toml [section.my_option_id] type = "date" @@ -1134,7 +1109,7 @@ class TimeOption(BaseInputOption): Ask for an hour in the form `"22:35"`. Renders as a date-picker in the web-admin and a regular prompt in CLI. - #### Examples + #### Example ```toml [section.my_option_id] type = "time" @@ -1167,7 +1142,7 @@ class EmailOption(BaseInputOption): """ Ask for an email. Validation made with [python-email-validator](https://github.com/JoshData/python-email-validator) - #### Examples + #### Example ```toml [section.my_option_id] type = "email" @@ -1187,7 +1162,7 @@ class WebPathOption(BaseStringOption): """ Ask for an web path (the part of an url after the domain). Used by default in app install to define from where the app will be accessible. - #### Examples + #### Example ```toml [section.my_option_id] type = "path" @@ -1235,7 +1210,7 @@ class URLOption(BaseStringOption): """ Ask for any url. - #### Examples + #### Example ```toml [section.my_option_id] type = "url" @@ -1259,7 +1234,7 @@ class FileOption(BaseInputOption): Ask for file. Renders a file prompt in the web-admin and ask for a path in CLI. - #### Examples + #### Example ```toml [section.my_option_id] type = "file" @@ -1387,7 +1362,7 @@ class SelectOption(BaseChoicesOption): Ask for value from a limited set of values. Renders as a regular `