Made a lot of improvements in web UI + have a working configuration s…

…ave for services and global config

src/ui/app/dependencies.py

@@ -10,5 +10,5 @@
 DB = UIDatabase(getLogger("UI"), log=False)
 DATA = UIData(Path(sep, "var", "tmp", "bunkerweb").joinpath("ui_data.json"))
 
-BW_CONFIG = Config(DB)
+BW_CONFIG = Config(DB, data=DATA)
 BW_INSTANCES_UTILS = InstancesUtils(DB)

src/ui/app/models/config.py

@@ -9,11 +9,14 @@
 from re import error as RegexError, search as re_search
 from typing import List, Literal, Optional, Set, Tuple, Union
 
+from app.utils import get_blacklisted_settings
+
 
 class Config:
-    def __init__(self, db) -> None:
+    def __init__(self, db, data) -> None:
         self.__settings = json_loads(Path(sep, "usr", "share", "bunkerweb", "settings.json").read_text(encoding="utf-8"))
         self.__db = db
+        self.__data = data
 
     def __gen_conf(
         self, global_conf: dict, services_conf: list[dict], *, check_changes: bool = True, changed_service: Optional[str] = None
@@ -104,7 +107,7 @@ def get_services(self, methods: bool = True, with_drafts: bool = False) -> list[
         """
         return self.__db.get_services_settings(methods=methods, with_drafts=with_drafts)
 
-    def check_variables(self, variables: dict, config: dict) -> dict:
+    def check_variables(self, variables: dict, config: dict, *, global_config: bool = False, threaded: bool = False) -> dict:
         """Testify that the variables passed are valid
 
         Parameters
@@ -117,47 +120,75 @@ def check_variables(self, variables: dict, config: dict) -> dict:
         int
             Return the error code
         """
+        self.__data.load_from_file()
         plugins_settings = self.get_plugins_settings()
         for k, v in variables.copy().items():
             check = False
 
-            if k.endswith("SCHEMA"):
-                variables.pop(k)
-                continue
-
             if k in plugins_settings:
                 setting = k
             else:
                 setting = k[0 : k.rfind("_")]  # noqa: E203
                 if setting not in plugins_settings or "multiple" not in plugins_settings[setting]:
-                    flash(f"Variable {k} is not valid.", "error")
+                    content = f"Variable {k} is not valid."
+                    if threaded:
+                        self.__data["TO_FLASH"].append({"content": content, "type": "error"})
+                    else:
+                        flash(content, "error")
                     variables.pop(k)
                     continue
 
-            if setting in ("AUTOCONF_MODE", "SWARM_MODE", "KUBERNETES_MODE", "IS_LOADING", "IS_DRAFT"):
-                flash(f"Variable {k} is not editable, ignoring it", "error")
+            if setting in get_blacklisted_settings(global_config):
+                message = f"Variable {k} is not editable, ignoring it"
+                if threaded:
+                    self.__data["TO_FLASH"].append({"content": message, "type": "error"})
+                else:
+                    flash(message, "error")
                 variables.pop(k)
                 continue
             elif setting not in config and plugins_settings[setting]["default"] == v:
                 variables.pop(k)
                 continue
             elif config[setting]["method"] not in ("default", "ui"):
-                flash(f"Variable {k} is not editable as is it managed by the {config[setting]['method']}, ignoring it", "error")
+                message = f"Variable {k} is not editable as is it managed by the {config[setting]['method']}, ignoring it"
+                if threaded:
+                    self.__data["TO_FLASH"].append({"content": message, "type": "error"})
+                else:
+                    flash(message, "error")
                 variables.pop(k)
                 continue
 
             try:
                 if re_search(plugins_settings[setting]["regex"], v):
                     check = True
             except RegexError as e:
-                flash(f"Invalid regex for setting {setting} : {plugins_settings[setting]['regex']}, ignoring regex check:{e}", "error")
+                message = f"Invalid regex for setting {setting} : {plugins_settings[setting]['regex']}, ignoring regex check:{e}"
+                if threaded:
+                    self.__data["TO_FLASH"].append({"content": message, "type": "error"})
+                else:
+                    flash(message, "error")
                 variables.pop(k)
                 continue
 
             if not check:
-                flash(f"Variable {k} is not valid.", "error")
+                message = f"Variable {k} is not valid."
+                if threaded:
+                    self.__data["TO_FLASH"].append({"content": message, "type": "error"})
+                else:
+                    flash(message, "error")
                 variables.pop(k)
 
+        for k in config:
+            if k in plugins_settings:
+                continue
+            setting = k[0 : k.rfind("_")]  # noqa: E203
+
+            if setting not in plugins_settings or "multiple" not in plugins_settings[setting]:
+                continue
+
+            if k not in variables:
+                variables[k] = plugins_settings[setting]["default"]
+
         return variables
 
     def new_service(self, variables: dict, is_draft: bool = False) -> Tuple[str, int]:

src/ui/app/models/totp.py

@@ -95,10 +95,12 @@ def generate_qrcode(self, username: str, totp: str) -> str:
 
     def get_last_counter(self, user: Users) -> Optional[int]:
         """Fetch stored last_counter from cache."""
+        DATA.load_from_file()
         return DATA.get("totp_last_counter", {}).get(user.get_id())
 
     def set_last_counter(self, user: Users, tmatch: TotpMatch) -> None:
         """Cache last_counter."""
+        DATA.load_from_file()
         if "totp_last_counter" not in DATA:
             DATA["totp_last_counter"] = {}
         DATA["totp_last_counter"][user.get_id()] = tmatch.counter

src/ui/app/routes/configs.py

@@ -5,7 +5,7 @@
 from flask import Blueprint, flash, redirect, render_template, request, url_for
 from flask_login import login_required
 
-from app.dependencies import BW_CONFIG, DATA, DB
+from app.dependencies import BW_CONFIG, DATA, DB  # TODO: remember about DATA.load_from_file()
 from app.utils import LOGGER, PLUGIN_NAME_RX, path_to_dict
 
 from app.routes.utils import handle_error, verify_data_in_form

src/ui/app/routes/global_config.py

@@ -1,6 +1,7 @@
 from contextlib import suppress
 from threading import Thread
 from time import time
+from typing import Dict
 
 from flask import Blueprint, flash, redirect, render_template, request, url_for
 from flask_login import login_required
@@ -19,68 +20,71 @@ def global_config_page():
     if request.method == "POST":
         if DB.readonly:
             return handle_error("Database is in read-only mode", "global_config")
+        DATA.load_from_file()
 
         # Check variables
         variables = request.form.to_dict().copy()
         del variables["csrf_token"]
 
-        # Edit check fields and remove already existing ones
-        config = DB.get_config(methods=True, with_drafts=True)
-        services = config["SERVER_NAME"]["value"].split(" ")
-        for variable, value in variables.copy().items():
-            setting = config.get(variable, {"value": None, "global": True})
-            if setting["global"] and value == setting["value"]:
-                del variables[variable]
-                continue
-
-        variables = BW_CONFIG.check_variables(variables, config)
-
-        if not variables:
-            return handle_error("The global configuration was not edited because no values were changed.", "global_config", True)
-
-        for variable, value in variables.copy().items():
-            for service in services:
-                setting = config.get(f"{service}_{variable}", None)
-                if setting and setting["global"] and (setting["value"] != value or setting["value"] == config.get(variable, {"value": None})["value"]):
-                    variables[f"{service}_{variable}"] = value
-
-        db_metadata = DB.get_metadata()
-
-        def update_global_config(threaded: bool = False):
+        def update_global_config(variables: Dict[str, str], threaded: bool = False):
             wait_applying()
 
-            manage_bunkerweb("global_config", variables, threaded=threaded)
-
-        if "PRO_LICENSE_KEY" in variables:
-            DATA["PRO_LOADING"] = True
+            # Edit check fields and remove already existing ones
+            config = DB.get_config(methods=True, with_drafts=True)
+            services = config["SERVER_NAME"]["value"].split(" ")
+            for variable, value in variables.copy().items():
+                setting = config.get(variable, {"value": None, "global": True})
+                if setting["global"] and value == setting["value"]:
+                    del variables[variable]
+                    continue
+
+            variables = BW_CONFIG.check_variables(variables, config, global_config=True, threaded=threaded)
+
+            if not variables:
+                content = "The global configuration was not edited because no values were changed."
+                if threaded:
+                    DATA["TO_FLASH"].append({"content": content, "type": "warning"})
+                else:
+                    flash(content, "warning")
+                DATA.update({"RELOADING": False, "CONFIG_CHANGED": False})
+                return
+
+            if "PRO_LICENSE_KEY" in variables:
+                DATA["PRO_LOADING"] = True
+
+            for variable, value in variables.copy().items():
+                for service in services:
+                    setting = config.get(f"{service}_{variable}", None)
+                    if setting and setting["global"] and (setting["value"] != value or setting["value"] == config.get(variable, {"value": None})["value"]):
+                        variables[f"{service}_{variable}"] = value
+
+            with suppress(BaseException):
+                if config["PRO_LICENSE_KEY"]["value"] != variables["PRO_LICENSE_KEY"]:
+                    if threaded:
+                        DATA["TO_FLASH"].append({"content": "Checking license key to upgrade.", "type": "success"})
+                    else:
+                        flash("Checking license key to upgrade.", "success")
 
-        if any(
-            v
-            for k, v in db_metadata.items()
-            if k in ("custom_configs_changed", "external_plugins_changed", "pro_plugins_changed", "plugins_config_changed", "instances_changed")
-        ):
-            DATA["RELOADING"] = True
-            DATA["LAST_RELOAD"] = time()
-            Thread(target=update_global_config, args=(True,)).start()
-        else:
-            update_global_config()
+            manage_bunkerweb("global_config", variables, threaded=threaded)
 
-        DATA["CONFIG_CHANGED"] = True
+        DATA.update({"RELOADING": True, "LAST_RELOAD": time(), "CONFIG_CHANGED": True})
+        Thread(target=update_global_config, args=(variables, True)).start()
 
-        with suppress(BaseException):
-            if config["PRO_LICENSE_KEY"]["value"] != variables["PRO_LICENSE_KEY"]:
-                flash("Checking license key to upgrade.", "success")
+        arguments = {}
+        if request.args.get("keywords"):
+            arguments["keywords"] = request.args["keywords"]
+        if request.args.get("type", "all") != "all":
+            arguments["type"] = request.args["type"]
 
         return redirect(
             url_for(
                 "loading",
-                next=url_for("global_config.global_config_page"),
+                next=url_for("global_config.global_config_page") + f"?{'&'.join([f'{k}={v}' for k, v in arguments.items()])}",
                 message="Saving global configuration",
             )
         )
 
     keywords = request.args.get("keywords", "")
     search_type = request.args.get("type", "all")
     global_config = DB.get_config(global_only=True, methods=True)
-    plugins = BW_CONFIG.get_plugins()
-    return render_template("global_config.html", config=global_config, plugins=plugins, keywords=keywords, type=search_type)
+    return render_template("global_config.html", config=global_config, keywords=keywords, type=search_type)

src/ui/app/routes/instances.py

@@ -75,6 +75,7 @@ def instances_action(action: Literal["ping", "reload", "stop", "delete"]):  # TO
     instances = request.form["instances"].split(",")
     if not instances:
         return handle_error("No instances selected.", "instances", True)
+    DATA.load_from_file()
 
     if action == "ping":
         succeed = []

src/ui/app/routes/plugins.py

@@ -21,7 +21,7 @@
 
 from common_utils import bytes_hash  # type: ignore
 
-from app.dependencies import BW_CONFIG, BW_INSTANCES_UTILS, DATA, DB
+from app.dependencies import BW_CONFIG, BW_INSTANCES_UTILS, DATA, DB  # TODO: remember about DATA.load_from_file()
 from app.utils import LOGGER, PLUGIN_NAME_RX, TMP_DIR
 
 from app.routes.utils import PLUGIN_ID_RX, PLUGIN_KEYS, error_message, handle_error, verify_data_in_form, wait_applying