From 904166be7cbafe14ac1080806cb82fe288485408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Tue, 20 Aug 2024 11:39:44 +0100 Subject: [PATCH] Made support for custom timezone simpler --- src/autoconf/Config.py | 5 +- src/autoconf/Dockerfile | 2 +- src/bw/Dockerfile | 2 +- src/common/core/backup/bwcli/restore.py | 4 +- src/common/core/backup/bwcli/save.py | 4 +- src/common/core/backup/jobs/backup-data.py | 4 +- src/common/core/backup/utils.py | 10 ++-- src/common/core/jobs/jobs/mmdb-asn.py | 4 +- src/common/core/jobs/jobs/mmdb-country.py | 4 +- .../core/pro/jobs/download-pro-plugins.py | 4 +- src/common/db/Database.py | 44 ++++++++-------- src/common/gen/Configurator.py | 1 + src/common/utils/common_utils.py | 14 +---- src/common/utils/jobs.py | 4 +- src/common/utils/logger.py | 52 +++---------------- src/scheduler/Dockerfile | 2 +- src/scheduler/JobScheduler.py | 7 ++- src/scheduler/main.py | 29 +++++++---- src/ui/Dockerfile | 2 +- src/ui/main.py | 13 ++--- src/ui/models.py | 6 +-- src/ui/pages/bans.py | 4 +- src/ui/pages/login.py | 4 +- src/ui/pages/utils.py | 6 +-- src/ui/ui_database.py | 11 ++-- 25 files changed, 90 insertions(+), 152 deletions(-) diff --git a/src/autoconf/Config.py b/src/autoconf/Config.py index 1007cfc12..2c452a3da 100644 --- a/src/autoconf/Config.py +++ b/src/autoconf/Config.py @@ -6,7 +6,6 @@ from time import sleep from typing import Any, Dict, List, Optional -from common_utils import get_timezone # type: ignore from Database import Database # type: ignore from logger import setup_logger # type: ignore @@ -83,9 +82,9 @@ def have_to_wait(self) -> bool: ) def wait_applying(self, startup: bool = False): - current_time = datetime.now(get_timezone()) + current_time = datetime.now() ready = False - while not ready and (datetime.now(get_timezone()) - current_time).seconds < 240: + while not ready and (datetime.now() - current_time).seconds < 240: db_metadata = self._db.get_metadata() if isinstance(db_metadata, str): if not startup: diff --git a/src/autoconf/Dockerfile b/src/autoconf/Dockerfile index 3a54ecfc7..1fa60bb7a 100644 --- a/src/autoconf/Dockerfile +++ b/src/autoconf/Dockerfile @@ -37,7 +37,7 @@ FROM python:3.12.5-alpine@sha256:c2f41e6a5a67bc39b95be3988dd19fbd05d1b82375c46d9 RUN umask 027 # Install bash and create autoconf user -RUN apk add --no-cache bash && \ +RUN apk add --no-cache bash tzdata && \ addgroup -g 101 autoconf && \ adduser -h /var/cache/autoconf -g autoconf -s /bin/sh -G autoconf -D -H -u 101 autoconf diff --git a/src/bw/Dockerfile b/src/bw/Dockerfile index 4ad6257c7..9f857314d 100644 --- a/src/bw/Dockerfile +++ b/src/bw/Dockerfile @@ -48,7 +48,7 @@ FROM nginx:1.26.2-alpine-slim@sha256:28967af9fa8d5e1c58a45feeb35e2f326bb6d99b120 RUN umask 027 # Install runtime dependencies -RUN apk add --no-cache openssl pcre bash python3 yajl geoip libxml2 libgd curl +RUN apk add --no-cache openssl pcre bash python3 yajl geoip libxml2 libgd curl tzdata # Fix CVEs RUN apk add --no-cache "busybox>=1.36.1-r17" "busybox-binsh>=1.36.1-r17" "ssl_client>=1.36.1-r17" # CVE-2023-42363 CVE-2023-42366 diff --git a/src/common/core/backup/bwcli/restore.py b/src/common/core/backup/bwcli/restore.py index 4e707646b..5b2059e52 100644 --- a/src/common/core/backup/bwcli/restore.py +++ b/src/common/core/backup/bwcli/restore.py @@ -13,8 +13,6 @@ from utils import acquire_db_lock, backup_database, BACKUP_DIR, DB_LOCK_FILE, LOGGER, restore_database -from common_utils import get_timezone # type: ignore - status = 0 try: @@ -51,7 +49,7 @@ sys_exit(1) LOGGER.info("Backing up the current database before restoring the backup ...") - current_time = datetime.now(get_timezone()) + current_time = datetime.now() tmp_backup_dir = Path(sep, "tmp", "bunkerweb", "backups") tmp_backup_dir.mkdir(parents=True, exist_ok=True) db = backup_database(current_time, backup_dir=tmp_backup_dir) diff --git a/src/common/core/backup/bwcli/save.py b/src/common/core/backup/bwcli/save.py index c270ea4bc..ec7ee4956 100644 --- a/src/common/core/backup/bwcli/save.py +++ b/src/common/core/backup/bwcli/save.py @@ -13,8 +13,6 @@ from utils import acquire_db_lock, backup_database, BACKUP_DIR, DB_LOCK_FILE, LOGGER -from common_utils import get_timezone # type: ignore - status = 0 try: @@ -41,7 +39,7 @@ LOGGER.info(f"Creating directory {directory} as it does not exist") directory.mkdir(parents=True, exist_ok=True) - backup_database(datetime.now(get_timezone()), backup_dir=directory) + backup_database(datetime.now(), backup_dir=directory) except SystemExit as se: status = se.code except: diff --git a/src/common/core/backup/jobs/backup-data.py b/src/common/core/backup/jobs/backup-data.py index 9381342fc..11a2cfbac 100644 --- a/src/common/core/backup/jobs/backup-data.py +++ b/src/common/core/backup/jobs/backup-data.py @@ -16,8 +16,6 @@ from jobs import Job # type: ignore from utils import backup_database -from common_utils import get_timezone # type: ignore - LOGGER = setup_logger("BACKUP", getenv("LOG_LEVEL", "INFO")) status = 0 @@ -37,7 +35,7 @@ if last_backup_date: last_backup_date = datetime.fromisoformat(last_backup_date) - current_time = datetime.now(get_timezone()) + current_time = datetime.now() backup_period = getenv("BACKUP_SCHEDULE", "daily") PERIOD_STAMPS = { "daily": timedelta(days=1).total_seconds(), diff --git a/src/common/core/backup/utils.py b/src/common/core/backup/utils.py index 1beeff828..f7ecbca65 100644 --- a/src/common/core/backup/utils.py +++ b/src/common/core/backup/utils.py @@ -16,7 +16,7 @@ if deps_path not in sys_path: sys_path.append(deps_path) -from common_utils import bytes_hash, get_timezone # type: ignore +from common_utils import bytes_hash # type: ignore from Database import Database # type: ignore from logger import setup_logger # type: ignore from model import Base # type: ignore @@ -30,7 +30,7 @@ def acquire_db_lock(): """Acquire the database lock to prevent concurrent access to the database.""" - current_time = datetime.now(get_timezone()) + current_time = datetime.now() while DB_LOCK_FILE.is_file() and DB_LOCK_FILE.stat().st_ctime + 30 > current_time.timestamp(): LOGGER.warning("Database is locked, waiting for it to be unlocked (timeout: 30s) ...") sleep(1) @@ -46,9 +46,9 @@ def backup_database(current_time: datetime, db: Database = None, backup_dir: Pat backup_file = backup_dir.joinpath(f"backup-{database}-{current_time.strftime('%Y-%m-%d_%H-%M-%S')}.zip") LOGGER.debug(f"Backup file path: {backup_file}") stderr = "Table 'db.test_" - current_time = datetime.now(get_timezone()) + current_time = datetime.now() - while "Table 'db.test_" in stderr and (datetime.now(get_timezone()) - current_time).total_seconds() < 10: + while "Table 'db.test_" in stderr and (datetime.now() - current_time).total_seconds() < 10: if database == "sqlite": match = DB_STRING_RX.search(db.database_uri) if not match: @@ -94,7 +94,7 @@ def backup_database(current_time: datetime, db: Database = None, backup_dir: Pat LOGGER.error(f"Failed to dump the database: {stderr}") sys_exit(1) - if (datetime.now(get_timezone()) - current_time).total_seconds() >= 10: + if (datetime.now() - current_time).total_seconds() >= 10: LOGGER.error("Failed to dump the database: Timeout reached") sys_exit(1) diff --git a/src/common/core/jobs/jobs/mmdb-asn.py b/src/common/core/jobs/jobs/mmdb-asn.py index 000a8639d..99af70e83 100644 --- a/src/common/core/jobs/jobs/mmdb-asn.py +++ b/src/common/core/jobs/jobs/mmdb-asn.py @@ -18,7 +18,7 @@ from requests import RequestException, Response, get from logger import setup_logger # type: ignore -from common_utils import bytes_hash, get_timezone, file_hash # type: ignore +from common_utils import bytes_hash, file_hash # type: ignore from jobs import Job # type: ignore LOGGER = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO")) @@ -62,7 +62,7 @@ def request_mmdb() -> Optional[Response]: if response and response.status_code == 200: skip_dl = response.content.find(bytes_hash(job_cache["data"], algorithm="sha1").encode()) != -1 - elif job_cache["last_update"] < (datetime.now(get_timezone()) - timedelta(weeks=1)).timestamp(): + elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp(): LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...") skip_dl = False diff --git a/src/common/core/jobs/jobs/mmdb-country.py b/src/common/core/jobs/jobs/mmdb-country.py index 9d2b999cc..346a3ad4a 100644 --- a/src/common/core/jobs/jobs/mmdb-country.py +++ b/src/common/core/jobs/jobs/mmdb-country.py @@ -18,7 +18,7 @@ from requests import RequestException, Response, get from logger import setup_logger # type: ignore -from common_utils import bytes_hash, get_timezone, file_hash # type: ignore +from common_utils import bytes_hash, file_hash # type: ignore from jobs import Job # type: ignore LOGGER = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO")) @@ -62,7 +62,7 @@ def request_mmdb() -> Optional[Response]: if response and response.status_code == 200: skip_dl = response.content.find(bytes_hash(job_cache["data"], algorithm="sha1").encode()) != -1 - elif job_cache["last_update"] < (datetime.now(get_timezone()) - timedelta(weeks=1)).timestamp(): + elif job_cache["last_update"] < (datetime.now() - timedelta(weeks=1)).timestamp(): LOGGER.warning("Unable to check if the cache file is the latest version from db-ip.com and file is older than 1 week, checking anyway...") skip_dl = False diff --git a/src/common/core/pro/jobs/download-pro-plugins.py b/src/common/core/pro/jobs/download-pro-plugins.py index 8434f1821..29801b3b7 100644 --- a/src/common/core/pro/jobs/download-pro-plugins.py +++ b/src/common/core/pro/jobs/download-pro-plugins.py @@ -23,7 +23,7 @@ from Database import Database # type: ignore from logger import setup_logger # type: ignore -from common_utils import bytes_hash, get_os_info, get_integration, get_timezone, get_version # type: ignore +from common_utils import bytes_hash, get_os_info, get_integration, get_version # type: ignore API_ENDPOINT = "https://api.bunkerweb.io" PREVIEW_ENDPOINT = "https://assets.bunkerity.com/bw-pro/preview" @@ -95,7 +95,7 @@ def install_plugin(plugin_path: Path, db, preview: bool = True) -> bool: try: db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) db_metadata = db.get_metadata() - current_date = datetime.now(get_timezone()) + current_date = datetime.now() pro_license_key = getenv("PRO_LICENSE_KEY", "").strip() LOGGER.info("Checking BunkerWeb Pro status...") diff --git a/src/common/db/Database.py b/src/common/db/Database.py index e6ea9f469..e2d4fcffd 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -43,7 +43,7 @@ if deps_path not in sys_path: sys_path.append(deps_path) -from common_utils import bytes_hash, get_timezone # type: ignore +from common_utils import bytes_hash # type: ignore from pymysql import install_as_MySQLdb from sqlalchemy import create_engine, event, MetaData as sql_metadata, func, join, select as db_select, text, inspect @@ -168,7 +168,7 @@ def __init__( DATABASE_RETRY_TIMEOUT = int(DATABASE_RETRY_TIMEOUT) - current_time = datetime.now(get_timezone()) + current_time = datetime.now() not_connected = True fallback = False @@ -185,7 +185,7 @@ def __init__( not_connected = False except (OperationalError, DatabaseError) as e: - if (datetime.now(get_timezone()) - current_time).total_seconds() > DATABASE_RETRY_TIMEOUT: + if (datetime.now() - current_time).total_seconds() > DATABASE_RETRY_TIMEOUT: if not fallback and self.database_uri_readonly: self.logger.error(f"Can't connect to database after {DATABASE_RETRY_TIMEOUT} seconds. Falling back to read-only database connection") self.sql_engine.dispose(close=True) @@ -241,7 +241,7 @@ def test_write(self): def retry_connection(self, *, readonly: bool = False, fallback: bool = False, log: bool = True, **kwargs) -> None: """Retry the connection to the database""" - self.last_connection_retry = datetime.now(get_timezone()) + self.last_connection_retry = datetime.now() if log: self.logger.debug(f"Retrying the connection to the database{' in read-only mode' if readonly else ''}{' with fallback' if fallback else ''} ...") @@ -476,7 +476,7 @@ def checked_changes( if not metadata: return "The metadata are not set yet, try again" - current_time = datetime.now(get_timezone()) + current_time = datetime.now() if "config" in changes: if not metadata.first_config_saved: @@ -536,7 +536,7 @@ def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tu db_ui_version = db_version self.logger.warning(f"Database version ({db_version}) is different from Bunkerweb version ({bunkerweb_version}), migrating ...") - current_time = datetime.now(get_timezone()) + current_time = datetime.now() error = True # ? Wait for the metadata to be available while error: @@ -545,7 +545,7 @@ def init_tables(self, default_plugins: List[dict], bunkerweb_version: str) -> Tu metadata.reflect(self.sql_engine) error = False except BaseException as e: - if (datetime.now(get_timezone()) - current_time).total_seconds() > 10: + if (datetime.now() - current_time).total_seconds() > 10: raise e sleep(1) @@ -1328,7 +1328,7 @@ def save_config(self, config: Dict[str, Any], method: str, changed: Optional[boo session.query(Custom_configs).filter(Custom_configs.service_id.in_(missing_ids)).delete() session.query(Jobs_cache).filter(Jobs_cache.service_id.in_(missing_ids)).delete() session.query(Metadata).filter_by(id=1).update( - {Metadata.custom_configs_changed: True, Metadata.last_custom_configs_change: datetime.now(get_timezone())} + {Metadata.custom_configs_changed: True, Metadata.last_custom_configs_change: datetime.now()} ) changed_services = True @@ -1672,7 +1672,7 @@ def save_custom_configs( metadata = session.query(Metadata).get(1) if metadata is not None: metadata.custom_configs_changed = True - metadata.last_custom_configs_change = datetime.now(get_timezone()) + metadata.last_custom_configs_change = datetime.now() try: session.add_all(to_put) @@ -1994,7 +1994,7 @@ def add_job_run(self, job_name: str, success: bool, start_date: datetime, end_da if self.readonly: return "The database is read-only, the changes will not be saved" - session.add(Jobs_runs(job_name=job_name, success=success, start_date=start_date, end_date=end_date or datetime.now(get_timezone()))) + session.add(Jobs_runs(job_name=job_name, success=success, start_date=start_date, end_date=end_date or datetime.now())) try: session.commit() @@ -2062,13 +2062,13 @@ def upsert_job_cache( service_id=service_id, file_name=file_name, data=data, - last_update=datetime.now(get_timezone()), + last_update=datetime.now(), checksum=checksum, ) ) else: cache.data = data - cache.last_update = datetime.now(get_timezone()) + cache.last_update = datetime.now() cache.checksum = checksum try: @@ -2858,10 +2858,10 @@ def update_external_plugins( if metadata is not None: if _type in ("external", "ui"): metadata.external_plugins_changed = True - metadata.last_external_plugins_change = datetime.now(get_timezone()) + metadata.last_external_plugins_change = datetime.now() elif _type == "pro": metadata.pro_plugins_changed = True - metadata.last_pro_plugins_change = datetime.now(get_timezone()) + metadata.last_pro_plugins_change = datetime.now() try: session.add_all(to_put) @@ -2902,10 +2902,10 @@ def delete_plugin(self, plugin_id: str, method: str) -> str: if metadata is not None: if method in ("external", "ui"): metadata.external_plugins_changed = True - metadata.last_external_plugins_change = datetime.now(get_timezone()) + metadata.last_external_plugins_change = datetime.now() elif method == "pro": metadata.pro_plugins_changed = True - metadata.last_pro_plugins_change = datetime.now(get_timezone()) + metadata.last_pro_plugins_change = datetime.now() try: session.commit() @@ -3127,7 +3127,7 @@ def add_instance(self, hostname: str, port: int, server_name: str, method: str, if db_instance is not None: return f"Instance {hostname} already exists, will not be added." - current_time = datetime.now(get_timezone()) + current_time = datetime.now() session.add( Instances( hostname=hostname, @@ -3145,7 +3145,7 @@ def add_instance(self, hostname: str, port: int, server_name: str, method: str, metadata = session.query(Metadata).get(1) if metadata is not None: metadata.instances_changed = True - metadata.last_instances_change = datetime.now(get_timezone()) + metadata.last_instances_change = datetime.now() try: session.commit() @@ -3172,7 +3172,7 @@ def delete_instance(self, hostname: str, changed: Optional[bool] = True) -> str: metadata = session.query(Metadata).get(1) if metadata is not None: metadata.instances_changed = True - metadata.last_instances_change = datetime.now(get_timezone()) + metadata.last_instances_change = datetime.now() try: session.commit() @@ -3194,7 +3194,7 @@ def update_instances(self, instances: List[Dict[str, Any]], method: str, changed if instance.get("hostname") is None: continue - current_time = datetime.now(get_timezone()) + current_time = datetime.now() to_put.append( Instances( hostname=instance["hostname"], @@ -3214,7 +3214,7 @@ def update_instances(self, instances: List[Dict[str, Any]], method: str, changed metadata = session.query(Metadata).get(1) if metadata is not None: metadata.instances_changed = True - metadata.last_instances_change = datetime.now(get_timezone()) + metadata.last_instances_change = datetime.now() try: session.add_all(to_put) @@ -3236,7 +3236,7 @@ def update_instance(self, hostname: str, status: str) -> str: return f"Instance {hostname} does not exist, will not be updated." db_instance.status = status - db_instance.last_seen = datetime.now(get_timezone()) + db_instance.last_seen = datetime.now() try: session.commit() diff --git a/src/common/gen/Configurator.py b/src/common/gen/Configurator.py index e8223262b..0033c99b1 100644 --- a/src/common/gen/Configurator.py +++ b/src/common/gen/Configurator.py @@ -209,6 +209,7 @@ def get_config(self, db=None) -> Dict[str, str]: "SHLVL", "SERVER_SOFTWARE", "NAMESPACE", + "TZ", ) ): self.__logger.warning(f"Ignoring variable {variable} : {err} - {value = !r}") diff --git a/src/common/utils/common_utils.py b/src/common/utils/common_utils.py index 8421cd493..f95c70d59 100644 --- a/src/common/utils/common_utils.py +++ b/src/common/utils/common_utils.py @@ -1,13 +1,10 @@ from hashlib import new as new_hash from io import BytesIO -from logging import error -from os import environ, getenv, sep +from os import getenv, sep from pathlib import Path from platform import machine from typing import Dict, Union -from pytz import UnknownTimeZoneError, timezone - def dict_to_frozenset(d): if isinstance(d, list): @@ -91,12 +88,3 @@ def bytes_hash(bio: Union[str, bytes, BytesIO], *, algorithm: str = "sha512") -> _hash.update(data) bio.seek(0, 0) return _hash.hexdigest() - - -def get_timezone(): - try: - return timezone(getenv("TZ", "UTC")) - except UnknownTimeZoneError as e: - environ["TZ"] = "UTC" - error(f"Invalid timezone: {e}, using UTC instead") - return timezone("UTC") diff --git a/src/common/utils/jobs.py b/src/common/utils/jobs.py index e2bcbaf49..9d38a148b 100644 --- a/src/common/utils/jobs.py +++ b/src/common/utils/jobs.py @@ -14,7 +14,7 @@ from traceback import format_exc from typing import Any, Dict, Literal, Optional, Tuple, Union -from common_utils import bytes_hash, file_hash, get_timezone +from common_utils import bytes_hash, file_hash LOCK = Lock() EXPIRE_TIME = { @@ -144,7 +144,7 @@ def is_cached_file( try: cache_info = self.get_cache(name, job_name=job_name, service_id=service_id, plugin_id=plugin_id, with_info=True, with_data=False) if isinstance(cache_info, dict): - current_time = datetime.now(get_timezone()).timestamp() + current_time = datetime.now().timestamp() if current_time < cache_info["last_update"]: return False is_cached = current_time - cache_info["last_update"] < EXPIRE_TIME[expire] diff --git a/src/common/utils/logger.py b/src/common/utils/logger.py index e93bf0e75..76996e3e9 100644 --- a/src/common/utils/logger.py +++ b/src/common/utils/logger.py @@ -1,41 +1,7 @@ -from datetime import datetime, timezone -from logging import ( - CRITICAL, - DEBUG, - ERROR, - INFO, - WARNING, - Formatter, - Logger, - _nameToLevel, - addLevelName, - getLogger, - setLoggerClass, - StreamHandler, -) +from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING, Logger, _nameToLevel, addLevelName, basicConfig, getLogger, setLoggerClass from os import getenv from typing import Optional, Union -from common_utils import get_timezone - - -class BwFormatter(Formatter): - """Overrides logging.Formatter to use an aware datetime object.""" - - def converter(self, timestamp): - """Convert timestamp to an aware datetime object in the desired timezone.""" - return datetime.fromtimestamp(timestamp, tz=timezone.utc).astimezone(get_timezone()) - - def formatTime(self, record, datefmt=None): - """Format the datetime according to the specified format or ISO 8601.""" - dt = self.converter(record.created) - if datefmt: - return dt.strftime(datefmt) - try: - return dt.isoformat(timespec="milliseconds") - except TypeError: - return dt.isoformat() - class BWLogger(Logger): """Custom logger class inheriting from the standard Logger.""" @@ -50,17 +16,11 @@ def __init__(self, name, level=INFO): # Set the default logging level based on environment variables default_level = _nameToLevel.get(getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper(), INFO) -# Create a custom formatter instance -formatter = BwFormatter(fmt="%(asctime)s [%(name)s] [%(process)d] [%(levelname)s] - %(message)s", datefmt="[%Y-%m-%d %H:%M:%S %z]") - -# Create a console handler and set the custom formatter -handler = StreamHandler() -handler.setFormatter(formatter) - -# Get the root logger and add the handler to it -root_logger = getLogger() -root_logger.setLevel(default_level) -root_logger.addHandler(handler) +basicConfig( + format="%(asctime)s [%(name)s] [%(process)d] [%(levelname)s] - %(message)s", + datefmt="[%Y-%m-%d %H:%M:%S %z]", + level=default_level, +) # Set the default logging level for specific SQLAlchemy components database_default_level = _nameToLevel.get(getenv("DATABASE_LOG_LEVEL", "WARNING").upper(), WARNING) diff --git a/src/scheduler/Dockerfile b/src/scheduler/Dockerfile index 9b2282e7a..b59d22026 100644 --- a/src/scheduler/Dockerfile +++ b/src/scheduler/Dockerfile @@ -41,7 +41,7 @@ FROM python:3.12.5-alpine@sha256:c2f41e6a5a67bc39b95be3988dd19fbd05d1b82375c46d9 RUN umask 027 # Install runtime dependencies and add scheduler user -RUN apk add --no-cache bash unzip libgcc libstdc++ libpq openssl libmagic mariadb-connector-c mariadb-client postgresql-client sqlite && \ +RUN apk add --no-cache bash unzip libgcc libstdc++ libpq openssl libmagic mariadb-connector-c mariadb-client postgresql-client sqlite tzdata && \ addgroup -g 101 scheduler && \ adduser -h /var/cache/nginx -g scheduler -s /bin/sh -G scheduler -D -H -u 101 scheduler diff --git a/src/scheduler/JobScheduler.py b/src/scheduler/JobScheduler.py index b8b0daf00..8a92461fa 100644 --- a/src/scheduler/JobScheduler.py +++ b/src/scheduler/JobScheduler.py @@ -27,7 +27,6 @@ if deps_path not in sys_path: sys_path.append(deps_path) -from common_utils import get_timezone # type: ignore from Database import Database # type: ignore from logger import setup_logger # type: ignore from ApiCaller import ApiCaller # type: ignore @@ -163,7 +162,7 @@ def __job_wrapper(self, path: str, plugin: str, name: str, file: str) -> int: self.__logger.info(f"Executing job {name} from plugin {plugin} ...") success = True ret = -1 - start_date = datetime.now(get_timezone()) + start_date = datetime.now() try: proc = run(join(path, "jobs", file), stdin=DEVNULL, stderr=STDOUT, env=self.__env, check=False) ret = proc.returncode @@ -172,7 +171,7 @@ def __job_wrapper(self, path: str, plugin: str, name: str, file: str) -> int: self.__logger.error(f"Exception while executing job {name} from plugin {plugin} :\n{format_exc()}") with self.__thread_lock: self.__job_success = False - end_date = datetime.now(get_timezone()) + end_date = datetime.now() if ret == 1: with self.__thread_lock: @@ -352,7 +351,7 @@ def try_database_readonly(self, force: bool = False) -> bool: except BaseException: self.db.readonly = True return True - elif not force and self.db.last_connection_retry and (datetime.now(get_timezone()) - self.db.last_connection_retry).total_seconds() > 30: + elif not force and self.db.last_connection_retry and (datetime.now() - self.db.last_connection_retry).total_seconds() > 30: return True if self.db.database_uri and self.db.readonly: diff --git a/src/scheduler/main.py b/src/scheduler/main.py index 52ddff1df..0fb7548ff 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -27,7 +27,7 @@ from dotenv import dotenv_values from schedule import every as schedule_every, run_pending -from common_utils import bytes_hash, dict_to_frozenset, get_integration, get_timezone # type: ignore +from common_utils import bytes_hash, dict_to_frozenset, get_integration # type: ignore from logger import setup_logger # type: ignore from Database import Database # type: ignore from JobScheduler import JobScheduler @@ -97,8 +97,8 @@ def handle_stop(signum, frame): - current_time = datetime.now(get_timezone()) - while APPLYING_CHANGES.is_set() and (datetime.now(get_timezone()) - current_time).seconds < 30: + current_time = datetime.now() + while APPLYING_CHANGES.is_set() and (datetime.now() - current_time).seconds < 30: LOGGER.warning("Waiting for the changes to be applied before stopping ...") sleep(1) @@ -373,8 +373,12 @@ def run_in_slave_mode(): # TODO: Refactor this feature continue sleep(5) + tz = getenv("TZ") + if tz: + env["TZ"] = tz + # Instantiate scheduler environment - SCHEDULER.env = env | {"TZ": getenv("TZ", "UTC"), "LOG_LEVEL": getenv("CUSTOM_LOG_LEVEL", env.get("LOG_LEVEL", "notice"))} + SCHEDULER.env = env | {"LOG_LEVEL": getenv("CUSTOM_LOG_LEVEL", env.get("LOG_LEVEL", "notice"))} threads = [ Thread(target=generate_custom_configs), @@ -561,9 +565,12 @@ def healthcheck_job(): env = SCHEDULER.db.get_config() env["DATABASE_URI"] = SCHEDULER.db.database_uri + tz = getenv("TZ") + if tz: + env["TZ"] = tz # Instantiate scheduler environment - SCHEDULER.env = env | {"TZ": getenv("TZ", "UTC"), "LOG_LEVEL": getenv("CUSTOM_LOG_LEVEL", env.get("LOG_LEVEL", "notice"))} + SCHEDULER.env = env | {"LOG_LEVEL": getenv("CUSTOM_LOG_LEVEL", env.get("LOG_LEVEL", "notice"))} threads = [] @@ -685,8 +692,6 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): LOGGER.info("Running plugins download jobs ...") - # Update the environment variables of the scheduler - SCHEDULER.env = env | {"TZ": getenv("TZ", "UTC"), "LOG_LEVEL": getenv("CUSTOM_LOG_LEVEL", env.get("LOG_LEVEL", "notice"))} if not SCHEDULER.run_single("download-plugins"): LOGGER.warning("download-plugins job failed at first start, plugins settings set by the user may not be up to date ...") if not SCHEDULER.run_single("download-pro-plugins"): @@ -729,6 +734,9 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): SCHEDULER.update_jobs() env = SCHEDULER.db.get_config() env["DATABASE_URI"] = SCHEDULER.db.database_uri + tz = getenv("TZ") + if tz: + env["TZ"] = tz LOGGER.info("Executing scheduler ...") @@ -900,7 +908,7 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): scheduler_first_start = False if not HEALTHY_PATH.is_file(): - HEALTHY_PATH.write_text(datetime.now(get_timezone()).isoformat(), encoding="utf-8") + HEALTHY_PATH.write_text(datetime.now().isoformat(), encoding="utf-8") APPLYING_CHANGES.clear() schedule_every(HEALTHCHECK_INTERVAL).seconds.do(healthcheck_job) @@ -913,7 +921,7 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): sleep(3 if SCHEDULER.db.readonly else 1) run_pending() SCHEDULER.run_pending() - current_time = datetime.now(get_timezone()) + current_time = datetime.now() while DB_LOCK_FILE.is_file() and DB_LOCK_FILE.stat().st_ctime + 30 > current_time.timestamp(): LOGGER.debug("Database is locked, waiting for it to be unlocked (timeout: 30s) ...") @@ -1043,6 +1051,9 @@ def check_plugin_changes(_type: Literal["external", "pro"] = "external"): CHANGES.append("config") env = SCHEDULER.db.get_config() env["DATABASE_URI"] = SCHEDULER.db.database_uri + tz = getenv("TZ") + if tz: + env["TZ"] = tz except: LOGGER.error(f"Exception while executing scheduler : {format_exc()}") diff --git a/src/ui/Dockerfile b/src/ui/Dockerfile index 960c376df..655a115a6 100644 --- a/src/ui/Dockerfile +++ b/src/ui/Dockerfile @@ -56,7 +56,7 @@ FROM python:3.12.5-alpine@sha256:c2f41e6a5a67bc39b95be3988dd19fbd05d1b82375c46d9 RUN umask 027 # Install runtime dependencies and add ui user -RUN apk add --no-cache bash unzip libmagic mariadb-connector-c mariadb-client postgresql-client sqlite && \ +RUN apk add --no-cache bash unzip libmagic mariadb-connector-c mariadb-client postgresql-client sqlite tzdata && \ addgroup -g 101 ui && \ adduser -h /var/cache/nginx -g ui -s /bin/bash -G ui -D -H -u 101 ui diff --git a/src/ui/main.py b/src/ui/main.py index ff9683495..ec08deda0 100644 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -18,8 +18,6 @@ from signal import SIGINT, signal, SIGTERM from time import time -from common_utils import get_timezone # type: ignore - from src.reverse_proxied import ReverseProxied from pages.bans import bans @@ -245,10 +243,7 @@ def before_request(): if ( DB.database_uri and DB.readonly - and ( - datetime.now(get_timezone()) - datetime.fromisoformat(DATA.get("LAST_DATABASE_RETRY", "1970-01-01T00:00:00")).replace(tzinfo=get_timezone()) - > timedelta(minutes=1) - ) + and (datetime.now() - datetime.fromisoformat(DATA.get("LAST_DATABASE_RETRY", "1970-01-01T00:00:00")).astimezone() > timedelta(minutes=1)) ): try: DB.retry_connection(pool_timeout=1) @@ -265,19 +260,19 @@ def before_request(): DB.retry_connection(fallback=True, pool_timeout=1) DB.retry_connection(fallback=True, log=False) DATA["READONLY_MODE"] = True - DATA["LAST_DATABASE_RETRY"] = DB.last_connection_retry.isoformat() if DB.last_connection_retry else datetime.now(get_timezone()).isoformat() + DATA["LAST_DATABASE_RETRY"] = DB.last_connection_retry.isoformat() if DB.last_connection_retry else datetime.now().isoformat() elif not DATA.get("READONLY_MODE", False) and request.method == "POST" and not ("/totp" in request.path or "/login" in request.path): try: DB.test_write() DATA["READONLY_MODE"] = False except BaseException: DATA["READONLY_MODE"] = True - DATA["LAST_DATABASE_RETRY"] = DB.last_connection_retry.isoformat() if DB.last_connection_retry else datetime.now(get_timezone()).isoformat() + DATA["LAST_DATABASE_RETRY"] = DB.last_connection_retry.isoformat() if DB.last_connection_retry else datetime.now().isoformat() else: try: DB.test_read() except BaseException: - DATA["LAST_DATABASE_RETRY"] = DB.last_connection_retry.isoformat() if DB.last_connection_retry else datetime.now(get_timezone()).isoformat() + DATA["LAST_DATABASE_RETRY"] = DB.last_connection_retry.isoformat() if DB.last_connection_retry else datetime.now().isoformat() DB.readonly = DATA.get("READONLY_MODE", False) diff --git a/src/ui/models.py b/src/ui/models.py index 65a2ce567..420a85db7 100644 --- a/src/ui/models.py +++ b/src/ui/models.py @@ -6,8 +6,6 @@ if deps_path not in sys_path: sys_path.append(deps_path) -from common_utils import get_timezone # type: ignore - from bcrypt import checkpw from flask_login import AnonymousUserMixin, UserMixin from sqlalchemy.orm import declarative_base, relationship @@ -29,8 +27,8 @@ class AnonymousUser(AnonymousUserMixin): last_login_ip = None login_count = 0 totp_secret = None - creation_date = datetime.now(get_timezone()) - update_date = datetime.now(get_timezone()) + creation_date = datetime.now() + update_date = datetime.now() list_roles = [] list_permissions = [] list_recovery_codes = [] diff --git a/src/ui/pages/bans.py b/src/ui/pages/bans.py index 0644b47cb..0fa6fb900 100644 --- a/src/ui/pages/bans.py +++ b/src/ui/pages/bans.py @@ -8,8 +8,6 @@ from flask_login import login_required from redis import Redis, Sentinel -from common_utils import get_timezone # type: ignore - from builder.bans import bans_builder # type: ignore from dependencies import BW_CONFIG, BW_INSTANCES_UTILS, DB @@ -170,7 +168,7 @@ def get_load_data(): ban_end = float(ban["ban_end"]) except ValueError: continue - ban_end = (datetime.fromtimestamp(ban_end) - datetime.now(get_timezone())).total_seconds() + ban_end = (datetime.fromtimestamp(ban_end) - datetime.now()).total_seconds() if redis_client: ok = redis_client.set(f"bans_ip_{ban['ip']}", dumps({"reason": reason, "date": time()})) diff --git a/src/ui/pages/login.py b/src/ui/pages/login.py index 948f024d1..616e8d9be 100644 --- a/src/ui/pages/login.py +++ b/src/ui/pages/login.py @@ -3,8 +3,6 @@ from flask import Blueprint, flash, redirect, render_template, request, session, url_for from flask_login import current_user, login_user -from common_utils import get_timezone # type: ignore - from dependencies import DB from utils import LOGGER @@ -29,7 +27,7 @@ def login_page(): session["user_agent"] = request.headers.get("User-Agent") session["totp_validated"] = False - ui_user.last_login_at = datetime.now(get_timezone()) + ui_user.last_login_at = datetime.now() ui_user.last_login_ip = request.remote_addr ui_user.login_count += 1 diff --git a/src/ui/pages/utils.py b/src/ui/pages/utils.py index de5e2e505..bf50d032f 100644 --- a/src/ui/pages/utils.py +++ b/src/ui/pages/utils.py @@ -10,8 +10,6 @@ from qrcode.main import QRCode from regex import compile as re_compile -from common_utils import get_timezone # type: ignore - from src.instance import Instance from dependencies import BW_CONFIG, DATA, DB @@ -25,9 +23,9 @@ def wait_applying(): - current_time = datetime.now(get_timezone()) + current_time = datetime.now() ready = False - while not ready and (datetime.now(get_timezone()) - current_time).seconds < 120: + while not ready and (datetime.now() - current_time).seconds < 120: db_metadata = DB.get_metadata() if isinstance(db_metadata, str): LOGGER.error(f"An error occurred when checking for changes in the database : {db_metadata}") diff --git a/src/ui/ui_database.py b/src/ui/ui_database.py index 8bd2673ff..48516dbe3 100644 --- a/src/ui/ui_database.py +++ b/src/ui/ui_database.py @@ -16,7 +16,6 @@ from sqlalchemy.orm import joinedload from sqlalchemy.exc import IntegrityError -from common_utils import get_timezone # type: ignore from Database import Database # type: ignore from model import Metadata # type: ignore @@ -48,7 +47,7 @@ def init_ui_tables(self, bunkerweb_version: str) -> Tuple[bool, str]: if db_version != bunkerweb_version: self.logger.warning(f"UI tables version ({db_version}) is different from BunkerWeb version ({bunkerweb_version}), migrating them ...") - current_time = datetime.now(get_timezone()) + current_time = datetime.now() error = True while error: try: @@ -56,7 +55,7 @@ def init_ui_tables(self, bunkerweb_version: str) -> Tuple[bool, str]: metadata.reflect(self.sql_engine) error = False except BaseException as e: - if (datetime.now(get_timezone()) - current_time).total_seconds() > 10: + if (datetime.now() - current_time).total_seconds() > 10: raise e sleep(1) @@ -205,7 +204,7 @@ def create_ui_user( return f"Role {role} doesn't exist" session.add(RolesUsers(user_name=username, role_name=role)) - current_time = datetime.now(get_timezone()) + current_time = datetime.now() session.add( Users( username=username, @@ -255,7 +254,7 @@ def update_ui_user( user.password = password.decode("utf-8") user.totp_secret = totp_secret user.method = method - user.update_date = datetime.now(get_timezone()) + user.update_date = datetime.now() try: session.commit() @@ -318,7 +317,7 @@ def create_ui_role(self, name: str, description: str, permissions: List[str]) -> if session.query(Roles).with_entities(Roles.name).filter_by(name=name).first(): return f"Role {name} already exists" - session.add(Roles(name=name, description=description, update_datetime=datetime.now(get_timezone()))) + session.add(Roles(name=name, description=description, update_datetime=datetime.now())) for permission in permissions: if not session.query(Permissions).with_entities(Permissions.name).filter_by(name=permission).first():