From 454ff755d65d2946b6b3dbab4968f21007bc8332 Mon Sep 17 00:00:00 2001 From: Arno Gobbin <32413451+a-gn@users.noreply.github.com> Date: Mon, 26 Aug 2024 19:53:00 +0200 Subject: [PATCH 1/3] add pydantic models for borg 1.x CLI (#8338) --- pyproject.toml | 1 + src/borg/public/__init__.py | 0 src/borg/public/cli_api/__init__.py | 0 src/borg/public/cli_api/v1.py | 169 ++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) create mode 100644 src/borg/public/__init__.py create mode 100644 src/borg/public/cli_api/__init__.py create mode 100644 src/borg/public/cli_api/v1.py diff --git a/pyproject.toml b/pyproject.toml index 3003d9bbbe..370f964481 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ llfuse = ["llfuse >= 1.3.8"] pyfuse3 = ["pyfuse3 >= 3.1.1"] nofuse = [] +pydantic = ["pydantic >= 2.8.2"] [project.urls] "Homepage" = "https://borgbackup.org/" diff --git a/src/borg/public/__init__.py b/src/borg/public/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/borg/public/cli_api/__init__.py b/src/borg/public/cli_api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/borg/public/cli_api/v1.py b/src/borg/public/cli_api/v1.py new file mode 100644 index 0000000000..6ce1be4077 --- /dev/null +++ b/src/borg/public/cli_api/v1.py @@ -0,0 +1,169 @@ +"""Pydantic models that can parse borg 1.x's CLI output. + +The two top-level models are: + +- `BorgLogLine`, which parses any line of borg's logging output, +- all `Borg*Result` classes, which parse the final JSON output of some borg commands. + +The different types of log lines are defined in the other models. +""" + +import json +import logging +import typing +from datetime import datetime +from pathlib import Path + +import pydantic + +_log = logging.getLogger(__name__) + + +class BaseBorgLogLine(pydantic.BaseModel): + def get_level(self) -> int: + """Get the log level for this line as a `logging` level value. + + If this is a log message with a levelname, use it. + Otherwise, progress messages get `DEBUG` level, and other messages get `INFO`. + """ + return logging.DEBUG + + +class ArchiveProgressLogLine(BaseBorgLogLine): + original_size: int + compressed_size: int + deduplicated_size: int + nfiles: int + path: Path + time: float + + +class FinishedArchiveProgress(BaseBorgLogLine): + """JSON object printed on stdout when an archive is finished.""" + + time: float + type: typing.Literal["archive_progress"] + finished: bool + + +class ProgressMessage(BaseBorgLogLine): + operation: int + msgid: typing.Optional[str] + finished: bool + message: typing.Optional[str] + time: float + + +class ProgressPercent(BaseBorgLogLine): + operation: int + msgid: str | None = pydantic.Field(None) + finished: bool + message: str | None = pydantic.Field(None) + current: float | None = pydantic.Field(None) + info: list[str] | None = pydantic.Field(None) + total: float | None = pydantic.Field(None) + time: float + + @pydantic.model_validator(mode="after") + def fields_depending_on_finished(self) -> typing.Self: + if self.finished: + if self.message is not None: + raise ValueError("message must be None if finished is True") + if self.current != self.total: + raise ValueError("current must be equal to total if finished is True") + if self.info is not None: + raise ValueError("info must be None if finished is True") + if self.total is not None: + raise ValueError("total must be None if finished is True") + else: + if self.message is None: + raise ValueError("message must not be None if finished is False") + if self.current is None: + raise ValueError("current must not be None if finished is False") + if self.info is None: + raise ValueError("info must not be None if finished is False") + if self.total is None: + raise ValueError("total must not be None if finished is False") + return self + + +class FileStatus(BaseBorgLogLine): + status: str + path: Path + + +class LogMessage(BaseBorgLogLine): + time: float + levelname: typing.Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + name: str + message: str + msgid: typing.Optional[str] + + def get_level(self) -> int: + try: + return getattr(logging, self.levelname) + except AttributeError: + _log.warning( + "could not find log level %s, giving the following message WARNING level: %s", + self.levelname, + json.dumps(self), + ) + return logging.WARNING + + +_BorgLogLinePossibleTypes = ( + ArchiveProgressLogLine | FinishedArchiveProgress | ProgressMessage | ProgressPercent | FileStatus | LogMessage +) + + +class BorgLogLine(pydantic.RootModel[_BorgLogLinePossibleTypes]): + """A log line from Borg with the `--log-json` argument.""" + + def get_level(self) -> int: + return self.root.get_level() + + +class _BorgArchive(pydantic.BaseModel): + """Basic archive attributes.""" + + name: str + id: str + start: datetime + + +class _BorgArchiveStatistics(pydantic.BaseModel): + """Statistics of an archive.""" + + original_size: int + compressed_size: int + deduplicated_size: int + nfiles: int + + +class _BorgLimitUsage(pydantic.BaseModel): + """Usage of borg limits by an archive.""" + + max_archive_size: float + + +class _BorgDetailedArchive(_BorgArchive): + """Archive attributes, as printed by `json info` or `json create`.""" + + end: datetime + duration: float + stats: _BorgArchiveStatistics + limits: _BorgLimitUsage + command_line: typing.List[str] + chunker_params: typing.Any | None = None + + +class BorgCreateResult(pydantic.BaseModel): + """JSON object printed at the end of `borg create`.""" + + archive: _BorgDetailedArchive + + +class BorgListResult(pydantic.BaseModel): + """JSON object printed at the end of `borg list`.""" + + archives: typing.List[_BorgArchive] From b5c7df6cdaf81ddb2548d85de3a1e99d72dfbe85 Mon Sep 17 00:00:00 2001 From: Arno Gobbin <32413451+a-gn@users.noreply.github.com> Date: Mon, 26 Aug 2024 20:10:42 +0200 Subject: [PATCH 2/3] add tests, can't run them yet (#8338) --- src/borg/helpers/progress.py | 11 +++++++++-- src/borg/public/cli_api/v1.py | 2 +- src/borg/testsuite/public/cli_api.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 src/borg/testsuite/public/cli_api.py diff --git a/src/borg/helpers/progress.py b/src/borg/helpers/progress.py index 8d6f8caff0..7f341990e8 100644 --- a/src/borg/helpers/progress.py +++ b/src/borg/helpers/progress.py @@ -1,6 +1,7 @@ import logging import json import time +import typing from ..logger import create_logger @@ -24,9 +25,15 @@ def __init__(self, msgid=None): self.id = self.operation_id() self.msgid = msgid - def make_json(self, *, finished=False, **kwargs): + def make_json(self, *, finished=False, override_time: typing.Optional[float] = None, **kwargs): kwargs.update( - dict(operation=self.id, msgid=self.msgid, type=self.JSON_TYPE, finished=finished, time=time.time()) + dict( + operation=self.id, + msgid=self.msgid, + type=self.JSON_TYPE, + finished=finished, + time=override_time or time.time(), + ) ) return json.dumps(kwargs) diff --git a/src/borg/public/cli_api/v1.py b/src/borg/public/cli_api/v1.py index 6ce1be4077..071120140c 100644 --- a/src/borg/public/cli_api/v1.py +++ b/src/borg/public/cli_api/v1.py @@ -1,4 +1,4 @@ -"""Pydantic models that can parse borg 1.x's CLI output. +"""Pydantic models that can parse borg 1.x's JSON output. The two top-level models are: diff --git a/src/borg/testsuite/public/cli_api.py b/src/borg/testsuite/public/cli_api.py new file mode 100644 index 0000000000..5281ce3e21 --- /dev/null +++ b/src/borg/testsuite/public/cli_api.py @@ -0,0 +1,20 @@ +import borg.public.cli_api.v1 as v1 +from borg.helpers.progress import ProgressIndicatorBase + + +def test_parse_progress_percent_unfinished(): + percent = ProgressIndicatorBase() + override_time = 4567.23 + json_output = percent.make_json(finished=False, current=10, override_time=override_time) + assert v1.ProgressPercent.model_validate_json(json_output) == v1.ProgressPercent( + operation=1, msgid=None, finished=False, message=None, current=10, info=None, total=None, time=4567.23 + ) + + +def test_parse_progress_percent_finished(): + percent = ProgressIndicatorBase() + override_time = 4567.23 + json_output = percent.make_json(finished=True, override_time=override_time) + assert v1.ProgressPercent.model_validate_json(json_output) == v1.ProgressPercent( + operation=1, msgid=None, finished=True, message=None, current=None, info=None, total=None, time=override_time + ) From 9216364439ab2ad8a2e88a9dba0b8b25d597defb Mon Sep 17 00:00:00 2001 From: Arno Gobbin <32413451+a-gn@users.noreply.github.com> Date: Sat, 31 Aug 2024 20:01:32 +0200 Subject: [PATCH 3/3] don't use Python 3.10 union syntax (#8338) --- src/borg/public/cli_api/v1.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/borg/public/cli_api/v1.py b/src/borg/public/cli_api/v1.py index 071120140c..cd28a07aca 100644 --- a/src/borg/public/cli_api/v1.py +++ b/src/borg/public/cli_api/v1.py @@ -56,12 +56,12 @@ class ProgressMessage(BaseBorgLogLine): class ProgressPercent(BaseBorgLogLine): operation: int - msgid: str | None = pydantic.Field(None) + msgid: typing.Optional[str] = pydantic.Field(None) finished: bool - message: str | None = pydantic.Field(None) - current: float | None = pydantic.Field(None) + message: typing.Optional[str] = pydantic.Field(None) + current: typing.Optional[float] = pydantic.Field(None) info: list[str] | None = pydantic.Field(None) - total: float | None = pydantic.Field(None) + total: typing.Optional[float] = pydantic.Field(None) time: float @pydantic.model_validator(mode="after") @@ -154,7 +154,7 @@ class _BorgDetailedArchive(_BorgArchive): stats: _BorgArchiveStatistics limits: _BorgLimitUsage command_line: typing.List[str] - chunker_params: typing.Any | None = None + chunker_params: typing.Optional[typing.Any] = None class BorgCreateResult(pydantic.BaseModel):