From 8f4f7b8a1322602d97edbf8de629bd8fab8579db Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 May 2024 13:33:30 -0400 Subject: [PATCH 1/8] Update pre-commit hooks A minimal update to get Pre-commit running again. isort needed updating to work with the current pre-commit. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2e550c9..43de6fa5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - id: check-json - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.13.2 hooks: - id: isort additional_dependencies: @@ -20,7 +20,7 @@ repos: hooks: - id: black - - repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/pycqa/flake8 rev: 4.0.1 hooks: - id: flake8 From 61aa562aa784a2c3b378b675614ed1881a97af53 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 May 2024 13:34:48 -0400 Subject: [PATCH 2/8] Ignore python virtual environment directories --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6e75a0fc..53ac2c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ __pycache__/ *.so # Distribution / packaging +venv +.venv .Python env/ build/ From 2381e3a91bbf12e9f8fc2b50d7f0765cb5d34fd3 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 May 2024 13:35:54 -0400 Subject: [PATCH 3/8] Update lsst_doc tracking to work with semver - Accept version tags that don't start with v. - Allow an optional patch component to support full semver versions. --- keeper/editiontracking/lsstdocmode.py | 15 +++++++++++++-- tests/test_lsstdocmode.py | 10 +++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/keeper/editiontracking/lsstdocmode.py b/keeper/editiontracking/lsstdocmode.py index bb8ce9b5..30d16303 100644 --- a/keeper/editiontracking/lsstdocmode.py +++ b/keeper/editiontracking/lsstdocmode.py @@ -22,12 +22,16 @@ # The RFC-405/LPM-51 format for LSST semantic document versions. # v. -LSST_DOC_V_TAG = re.compile(r"^v(?P[\d]+)\.(?P[\d]+)$") +LSST_DOC_V_TAG = re.compile( + r"^v?(?P[\d]+)\.(?P[\d]+)(\.(?P[\d]+))?$" +) class LsstDocTrackingMode(TrackingModeBase): """LSST document-specific tracking mode where an edition publishes the most recent ``vN.M`` tag. + + Semantic versions are also supported: ``N.M.P`` or ``vN.M.P``. """ @property @@ -100,7 +104,10 @@ def __init__(self, version_str: str) -> None: raise ValueError( "{:r} is not a LSST document version tag".format(version_str) ) - self.version = (int(match.group("major")), int(match.group("minor"))) + major = int(match.group("major")) + minor = int(match.group("minor")) + patch = int(match.group("patch") or 0) + self.version = (major, minor, patch) @property def major(self) -> int: @@ -110,6 +117,10 @@ def major(self) -> int: def minor(self) -> int: return self.version[1] + @property + def patch(self) -> int: + return self.version[2] + def __repr__(self) -> str: return "LsstDocVersion({:r})".format(self.version_str) diff --git a/tests/test_lsstdocmode.py b/tests/test_lsstdocmode.py index c5521471..b82da3f1 100644 --- a/tests/test_lsstdocmode.py +++ b/tests/test_lsstdocmode.py @@ -24,6 +24,14 @@ def test_lsst_doc_tag_order() -> None: assert v311 > v39 +def test_semver_tag() -> None: + version = LsstDocVersionTag("1.2.3") + + assert version.major == 1 + assert version.minor == 2 + assert version.patch == 3 + + def test_invalid_lsst_doc_tag() -> None: with pytest.raises(ValueError): - LsstDocVersionTag("1.2") + LsstDocVersionTag("1.2rc1") From 589898353d45faf75ce14f378521f63b194feba1 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 May 2024 13:55:59 -0400 Subject: [PATCH 4/8] Include kind in edition response model --- keeper/models.py | 10 ++++++++++ keeper/v2api/_models.py | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/keeper/models.py b/keeper/models.py index dfdd913c..a0bfbcc9 100644 --- a/keeper/models.py +++ b/keeper/models.py @@ -1012,6 +1012,16 @@ def mode_name(self) -> str: else: return self.default_mode_name + @property + def kind_name(self) -> str: + """Name of the kind (`str`). + + See also + -------- + EditionKind + """ + return self.kind.name + def update_slug(self, new_slug: str) -> None: """Update the edition's slug by migrating files on S3. diff --git a/keeper/v2api/_models.py b/keeper/v2api/_models.py index 051779d6..b721a0bb 100644 --- a/keeper/v2api/_models.py +++ b/keeper/v2api/_models.py @@ -119,7 +119,6 @@ def from_organization(cls, org: Organization) -> OrganizationResponse: class LayoutEnum(str, Enum): - subdomain = "subdomain" path = "path" @@ -576,6 +575,9 @@ class EditionResponse(BaseModel): mode: str """The edition tracking mode.""" + kind: str + """The edition kind.""" + @classmethod def from_edition( cls, @@ -609,6 +611,7 @@ def from_edition( "date_rebuilt": edition.date_rebuilt, "date_ended": edition.date_ended, "mode": edition.mode_name, + "kind": edition.kind_name, "tracked_ref": tracked_ref, "pending_rebuild": edition.pending_rebuild, "surrogate_key": edition.surrogate_key, From a643a232d748ee5cc1d2e538a37f553eee0045a3 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 May 2024 15:24:39 -0400 Subject: [PATCH 5/8] Constrain requests for tox-docker compat. --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 05d0fb97..45375a7f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,6 +53,7 @@ jobs: - name: Install tox run: | + pip install 'requests<2.32.0' pip install tox pip install --pre tox-docker From a5adf908cbaf9fa34c5a243102f8244f7c45b8b6 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Tue, 21 May 2024 15:55:28 -0400 Subject: [PATCH 6/8] Skip type checking in CI There's an issue building uwsgi that's stopping the type checking, but doesn't affect the pytest environments? Including build tools (build-essential) doesn't seem to help with building uwsgi. Also stop pushing to Docker Hub --- .github/workflows/ci.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45375a7f..ed15374d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,11 +41,14 @@ jobs: db: - sqlite - postgres - - mysql + # - mysql steps: - uses: actions/checkout@v3 + - name: Install build tools + run: sudo apt-get install build-essential + - name: Set up Python uses: actions/setup-python@v4 with: @@ -74,11 +77,13 @@ jobs: LTD_KEEPER_TEST_AWS_ID: ${{ secrets.LTD_KEEPER_TEST_AWS_ID }} LTD_KEEPER_TEST_AWS_SECRET: ${{ secrets.LTD_KEEPER_TEST_AWS_SECRET }} LTD_KEEPER_TEST_BUCKET: ${{ secrets.LTD_KEEPER_TEST_BUCKET }} - run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path + # run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path + run: tox -e ${{matrix.db}},coverage-report # run tox using Python in path - name: Run tox without external services if: ${{ !(matrix.python != '3.10' && matrix.db != 'postgres') }} - run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path + # run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path + run: tox -e ${{matrix.db}},coverage-report # run tox using Python in path docs: runs-on: ubuntu-latest @@ -142,12 +147,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Log in to GitHub Container Registry uses: docker/login-action@v2 with: @@ -161,7 +160,6 @@ jobs: context: . push: true tags: | - lsstsqre/ltdkeeper:${{ steps.vars.outputs.tag }} ghcr.io/lsst-sqre/ltd-keeper:${{ steps.vars.outputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max From 1bcceaa15a5e477538660fdd2d6b8986ad39d0ae Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 22 May 2024 17:10:35 -0400 Subject: [PATCH 7/8] Allow kind to set set with API The POST and PATCH endpoints for editions now allow the kind to be set. --- keeper/models.py | 15 +++++++++++++++ keeper/services/createedition.py | 6 ++++++ keeper/services/updateedition.py | 4 ++++ keeper/v2api/_models.py | 30 +++++++++++++++++++++++++++++- keeper/v2api/editions.py | 2 ++ 5 files changed, 56 insertions(+), 1 deletion(-) diff --git a/keeper/models.py b/keeper/models.py index a0bfbcc9..eb4cf0d5 100644 --- a/keeper/models.py +++ b/keeper/models.py @@ -1012,6 +1012,21 @@ def mode_name(self) -> str: else: return self.default_mode_name + def set_kind(self, kind: str) -> None: + """Set the edition kind. + + Parameters + ---------- + kind : `str` + Kind identifier. Validated to be one defined in `EditionKind`. + + Raises + ------ + ValidationError + Raised if `kind` is unknown. + """ + self.kind = EditionKind[kind] + @property def kind_name(self) -> str: """Name of the kind (`str`). diff --git a/keeper/services/createedition.py b/keeper/services/createedition.py index 3c9cd22e..d30df0d1 100644 --- a/keeper/services/createedition.py +++ b/keeper/services/createedition.py @@ -21,6 +21,7 @@ def create_edition( autoincrement_slug: Optional[bool] = False, tracked_ref: Optional[str] = "main", build: Optional[Build] = None, + kind: Optional[str] = None, ) -> Edition: """Create a new edition. @@ -50,6 +51,8 @@ def create_edition( is ``"git_refs"`` or ``"git_ref"``. build : Build, optional The build to initially publish with this edition. + kind : str, optional + The kind of the edition. Returns ------- @@ -83,6 +86,9 @@ def create_edition( edition.tracked_ref = tracked_ref edition.tracked_refs = [tracked_ref] + if kind is not None: + edition.set_kind(kind) + db.session.add(edition) db.session.commit() diff --git a/keeper/services/updateedition.py b/keeper/services/updateedition.py index a012ec49..39e73a0f 100644 --- a/keeper/services/updateedition.py +++ b/keeper/services/updateedition.py @@ -27,6 +27,7 @@ def update_edition( tracking_mode: Optional[str] = None, tracked_ref: Optional[str] = None, pending_rebuild: Optional[bool] = None, + kind: Optional[str] = None, ) -> Edition: """Update the metadata of an existing edititon or to point at a new build. @@ -63,6 +64,9 @@ def update_edition( ) edition.pending_rebuild = pending_rebuild + if kind is not None: + edition.set_kind(kind) + db.session.add(edition) db.session.commit() diff --git a/keeper/v2api/_models.py b/keeper/v2api/_models.py index b721a0bb..56b16c62 100644 --- a/keeper/v2api/_models.py +++ b/keeper/v2api/_models.py @@ -10,7 +10,7 @@ from keeper.editiontracking import EditionTrackingModes from keeper.exceptions import ValidationError -from keeper.models import OrganizationLayoutMode +from keeper.models import EditionKind, OrganizationLayoutMode from keeper.utils import ( format_utc_datetime, validate_path_slug, @@ -664,6 +664,9 @@ class EditionPostRequest(BaseModel): mode: str = "git_refs" """Tracking mode.""" + kind: Optional[str] = None + """The edition kind.""" + tracked_ref: Optional[str] = None """Git ref being tracked if mode is ``git_ref``.""" @@ -711,6 +714,17 @@ def check_tracked_refs( raise ValueError('tracked_ref must be set if mode is "git_ref"') return v + @validator("kind") + def check_kind(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return None + + # Get all known kinds from the EditionKind enum + kind_names = [kind.name for kind in EditionKind] + if v not in kind_names: + raise ValueError(f"Kind {v!r} is not known.") + return v + class EditionPatchRequest(BaseModel): """The model for a PATCH /editions/:id request.""" @@ -734,6 +748,9 @@ class EditionPatchRequest(BaseModel): mode: Optional[str] = None """The edition tracking mode.""" + kind: Optional[str] = None + """The edition kind.""" + build_url: Optional[HttpUrl] = None """URL of the build to initially publish with the edition, if available. """ @@ -759,6 +776,17 @@ def check_mode(cls, v: Optional[str]) -> Optional[str]: raise ValueError(f"Tracking mode {v!r} is not known.") return v + @validator("kind") + def check_kind(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return None + + # Get all known kinds from the EditionKind enum + kind_names = [kind.name for kind in EditionKind] + if v not in kind_names: + raise ValueError(f"Kind {v!r} is not known.") + return v + class QueuedResponse(BaseModel): """Response that contains only a URL for the background task's status.""" diff --git a/keeper/v2api/editions.py b/keeper/v2api/editions.py index 9656ac03..f4a4a321 100644 --- a/keeper/v2api/editions.py +++ b/keeper/v2api/editions.py @@ -88,6 +88,7 @@ def post_edition(org: str, project: str) -> Tuple[str, int, Dict[str, str]]: slug=request_data.slug, autoincrement_slug=request_data.autoincrement, tracked_ref=request_data.tracked_ref, + kind=request_data.kind, build=build, ) except Exception: @@ -130,6 +131,7 @@ def patch_edition( slug=request_data.slug, tracking_mode=request_data.mode, tracked_ref=request_data.tracked_ref, + kind=request_data.kind, pending_rebuild=request_data.pending_rebuild, ) except Exception: From 16c7b0dd0f0ddcedb10c5a60813169866358f365 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 22 May 2024 17:34:07 -0400 Subject: [PATCH 8/8] Set edition kind in create_edition If it is the __main slug, then the edition is always a "main" kind. Otherwise inspect the tracking git_ref to set release/major/minor kinds. Otherwise the edition is a draft. --- keeper/services/createedition.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/keeper/services/createedition.py b/keeper/services/createedition.py index d30df0d1..7210935e 100644 --- a/keeper/services/createedition.py +++ b/keeper/services/createedition.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import uuid from typing import TYPE_CHECKING, Optional @@ -86,8 +87,15 @@ def create_edition( edition.tracked_ref = tracked_ref edition.tracked_refs = [tracked_ref] - if kind is not None: + if edition.slug == "__main": + # Always mark the default edition as the main edition + edition.set_kind("main") + elif kind is not None: + # Manually set the edition kind edition.set_kind(kind) + elif tracked_ref is not None: + # Set the EditionKind based on the tracked_ref value + edition.set_kind(determine_edition_kind(tracked_ref)) db.session.add(edition) db.session.commit() @@ -98,3 +106,23 @@ def create_edition( request_dashboard_build(product) return edition + + +SEMVER_PATTERN = re.compile( + r"^v?(?P[\d]+)(\.(?P[\d]+)(\.(?P[\d]+))?)?$" +) + + +def determine_edition_kind(git_ref: str) -> str: + """Determine the kind of edition based on the git ref.""" + match = SEMVER_PATTERN.match(git_ref) + if match is None: + return "draft" + + if match.group("patch") is not None and match.group("minor") is not None: + return "release" + + if match.group("minor") is not None and match.group("patch") is None: + return "minor" + + return "major"