Merge pull request #96 from lsst-sqre/u/jsickcodes/path-based-publish…

…ed-urls

Add support for path-based project layouts

keeper/models.py

@@ -478,7 +478,7 @@ class Product(db.Model):  # type: ignore
     root_domain = db.Column(db.Unicode(255), nullable=False)
     """Root domain name serving docs (e.g., lsst.io)."""
 
-    root_fastly_domain = db.Column(db.Unicode(255), nullable=False)
+    root_fastly_domain = db.Column(db.Unicode(255), nullable=True)
     """Fastly CDN domain name (without doc's domain prepended)."""
 
     bucket_name = db.Column(db.Unicode(255), nullable=True)
@@ -517,10 +517,14 @@ def domain(self) -> str:
         (E.g. ``product.lsst.io`` if ``product`` is the slug and ``lsst.io``
         is the ``root_domain``.)
         """
-        return ".".join((self.slug, self.root_domain))
+        root_domain = self.organization.root_domain
+        if self.organization.layout == OrganizationLayoutMode.subdomain:
+            return ".".join((self.slug, self.root_domain))
+        else:
+            return root_domain
 
     @property
-    def fastly_domain(self) -> str:
+    def fastly_domain(self) -> Optional[str]:
         """Domain where Fastly serves content from for this product."""
         # Note that in non-ssl contexts fastly wants you to prepend the domain
         # to fastly's origin domain. However we don't do this with TLS.
@@ -529,8 +533,21 @@ def fastly_domain(self) -> str:
 
     @property
     def published_url(self) -> str:
-        """URL where this product is published to the end-user."""
-        parts = ("https", self.domain, "", "", "", "")
+        """URL where this product is published to the end-user.
+
+        This domain *does not* end with a trailing /.
+        """
+        layout_mode = self.organization.layout
+        if layout_mode == OrganizationLayoutMode.path:
+            # Sub-path based layout
+            if self.organization.root_path_prefix.endswith("/"):
+                path = f"{self.organization.root_path_prefix}{self.slug}"
+            else:
+                path = f"{self.organization.root_path_prefix}/{self.slug}"
+            parts = ("https", self.domain, path, "", "", "")
+        else:
+            # Domain-based layout
+            parts = ("https", self.domain, "", "", "", "")
         return urllib.parse.urlunparse(parts)
 
     @property
@@ -649,15 +666,10 @@ def bucket_root_dirname(self) -> str:
     @property
     def published_url(self) -> str:
         """URL where this build is published to the end-user."""
-        parts = (
-            "https",
-            self.product.domain,
-            "/builds/{0}".format(self.slug),
-            "",
-            "",
-            "",
-        )
-        return urllib.parse.urlunparse(parts)
+        product_root_url = self.product.published_url
+        if not product_root_url.endswith("/"):
+            product_root_url = f"{product_root_url}/"
+        return f"{product_root_url}builds/{self.slug}"
 
     def register_uploaded_build(self) -> None:
         """Register that a build is uploaded and determine what editions should
@@ -799,19 +811,14 @@ def bucket_root_dirname(self) -> str:
     @property
     def published_url(self) -> str:
         """URL where this edition is published to the end-user."""
+        product_root_url = self.product.published_url
         if self.slug == "main":
-            # Special case for main; published at product's root
-            parts = ("https", self.product.domain, "", "", "", "")
+            # Special case for main; published at the product's base path
+            return product_root_url
         else:
-            parts = (
-                "https",
-                self.product.domain,
-                "/v/{0}".format(self.slug),
-                "",
-                "",
-                "",
-            )
-        return urllib.parse.urlunparse(parts)
+            if not product_root_url.endswith("/"):
+                product_root_url = f"{product_root_url}/"
+            return f"{product_root_url}v/{self.slug}"
 
     def should_rebuild(self, build: Build) -> bool:
         """Determine whether the edition should be rebuilt to show a certain

keeper/services/createproduct.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING, Optional, Tuple
 
 import keeper.route53
-from keeper.models import Product, db
+from keeper.models import OrganizationLayoutMode, Product, db
 
 from .createedition import create_edition
 from .requestdashboardbuild import request_dashboard_build
@@ -61,7 +61,8 @@ def create_product(
     product.root_fastly_domain = org.fastly_domain
     product.bucket_name = org.bucket_name
 
-    configure_subdomain(product)
+    if org.layout == OrganizationLayoutMode.subdomain:
+        configure_subdomain(product)
 
     db.session.add(product)
     db.session.flush()  # Because Edition._validate_slug does not autoflush
@@ -91,6 +92,10 @@ def configure_subdomain(product: Product) -> None:
     organization = product.organization
     aws_id = organization.aws_id
     aws_secret = organization.get_aws_secret_key()
+    if product.fastly_domain is None:
+        raise RuntimeError(
+            "Fastly domain is not set on subdomain-based layout."
+        )
     if aws_id is not None and aws_secret is not None:
         keeper.route53.create_cname(
             product.domain,

migrations/versions/8c431c5e70a8_v2_tables.py

@@ -147,6 +147,8 @@ def upgrade():
     op.execute("UPDATE products SET organization_id = 1")
     # Make products.organization_id non-nullable
     op.alter_column("products", "organization_id", nullable=False)
+    # Make root_fastly_domain nullable
+    op.alter_column("products", "root_fastly_domain", nullable=True)
 
 
 def downgrade():
@@ -169,3 +171,6 @@ def downgrade():
     op.drop_table("tags")
     op.drop_table("dashboardtemplates")
     op.drop_table("organizations")
+
+    # Make root_fastly_domain non-nullable
+    op.alter_column("products", "root_fastly_domain", nullable=False)

tests/v2api/test_project_path_layout.py

@@ -0,0 +1,165 @@
+"""Test /v2/ APIs for projects with a path-based layout."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from mock import MagicMock
+
+from keeper.testutils import MockTaskQueue
+
+if TYPE_CHECKING:
+    from unittest.mock import Mock
+
+    from keeper.testutils import TestClient
+
+
+def test_projects(client: TestClient, mocker: Mock) -> None:
+    task_queue = mocker.patch(
+        "keeper.taskrunner.inspect_task_queue", return_value=None
+    )
+    task_queue = MockTaskQueue(mocker)  # noqa
+
+    mock_presigned_url = {
+        "url": "https://example.com",
+        "fields": {"key": "a/b/${filename}"},
+    }
+    presign_post_mock = mocker.patch(
+        "keeper.services.createbuild.presign_post_url_for_prefix",
+        new=MagicMock(return_value=mock_presigned_url),
+    )
+    presign_post_mock = mocker.patch(
+        "keeper.services.createbuild.presign_post_url_for_directory_object",
+        new=MagicMock(return_value=mock_presigned_url),
+    )
+    s3_session_mock = mocker.patch(
+        "keeper.services.createbuild.open_s3_session"
+    )
+
+    # Create a default organization ===========================================
+    mocker.resetall()
+
+    request_data = {
+        "slug": "test1",
+        "title": "Test 1",
+        "layout": "path",
+        "domain": "www.example.org",
+        "path_prefix": "/orgprefix",
+        "bucket_name": "test-bucket",
+        "fastly_support": False,
+    }
+    r = client.post("/v2/orgs", request_data)
+    task_queue.apply_task_side_effects()
+    assert r.status == 201
+    org1_url = r.headers["Location"]
+    org1_projects_url = r.json["projects_url"]
+
+    # Get project list (should be empty) ======================================
+    mocker.resetall()
+
+    r = client.get(org1_projects_url)
+    assert r.status == 200
+    assert len(r.json) == 0
+
+    # Create a project ========================================================
+    mocker.resetall()
+
+    request_data = {
+        "slug": "alpha",
+        "title": "Alpha",
+        "source_repo_url": "https://github.com/example/alpha",
+        "default_edition_mode": "git_refs",
+    }
+    r = client.post(org1_projects_url, request_data)
+    task_queue.apply_task_side_effects()
+    assert r.status == 201
+    project1_url = r.headers["Location"]
+    project1_data = r.json
+
+    assert project1_data["organization_url"] == org1_url
+    assert project1_data["self_url"] == project1_url
+    assert project1_data["published_url"] == (
+        "https://www.example.org/orgprefix/alpha"
+    )
+    project1_builds_url = project1_data["builds_url"]
+    project1_editions_url = project1_data["editions_url"]
+    project1_default_edition_url = project1_data["default_edition"]["self_url"]
+
+    # Get project list again ==================================================
+    mocker.resetall()
+
+    r = client.get(org1_projects_url)
+    assert r.status == 200
+    assert r.json[0] == project1_data
+
+    # Create a build ==========================================================
+    mocker.resetall()
+
+    build1_request = {"git_ref": "master"}
+    r = client.post(project1_builds_url, build1_request)
+    task_queue.apply_task_side_effects()
+    s3_session_mock.assert_called_once()
+    presign_post_mock.assert_called_once()
+    assert r.status == 201
+    assert r.json["project_url"] == project1_url
+    assert r.json["date_created"] is not None
+    assert r.json["date_ended"] is None
+    assert r.json["uploaded"] is False
+    assert r.json["published_url"] == (
+        "https://www.example.org/orgprefix/alpha/builds/1"
+    )
+    assert "post_prefix_urls" in r.json
+    assert "post_dir_urls" in r.json
+    assert len(r.json["surrogate_key"]) == 32  # should be a uuid4 -> hex
+    build1_url = r.headers["Location"]
+
+    # ========================================================================
+    # List builds
+    mocker.resetall()
+
+    r = client.get(project1_builds_url)
+    assert len(r.json) == 1
+
+    # ========================================================================
+    # Register upload
+    mocker.resetall()
+
+    r = client.patch(build1_url, {"uploaded": True})
+    task_queue.apply_task_side_effects()
+    assert r.status == 202
+
+    task_queue.assert_launched_once()
+    task_queue.assert_edition_build_v2(
+        project1_default_edition_url,
+        build1_url,
+    )
+    task_queue.assert_dashboard_build_v2(project1_url)
+
+    r = client.get(build1_url)
+    assert r.json["uploaded"] is True
+
+    # =========================================================================
+    # Get the default edition
+    mocker.resetall()
+    r = client.get(project1_default_edition_url)
+    assert r.status == 200
+    data = r.json
+
+    assert data["build_url"] == build1_url
+    assert data["published_url"] == ("https://www.example.org/orgprefix/alpha")
+
+    # =========================================================================
+    # Create a new edition
+    mocker.resetall()
+    edition2_data = {
+        "slug": "another-edition",
+        "title": "Another edition",
+        "build_url": build1_url,
+        "mode": "manual",
+    }
+    r = client.post(project1_editions_url, edition2_data)
+    task_queue.apply_task_side_effects()
+    assert r.status == 202
+    assert r.json["published_url"] == (
+        "https://www.example.org/orgprefix/alpha/v/another-edition"
+    )