From 79b284c6c2b05decea9d72cc3cd6ac95b4d3ecfa Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Tue, 16 Jul 2024 22:54:49 -0400 Subject: [PATCH 01/33] add pop table and run migrations --- .../versions/cb7cd5fd35d8_pop_table.py | 61 +++++++++++++++++++ backend/app/models.py | 16 ++++- backend/requirements.txt | 6 +- 3 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py diff --git a/backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py b/backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py new file mode 100644 index 00000000..8cb3627b --- /dev/null +++ b/backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py @@ -0,0 +1,61 @@ +"""pop table + +Revision ID: cb7cd5fd35d8 +Revises: +Create Date: 2024-07-16 22:47:09.900729 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import text +from sqlmodel.sql import sqltypes +import geoalchemy2 + + +# revision identifiers, used by Alembic. +revision: str = "cb7cd5fd35d8" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) + op.create_table( + "population", + sa.Column("path", sqltypes.AutoString(), nullable=False), + sa.Column("area_land", sa.Integer(), nullable=False), + sa.Column("area_water", sa.Integer(), nullable=False), + sa.Column("other_pop", sa.Integer(), nullable=False), + sa.Column("total_pop", sa.Integer(), nullable=False), + sa.Column( + "geography", + geoalchemy2.types.Geometry( + geometry_type="POLYGON", + srid=4269, + from_text="ST_GeomFromEWKT", + name="geometry", + ), + nullable=True, + ), + sa.PrimaryKeyConstraint("path"), + ) + # NOTE: geospatial indices are created by default + # op.create_index('idx_population_geography', 'population', ['geography'], unique=False, postgresql_using='gist') + op.create_index(op.f("ix_population_path"), "population", ["path"], unique=True) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_population_path"), table_name="population") + op.drop_index( + "idx_population_geography", table_name="population", postgresql_using="gist" + ) + op.drop_table("population") + op.execute(text("DROP EXTENSION IF EXISTS postgis")) + # ### end Alembic commands ### diff --git a/backend/app/models.py b/backend/app/models.py index 366932e2..0ead70e9 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,7 +1,8 @@ from datetime import datetime -from typing import Optional, Dict +from typing import AnyStr, Optional, Dict from pydantic import BaseModel, Field as PydanticField -from sqlmodel import Field, SQLModel, UUID, TIMESTAMP, text +from sqlmodel import Field, SQLModel, UUID, TIMESTAMP, text, Column +from geoalchemy2 import Geometry # Postgres @@ -32,6 +33,17 @@ class TimeStampMixin(SQLModel): ) +class Population(SQLModel, table=True): + path: str = Field(unique=True, nullable=False, index=True, primary_key=True) + area_land: int + area_water: int + other_pop: int + total_pop: int + geography: AnyStr = Field( + sa_column=Column(Geometry(geometry_type="POLYGON", srid=4269)) + ) + + # MongoDB PLAN_COLLECTION_NAME = "plans" diff --git a/backend/requirements.txt b/backend/requirements.txt index 64b5cb6e..e7e659a4 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -23,6 +23,7 @@ dnspython==2.6.1 # via pymongo duckdb==0.10.1 fastapi==0.110.0 +geoalchemy2==0.15.2 h11==0.14.0 # via # httpcore @@ -45,7 +46,9 @@ numpy==1.26.4 # pandas # pyarrow packaging==24.0 - # via pytest + # via + # geoalchemy2 + # pytest pandas==2.2.1 pluggy==1.5.0 # via pytest @@ -83,6 +86,7 @@ sniffio==1.3.1 sqlalchemy==2.0.29 # via # alembic + # geoalchemy2 # sqlmodel sqlmodel==0.0.16 starlette==0.36.3 From 391c3d243f4438e48291997127c43c294aea393e Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Tue, 16 Jul 2024 23:15:28 -0400 Subject: [PATCH 02/33] bleh --- backend/app/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/models.py b/backend/app/models.py index 0ead70e9..638dedee 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import AnyStr, Optional, Dict +from typing import Any, Optional, Dict from pydantic import BaseModel, Field as PydanticField from sqlmodel import Field, SQLModel, UUID, TIMESTAMP, text, Column from geoalchemy2 import Geometry @@ -39,7 +39,7 @@ class Population(SQLModel, table=True): area_water: int other_pop: int total_pop: int - geography: AnyStr = Field( + geography: Any = Field( sa_column=Column(Geometry(geometry_type="POLYGON", srid=4269)) ) From 48c75ec4d56b7a049c0c15889d8071de4d074647 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 00:43:42 -0400 Subject: [PATCH 03/33] spin up and down test db --- backend/app/test_main.py | 55 +++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/backend/app/test_main.py b/backend/app/test_main.py index eb3c8edc..ddb39de6 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -1,14 +1,41 @@ +import os from pymongo.results import InsertOneResult import pytest from fastapi.testclient import TestClient -from sqlmodel import Session, SQLModel, create_engine -from sqlmodel.pool import StaticPool +from sqlmodel import Session, create_engine from app.main import app, get_session from app.core.db import get_mongo_database +from pydantic_core import MultiHostUrl +from sqlalchemy import text +from sqlalchemy.exc import OperationalError, ProgrammingError +import subprocess + client = TestClient(app) +POSTGRES_TEST_DB = "districtr_test" +POSTGRES_TEST_SCHEME = "postgresql+psycopg" +POSTGRES_TEST_USER = "postgres" +POSTGRES_TEST_HOST = "localhost" +POSTGRES_TEST_PORT = 5432 + +my_env = os.environ.copy() + +my_env["POSTGRES_DB"] = POSTGRES_TEST_DB +my_env["POSTGRES_SCHEME"] = POSTGRES_TEST_SCHEME +my_env["POSTGRES_USER"] = POSTGRES_TEST_USER +my_env["POSTGRES_SERVER"] = POSTGRES_TEST_HOST +my_env["POSTGRES_PORT"] = str(POSTGRES_TEST_PORT) + +TEST_SQLALCHEMY_DATABASE_URI = MultiHostUrl.build( + scheme=POSTGRES_TEST_SCHEME, + username=POSTGRES_TEST_USER, + host=POSTGRES_TEST_HOST, + port=POSTGRES_TEST_PORT, + path=POSTGRES_TEST_DB, +) + def test_read_main(): response = client.get("/") @@ -26,12 +53,28 @@ def test_get_session(): ## Test DB +@pytest.fixture(scope="session", autouse=True, name="engine") +def engine_fixture(request): + _engine = create_engine("postgresql://postgres@/postgres") + conn = _engine.connect() + conn.execute(text("commit")) + try: + conn.execute(text(f"CREATE DATABASE {POSTGRES_TEST_DB}")) + except (OperationalError, ProgrammingError): + pass + + subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) + + def teardown(): + conn.execute(text(f"DROP DATABASE {POSTGRES_TEST_DB}")) + conn.close() + + request.addfinalizer(teardown) + + @pytest.fixture(name="session") def session_fixture(): - engine = create_engine( - "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool - ) - SQLModel.metadata.create_all(engine) + engine = create_engine(str(TEST_SQLALCHEMY_DATABASE_URI), echo=True) with Session(engine) as session: yield session From 5bb4bb265c6c4f5a8fcc6e29ce60274c3d3a8aa6 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 00:49:27 -0400 Subject: [PATCH 04/33] try this --- .github/workflows/test-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index c8b49be7..b2c5b6e1 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -18,7 +18,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements.txt --no-cache-dir working-directory: backend - name: Lint with Ruff run: | From e44f6919e76059a9b8b25990de3a485842a2cfb1 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 00:54:57 -0400 Subject: [PATCH 05/33] run on workflow update too --- .github/workflows/test-backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index b2c5b6e1..fbed7676 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -4,6 +4,7 @@ on: push: paths: - "backend/**" + - ".github/workflows/test-backend.yml" jobs: build: From 4ab8178821bd73d45910bc0ef51ffa2d151902d5 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 00:59:40 -0400 Subject: [PATCH 06/33] add postgres --- .github/workflows/test-backend.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index fbed7676..1f46756d 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -31,6 +31,8 @@ jobs: run: docker network create test_network - name: Start MongoDB run: docker run -d --network test_network --name mongo -p 27017:27017 mongo:latest + - name: Start Postgres + run: docker run -d --network test_network --name postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres -p 5432:5432 postgres:16.3-alpine3.20 - name: Build app image run: cp .env.dev .env && docker build -t districtr . working-directory: backend From a1ee61a82c1835e582856a4241695a883544820f Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 09:55:06 -0400 Subject: [PATCH 07/33] try this --- .github/workflows/test-backend.yml | 45 ++++++++++++++++++++++-------- backend/app/core/config.py | 2 +- backend/app/test_main.py | 34 ++++++++++++---------- backend/test.sh | 1 + 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 1f46756d..19583e16 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -7,34 +7,55 @@ on: - ".github/workflows/test-backend.yml" jobs: - build: + container-job: runs-on: ubuntu-latest + + container: python:3.12-slim-bullseye + + services: + postgres: + image: postgres:16.3-alpine3.20 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - - uses: actions/checkout@v4 + - name: Checkout repo code + uses: actions/checkout@v4 + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.12" cache: "pip" + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt --no-cache-dir working-directory: backend + - name: Lint with Ruff run: | pip install ruff ruff --output-format=github . continue-on-error: true working-directory: backend - - name: Create docker network - run: docker network create test_network - - name: Start MongoDB - run: docker run -d --network test_network --name mongo -p 27017:27017 mongo:latest - - name: Start Postgres - run: docker run -d --network test_network --name postgres -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres -p 5432:5432 postgres:16.3-alpine3.20 - - name: Build app image - run: cp .env.dev .env && docker build -t districtr . + + - name: Run migrations and test + run: alembic upgrade head && ./test.sh working-directory: backend - - name: Test - run: docker run --network test_network -e MONGODB_SERVER=mongo --rm districtr ./test.sh + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_HOST: postgres + ENVIRONMENT: test diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 72a01c45..ba03d024 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -32,7 +32,7 @@ class Settings(BaseSettings): # 60 minutes * 24 hours * 8 days = 8 days ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8 DOMAIN: str = "localhost" - ENVIRONMENT: Literal["local", "staging", "production"] = "local" + ENVIRONMENT: Literal["local", "staging", "production", "test"] = "local" @computed_field # type: ignore[misc] @property diff --git a/backend/app/test_main.py b/backend/app/test_main.py index ddb39de6..09129307 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -14,25 +14,23 @@ client = TestClient(app) +ENVIRONMENT = os.environ.get("ENVIRONMENT") POSTGRES_TEST_DB = "districtr_test" -POSTGRES_TEST_SCHEME = "postgresql+psycopg" -POSTGRES_TEST_USER = "postgres" -POSTGRES_TEST_HOST = "localhost" -POSTGRES_TEST_PORT = 5432 +POSTGRES_SCHEME = "postgresql+psycopg" +POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres") +POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "postgres") +POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") +POSTGRES_PORT = os.environ.get("POSTGRES_PORT", 5432) my_env = os.environ.copy() my_env["POSTGRES_DB"] = POSTGRES_TEST_DB -my_env["POSTGRES_SCHEME"] = POSTGRES_TEST_SCHEME -my_env["POSTGRES_USER"] = POSTGRES_TEST_USER -my_env["POSTGRES_SERVER"] = POSTGRES_TEST_HOST -my_env["POSTGRES_PORT"] = str(POSTGRES_TEST_PORT) TEST_SQLALCHEMY_DATABASE_URI = MultiHostUrl.build( - scheme=POSTGRES_TEST_SCHEME, - username=POSTGRES_TEST_USER, - host=POSTGRES_TEST_HOST, - port=POSTGRES_TEST_PORT, + scheme=POSTGRES_SCHEME, + username=POSTGRES_USER, + host=POSTGRES_HOST, + port=int(POSTGRES_PORT), path=POSTGRES_TEST_DB, ) @@ -55,15 +53,21 @@ def test_get_session(): @pytest.fixture(scope="session", autouse=True, name="engine") def engine_fixture(request): - _engine = create_engine("postgresql://postgres@/postgres") + url = f"{POSTGRES_SCHEME}://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}/postgres" + _engine = create_engine(url) conn = _engine.connect() conn.execute(text("commit")) try: - conn.execute(text(f"CREATE DATABASE {POSTGRES_TEST_DB}")) + if conn.in_transaction(): + conn.rollback() + conn.execution_options(isolation_level="AUTOCOMMIT").execute( + text(f"CREATE DATABASE {POSTGRES_TEST_DB}") + ) except (OperationalError, ProgrammingError): pass - subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) + if ENVIRONMENT != "test": + subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) def teardown(): conn.execute(text(f"DROP DATABASE {POSTGRES_TEST_DB}")) diff --git a/backend/test.sh b/backend/test.sh index 90c65a27..64c311ce 100755 --- a/backend/test.sh +++ b/backend/test.sh @@ -1,3 +1,4 @@ #!/bin/bash +alembic upgrade head python -c 'from app.core.db import create_collections; create_collections()' pytest -v From c01f25312ceebbc12eb525fc73e829b081a178b5 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 10:00:10 -0400 Subject: [PATCH 08/33] lsp release --- .github/workflows/test-backend.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 19583e16..c4f8efec 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -31,6 +31,9 @@ jobs: - name: Checkout repo code uses: actions/checkout@v4 + - name: Install lsb-release + run: apt-get update && apt-get install -y lsb-release && apt-get clean all + - name: Set up Python uses: actions/setup-python@v5 with: From 0d5026d86eafa53d3688c74a8c989fece8481fdd Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 22:29:03 -0400 Subject: [PATCH 09/33] on workflow dispatch for testing --- .github/workflows/test-backend.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index c4f8efec..6762ec4e 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -1,16 +1,18 @@ name: Test backend -on: - push: - paths: - - "backend/**" - - ".github/workflows/test-backend.yml" +# on: +# push: +# paths: +# - "backend/**" +# - ".github/workflows/test-backend.yml" + +on: workflow_dispatch jobs: container-job: runs-on: ubuntu-latest - container: python:3.12-slim-bullseye + container: python:3.12 services: postgres: @@ -31,9 +33,6 @@ jobs: - name: Checkout repo code uses: actions/checkout@v4 - - name: Install lsb-release - run: apt-get update && apt-get install -y lsb-release && apt-get clean all - - name: Set up Python uses: actions/setup-python@v5 with: From 0414f30332aba0ccc8a1802a1e161cdb1a35d73f Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 22:33:14 -0400 Subject: [PATCH 10/33] rm pip cache --- .github/workflows/test-backend.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 6762ec4e..7cb1ee8a 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -37,7 +37,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" - cache: "pip" - name: Install dependencies run: | From afe219397e61980b33daf197ff6db230e988f29d Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 22:40:30 -0400 Subject: [PATCH 11/33] add env vars --- .github/workflows/test-backend.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 7cb1ee8a..02efcc8a 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -18,9 +18,12 @@ jobs: postgres: image: postgres:16.3-alpine3.20 env: + POSTGRES_SCHEME: postgresql+psycopg POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres + POSTGRES_SERVER: postgres + POSTGRES_PORT: 5432 ports: - 5432:5432 options: >- @@ -55,8 +58,10 @@ jobs: run: alembic upgrade head && ./test.sh working-directory: backend env: + POSTGRES_SCHEME: postgresql+psycopg POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - POSTGRES_HOST: postgres + POSTGRES_SERVER: postgres + POSTGRES_PORT: 5432 ENVIRONMENT: test From d1347d1793e816d7aa427a671218f50c15750d13 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 22:48:17 -0400 Subject: [PATCH 12/33] use postgis image --- .github/workflows/test-backend.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 02efcc8a..4df25308 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -10,13 +10,13 @@ on: workflow_dispatch jobs: container-job: - runs-on: ubuntu-latest + runs-on: debian-latest container: python:3.12 services: postgres: - image: postgres:16.3-alpine3.20 + image: postgis/postgis:16-3.4 env: POSTGRES_SCHEME: postgresql+psycopg POSTGRES_USER: postgres From ff3a0560aac72cad145cb5164e92e9775012f4fc Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 22:49:40 -0400 Subject: [PATCH 13/33] run on ubuntu --- .github/workflows/test-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 4df25308..05991fa2 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -10,7 +10,7 @@ on: workflow_dispatch jobs: container-job: - runs-on: debian-latest + runs-on: ubuntu-latest container: python:3.12 From 5164dba68a2d2d05bdb3902e81593cc059965e6b Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:12:22 -0400 Subject: [PATCH 14/33] get rid of mongo --- .github/workflows/test-backend.yml | 22 +++------- backend/.env.dev | 6 --- backend/.env.production | 8 ---- backend/.env.test | 15 +++++++ backend/README.md | 25 ------------ backend/app/core/config.py | 21 ---------- backend/app/core/db.py | 40 ------------------ backend/app/main.py | 65 ++++++------------------------ backend/app/models.py | 52 +----------------------- backend/app/test_main.py | 48 ++-------------------- backend/cli.py | 21 ---------- backend/test.sh | 1 - 12 files changed, 37 insertions(+), 287 deletions(-) create mode 100644 backend/.env.test diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 05991fa2..e61dcdf9 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -1,12 +1,10 @@ name: Test backend -# on: -# push: -# paths: -# - "backend/**" -# - ".github/workflows/test-backend.yml" - -on: workflow_dispatch +on: + push: + paths: + - "backend/**" + - ".github/workflows/test-backend.yml" jobs: container-job: @@ -55,13 +53,5 @@ jobs: working-directory: backend - name: Run migrations and test - run: alembic upgrade head && ./test.sh + run: cp .env.test .env && ./test.sh working-directory: backend - env: - POSTGRES_SCHEME: postgresql+psycopg - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - POSTGRES_SERVER: postgres - POSTGRES_PORT: 5432 - ENVIRONMENT: test diff --git a/backend/.env.dev b/backend/.env.dev index ff1fccba..fc016777 100644 --- a/backend/.env.dev +++ b/backend/.env.dev @@ -13,9 +13,3 @@ POSTGRES_PASSWORD={fill-me} POSTGRES_DB=districtr POSTGRES_SERVER=localhost POSTGRES_PORT=5432 - -# MongoDB -MONGODB_SCHEME=mongodb -MONGODB_SERVER=localhost -MONGODB_PORT=27017 -MONGODB_DB=districtr diff --git a/backend/.env.production b/backend/.env.production index ac81459e..e33e511f 100644 --- a/backend/.env.production +++ b/backend/.env.production @@ -15,11 +15,3 @@ POSTGRES_PASSWORD={fill-me} POSTGRES_DB={fill-me} POSTGRES_SERVER={fill-me}.flycast POSTGRES_PORT=5432 - -# MongoDB -MONGODB_SCHEME=mongodb+srv -MONGODB_SERVER=tbd -MONGODB_PORT=tbd -MONGODB_DB=districtr -MONGODB_USER=tbd -MONGODB_PASSWORD=tbd diff --git a/backend/.env.test b/backend/.env.test new file mode 100644 index 00000000..8b3b4002 --- /dev/null +++ b/backend/.env.test @@ -0,0 +1,15 @@ +# Backend +DOMAIN=postgres +ENVIRONMENT=test +PROJECT_NAME="Districtr v2 backend" +BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173" +SECRET_KEY=mysupersecretkey + +# Postgres +DATABASE_URL=postgresql+psycopg://postgres:postgres@postgres:5432/postgres +POSTGRES_SCHEME=postgresql+psycopg +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_SERVER=postgres +POSTGRES_PORT=5432 diff --git a/backend/README.md b/backend/README.md index 0c01dbf2..e003a92c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -87,31 +87,6 @@ Or with full coverage report: `coverage run --source=app -m pytest -v && coverage html && open htmlcov/index.html` -### MongoDB - -#### MacOS - -Follow [install instructions](https://github.com/mongodb/homebrew-brew). - -#### Linux - -See [Install MongoDB Community Edition on Linux](https://www.mongodb.com/docs/manual/administration/install-on-linux/) - -#### Set-up test database - -1. `brew services start mongodb-community` on Mac to start the server. TBD other platforms. Stop the server with `brew services stop mongodb-community`. -1. `mongosh` -1. `use districtr` to create a new database in `/usr/local/var/mongodb` (intel) or `/opt/homebrew/var/mongodb` (Apple silicon). Connects to the db if it already exists. - -More info [here](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-os-x/). - -Create collections: -1. `python cli.py create_collections` - -Optionally you can create or update individual collections with `python cli.py create_collections -c {{ collection_name_1 }} -c {{ collection_name_2 }}`. - -Confirm in `mongosh` with `use districtr` followed by `show collections` or `python cli.py list-collections`. - ### Useful reference apps - [full-stack-fastapi-template](https://github.com/tiangolo/full-stack-fastapi-template/tree/master) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index ba03d024..82e4fde6 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -69,27 +69,6 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: path=self.POSTGRES_DB, ) - # MongoDB - - MONGODB_SCHEME: str = "mongodb+srv" - MONGODB_SERVER: str | int = "" - MONGODB_PORT: int = 27017 - MONGODB_USER: str = "" - MONGODB_PASSWORD: str = "" - MONGODB_DB: str = "districtr" - - @computed_field # type: ignore[misc] - @property - def MONGODB_URI(self) -> str: - if self.ENVIRONMENT == "local": - return f"{self.MONGODB_SCHEME}://{self.MONGODB_SERVER}:{self.MONGODB_PORT}/{self.MONGODB_DB}" - - assert ( - self.MONGODB_USER and self.MONGODB_PASSWORD - ), f"MONGODB_SERVER, MONGODB_USER, and MONGODB_PASSWORD must be set. Got server `{self.MONGODB_SERVER}` and user `{self.MONGODB_USER}`." - - return f"{self.MONGODB_SCHEME}://{self.MONGODB_USER}:{self.MONGODB_PASSWORD}@{self.MONGODB_SERVER}:{self.MONGODB_PORT}/{self.MONGODB_DB}" - # Security def _check_default_secret(self, var_name: str, value: str | None) -> None: diff --git a/backend/app/core/db.py b/backend/app/core/db.py index 2ce3d6cd..dcfac915 100644 --- a/backend/app/core/db.py +++ b/backend/app/core/db.py @@ -1,45 +1,5 @@ from sqlmodel import create_engine -from pymongo import MongoClient -from pymongo.database import Database from app.core.config import settings -from app.models import PLAN_COLLECTION_NAME, PLAN_COLLECTION_SCHEMA engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), echo=True) - - -COLLECTIONS = {PLAN_COLLECTION_NAME: PLAN_COLLECTION_SCHEMA} - - -def get_mongo_database() -> Database: - """ - Get MongoDB database. - - Returns: - pymongo.database.Database: MongoDB database - """ - client = MongoClient(settings.MONGODB_URI) - return client[settings.MONGODB_DB] - - -def create_collections(collections: list[str] | None) -> None: - """ - Create collections in MongoDB if they do not exist, otherwise apply migrations. - - Args: - collections (list[str]): List of collection names - """ - db = get_mongo_database() - all_collections = list(COLLECTIONS.keys()) - - if not collections: - collections = all_collections - - for collection_name in collections: - collection_schema = COLLECTIONS[collection_name] - - if collection_name not in db.list_collection_names(): - db.create_collection(collection_name, validator=collection_schema) - print(f"Collection {collection_name} created") - else: - db.command("collMod", collection_name, validator=collection_schema) diff --git a/backend/app/main.py b/backend/app/main.py index 4133255b..3b012864 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,13 +1,11 @@ from fastapi import FastAPI, status, Depends, HTTPException -from pymongo.database import Database +from sqlalchemy import text from sqlmodel import Session from starlette.middleware.cors import CORSMiddleware import logging -from bson import ObjectId -from app.core.db import engine, get_mongo_database +from app.core.db import engine from app.core.config import settings -from app.models import AssignmentsCreate, AssignmentsPublic, AssignmentsUpdate app = FastAPI() @@ -32,57 +30,18 @@ def get_session(): yield session -def get_mongodb_client(): - yield get_mongo_database() - - @app.get("/") async def root(): return {"message": "Hello World"} -@app.get("/plan/{plan_id}") -async def get_plan(plan_id: str, mongodb: Database = Depends(get_mongodb_client)): - plan = mongodb.plans.find_one({"_id": ObjectId(plan_id)}) - - if not plan: - raise HTTPException(status_code=404, detail="Plan not found") - - return str(plan) - - -@app.post( - "/plan", - status_code=status.HTTP_201_CREATED, - response_model=AssignmentsPublic, -) -async def create_plan( - *, data: AssignmentsCreate, mongodb: Database = Depends(get_mongodb_client) -): - db_plan = AssignmentsCreate.model_validate(data) - plan = mongodb.plans.insert_one(db_plan.model_dump()) - return AssignmentsPublic(_id=str(plan.inserted_id)) - - -@app.put( - "/plan/{plan_id}", - status_code=status.HTTP_200_OK, - response_model=AssignmentsUpdate, -) -async def update_plan( - *, - plan_id: str, - data: AssignmentsCreate, - mongodb: Database = Depends(get_mongodb_client), -): - db_plan = AssignmentsCreate.model_validate(data) - new_assignments = {f"assignments.{k}": v for k, v in db_plan.assignments.items()} - result = mongodb.plans.update_many( - {"_id": ObjectId(plan_id)}, {"$set": new_assignments}, upsert=True - ) - return AssignmentsUpdate( - acknowledged=result.acknowledged, - upserted_id=result.upserted_id, - matched_count=result.matched_count, - modified_count=result.modified_count, - ) +@app.get("/db_is_alive") +async def db_is_alive(session: Session = Depends(get_session)): + try: + session.execute(text("SELECT 1")) + return {"message": "DB is alive"} + except Exception as e: + logger.error(e) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="DB is unreachable" + ) diff --git a/backend/app/models.py b/backend/app/models.py index 638dedee..de82fd8d 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,11 +1,8 @@ from datetime import datetime -from typing import Any, Optional, Dict -from pydantic import BaseModel, Field as PydanticField +from typing import Any, Optional from sqlmodel import Field, SQLModel, UUID, TIMESTAMP, text, Column from geoalchemy2 import Geometry -# Postgres - class UUIDType(UUID): def __init__(self, *args, **kwargs): @@ -42,50 +39,3 @@ class Population(SQLModel, table=True): geography: Any = Field( sa_column=Column(Geometry(geometry_type="POLYGON", srid=4269)) ) - - -# MongoDB - -PLAN_COLLECTION_NAME = "plans" - -PLAN_COLLECTION_SCHEMA = { - "$jsonSchema": { - "bsonType": "object", - "required": ["assignments"], - "properties": { - "assignments": { - "bsonType": "object", - "additionalProperties": {"bsonType": "int"}, - } - }, - } -} - - -class Assignments(BaseModel): - assignments: Dict[str, int] = PydanticField(description="Assignments dictionary") - - -class AssignmentsCreate(Assignments): - pass - - -class AssignmentsUpdate(BaseModel): - """ - { - acknowledged: true, - insertedId: null, - matchedCount: 0, - modifiedCount: 0, - upsertedCount: 0 - } - """ - - acknowledged: bool = PydanticField(description="Acknowledged") - upserted_id: Optional[str] = PydanticField(description="Inserted ID") - matched_count: int = PydanticField(description="Matched count") - modified_count: int = PydanticField(description="Modified count") - - -class AssignmentsPublic(BaseModel): - id: str = PydanticField(alias="_id", description="Assignment ID") diff --git a/backend/app/test_main.py b/backend/app/test_main.py index 09129307..2cdaf2b1 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -1,11 +1,9 @@ import os -from pymongo.results import InsertOneResult import pytest from fastapi.testclient import TestClient from sqlmodel import Session, create_engine from app.main import app, get_session -from app.core.db import get_mongo_database from pydantic_core import MultiHostUrl from sqlalchemy import text from sqlalchemy.exc import OperationalError, ProgrammingError @@ -98,47 +96,7 @@ def get_auth_result_override(): app.dependency_overrides.clear() -@pytest.fixture(name="plan") -def plan_id_fixture() -> InsertOneResult: - db = get_mongo_database() - return db.plans.insert_one({"assignments": {"06067001101": 1}}) - - -def test_get_plan(client: TestClient, plan: InsertOneResult): - response = client.get(f"/plan/{plan.inserted_id}") - assert response.status_code == 200 - - -def test_create_plan(client: TestClient): - response = client.post("/plan", json={"assignments": {"06067001101": 1}}) - assert response.status_code == 201 - - -def test_update_add_to_plan(client: TestClient, plan: InsertOneResult): - response = client.put( - f"/plan/{plan.inserted_id}", json={"assignments": {"06067001102": 1}} - ) - assert response.status_code == 200 - data = response.json() - assert data["modified_count"] == 1 - assert data["upserted_id"] is None - - -def test_update_update_and_add(client: TestClient, plan: InsertOneResult): - response = client.put( - f"/plan/{plan.inserted_id}", - json={"assignments": {"06067001101": 2, "06067001102": 1}}, - ) +def test_db_is_alive(client): + response = client.get("/db_is_alive") assert response.status_code == 200 - data = response.json() - assert data["modified_count"] == 1 - assert data["matched_count"] == 1 - assert data["acknowledged"] is True - assert data["upserted_id"] is None - - -def test_get_missing_plan(client: TestClient): - response = client.get("/plan/6680e7d8b65f636e1a966c3e") - data = response.json() - assert response.status_code == 404 - assert data["detail"] == "Plan not found" + assert response.json() == {"message": "DB is alive"} diff --git a/backend/cli.py b/backend/cli.py index 827ea8aa..741ed015 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1,5 +1,4 @@ import click -from app.core.db import create_collections as _create_collections, get_mongo_database @click.group() @@ -7,25 +6,5 @@ def cli(): pass -@cli.command("list-collections") -def list_collections(): - db = get_mongo_database() - collections = db.list_collection_names() - print(collections) - - -@cli.command("create-collections") -@click.option("--collections", "-c", help="Collection name", multiple=True) -def create_collections(collections: tuple[str]): - """ - Create collections in MongoDB if they do not exist, otherwise apply migrations. - - Args: - collections (tuple[str]): Collection names. - Pass multiple collection names with `python cli.py create-collections -c collection1 -c collection2`. - """ - _create_collections(list(collections)) - - if __name__ == "__main__": cli() diff --git a/backend/test.sh b/backend/test.sh index 64c311ce..2c959736 100755 --- a/backend/test.sh +++ b/backend/test.sh @@ -1,4 +1,3 @@ #!/bin/bash alembic upgrade head -python -c 'from app.core.db import create_collections; create_collections()' pytest -v From f521b1cd50617d796b6304a19141a73563d2b8f6 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:30:21 -0400 Subject: [PATCH 15/33] close remaining connections --- backend/app/test_main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/app/test_main.py b/backend/app/test_main.py index 2cdaf2b1..f40e07e1 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -49,7 +49,7 @@ def test_get_session(): ## Test DB -@pytest.fixture(scope="session", autouse=True, name="engine") +@pytest.fixture(scope="session", name="engine") def engine_fixture(request): url = f"{POSTGRES_SCHEME}://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}/postgres" _engine = create_engine(url) @@ -68,15 +68,23 @@ def engine_fixture(request): subprocess.run(["alembic", "upgrade", "head"], check=True, env=my_env) def teardown(): + close_connections_query = f""" + SELECT pg_terminate_backend(pg_stat_activity.pid) + FROM pg_stat_activity + WHERE pg_stat_activity.datname = '{POSTGRES_TEST_DB}' + AND pid <> pg_backend_pid(); + """ + conn.execute(text(close_connections_query)) conn.execute(text(f"DROP DATABASE {POSTGRES_TEST_DB}")) conn.close() request.addfinalizer(teardown) + return create_engine(str(TEST_SQLALCHEMY_DATABASE_URI), echo=True) + @pytest.fixture(name="session") -def session_fixture(): - engine = create_engine(str(TEST_SQLALCHEMY_DATABASE_URI), echo=True) +def session_fixture(engine): with Session(engine) as session: yield session From 00486b61f4ef77e655e4fa586208c56890e518cd Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:33:56 -0400 Subject: [PATCH 16/33] try doing env vars this way --- .github/workflows/test-backend.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index e61dcdf9..62785fa0 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -53,5 +53,17 @@ jobs: working-directory: backend - name: Run migrations and test - run: cp .env.test .env && ./test.sh + run: ./test.sh working-directory: backend + env: + DOMAIN: postgres + ENVIRONMENT: test + PROJECT_NAME: Districtr v2 backend + BACKEND_CORS_ORIGINS: "http://localhost,http://localhost:5173" + SECRET_KEY: mysupersecretkey + POSTGRES_SCHEME: postgresql+psycopg + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_SERVER: postgres + POSTGRES_PORT: 5432 From 02f4e3c22878b47034a6b9d5f80d60860207445e Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:39:58 -0400 Subject: [PATCH 17/33] poll for readiness clean up env --- .github/workflows/test-backend.yml | 19 ++++++++++++++----- backend/requirements.txt | 2 -- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 62785fa0..786680f2 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -45,12 +45,21 @@ jobs: pip install -r requirements.txt --no-cache-dir working-directory: backend - - name: Lint with Ruff + - name: Wait for PostgreSQL to be ready run: | - pip install ruff - ruff --output-format=github . - continue-on-error: true - working-directory: backend + echo "Waiting for PostgreSQL to be ready..." + for i in {1..60}; do + if pg_isready -h postgres -p 5432; then + echo "PostgreSQL is ready!" + break + fi + echo "PostgreSQL is not ready yet... retrying in 1 second." + sleep 1 + done + if ! pg_isready -h postgres -p 5432; then + echo "PostgreSQL did not become ready in time." >&2 + exit 1 + fi - name: Run migrations and test run: ./test.sh diff --git a/backend/requirements.txt b/backend/requirements.txt index e7e659a4..8f8dfc49 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -20,7 +20,6 @@ coverage==7.5.1 # via pytest-cov cryptography==42.0.5 dnspython==2.6.1 - # via pymongo duckdb==0.10.1 fastapi==0.110.0 geoalchemy2==0.15.2 @@ -66,7 +65,6 @@ pydantic-core==2.16.3 # via pydantic pydantic-settings==2.2.1 pyjwt==2.8.0 -pymongo==4.8.0 pytest==8.2.0 # via pytest-cov pytest-cov==5.0.0 From 2d6ed4adb9619df2f1cb4c5e537e46eddbf8060c Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:46:47 -0400 Subject: [PATCH 18/33] remove that --- .github/workflows/test-backend.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 786680f2..58be1ad8 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -45,22 +45,6 @@ jobs: pip install -r requirements.txt --no-cache-dir working-directory: backend - - name: Wait for PostgreSQL to be ready - run: | - echo "Waiting for PostgreSQL to be ready..." - for i in {1..60}; do - if pg_isready -h postgres -p 5432; then - echo "PostgreSQL is ready!" - break - fi - echo "PostgreSQL is not ready yet... retrying in 1 second." - sleep 1 - done - if ! pg_isready -h postgres -p 5432; then - echo "PostgreSQL did not become ready in time." >&2 - exit 1 - fi - - name: Run migrations and test run: ./test.sh working-directory: backend From d195ad4c06f300833b53b7313b11361807ba9425 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:50:56 -0400 Subject: [PATCH 19/33] update env var name --- backend/app/test_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app/test_main.py b/backend/app/test_main.py index f40e07e1..ee6d37f5 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -17,7 +17,7 @@ POSTGRES_SCHEME = "postgresql+psycopg" POSTGRES_USER = os.environ.get("POSTGRES_USER", "postgres") POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "postgres") -POSTGRES_HOST = os.environ.get("POSTGRES_HOST", "localhost") +POSTGRES_SERVER = os.environ.get("POSTGRES_SERVER", "localhost") POSTGRES_PORT = os.environ.get("POSTGRES_PORT", 5432) my_env = os.environ.copy() @@ -27,7 +27,7 @@ TEST_SQLALCHEMY_DATABASE_URI = MultiHostUrl.build( scheme=POSTGRES_SCHEME, username=POSTGRES_USER, - host=POSTGRES_HOST, + host=POSTGRES_SERVER, port=int(POSTGRES_PORT), path=POSTGRES_TEST_DB, ) @@ -51,7 +51,7 @@ def test_get_session(): @pytest.fixture(scope="session", name="engine") def engine_fixture(request): - url = f"{POSTGRES_SCHEME}://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}/postgres" + url = f"{POSTGRES_SCHEME}://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_SERVER}/postgres" _engine = create_engine(url) conn = _engine.connect() conn.execute(text("commit")) From 9fb2676ca3d06beaa789e27a6732fc2f6e64a2a3 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Wed, 17 Jul 2024 23:53:14 -0400 Subject: [PATCH 20/33] add password --- backend/app/test_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/test_main.py b/backend/app/test_main.py index ee6d37f5..ce7111c1 100644 --- a/backend/app/test_main.py +++ b/backend/app/test_main.py @@ -30,6 +30,7 @@ host=POSTGRES_SERVER, port=int(POSTGRES_PORT), path=POSTGRES_TEST_DB, + password=POSTGRES_PASSWORD, ) From b1168ffe3bca32aa54cc2266fa202ec5e2df34b6 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 09:29:16 -0400 Subject: [PATCH 21/33] handle db url if provided --- backend/app/core/config.py | 4 ++++ backend/cli.py | 9 +++++++++ backend/fly.toml | 21 +++++++++++---------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 82e4fde6..f6a6e173 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -56,10 +56,14 @@ def server_host(self) -> str: POSTGRES_USER: str POSTGRES_PASSWORD: str POSTGRES_DB: str = "" + DATABASE_URL: str | None = None @computed_field # type: ignore[misc] @property def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: + if self.DATABASE_URL: + return MultiHostUrl(self.DATABASE_URL) + return MultiHostUrl.build( scheme=self.POSTGRES_SCHEME, username=self.POSTGRES_USER, diff --git a/backend/cli.py b/backend/cli.py index 741ed015..3996b03b 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1,4 +1,7 @@ import click +# import subprocess +# from pathlib import Path +# from urllib.parse import urlparse @click.group() @@ -6,5 +9,11 @@ def cli(): pass +@cli.command() +@click.option("--gpkg", "-g", help="Path or URL to GeoPackage file") +def import_gerrydb_view(): + print("Importing GerryDB view...") + + if __name__ == "__main__": cli() diff --git a/backend/fly.toml b/backend/fly.toml index 58ad5b56..6594b25c 100644 --- a/backend/fly.toml +++ b/backend/fly.toml @@ -1,4 +1,5 @@ - +# fly.toml app configuration file generated for districtr-v2-api on 2024-07-18T09:19:04-04:00 +# # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # @@ -8,14 +9,14 @@ primary_region = 'ewr' [build] [http_service] -internal_port = 8080 -force_https = true -auto_stop_machines = true -auto_start_machines = true -min_machines_running = 0 -processes = ['app'] + internal_port = 8080 + force_https = true + auto_stop_machines = true + auto_start_machines = true + min_machines_running = 0 + processes = ['app'] [[vm]] -memory = '1gb' -cpu_kind = 'shared' -cpus = 1 + memory = '1gb' + cpu_kind = 'shared' + cpus = 1 From 4147d69ec7dff52840e0bccac1a76f39d08cd2d0 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 09:35:27 -0400 Subject: [PATCH 22/33] fly deploy on merge to main --- .github/workflows/fly.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/fly.yml diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml new file mode 100644 index 00000000..4b14c1a9 --- /dev/null +++ b/.github/workflows/fly.yml @@ -0,0 +1,19 @@ +name: Fly Deploy +on: + push: + branches: + - main + paths: + - "backend/**" +jobs: + deploy: + name: Deploy app + runs-on: ubuntu-latest + concurrency: deploy-group + steps: + - uses: actions/checkout@v3 + - uses: superfly/flyctl-actions/setup-flyctl@master + - run: flyctl deploy --remote-only + working-directory: backend + env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} From 96794102b4dca888075cbb0eff43e143b9057142 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 11:40:58 -0400 Subject: [PATCH 23/33] alembic upgrade with db url if present --- .github/workflows/test-backend.yml | 2 +- backend/app/alembic/env.py | 5 +++++ backend/fly.toml | 4 +--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 58be1ad8..f2f0f0e5 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -10,7 +10,7 @@ jobs: container-job: runs-on: ubuntu-latest - container: python:3.12 + container: python:3.12.2-slim-bullseye services: postgres: diff --git a/backend/app/alembic/env.py b/backend/app/alembic/env.py index c65c4665..65ac97cc 100644 --- a/backend/app/alembic/env.py +++ b/backend/app/alembic/env.py @@ -34,6 +34,11 @@ def get_url(): + database_url = os.getenv("DATABASE_URL", None) + + if database_url: + return database_url + user = os.getenv("POSTGRES_USER", "postgres") password = os.getenv("POSTGRES_PASSWORD", "") server = os.getenv("POSTGRES_SERVER", "db") diff --git a/backend/fly.toml b/backend/fly.toml index 6594b25c..7a25693e 100644 --- a/backend/fly.toml +++ b/backend/fly.toml @@ -1,4 +1,4 @@ -# fly.toml app configuration file generated for districtr-v2-api on 2024-07-18T09:19:04-04:00 +# fly.toml app configuration file generated for districtr-v2-api on 2024-07-18T11:06:14-04:00 # # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # @@ -6,8 +6,6 @@ app = 'districtr-v2-api' primary_region = 'ewr' -[build] - [http_service] internal_port = 8080 force_https = true From 06ce80ccb4b42e8ded21cb49bf9a31ec92fdf17a Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 11:43:16 -0400 Subject: [PATCH 24/33] revert gha change --- .github/workflows/test-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index f2f0f0e5..58be1ad8 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -10,7 +10,7 @@ jobs: container-job: runs-on: ubuntu-latest - container: python:3.12.2-slim-bullseye + container: python:3.12 services: postgres: From 632b8b9bee8223a47082fe6ea8bdd39ba8456d27 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 11:47:05 -0400 Subject: [PATCH 25/33] expose test results --- .github/workflows/test-backend.yml | 4 ++-- backend/test.sh | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100755 backend/test.sh diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 58be1ad8..e300c9ef 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -45,8 +45,8 @@ jobs: pip install -r requirements.txt --no-cache-dir working-directory: backend - - name: Run migrations and test - run: ./test.sh + - name: Run tests + run: pytest -v working-directory: backend env: DOMAIN: postgres diff --git a/backend/test.sh b/backend/test.sh deleted file mode 100755 index 2c959736..00000000 --- a/backend/test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -alembic upgrade head -pytest -v From c98ba20c20adc5186b5fbe1b788ad922384cad16 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 20:06:31 -0400 Subject: [PATCH 26/33] add sentry monitoring to backend --- backend/app/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/app/main.py b/backend/app/main.py index 3b012864..2a964cb9 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -4,9 +4,17 @@ from starlette.middleware.cors import CORSMiddleware import logging +import sentry_sdk from app.core.db import engine from app.core.config import settings +if settings.ENVIRONMENT == "production": + sentry_sdk.init( + dsn="https://b14aae02017e3a9c425de4b22af7dd0c@o4507623009091584.ingest.us.sentry.io/4507623009746944", + traces_sample_rate=1.0, + profiles_sample_rate=1.0, + ) + app = FastAPI() logger = logging.getLogger(__name__) From 4c53044ec0c0e2513759150b151011b37023be08 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Thu, 18 Jul 2024 22:43:49 -0400 Subject: [PATCH 27/33] set up boto3 for r2 --- backend/app/core/config.py | 24 ++++++++++++++++++++++++ backend/app/models.py | 3 ++- backend/cli.py | 22 ++++++++++++++++++++-- backend/requirements.txt | 24 +++++++++++++++++++++--- 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index f6a6e173..9a2561c6 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -1,5 +1,6 @@ import secrets import warnings +import boto3 from functools import lru_cache from typing import Annotated, Any, Literal @@ -93,6 +94,29 @@ def _enforce_non_default_secrets(self) -> Self: return self + # R2 + + R2_BUCKET_NAME: str | None = None + ACCOUNT_ID: str | None = None + AWS_ACCESS_KEY_ID: str | None = None + AWS_SECRET_ACCESS_KEY: str | None = None + + def get_s3_client(self): + if ( + not self.ACCOUNT_ID + or not self.AWS_ACCESS_KEY_ID + or not self.AWS_SECRET_ACCESS_KEY + ): + return None + + return boto3.client( + service_name="s3", + endpoint_url=f"https://{self.ACCOUNT_ID}.r2.cloudflarestorage.com", + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + region_name="auto", + ) + @lru_cache() def get_settings(): diff --git a/backend/app/models.py b/backend/app/models.py index de82fd8d..6f2656da 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -31,7 +31,8 @@ class TimeStampMixin(SQLModel): class Population(SQLModel, table=True): - path: str = Field(unique=True, nullable=False, index=True, primary_key=True) + name: str = Field(nullable=False, index=True) + path: str = Field(unique=False, nullable=False, index=True, primary_key=False) area_land: int area_water: int other_pop: int diff --git a/backend/cli.py b/backend/cli.py index 3996b03b..34b3ef51 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1,18 +1,36 @@ import click +import logging +from app.core.config import settings # import subprocess # from pathlib import Path # from urllib.parse import urlparse +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + @click.group() def cli(): pass -@cli.command() +@cli.command("import-gerrydb-view") @click.option("--gpkg", "-g", help="Path or URL to GeoPackage file") -def import_gerrydb_view(): +def import_gerrydb_view(gpkg: str): print("Importing GerryDB view...") + s3 = settings.get_s3_client() + + if not s3: + raise ValueError("S3 client is not available") + + object_information = s3.head_object(Bucket=settings.R2_BUCKET_NAME, Key=gpkg) + + if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: + raise ValueError( + f"GeoPackage file {gpkg} not found in S3 bucket {settings.R2_BUCKET_NAME}" + ) + + logger.info("Importing GerryDB view. Got response:\n%s", object_information) if __name__ == "__main__": diff --git a/backend/requirements.txt b/backend/requirements.txt index 8f8dfc49..c8bb76dc 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,10 +8,16 @@ anyio==4.3.0 # httpx # starlette # watchfiles -certifi==2024.2.2 +boto3==1.34.145 +botocore==1.34.145 + # via + # boto3 + # s3transfer +certifi==2024.7.4 # via # httpcore # httpx + # sentry-sdk cffi==1.16.0 # via cryptography click==8.1.7 @@ -36,6 +42,10 @@ idna==3.6 # httpx iniconfig==2.0.0 # via pytest +jmespath==1.0.1 + # via + # boto3 + # botocore mako==1.3.2 # via alembic markupsafe==2.1.5 @@ -69,12 +79,17 @@ pytest==8.2.0 # via pytest-cov pytest-cov==5.0.0 python-dateutil==2.9.0.post0 - # via pandas + # via + # botocore + # pandas python-dotenv==1.0.1 # via pydantic-settings pytz==2024.1 # via pandas pyyaml==6.0.1 +s3transfer==0.10.2 + # via boto3 +sentry-sdk==2.10.0 six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -100,7 +115,10 @@ typing-extensions==4.10.0 # sqlalchemy tzdata==2024.1 # via pandas -urllib3==2.2.1 +urllib3==2.2.2 + # via + # botocore + # sentry-sdk uvicorn==0.29.0 uvloop==0.19.0 watchfiles==0.21.0 From 739947cb058b27a623da4c1da993685e9728f108 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Fri, 19 Jul 2024 00:07:51 -0400 Subject: [PATCH 28/33] works with arbitrary files but failing on r2 --- backend/app/core/config.py | 1 + backend/cli.py | 70 ++++++++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 9a2561c6..b7c2a748 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -98,6 +98,7 @@ def _enforce_non_default_secrets(self) -> Self: R2_BUCKET_NAME: str | None = None ACCOUNT_ID: str | None = None + AWS_S3_ENDPOINT: str | None = None AWS_ACCESS_KEY_ID: str | None = None AWS_SECRET_ACCESS_KEY: str | None = None diff --git a/backend/cli.py b/backend/cli.py index 34b3ef51..1028f5fd 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1,9 +1,9 @@ +import os import click import logging from app.core.config import settings -# import subprocess -# from pathlib import Path -# from urllib.parse import urlparse +import subprocess +from urllib.parse import urlparse logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -15,22 +15,64 @@ def cli(): @cli.command("import-gerrydb-view") -@click.option("--gpkg", "-g", help="Path or URL to GeoPackage file") -def import_gerrydb_view(gpkg: str): +@click.option("--layer", "-n", help="layer of the view", required=True) +@click.option("--gpkg", "-g", help="Path or URL to GeoPackage file", required=True) +def import_gerrydb_view(layer, gpkg: str): print("Importing GerryDB view...") - s3 = settings.get_s3_client() - if not s3: - raise ValueError("S3 client is not available") + url = urlparse(gpkg) + logger.info("URL: %s", url) - object_information = s3.head_object(Bucket=settings.R2_BUCKET_NAME, Key=gpkg) + kwargs = {} - if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: - raise ValueError( - f"GeoPackage file {gpkg} not found in S3 bucket {settings.R2_BUCKET_NAME}" - ) + if url.scheme == "s3": + s3 = settings.get_s3_client() - logger.info("Importing GerryDB view. Got response:\n%s", object_information) + if not s3: + raise ValueError("S3 client is not available") + + file_name = url.path.lstrip("/") + object_information = s3.head_object(Bucket=url.netloc, Key=file_name) + + if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: + raise ValueError( + f"GeoPackage file {gpkg} not found in S3 bucket {url.netloc}" + ) + + logger.info("Importing GerryDB view. Got response:\n%s", object_information) + + path = f"/vsis3/{url.netloc}/{file_name}" + + kwargs["env"] = { + **os.environ, + "AWS_S3_ENDPOINT": settings.AWS_S3_ENDPOINT, + "AWS_ACCESS_KEY_ID": settings.AWS_ACCESS_KEY_ID, + "AWS_SECRET_ACCESS_KEY": settings.AWS_SECRET_ACCESS_KEY, + } + else: + path = gpkg + + result = subprocess.run( + args=[ + "ogr2ogr", + "-f", + "PostgreSQL", + f"PG:host={settings.POSTGRES_SERVER} port={settings.POSTGRES_PORT} dbname={settings.POSTGRES_DB} user={settings.POSTGRES_USER} password={settings.POSTGRES_PASSWORD}", + path, + layer, # must match layer name in gpkg + "-lco", + "OVERWRITE=yes", + "-nln", + layer, + ], + **kwargs, + ) + + if result.returncode != 0: + logger.error("ogr2ogr failed. Got %s", result) + raise ValueError(f"ogr2ogr failed with return code {result.returncode}") + + logger.info("GerryDB view imported successfully") if __name__ == "__main__": From d541d8930ac73b0534cb8f29ee69c1a041a9990a Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sat, 20 Jul 2024 10:05:52 -0400 Subject: [PATCH 29/33] config fly volumes --- backend/.env.dev | 3 +++ backend/.env.production | 3 +++ backend/.env.test | 3 +++ backend/app/core/config.py | 4 ++++ backend/fly.toml | 16 +++++++++++++++- 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/backend/.env.dev b/backend/.env.dev index fc016777..f76423ab 100644 --- a/backend/.env.dev +++ b/backend/.env.dev @@ -13,3 +13,6 @@ POSTGRES_PASSWORD={fill-me} POSTGRES_DB=districtr POSTGRES_SERVER=localhost POSTGRES_PORT=5432 + +# Volumes +VOLUME_PATH=/data diff --git a/backend/.env.production b/backend/.env.production index e33e511f..688e1c75 100644 --- a/backend/.env.production +++ b/backend/.env.production @@ -15,3 +15,6 @@ POSTGRES_PASSWORD={fill-me} POSTGRES_DB={fill-me} POSTGRES_SERVER={fill-me}.flycast POSTGRES_PORT=5432 + +# Volumes +VOLUME_PATH=/data diff --git a/backend/.env.test b/backend/.env.test index 8b3b4002..490f4626 100644 --- a/backend/.env.test +++ b/backend/.env.test @@ -13,3 +13,6 @@ POSTGRES_PASSWORD=postgres POSTGRES_DB=postgres POSTGRES_SERVER=postgres POSTGRES_PORT=5432 + +# Volumes +VOLUME_PATH=/data diff --git a/backend/app/core/config.py b/backend/app/core/config.py index b7c2a748..2ef0fd07 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -94,6 +94,10 @@ def _enforce_non_default_secrets(self) -> Self: return self + # Volumes + + VOLUME_PATH: str = "/data" + # R2 R2_BUCKET_NAME: str | None = None diff --git a/backend/fly.toml b/backend/fly.toml index 7a25693e..e9819949 100644 --- a/backend/fly.toml +++ b/backend/fly.toml @@ -1,4 +1,4 @@ -# fly.toml app configuration file generated for districtr-v2-api on 2024-07-18T11:06:14-04:00 +# fly.toml app configuration file generated for districtr-v2-api on 2024-07-20T10:05:11-04:00 # # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # @@ -6,6 +6,20 @@ app = 'districtr-v2-api' primary_region = 'ewr' +[env] + BACKEND_CORS_ORIGINS = 'http://localhost:3000' + DOMAIN = 'http://localhost' + ENVIRONMENT = 'production' + PROJECT_NAME = 'Districtr v2 backend' + R2_BUCKET_NAME = 'districtr-v2-dev' + VOLUME_PATH = '/data' + +[[mounts]] + source = 'gerrydb_views' + destination = '/data' + initial_size = '10gb' + processes = ['app'] + [http_service] internal_port = 8080 force_https = true From 8acb42ba5ab1a6046875484b773c02422cf6f909 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sat, 20 Jul 2024 10:34:47 -0400 Subject: [PATCH 30/33] just download the file --- backend/cli.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 1028f5fd..790ba323 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -17,14 +17,17 @@ def cli(): @cli.command("import-gerrydb-view") @click.option("--layer", "-n", help="layer of the view", required=True) @click.option("--gpkg", "-g", help="Path or URL to GeoPackage file", required=True) -def import_gerrydb_view(layer, gpkg: str): +@click.option("--replace", "-f", help="Replace the file if it exists", is_flag=True) +@click.option("--rm", "-r", help="Delete file after loading to postgres", is_flag=True) +def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): + if layer == "": + raise ValueError("Layer name is required") + print("Importing GerryDB view...") url = urlparse(gpkg) logger.info("URL: %s", url) - kwargs = {} - if url.scheme == "s3": s3 = settings.get_s3_client() @@ -32,6 +35,7 @@ def import_gerrydb_view(layer, gpkg: str): raise ValueError("S3 client is not available") file_name = url.path.lstrip("/") + logger.info("File name: %s", file_name) object_information = s3.head_object(Bucket=url.netloc, Key=file_name) if object_information["ResponseMetadata"]["HTTPStatusCode"] != 200: @@ -41,14 +45,14 @@ def import_gerrydb_view(layer, gpkg: str): logger.info("Importing GerryDB view. Got response:\n%s", object_information) - path = f"/vsis3/{url.netloc}/{file_name}" + # Download to settings.VOLUME_PATH + path = os.path.join(settings.VOLUME_PATH, file_name) - kwargs["env"] = { - **os.environ, - "AWS_S3_ENDPOINT": settings.AWS_S3_ENDPOINT, - "AWS_ACCESS_KEY_ID": settings.AWS_ACCESS_KEY_ID, - "AWS_SECRET_ACCESS_KEY": settings.AWS_SECRET_ACCESS_KEY, - } + if os.path.exists(path) and not replace: + logger.info("File already exists. Skipping download.") + else: + logger.info("Downloading file...") + s3.download_file(url.netloc, file_name, path) else: path = gpkg @@ -65,7 +69,6 @@ def import_gerrydb_view(layer, gpkg: str): "-nln", layer, ], - **kwargs, ) if result.returncode != 0: @@ -74,6 +77,10 @@ def import_gerrydb_view(layer, gpkg: str): logger.info("GerryDB view imported successfully") + if rm: + os.remove(path) + logger.info("Deleted file %s", path) + if __name__ == "__main__": cli() From 61f5255d9240e96d08a7d30c251acbcd6ef7a7a8 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sat, 20 Jul 2024 11:31:35 -0400 Subject: [PATCH 31/33] track uploaded tables --- .../966d8d72887e_add_gerrydb_schema.py | 57 +++++++++++++++++ .../versions/cb7cd5fd35d8_pop_table.py | 61 ------------------- backend/app/models.py | 20 +++--- backend/cli.py | 39 +++++++++++- 4 files changed, 103 insertions(+), 74 deletions(-) create mode 100644 backend/app/alembic/versions/966d8d72887e_add_gerrydb_schema.py delete mode 100644 backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py diff --git a/backend/app/alembic/versions/966d8d72887e_add_gerrydb_schema.py b/backend/app/alembic/versions/966d8d72887e_add_gerrydb_schema.py new file mode 100644 index 00000000..7ca386d7 --- /dev/null +++ b/backend/app/alembic/versions/966d8d72887e_add_gerrydb_schema.py @@ -0,0 +1,57 @@ +"""add gerrydb schema + +Revision ID: 966d8d72887e +Revises: +Create Date: 2024-07-20 10:50:48.136439 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +import sqlmodel.sql.sqltypes +from alembic import op +from app.models import UUIDType + + +# revision identifiers, used by Alembic. +revision: str = "966d8d72887e" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.execute(sa.text("CREATE EXTENSION IF NOT EXISTS postgis")) + op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS gerrydb")) + op.create_table( + "gerrydbtable", + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("uuid", UUIDType(), nullable=True), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("name"), + sa.UniqueConstraint("uuid"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("gerrydbtable") + op.execute(sa.text("DROP SCHEMA IF EXISTS gerrydb CASCADE")) + op.execute(sa.text("DROP EXTENSION IF EXISTS postgis")) + # ### end Alembic commands ### diff --git a/backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py b/backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py deleted file mode 100644 index 8cb3627b..00000000 --- a/backend/app/alembic/versions/cb7cd5fd35d8_pop_table.py +++ /dev/null @@ -1,61 +0,0 @@ -"""pop table - -Revision ID: cb7cd5fd35d8 -Revises: -Create Date: 2024-07-16 22:47:09.900729 - -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.sql import text -from sqlmodel.sql import sqltypes -import geoalchemy2 - - -# revision identifiers, used by Alembic. -revision: str = "cb7cd5fd35d8" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.execute(text("CREATE EXTENSION IF NOT EXISTS postgis")) - op.create_table( - "population", - sa.Column("path", sqltypes.AutoString(), nullable=False), - sa.Column("area_land", sa.Integer(), nullable=False), - sa.Column("area_water", sa.Integer(), nullable=False), - sa.Column("other_pop", sa.Integer(), nullable=False), - sa.Column("total_pop", sa.Integer(), nullable=False), - sa.Column( - "geography", - geoalchemy2.types.Geometry( - geometry_type="POLYGON", - srid=4269, - from_text="ST_GeomFromEWKT", - name="geometry", - ), - nullable=True, - ), - sa.PrimaryKeyConstraint("path"), - ) - # NOTE: geospatial indices are created by default - # op.create_index('idx_population_geography', 'population', ['geography'], unique=False, postgresql_using='gist') - op.create_index(op.f("ix_population_path"), "population", ["path"], unique=True) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f("ix_population_path"), table_name="population") - op.drop_index( - "idx_population_geography", table_name="population", postgresql_using="gist" - ) - op.drop_table("population") - op.execute(text("DROP EXTENSION IF EXISTS postgis")) - # ### end Alembic commands ### diff --git a/backend/app/models.py b/backend/app/models.py index 6f2656da..f7c4b5b2 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1,7 +1,6 @@ from datetime import datetime -from typing import Any, Optional +from typing import Optional from sqlmodel import Field, SQLModel, UUID, TIMESTAMP, text, Column -from geoalchemy2 import Geometry class UUIDType(UUID): @@ -30,13 +29,10 @@ class TimeStampMixin(SQLModel): ) -class Population(SQLModel, table=True): - name: str = Field(nullable=False, index=True) - path: str = Field(unique=False, nullable=False, index=True, primary_key=False) - area_land: int - area_water: int - other_pop: int - total_pop: int - geography: Any = Field( - sa_column=Column(Geometry(geometry_type="POLYGON", srid=4269)) - ) +class GerryDBTableBase(TimeStampMixin, SQLModel): + id: int = Field(default=None, primary_key=True) + + +class GerryDBTable(GerryDBTableBase, table=True): + uuid: str = Field(sa_column=Column(UUIDType, unique=True)) + name: str = Field(nullable=False, unique=True) diff --git a/backend/cli.py b/backend/cli.py index 790ba323..d016431b 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1,14 +1,21 @@ import os import click import logging +from app.main import get_session from app.core.config import settings import subprocess from urllib.parse import urlparse +from sqlalchemy import text +from uuid import uuid4 +# from fastapi import Depends logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) +GERRY_DB_SCHEMA = "gerrydb" + + @click.group() def cli(): pass @@ -67,7 +74,7 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): "-lco", "OVERWRITE=yes", "-nln", - layer, + f"{GERRY_DB_SCHEMA}.{layer}", # Forced that the layer is imported into the gerrydb schema ], ) @@ -81,6 +88,36 @@ def import_gerrydb_view(layer: str, gpkg: str, replace: bool, rm: bool): os.remove(path) logger.info("Deleted file %s", path) + print("GerryDB view imported successfully") + + _session = get_session() + session = next(_session) + + uuid = str(uuid4()) + + upsert_query = text(""" + INSERT INTO gerrydbtable (uuid, name, updated_at) + VALUES (:uuid, :name, now()) + ON CONFLICT (name) + DO UPDATE SET + updated_at = now() + """) + + try: + session.execute( + upsert_query, + { + "uuid": uuid, + "name": layer, + }, + ) + session.commit() + logger.info("GerryDB view upserted successfully.") + except Exception as e: + session.rollback() + logger.error("Failed to upsert GerryDB view. Got %s", e) + raise ValueError(f"Failed to upsert GerryDB view. Got {e}") + if __name__ == "__main__": cli() From a2684d40cebb982874ff3daa2071c5ff271c247c Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sat, 20 Jul 2024 11:51:02 -0400 Subject: [PATCH 32/33] add gdal --- backend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 5e746b37..7876fa5a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,7 +7,7 @@ ENV APP_HOME /app WORKDIR $APP_HOME RUN apt-get update && \ - apt-get install -y openssh-client libpq-dev postgresql && \ + apt-get install -y openssh-client libpq-dev postgresql libpq-dev gdal-bin libgdal-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From 6af14aecfe623288a38003b49eae6cba97245d55 Mon Sep 17 00:00:00 2001 From: Raphael Paul Laude Date: Sat, 20 Jul 2024 12:09:20 -0400 Subject: [PATCH 33/33] fill in missing attributes --- backend/app/core/config.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 2ef0fd07..12c11669 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -52,10 +52,10 @@ def server_host(self) -> str: # Postgres POSTGRES_SCHEME: str - POSTGRES_SERVER: str - POSTGRES_PORT: int = 5432 - POSTGRES_USER: str - POSTGRES_PASSWORD: str + POSTGRES_SERVER: str | None + POSTGRES_PORT: int | None = 5432 + POSTGRES_USER: str | None + POSTGRES_PASSWORD: str | None POSTGRES_DB: str = "" DATABASE_URL: str | None = None @@ -63,7 +63,19 @@ def server_host(self) -> str: @property def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn: if self.DATABASE_URL: - return MultiHostUrl(self.DATABASE_URL) + db_uri = MultiHostUrl(self.DATABASE_URL) + (host,) = db_uri.hosts() + + self.POSTGRES_SCHEME = db_uri.scheme + self.POSTGRES_PORT = host["port"] + self.POSTGRES_USER = host["username"] + self.POSTGRES_PASSWORD = host["password"] + self.POSTGRES_SERVER = host["host"] + + if db_uri.path: + self.POSTGRES_DB = db_uri.path.lstrip("/") + + return db_uri return MultiHostUrl.build( scheme=self.POSTGRES_SCHEME,