Merge pull request #1 from districtr/simple-pop-table-for-testing

.github/workflows/fly.yml

@@ -0,0 +1,19 @@
+name: Fly Deploy
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "backend/**"
+jobs:
+  deploy:
+    name: Deploy app
+    runs-on: ubuntu-latest
+    concurrency: deploy-group
+    steps:
+      - uses: actions/checkout@v3
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+      - run: flyctl deploy --remote-only
+        working-directory: backend
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

.github/workflows/test-backend.yml

@@ -4,34 +4,59 @@ on:
   push:
     paths:
       - "backend/**"
+      - ".github/workflows/test-backend.yml"
 
 jobs:
-  build:
+  container-job:
     runs-on: ubuntu-latest
+
+    container: python:3.12
+
+    services:
+      postgres:
+        image: postgis/postgis:16-3.4
+        env:
+          POSTGRES_SCHEME: postgresql+psycopg
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_SERVER: postgres
+          POSTGRES_PORT: 5432
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repo code
+        uses: actions/checkout@v4
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
-          cache: "pip"
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-        working-directory: backend
-      - name: Lint with Ruff
-        run: |
-          pip install ruff
-          ruff --output-format=github .
-        continue-on-error: true
+          pip install -r requirements.txt --no-cache-dir
         working-directory: backend
-      - name: Create docker network
-        run: docker network create test_network
-      - name: Start MongoDB
-        run: docker run -d --network test_network --name mongo -p 27017:27017 mongo:latest
-      - name: Build app image
-        run: cp .env.dev .env && docker build -t districtr .
+
+      - name: Run tests
+        run: pytest -v
         working-directory: backend
-      - name: Test
-        run: docker run --network test_network -e MONGODB_SERVER=mongo --rm districtr ./test.sh
+        env:
+          DOMAIN: postgres
+          ENVIRONMENT: test
+          PROJECT_NAME: Districtr v2 backend
+          BACKEND_CORS_ORIGINS: "http://localhost,http://localhost:5173"
+          SECRET_KEY: mysupersecretkey
+          POSTGRES_SCHEME: postgresql+psycopg
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+          POSTGRES_SERVER: postgres
+          POSTGRES_PORT: 5432

backend/.env.dev

@@ -14,8 +14,5 @@ POSTGRES_DB=districtr
 POSTGRES_SERVER=localhost
 POSTGRES_PORT=5432
 
-# MongoDB
-MONGODB_SCHEME=mongodb
-MONGODB_SERVER=localhost
-MONGODB_PORT=27017
-MONGODB_DB=districtr
+# Volumes
+VOLUME_PATH=/data

backend/.env.production

@@ -16,10 +16,5 @@ POSTGRES_DB={fill-me}
 POSTGRES_SERVER={fill-me}.flycast
 POSTGRES_PORT=5432
 
-# MongoDB
-MONGODB_SCHEME=mongodb+srv
-MONGODB_SERVER=tbd
-MONGODB_PORT=tbd
-MONGODB_DB=districtr
-MONGODB_USER=tbd
-MONGODB_PASSWORD=tbd
+# Volumes
+VOLUME_PATH=/data

backend/.env.test

@@ -0,0 +1,18 @@
+# Backend
+DOMAIN=postgres
+ENVIRONMENT=test
+PROJECT_NAME="Districtr v2 backend"
+BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173"
+SECRET_KEY=mysupersecretkey
+
+# Postgres
+DATABASE_URL=postgresql+psycopg://postgres:postgres@postgres:5432/postgres
+POSTGRES_SCHEME=postgresql+psycopg
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+POSTGRES_SERVER=postgres
+POSTGRES_PORT=5432
+
+# Volumes
+VOLUME_PATH=/data

backend/Dockerfile

@@ -7,7 +7,7 @@ ENV APP_HOME /app
 WORKDIR $APP_HOME
 
 RUN apt-get update && \
-    apt-get install -y openssh-client libpq-dev postgresql && \
+    apt-get install -y openssh-client libpq-dev postgresql libpq-dev gdal-bin libgdal-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

backend/README.md

@@ -87,31 +87,6 @@ Or with full coverage report:
 
 `coverage run --source=app -m pytest -v && coverage html && open htmlcov/index.html`
 
-### MongoDB
-
-#### MacOS
-
-Follow [install instructions](https://github.com/mongodb/homebrew-brew).
-
-#### Linux
-
-See [Install MongoDB Community Edition on Linux](https://www.mongodb.com/docs/manual/administration/install-on-linux/)
-
-#### Set-up test database
-
-1. `brew services start mongodb-community` on Mac to start the server. TBD other platforms. Stop the server with `brew services stop mongodb-community`.
-1. `mongosh`
-1. `use districtr` to create a new database in `/usr/local/var/mongodb` (intel) or `/opt/homebrew/var/mongodb` (Apple silicon). Connects to the db if it already exists.
-
-More info [here](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-os-x/).
-
-Create collections:
-1. `python cli.py create_collections`
-
-Optionally you can create or update individual collections with `python cli.py create_collections -c {{ collection_name_1 }} -c {{ collection_name_2 }}`.
-
-Confirm in `mongosh` with `use districtr` followed by `show collections` or `python cli.py list-collections`.
-
 ### Useful reference apps
 
 - [full-stack-fastapi-template](https://github.com/tiangolo/full-stack-fastapi-template/tree/master)

backend/app/alembic/env.py

@@ -34,6 +34,11 @@
 
 
 def get_url():
+    database_url = os.getenv("DATABASE_URL", None)
+
+    if database_url:
+        return database_url
+
     user = os.getenv("POSTGRES_USER", "postgres")
     password = os.getenv("POSTGRES_PASSWORD", "")
     server = os.getenv("POSTGRES_SERVER", "db")

backend/app/alembic/versions/966d8d72887e_add_gerrydb_schema.py

@@ -0,0 +1,57 @@
+"""add gerrydb schema
+
+Revision ID: 966d8d72887e
+Revises:
+Create Date: 2024-07-20 10:50:48.136439
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from alembic import op
+from app.models import UUIDType
+
+
+# revision identifiers, used by Alembic.
+revision: str = "966d8d72887e"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute(sa.text("CREATE EXTENSION IF NOT EXISTS postgis"))
+    op.execute(sa.text("CREATE SCHEMA IF NOT EXISTS gerrydb"))
+    op.create_table(
+        "gerrydbtable",
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("CURRENT_TIMESTAMP"),
+            nullable=False,
+        ),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("uuid", UUIDType(), nullable=True),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name"),
+        sa.UniqueConstraint("uuid"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("gerrydbtable")
+    op.execute(sa.text("DROP SCHEMA IF EXISTS gerrydb CASCADE"))
+    op.execute(sa.text("DROP EXTENSION IF EXISTS postgis"))
+    # ### end Alembic commands ###

backend/app/core/config.py

@@ -1,5 +1,6 @@
 import secrets
 import warnings
+import boto3
 from functools import lru_cache
 from typing import Annotated, Any, Literal
 
@@ -32,7 +33,7 @@ class Settings(BaseSettings):
     # 60 minutes * 24 hours * 8 days = 8 days
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
     DOMAIN: str = "localhost"
-    ENVIRONMENT: Literal["local", "staging", "production"] = "local"
+    ENVIRONMENT: Literal["local", "staging", "production", "test"] = "local"
 
     @computed_field  # type: ignore[misc]
     @property
@@ -51,15 +52,31 @@ def server_host(self) -> str:
     # Postgres
 
     POSTGRES_SCHEME: str
-    POSTGRES_SERVER: str
-    POSTGRES_PORT: int = 5432
-    POSTGRES_USER: str
-    POSTGRES_PASSWORD: str
+    POSTGRES_SERVER: str | None
+    POSTGRES_PORT: int | None = 5432
+    POSTGRES_USER: str | None
+    POSTGRES_PASSWORD: str | None
     POSTGRES_DB: str = ""
+    DATABASE_URL: str | None = None
 
     @computed_field  # type: ignore[misc]
     @property
     def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn:
+        if self.DATABASE_URL:
+            db_uri = MultiHostUrl(self.DATABASE_URL)
+            (host,) = db_uri.hosts()
+
+            self.POSTGRES_SCHEME = db_uri.scheme
+            self.POSTGRES_PORT = host["port"]
+            self.POSTGRES_USER = host["username"]
+            self.POSTGRES_PASSWORD = host["password"]
+            self.POSTGRES_SERVER = host["host"]
+
+            if db_uri.path:
+                self.POSTGRES_DB = db_uri.path.lstrip("/")
+
+            return db_uri
+
         return MultiHostUrl.build(
             scheme=self.POSTGRES_SCHEME,
             username=self.POSTGRES_USER,
@@ -69,27 +86,6 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn:
             path=self.POSTGRES_DB,
         )
 
-    # MongoDB
-
-    MONGODB_SCHEME: str = "mongodb+srv"
-    MONGODB_SERVER: str | int = ""
-    MONGODB_PORT: int = 27017
-    MONGODB_USER: str = ""
-    MONGODB_PASSWORD: str = ""
-    MONGODB_DB: str = "districtr"
-
-    @computed_field  # type: ignore[misc]
-    @property
-    def MONGODB_URI(self) -> str:
-        if self.ENVIRONMENT == "local":
-            return f"{self.MONGODB_SCHEME}://{self.MONGODB_SERVER}:{self.MONGODB_PORT}/{self.MONGODB_DB}"
-
-        assert (
-            self.MONGODB_USER and self.MONGODB_PASSWORD
-        ), f"MONGODB_SERVER, MONGODB_USER, and MONGODB_PASSWORD must be set. Got server `{self.MONGODB_SERVER}` and user `{self.MONGODB_USER}`."
-
-        return f"{self.MONGODB_SCHEME}://{self.MONGODB_USER}:{self.MONGODB_PASSWORD}@{self.MONGODB_SERVER}:{self.MONGODB_PORT}/{self.MONGODB_DB}"
-
     # Security
 
     def _check_default_secret(self, var_name: str, value: str | None) -> None:
@@ -110,6 +106,34 @@ def _enforce_non_default_secrets(self) -> Self:
 
         return self
 
+    # Volumes
+
+    VOLUME_PATH: str = "/data"
+
+    # R2
+
+    R2_BUCKET_NAME: str | None = None
+    ACCOUNT_ID: str | None = None
+    AWS_S3_ENDPOINT: str | None = None
+    AWS_ACCESS_KEY_ID: str | None = None
+    AWS_SECRET_ACCESS_KEY: str | None = None
+
+    def get_s3_client(self):
+        if (
+            not self.ACCOUNT_ID
+            or not self.AWS_ACCESS_KEY_ID
+            or not self.AWS_SECRET_ACCESS_KEY
+        ):
+            return None
+
+        return boto3.client(
+            service_name="s3",
+            endpoint_url=f"https://{self.ACCOUNT_ID}.r2.cloudflarestorage.com",
+            aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
+            region_name="auto",
+        )
+
 
 @lru_cache()
 def get_settings():