Merge pull request #20 from uchicago-dsi/basic-backend-setup

Setup FastAPI backend with postgres and mongodb

.github/workflows/test-backend.yml

@@ -0,0 +1,37 @@
+name: Test backend
+
+on:
+  push:
+    paths:
+      - "backend/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+        working-directory: backend
+      - name: Lint with Ruff
+        run: |
+          pip install ruff
+          ruff --output-format=github .
+        continue-on-error: true
+        working-directory: backend
+      - name: Create docker network
+        run: docker network create test_network
+      - name: Start MongoDB
+        run: docker run -d --network test_network --name mongo -p 27017:27017 mongo:latest
+      - name: Build app image
+        run: cp .env.dev .env && docker build -t districtr .
+        working-directory: backend
+      - name: Test
+        run: docker run --network test_network -e MONGODB_SERVER=mongo --rm districtr ./test.sh

.gitignore

@@ -8,6 +8,9 @@
 
 # testing
 /coverage
+.coverage*
+htmlcov/
+.pytest_cache
 
 # next.js
 /.next/

backend/.dockerignore

@@ -0,0 +1 @@
+fly.toml

backend/.env.dev

@@ -0,0 +1,21 @@
+# Backend
+DOMAIN=localhost
+ENVIRONMENT=local
+PROJECT_NAME="Districtr v2 backend"
+BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173"
+SECRET_KEY={fill-me}
+
+# Postgres
+DATABASE_URL=postgresql+psycopg://postgres:{fill-me}@localhost:5432/districtr
+POSTGRES_SCHEME=postgresql+psycopg
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD={fill-me}
+POSTGRES_DB=districtr
+POSTGRES_SERVER=localhost
+POSTGRES_PORT=5432
+
+# MongoDB
+MONGODB_SCHEME=mongodb
+MONGODB_SERVER=localhost
+MONGODB_PORT=27017
+MONGODB_DB=districtr

backend/.env.production

@@ -0,0 +1,25 @@
+# This is just for example / reference. Actual secrets will be managed through hosting provider.
+
+# Backend
+DOMAIN="http://localhost" # todo
+ENVIRONMENT=production
+PROJECT_NAME="Districtr v2 backend"
+BACKEND_CORS_ORIGINS="http://localhost" # todo
+SECRET_KEY={fill-me}
+
+# Postgres
+DATABASE_URL=postgresql://{fill-me}:{fill-me}@{fill-me}.flycast:5432/{fill-me}?sslmode=disable
+POSTGRES_SCHEME=postgresql+psycopg
+POSTGRES_USER={fill-me}
+POSTGRES_PASSWORD={fill-me}
+POSTGRES_DB={fill-me}
+POSTGRES_SERVER={fill-me}.flycast
+POSTGRES_PORT=5432
+
+# MongoDB
+MONGODB_SCHEME=mongodb+srv
+MONGODB_SERVER=tbd
+MONGODB_PORT=tbd
+MONGODB_DB=districtr
+MONGODB_USER=tbd
+MONGODB_PASSWORD=tbd

backend/Dockerfile

@@ -0,0 +1,19 @@
+ARG PYTHON_VERSION=3.12.2-slim-bullseye
+FROM python:${PYTHON_VERSION}
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED True
+ENV APP_HOME /app
+WORKDIR $APP_HOME
+
+RUN apt-get update && \
+    apt-get install -y openssh-client libpq-dev postgresql && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . ./
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]

backend/README.md

@@ -0,0 +1,117 @@
+# Backend
+
+## Running Locally
+
+`uvicorn app.main:app --reload`
+
+And if you want to exclude somewhat noisy changes in virtualenv:
+
+`uvicorn app.main:app --reload --reload-exclude '.venv/**/*.py'`
+
+## Shipping
+
+Don't forget to update requirements in case you added a new package with `uv pip freeze | uv pip compile - -o requirements.txt`
+
+If you haven't set up your venv see [here](./README.md#python-install).
+
+### Build locally
+
+- `docker build -t districtr .`
+- `docker run --rm -i districtr`
+
+### Deploy
+
+Automatic using GHA [fly.yml](../.github/workflows/fly.yml).
+
+Handy fly commands:
+
+- `fly status`
+- `fly machines start`
+- `fly ssh console`
+
+## Migrations
+
+### Revisions
+
+`alembic revision --autogenerate -m "{{ migration_name }}"`
+Then make sure the migration is what you want.
+
+`alembic upgrade head`
+
+To do this on production, run `fly postgres connect -a {{ database-id }}` then the above commands.
+
+### Downgrading
+
+`alembic downgrade -1`
+
+### Misc
+
+`alembic history`
+
+## Development
+
+- Try to maximize the use of url parameters for handling state in the frontend. On the backend, this means that practically we should add slugs or UUIDs to models if there might be a page associated with that concept.
+
+### Developer set-up
+
+1. `cp .env.dev .env`
+1. Fill out the .env with your local postgres credentials. Follow section [Postgres set-up](./README.md#postgres-set-up) for more details on creating your local database.
+1. Make sure you've set up your pre-commits with `pre-commit install` as documented [here](../README.md#python).
+
+### Python install
+
+Dependencies are managed with uv as noted in the root README. Follow set-up instructions [there](../README.md#python).
+
+Set-up virtual environment and install dependencies:
+
+1. `uv venv`
+1. `source venv/bin/activate` on UNIX machines or `venv\Scripts\activate` on Windows.
+1. `uv pip install -r requirements.txt`
+
+### Postgres set-up
+
+[Install postgres](https://www.postgresql.org/download/) based on your OS.
+
+1. `psql`
+1. `CREATE DATABASE districtr;`
+1. `\c districtr`
+1. `\dt` should yield no tables / make sure db is empty.
+1. `\q`
+1. `alembic upgrade head`
+
+### Testing
+
+`pytest --cov=app --cov-report=html`
+
+Or with full coverage report:
+
+`coverage run --source=app -m pytest -v && coverage html && open htmlcov/index.html`
+
+### MongoDB
+
+#### MacOS
+
+Follow [install instructions](https://github.com/mongodb/homebrew-brew).
+
+#### Linux
+
+See [Install MongoDB Community Edition on Linux](https://www.mongodb.com/docs/manual/administration/install-on-linux/)
+
+#### Set-up test database
+
+1. `brew services start mongodb-community` on Mac to start the server. TBD other platforms. Stop the server with `brew services stop mongodb-community`.
+1. `mongosh`
+1. `use districtr` to create a new database in `/usr/local/var/mongodb` (intel) or `/opt/homebrew/var/mongodb` (Apple silicon). Connects to the db if it already exists.
+
+More info [here](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-os-x/).
+
+Create collections:
+1. `python cli.py create_collections`
+
+Optionally you can create or update individual collections with `python cli.py create_collections -c {{ collection_name_1 }} -c {{ collection_name_2 }}`.
+
+Confirm in `mongosh` with `use districtr` followed by `show collections` or `python cli.py list-collections`.
+
+### Useful reference apps
+
+- [full-stack-fastapi-template](https://github.com/tiangolo/full-stack-fastapi-template/tree/master)

backend/alembic.ini

@@ -0,0 +1,113 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = app/alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

backend/app/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.