diff --git a/.gitignore b/.gitignore index 4708bace..e7b9d836 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ scratch/ PG:* *.dev +sample_data/* \ No newline at end of file diff --git a/README.md b/README.md index 9454bcc4..a81108fb 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,14 @@ The Districtr reboot monorepo. - [`pipelines`](pipelines/): Data pipelines, ETL. Not a main focus of the reboot. For now, will mostly contain scratch data transformation scripts before being integrated into the backend CLI. - [`prototypes`](prototypes/): Prototypes conducted as part of the reboot. +## Quickstart + +The backend (Python), frontend (NextJS), and database (postgres) can be run locally using Docker. + +- Install and configure [Docker](https://www.docker.com/) for your machine +- From the repo root, run `docker-compose up --build` +- To load in data, add data to a folder `sample_data` in the repo root, and in `docker-compose.yml` set `services > backend > environment > LOAD_GERRY_DB_DATA` to `true`. You can change where the script looks for available data with the `GPKG_DATA_DIR` variable. + ## Districtr reboot architecture After experimenting with various technologies (see [`prototypes`](prototypes/)) we landed on the following architecture for the Districtr reboot: @@ -16,11 +24,13 @@ After experimenting with various technologies (see [`prototypes`](prototypes/)) ![Districtr architecture](docs/images/districtr-architecture.png "Districtr architecture") The redesign aims to principally to address three key pain points in the Districtr application’s performance and maintainability: + 1. Slow tile rendering 1. Cumbersome use of tiles as global state for tile rendering and most metric calculation 1. Complexity and poor interoperability in architecture without slow copies And two key feature additions + 1. Block “shattering” 1. A headless CMS (this will be added in a later phase of work / is not currently a focus of the reboot) diff --git a/app/.env.docker b/app/.env.docker new file mode 100644 index 00000000..4e3f04c6 --- /dev/null +++ b/app/.env.docker @@ -0,0 +1,2 @@ +NEXT_PUBLIC_API_URL=http://localhost:8000 +NEXT_PUBLIC_S3_BUCKET_URL=https://pub-fa71193941a74e14a38eee99f30f53d9.r2.dev diff --git a/app/Dockerfile.dev b/app/Dockerfile.dev new file mode 100644 index 00000000..2e4fdf66 --- /dev/null +++ b/app/Dockerfile.dev @@ -0,0 +1,20 @@ +# Use official Node.js image as the base image +FROM node:18-alpine + +# Set working directory inside the container +WORKDIR /app + +# Copy package.json and package-lock.json to install dependencies first +COPY package.json package-lock.json ./ + +# Install dependencies +RUN npm install + +# Copy the rest of the application code +COPY . . + +# Expose the Next.js dev server port +EXPOSE 3000 + +# Start the Next.js application in development mode +CMD ["npm", "run", "dev"] \ No newline at end of file diff --git a/app/src/app/components/sidebar/ColorPicker.jsx b/app/src/app/components/sidebar/ColorPicker.jsx index 0af44efb..a9c1ca7a 100644 --- a/app/src/app/components/sidebar/ColorPicker.jsx +++ b/app/src/app/components/sidebar/ColorPicker.jsx @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { palette, color10 } from "../../constants/colors"; +import { _colorScheme } from "../../constants/colors"; import { Button } from "@radix-ui/themes"; import { styled } from "@stitches/react"; import * as RadioGroup from "@radix-ui/react-radio-group"; @@ -22,7 +22,7 @@ export function ColorPicker() { accumulatedGeoids: state.accumulatedGeoids, resetAccumulatedBlockPopulations: state.resetAccumulatedBlockPopulations, })); - const colorArray = color10; + const colorArray = _colorScheme; if (!colorArray) return null; const handleRadioChange = (value) => { console.log( diff --git a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx index dfcf3ff6..58f9ffac 100644 --- a/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx +++ b/app/src/app/components/sidebar/charts/HorizontalBarChart.tsx @@ -9,7 +9,7 @@ import { YAxis, Cell, } from "recharts"; -import { color10 } from "@/app/constants/colors"; +import { colorScheme } from "@/app/constants/colors"; type TooltipInput = { active?: boolean; @@ -58,7 +58,8 @@ export const HorizontalBar = () => { { {mapMetrics.data .sort((a, b) => a.zone - b.zone) .map((entry, index) => ( - + ))} diff --git a/app/src/app/constants/colors.ts b/app/src/app/constants/colors.ts index 5c74147b..d4249155 100644 --- a/app/src/app/constants/colors.ts +++ b/app/src/app/constants/colors.ts @@ -27,63 +27,47 @@ import { sky, } from "@radix-ui/colors"; -export const palette = { - colors: { - ...(tomato as object), - ...(red as object), - ...(ruby as object), - ...(crimson as object), - ...(pink as object), - ...(plum as object), - ...(purple as object), - ...(violet as object), - ...(iris as object), - ...(indigo as object), - ...(blue as object), - ...(cyan as object), - ...(teal as object), - ...(jade as object), - ...(green as object), - ...(grass as object), - ...(orange as object), - ...(amber as object), - ...(yellow as object), - ...(gold as object), - ...(brown as object), - ...(bronze as object), - ...(gray as object), - ...(mint as object), - ...(lime as object), - ...(sky as object), - } as { [key: string]: { [key: string]: string } }, -}; - -// bright colors! -export const color10 = [ - tomato.tomato10, - red.red10, - ruby.ruby10, - crimson.crimson10, - pink.pink10, - plum.plum10, - purple.purple10, - violet.violet10, - iris.iris10, - indigo.indigo10, - blue.blue10, - cyan.cyan10, - teal.teal10, - jade.jade10, - green.green10, - grass.grass10, - orange.orange10, - amber.amber10, - yellow.yellow10, - gold.gold10, - brown.brown10, - bronze.bronze10, - gray.gray10, - mint.mint10, - lime.lime10, - sky.sky10, +export const colorScheme = [ + "#0099cd", + "#ffca5d", + "#00cd99", + "#99cd00", + "#cd0099", + "#aa44ef", // lighter, req from San Diego + // Color brewer: + "#8dd3c7", + "#bebada", + "#fb8072", + "#80b1d3", + "#fdb462", + "#b3de69", + "#fccde5", + "#bc80bd", + "#ccebc5", + "#ffed6f", + "#ffffb3", + // other color brewer scheme: + "#a6cee3", + "#1f78b4", + "#b2df8a", + "#33a02c", + "#fb9a99", + "#e31a1c", + "#fdbf6f", + "#ff7f00", + "#cab2d6", + "#6a3d9a", + "#b15928", + // random material design colors: + "#64ffda", + "#00B8D4", + "#A1887F", + "#76FF03", + "#DCE775", + "#B388FF", + "#FF80AB", + "#D81B60", + "#26A69A", + "#FFEA00", + "#6200EA", ]; diff --git a/app/src/app/constants/layers.ts b/app/src/app/constants/layers.ts index bc0d12e5..5d79f404 100644 --- a/app/src/app/constants/layers.ts +++ b/app/src/app/constants/layers.ts @@ -3,7 +3,7 @@ import { MutableRefObject } from "react"; import { Map } from "maplibre-gl"; import { getBlocksSource } from "./sources"; import { gerryDBView } from "../api/apiHandlers"; -import { color10 } from "./colors"; +import { colorScheme } from "./colors"; export const BLOCK_SOURCE_ID = "blocks"; export const BLOCK_LAYER_ID = "blocks"; @@ -23,10 +23,13 @@ export const COUNTY_LAYER_IDS: string[] = [ export const LABELS_BREAK_LAYER_ID = "places_subplace"; const colorStyleBaseline: any[] = ["case"]; -export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = color10.reduce((val, color, i) => { - val.push(["==", ["feature-state", "zone"], i + 1], color); // 1-indexed per mapStore.ts - return val; -}, colorStyleBaseline); +export const ZONE_ASSIGNMENT_STYLE_DYNAMIC = _colorScheme.reduce( + (val, color, i) => { + val.push(["==", ["feature-state", "zone"], i + 1], color); // 1-indexed per mapStore.ts + return val; + }, + colorStyleBaseline +); ZONE_ASSIGNMENT_STYLE_DYNAMIC.push("#cecece"); // cast the above as an ExpressionSpecification @@ -35,7 +38,7 @@ export const ZONE_ASSIGNMENT_STYLE: ExpressionSpecification = ZONE_ASSIGNMENT_STYLE_DYNAMIC; export function getBlocksLayerSpecification( - sourceLayer: string, + sourceLayer: string ): LayerSpecification { return { id: BLOCK_LAYER_ID, @@ -58,7 +61,7 @@ export function getBlocksLayerSpecification( } export function getBlocksHoverLayerSpecification( - sourceLayer: string, + sourceLayer: string ): LayerSpecification { return { id: BLOCK_HOVER_LAYER_ID, @@ -70,7 +73,7 @@ export function getBlocksHoverLayerSpecification( }, paint: { "fill-opacity": [ - "case", + "case", // zone is selected and hover is true and hover is not null [ "all", @@ -81,7 +84,7 @@ export function getBlocksHoverLayerSpecification( // @ts-ignore ["!", ["==", ["feature-state", "hover"], null]], //< desired behavior but typerror ["boolean", ["feature-state", "hover"], true], - ] + ], ], 0.9, // zone is selected and hover is false, and hover is not null @@ -94,7 +97,7 @@ export function getBlocksHoverLayerSpecification( // @ts-ignore ["!", ["==", ["feature-state", "hover"], null]], //< desired behavior but typerror ["boolean", ["feature-state", "hover"], false], - ] + ], ], 0.7, // zone is selected, fallback, regardless of hover state @@ -102,8 +105,9 @@ export function getBlocksHoverLayerSpecification( ["!", ["==", ["feature-state", "zone"], null]], //< desired behavior but typerror 0.7, // hover is true, fallback, regardless of zone state - ["boolean", ["feature-state", "hover"], false], 0.6, - 0.2 + ["boolean", ["feature-state", "hover"], false], + 0.6, + 0.2, ], "fill-color": ZONE_ASSIGNMENT_STYLE || "#000000", }, @@ -112,18 +116,18 @@ export function getBlocksHoverLayerSpecification( const addBlockLayers = ( map: MutableRefObject, - gerryDBView: gerryDBView, + gerryDBView: gerryDBView ) => { const blockSource = getBlocksSource(gerryDBView.tiles_s3_path); removeBlockLayers(map); map.current?.addSource(BLOCK_SOURCE_ID, blockSource); map.current?.addLayer( getBlocksLayerSpecification(gerryDBView.name), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); map.current?.addLayer( getBlocksHoverLayerSpecification(gerryDBView.name), - LABELS_BREAK_LAYER_ID, + LABELS_BREAK_LAYER_ID ); }; diff --git a/backend/.env.docker b/backend/.env.docker new file mode 100644 index 00000000..e1ab04d0 --- /dev/null +++ b/backend/.env.docker @@ -0,0 +1,19 @@ +# Backend +DOMAIN=localhost +ENVIRONMENT=local +PROJECT_NAME="Districtr v2 backend" +BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,http://localhost:3000,http://127.0.0.1:3000" +SECRET_KEY="super-secret" + +# Postgres +DATABASE_URL=postgresql+psycopg://postgres:postgres@db:5432/districtr +POSTGRES_SCHEME=postgresql+psycopg +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=districtr +POSTGRES_SERVER=db # Use the service name `db` from docker-compose +POSTGRES_PORT=5432 + +# Volumes +VOLUME_PATH=/data + diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev new file mode 100644 index 00000000..478c414b --- /dev/null +++ b/backend/Dockerfile.dev @@ -0,0 +1,20 @@ +# Use official Python image with version >= 3.11 +FROM python:3.12-slim + +# Set working directory inside the container +WORKDIR /districtr-backend + +# Copy requirements file first and install dependencies +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the backend code into the container +COPY . . + +# Install PostGIS extension and PostgreSQL client for database operations +RUN apt-get update && apt-get install -y postgresql-client libpq-dev gdal-bin + +# Command to run the server +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--reload"] diff --git a/backend/scripts/load_data.py b/backend/scripts/load_data.py new file mode 100755 index 00000000..cc27a2b5 --- /dev/null +++ b/backend/scripts/load_data.py @@ -0,0 +1,66 @@ +import subprocess +import sqlalchemy as sa +from os import environ +from glob import glob + +# Optionally, set a data directory to load in +DATA_DIR = environ.get("GPKG_DATA_DIR", "sample_data") +# flag to load data, by default, will load data +LOAD_DATA = environ.get("LOAD_GERRY_DB_DATA", "true") + + +def update_tile_column(engine): + """ + Update the 'tiles_s3_path' column in the 'gerrydbtable' of the public schema. + + This function connects to the database using the provided SQLAlchemy engine + and executes an UPDATE query. It sets the 'tiles_s3_path' column to a + concatenated string based on the 'name' column. + + Args: + engine (sqlalchemy.engine.Engine): SQLAlchemy engine instance for database connection. + + Prints: + Success message with the number of updated rows or an error message if the update fails. + + Raises: + SQLAlchemyError: If there's an error during the database operation. + """ + print("UPDATING GERRYDB COLUMN") + with engine.connect() as connection: + try: + result = connection.execute( + sa.text( + "UPDATE public.gerrydbtable SET tiles_s3_path = CONCAT('tilesets/', name, '.pmtiles')" + ) + ) + updated_rows = result.rowcount + print(f"Successfully updated {updated_rows} rows in gerrydbtable.") + connection.commit() + except sa.exc.SQLAlchemyError as e: + print(f"Error updating gerrydbtable: {str(e)}") + connection.rollback() + + +def load_sample_data(): + """ + Load sample data from the specified data directory. + + This function iterates through all files with a '.gpkg' extension in the + specified data directory, and for each file, it runs a script to load the + GerryDB view. + + Args: + None + Returns: + None + """ + for gpkg in glob(f"{DATA_DIR}/*.gpkg"): + subprocess.run(["bash", "./scripts/load_gerrydb_view.sh", gpkg]) + + +if __name__ == "__main__": + if LOAD_DATA == "true": + load_sample_data() + engine = sa.create_engine(environ.get("DATABASE_URL")) + update_tile_column(engine) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..fe52aa45 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +services: + backend: + build: + context: ./backend + dockerfile: Dockerfile.dev + container_name: backend + volumes: + - ./backend/app:/districtr-backend/app # Adjust volumes to point to the backend folder + - ./backend/scripts:/districtr-backend/scripts # Ensure scripts are mounted from backend directory + - ./sample_data:/districtr-backend/sample_data + + env_file: + - ./backend/.env.docker # Ensure the env file is also from the backend folder + environment: + LOAD_GERRY_DB_DATA: false + GPKG_DATA_DIR: sample_data + depends_on: + db: + condition: service_healthy + command: bash -c " + until alembic upgrade head; do + echo 'Alembic failed, retrying in 5 seconds...' + sleep 5; + done && + python3 scripts/load_data.py && + uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload --reload-exclude '.venv/**/*.py' + " + ports: + - "8000:8000" + + db: + image: postgis/postgis:15-3.3-alpine + container_name: postgres_db + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: districtr + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + retries: 5 + + frontend: + build: + context: ./app + dockerfile: Dockerfile.dev + container_name: frontend + volumes: + - ./app:/app # Mount the app folder for hot reloading + - ./app/node_modules:/app/node_modules # Bind mount for node_modules + + env_file: + - ./app/.env.docker + ports: + - "3000:3000" # Expose Next.js on port 3000 + command: sh -c "npm install && npm run dev" + + pre-commit: + image: python:3.9 # or whichever version you prefer + volumes: + - .:/app + working_dir: /app + command: sh -c "pip install pre-commit && pre-commit run --all-files" + +volumes: + postgres_data: \ No newline at end of file