diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b01a74cc..22dc13df 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,8 +7,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' - name: "Upgrade pip" run: "pip install --upgrade pip" + - name: "Print python version" + run: "python --version" - name: "Install package" run: pip install ".[dev]" - name: "Run lint checks" diff --git a/.github/workflows/no_debug_allowed.yaml b/.github/workflows/no_debug_allowed.yaml index 89f07f33..90ce5f7e 100644 --- a/.github/workflows/no_debug_allowed.yaml +++ b/.github/workflows/no_debug_allowed.yaml @@ -7,6 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' - name: "Upgrade pip" run: "pip install --upgrade pip" - name: "Install package" diff --git a/.github/workflows/no_forgoten_migrations.yaml b/.github/workflows/no_forgoten_migrations.yaml new file mode 100644 index 00000000..e7e5e82b --- /dev/null +++ b/.github/workflows/no_forgoten_migrations.yaml @@ -0,0 +1,44 @@ +name: Make sure to run manage.py makemigrations if you change models + +on: [pull_request] + +jobs: + is-migration-diff-clean: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:latest + env: + POSTGRES_DB: nycmesh-dev + POSTGRES_USER: nycmesh + POSTGRES_PASSWORD: abcd1234 + POSTGRES_PORT: 5432 + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: "Upgrade pip" + run: "pip install --upgrade pip" + - name: "Install package" + run: pip install ".[dev]" + - name: "You forgot to run manage.py makemigrations for model changes" + env: + DB_NAME: nycmesh-dev + DB_USER: nycmesh + DB_PASSWORD: abcd1234 + DB_HOST: localhost + DB_PORT: 5432 + DJANGO_SECRET_KEY: k7j&!u07c%%97s!^a_6%mh_wbzo*$hl4lj_6c2ee6dk)y9!k88 + run: | + python src/manage.py makemigrations meshapi --dry-run # Run extra time for debug output + python src/manage.py makemigrations meshapi --dry-run | grep "No changes detected in app 'meshapi'" + diff --git a/.github/workflows/run_django_tests.yaml b/.github/workflows/run_django_tests.yaml index bf8f27a6..59bbcb16 100644 --- a/.github/workflows/run_django_tests.yaml +++ b/.github/workflows/run_django_tests.yaml @@ -25,13 +25,17 @@ jobs: image: pelias/parser:latest ports: - 6800:3000 - strategy: - max-parallel: 4 - matrix: - python-version: [3.11] + redis: + image: redis + ports: + - 6379:6379 steps: - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' - name: "Upgrade pip" run: "pip install --upgrade pip" - name: "Install package" diff --git a/docker-compose.yaml b/docker-compose.yaml index febf9d6a..832923ca 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -19,6 +19,16 @@ services: volumes: - postgres_data:/var/lib/postgresql/data/ + redis: + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + networks: + - api + ports: + - 6379:6379 + image: + redis + pelias: networks: - api @@ -28,6 +38,8 @@ services: depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy healthcheck: test: curl http://127.0.0.1:8081/api/v1 interval: 2s diff --git a/entrypoint.sh b/entrypoint.sh index f94efb21..a9ab470d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,6 +8,14 @@ done echo 'DB started' +# It's okay to start Celery in the background and continue without waiting, even though "migrate" +# might make DB changes we want to notify for since tasks are queued by Django Webhook and +# are executed as soon as celery starts +# FIXME: This makes testing locally a bit awkward, since this isn't started by "manage.py runserver" +# maybe there's a way to do this better? +echo 'Staring Celery Worker...' +celery -A meshdb worker -l INFO --detach + echo 'Running Migrations...' python manage.py migrate diff --git a/pyproject.toml b/pyproject.toml index 90ff3f38..7b5d62f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,10 @@ name = "nycmesh-meshdb" version = "0.1" dependencies = [ + "celery[redis]==5.3.*", "django==4.2.*", "djangorestframework==3.14.*", + "django-webhook>=0.0.7,<0.1.0", "psycopg2-binary==2.9.*", "gunicorn==21.2.*", "python-dotenv==1.0.*", @@ -26,7 +28,8 @@ dev = [ "black == 23.7.*", "isort == 5.12.*", "coverage == 7.3.*", - "mypy == 1.5.*" + "mypy == 1.5.*", + "flask == 3.0.*", ] [project.scripts] diff --git a/src/meshapi/migrations/0002_remove_installer_group_ptr_remove_readonly_group_ptr_and_more.py b/src/meshapi/migrations/0002_remove_installer_group_ptr_remove_readonly_group_ptr_and_more.py new file mode 100644 index 00000000..e6e1b361 --- /dev/null +++ b/src/meshapi/migrations/0002_remove_installer_group_ptr_remove_readonly_group_ptr_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.9 on 2024-01-28 01:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("meshapi", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="installer", + name="group_ptr", + ), + migrations.RemoveField( + model_name="readonly", + name="group_ptr", + ), + migrations.DeleteModel( + name="Admin", + ), + migrations.DeleteModel( + name="Installer", + ), + migrations.DeleteModel( + name="ReadOnly", + ), + ] diff --git a/src/meshapi/migrations/0003_alter_building_building_status_and_more.py b/src/meshapi/migrations/0003_alter_building_building_status_and_more.py new file mode 100644 index 00000000..618010ed --- /dev/null +++ b/src/meshapi/migrations/0003_alter_building_building_status_and_more.py @@ -0,0 +1,101 @@ +# Generated by Django 4.2.9 on 2024-01-28 02:49 + +from django.db import migrations, models + + +def migrate_building_status_int_to_str(apps, _): + Building = apps.get_model("meshapi", "Building") + + old_to_new_status_mapping = {0: "Inactive", 1: "Active"} + + for building in Building.objects.all(): + building.building_status = old_to_new_status_mapping[building.building_status_old] + building.save() + + +def migrate_install_status_int_to_str(apps, _): + Install = apps.get_model("meshapi", "Install") + + old_to_new_status_mapping = { + 0: "Open", + 1: "Scheduled", + 2: "NN Assigned", + 3: "Blocked", + 4: "Active", + 5: "Inactive", + 6: "Closed", + } + + for install in Install.objects.all(): + install.install_status = old_to_new_status_mapping[install.install_status_old] + install.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("meshapi", "0002_remove_installer_group_ptr_remove_readonly_group_ptr_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="building", + old_name="building_status", + new_name="building_status_old", + ), + migrations.AddField( + model_name="building", + name="building_status", + field=models.TextField(choices=[("Inactive", "Inactive"), ("Active", "Active")], default="Inactive"), + ), + migrations.RunPython(migrate_building_status_int_to_str), + migrations.RemoveField( + model_name="building", + name="building_status_old", + ), + migrations.AlterField( + model_name="building", + name="building_status", + field=models.TextField(choices=[("Inactive", "Inactive"), ("Active", "Active")]), + ), + migrations.RenameField( + model_name="install", + old_name="install_status", + new_name="install_status_old", + ), + migrations.AddField( + model_name="install", + name="install_status", + field=models.TextField( + choices=[ + ("Open", "Open"), + ("Scheduled", "Scheduled"), + ("NN Assigned", "Nn Assigned"), + ("Blocked", "Blocked"), + ("Active", "Active"), + ("Inactive", "Inactive"), + ("Closed", "Closed"), + ], + default="Inactive", + ), + ), + migrations.RunPython(migrate_install_status_int_to_str), + migrations.RemoveField( + model_name="install", + name="install_status_old", + ), + migrations.AlterField( + model_name="install", + name="install_status", + field=models.TextField( + choices=[ + ("Open", "Open"), + ("Scheduled", "Scheduled"), + ("NN Assigned", "Nn Assigned"), + ("Blocked", "Blocked"), + ("Active", "Active"), + ("Inactive", "Inactive"), + ("Closed", "Closed"), + ] + ), + ), + ] diff --git a/src/meshapi/models.py b/src/meshapi/models.py index 6a768047..fef39d20 100644 --- a/src/meshapi/models.py +++ b/src/meshapi/models.py @@ -9,12 +9,12 @@ class Building(models.Model): - class BuildingStatus(models.IntegerChoices): - INACTIVE = 0 - ACTIVE = 1 + class BuildingStatus(models.TextChoices): + INACTIVE = "Inactive" + ACTIVE = "Active" bin = models.IntegerField(blank=True, null=True) - building_status = models.IntegerField(choices=BuildingStatus.choices) + building_status = models.TextField(choices=BuildingStatus.choices) street_address = models.TextField(blank=True, null=True) city = models.TextField(blank=True, null=True) state = models.TextField(blank=True, null=True) @@ -54,14 +54,14 @@ def __str__(self): class Install(models.Model): - class InstallStatus(models.IntegerChoices): - OPEN = 0 - SCHEDULED = 1 - NN_ASSIGNED = 2 - BLOCKED = 3 - ACTIVE = 4 - INACTIVE = 5 - CLOSED = 6 + class InstallStatus(models.TextChoices): + OPEN = "Open" + SCHEDULED = "Scheduled" + NN_ASSIGNED = "NN Assigned" + BLOCKED = "Blocked" + ACTIVE = "Active" + INACTIVE = "Inactive" + CLOSED = "Closed" # Install Number (generated when form is submitted) install_number = models.AutoField( @@ -78,7 +78,7 @@ class InstallStatus(models.IntegerChoices): ) # Summary status of install - install_status = models.IntegerField(choices=InstallStatus.choices) + install_status = models.TextField(choices=InstallStatus.choices) # OSTicket ID ticket_id = models.IntegerField(blank=True, null=True) diff --git a/src/meshapi/tests/sample_data.py b/src/meshapi/tests/sample_data.py index 5a430eab..f67b5914 100644 --- a/src/meshapi/tests/sample_data.py +++ b/src/meshapi/tests/sample_data.py @@ -9,7 +9,7 @@ sample_building = { "bin": 8888, - "building_status": 1, + "building_status": "Active", "street_address": "3333 Chom St", "city": "Brooklyn", "state": "NY", diff --git a/src/meshapi/tests/test_webhooks.py b/src/meshapi/tests/test_webhooks.py new file mode 100644 index 00000000..c61dacdf --- /dev/null +++ b/src/meshapi/tests/test_webhooks.py @@ -0,0 +1,96 @@ +import multiprocessing +import queue + +from flask import Flask, Response, request + +multiprocessing.set_start_method("fork") + +import django_webhook.models +from celery.contrib.testing.worker import start_worker +from django.test import TransactionTestCase +from django_webhook.models import Webhook, WebhookTopic + +from meshdb.celery import app as celery_app + +from ..models import Building, Member +from .sample_data import sample_building, sample_member + +HTTP_CALL_WAITING_TIME = 2 # Seconds + + +def dummy_webhook_listener(http_requests_queue): + flask_app = Flask(__name__) + + @flask_app.route("/webhook", methods=["POST"]) + def respond(): + http_requests_queue.put(request.json) + return Response(status=200) + + flask_app.run(host="127.0.0.1", port=8089, debug=False) + + +class TestMemberWebhook(TransactionTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # Start the celery worker inside the test case + cls.celery_worker = start_worker(celery_app, perform_ping_check=False) + cls.celery_worker.__enter__() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + cls.celery_worker.__exit__(None, None, None) + + def setUp(self): + # Create a simple HTTP listener using flask + self.http_requests_queue = multiprocessing.Queue() + self.app_process = multiprocessing.Process( + target=dummy_webhook_listener, + args=(self.http_requests_queue,), + ) + self.app_process.start() + + # Load the possible webhook topics from the models, normally this happens + # at migration time but the test DB is odd + django_webhook.models.populate_topics_from_settings() + + # Create the webhook in Django + # (this would be done by an admin via the UI in prod) + webhook = Webhook(url="http://localhost:8089/webhook") + topics = [ + WebhookTopic.objects.get(name="meshapi.Member/create"), + ] + webhook.save() + webhook.topics.set(topics) + webhook.save() + + def tearDown(self) -> None: + self.app_process.terminate() + + def test_member(self): + # Create new member triggers webhook + member_obj = Member(**sample_member) + member_obj.save() + + try: + flask_request = self.http_requests_queue.get(timeout=HTTP_CALL_WAITING_TIME) + except queue.Empty as e: + raise RuntimeError("HTTP server not called...") from e + + assert flask_request["topic"] == "meshapi.Member/create" + for key, value in sample_member.items(): + assert flask_request["object"][key] == value + assert flask_request["object_type"] == "meshapi.Member" + assert flask_request["webhook_uuid"] + + def test_building(self): + # Create new building doesn't trigger webhook (they're not subscribed) + building_obj = Building(**sample_building) + building_obj.save() + + try: + self.http_requests_queue.get(timeout=HTTP_CALL_WAITING_TIME) + assert False, "HTTP server shouldn't have been called" + except queue.Empty: + pass diff --git a/src/meshdb/__init__.py b/src/meshdb/__init__.py index e69de29b..5568b6d7 100644 --- a/src/meshdb/__init__.py +++ b/src/meshdb/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/src/meshdb/celery.py b/src/meshdb/celery.py new file mode 100644 index 00000000..57c27038 --- /dev/null +++ b/src/meshdb/celery.py @@ -0,0 +1,18 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings") + +# Use the docker-hosted Redis container as the backend for Celery +app = Celery("meshdb", broker="redis://localhost:6379/0") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() diff --git a/src/meshdb/settings.py b/src/meshdb/settings.py index cf7d9f64..0e6491b2 100644 --- a/src/meshdb/settings.py +++ b/src/meshdb/settings.py @@ -10,8 +10,8 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ -from pathlib import Path import os +from pathlib import Path from dotenv import load_dotenv @@ -73,6 +73,7 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + "django_webhook", "rest_framework", "rest_framework.authtoken", "meshapi", @@ -178,3 +179,18 @@ "rest_framework.authentication.TokenAuthentication", ], } + +# Allow-list models which the admin can select to send webhooks for +DJANGO_WEBHOOK = dict( + MODELS=[ + "meshapi.Building", + "meshapi.Member", + "meshapi.Install", + "meshapi.Link", + "meshapi.Sector", + ], + # This breaks tests, and our write volumes are so low that this performance + # impact should be negligible (it's an extra DB call on any model change) + # If this is a problem in the future, look into setting this only during testing + USE_CACHE=False, +)