From b0d902dc09e8cc94c8a20b2018cc827f49f14114 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 7 Mar 2024 18:13:00 -0500 Subject: [PATCH 01/11] Use a custom JobNotFoundError Now when a user requests an unknown job, we respond with a structured error with type "unknown_job". --- changelog.d/20240307_175948_jsick_DM_43226.md | 17 +++++++++++++++++ src/noteburst/exceptions.py | 16 +++++++++++++++- src/noteburst/handlers/v1/handlers.py | 16 +++++++++++++++- src/noteburst/main.py | 3 +++ tests/handlers/v1_test.py | 9 +++++++++ 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20240307_175948_jsick_DM_43226.md diff --git a/changelog.d/20240307_175948_jsick_DM_43226.md b/changelog.d/20240307_175948_jsick_DM_43226.md new file mode 100644 index 0000000..dfc4175 --- /dev/null +++ b/changelog.d/20240307_175948_jsick_DM_43226.md @@ -0,0 +1,17 @@ + + +### Backwards-incompatible changes + +- + +### New features + +- Add formatted errors when a job is not found for the `GET /v1/notebooks/:job_id` endpoint. + +### Bug fixes + +- + +### Other changes + +- diff --git a/src/noteburst/exceptions.py b/src/noteburst/exceptions.py index d7d6716..7bc09e5 100644 --- a/src/noteburst/exceptions.py +++ b/src/noteburst/exceptions.py @@ -4,7 +4,10 @@ from typing import Self -__all__ = ["TaskError", "NbexecTaskError"] +from fastapi import status +from safir.fastapi import ClientRequestError + +__all__ = ["TaskError", "NbexecTaskError", "NoteburstClientRequestError"] class TaskError(Exception): @@ -27,3 +30,14 @@ class NbexecTaskError(TaskError): """Error related to a notebook execution task (nbexec).""" task_name = "nbexec" + + +class NoteburstClientRequestError(ClientRequestError): + """Error related to the API client.""" + + +class JobNotFoundError(NoteburstClientRequestError): + """Error raised when a notebook execution job is not found.""" + + error = "unknown_job" + status_code = status.HTTP_404_NOT_FOUND diff --git a/src/noteburst/handlers/v1/handlers.py b/src/noteburst/handlers/v1/handlers.py index 3f97f76..71e7d67 100644 --- a/src/noteburst/handlers/v1/handlers.py +++ b/src/noteburst/handlers/v1/handlers.py @@ -5,9 +5,12 @@ import structlog from arq.jobs import JobStatus from fastapi import APIRouter, Depends, Query, Request, Response -from safir.arq import ArqQueue +from safir.arq import ArqQueue, JobNotFound from safir.dependencies.arq import arq_dependency from safir.dependencies.gafaelfawr import auth_logger_dependency +from safir.models import ErrorLocation, ErrorModel + +from noteburst.exceptions import JobNotFoundError from .models import NotebookResponse, PostNotebookRequest @@ -75,6 +78,7 @@ async def post_nbexec( summary="Get information about a notebook execution job", response_model=NotebookResponse, response_model_exclude_none=True, + responses={404: {"description": "Not found", "model": ErrorModel}}, ) async def get_nbexec_job( *, @@ -123,6 +127,10 @@ async def get_nbexec_job( """ try: job_metadata = await arq_queue.get_job_metadata(job_id) + except JobNotFound: + raise JobNotFoundError( + "Job not found", location=ErrorLocation.path, field_path=["job_id"] + ) from None except Exception: logger.exception( "Error getting nbexec job metadata", @@ -139,6 +147,12 @@ async def get_nbexec_job( if result and job_metadata.status == JobStatus.complete: try: job_result = await arq_queue.get_job_result(job_id) + except JobNotFound: + raise JobNotFoundError( + "Job not found", + location=ErrorLocation.path, + field_path=["job_id"], + ) from None except Exception: logger.exception( "Error getting nbexec job result", diff --git a/src/noteburst/main.py b/src/noteburst/main.py index d562c5e..84e088e 100644 --- a/src/noteburst/main.py +++ b/src/noteburst/main.py @@ -17,6 +17,7 @@ from fastapi.openapi.utils import get_openapi from safir.dependencies.arq import arq_dependency from safir.dependencies.http_client import http_client_dependency +from safir.fastapi import ClientRequestError, client_request_error_handler from safir.logging import configure_logging, configure_uvicorn_logging from safir.middleware.x_forwarded import XForwardedMiddleware @@ -70,6 +71,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # Add middleware app.add_middleware(XForwardedMiddleware) +app.exception_handler(ClientRequestError)(client_request_error_handler) + def create_openapi() -> str: """Create the OpenAPI spec for static documentation.""" diff --git a/tests/handlers/v1_test.py b/tests/handlers/v1_test.py index 3b21bb3..ef3dbdb 100644 --- a/tests/handlers/v1_test.py +++ b/tests/handlers/v1_test.py @@ -81,3 +81,12 @@ async def test_post_nbexec( assert data["status"] == "complete" assert data["success"] is True assert data["ipynb"] == sample_ipynb_executed + + # Request a job that doesn't exist + response = await client.get("/noteburst/v1/notebooks/unknown") + assert response.status_code == 404 + data = response.json() + print(data) + assert data["detail"][0]["type"] == "unknown_job" + assert data["detail"][0]["loc"] == ["path", "job_id"] + assert data["detail"][0]["msg"] == "Job not found" From 5c9dd88ff46b84d48e68f97bfceafd5f7036c589 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Thu, 7 Mar 2024 19:11:42 -0500 Subject: [PATCH 02/11] Add SlackRouteErrorHandler This error handler will report uncaught exceptions based on NoteburstError (and Safir's SlackException) to the Slack webhook. --- src/noteburst/config.py | 10 ++++++++++ src/noteburst/exceptions.py | 16 +++++++++++++++- src/noteburst/handlers/v1/handlers.py | 3 ++- src/noteburst/main.py | 9 +++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/noteburst/config.py b/src/noteburst/config.py index 3345e50..6a3de17 100644 --- a/src/noteburst/config.py +++ b/src/noteburst/config.py @@ -138,6 +138,16 @@ class Config(BaseSettings): ), ] = ArqMode.production + slack_webhook_url: Annotated[ + HttpUrl | None, + Field( + alias="NOTEBURST_SLACK_WEBHOOK_URL", + description=( + "Webhook URL for sending error messages to a Slack channel." + ), + ), + ] = None + @property def arq_redis_settings(self) -> RedisSettings: """Create a Redis settings instance for arq.""" diff --git a/src/noteburst/exceptions.py b/src/noteburst/exceptions.py index 7bc09e5..b758ba1 100644 --- a/src/noteburst/exceptions.py +++ b/src/noteburst/exceptions.py @@ -6,8 +6,14 @@ from fastapi import status from safir.fastapi import ClientRequestError +from safir.slack.blockkit import SlackException -__all__ = ["TaskError", "NbexecTaskError", "NoteburstClientRequestError"] +__all__ = [ + "TaskError", + "NbexecTaskError", + "NoteburstClientRequestError", + "NoteburstError", +] class TaskError(Exception): @@ -41,3 +47,11 @@ class JobNotFoundError(NoteburstClientRequestError): error = "unknown_job" status_code = status.HTTP_404_NOT_FOUND + + +class NoteburstError(SlackException): + """Base class for internal Noteburst exceptions on the FastAPI side. + + This exception derives from SlackException so that uncaught internal + exceptions are reported to Slack. + """ diff --git a/src/noteburst/handlers/v1/handlers.py b/src/noteburst/handlers/v1/handlers.py index 71e7d67..a599fc0 100644 --- a/src/noteburst/handlers/v1/handlers.py +++ b/src/noteburst/handlers/v1/handlers.py @@ -9,12 +9,13 @@ from safir.dependencies.arq import arq_dependency from safir.dependencies.gafaelfawr import auth_logger_dependency from safir.models import ErrorLocation, ErrorModel +from safir.slack.webhook import SlackRouteErrorHandler from noteburst.exceptions import JobNotFoundError from .models import NotebookResponse, PostNotebookRequest -v1_router = APIRouter(tags=["v1"]) +v1_router = APIRouter(tags=["v1"], route_class=SlackRouteErrorHandler) """FastAPI router for the /v1/ REST API""" diff --git a/src/noteburst/main.py b/src/noteburst/main.py index 84e088e..a92ec82 100644 --- a/src/noteburst/main.py +++ b/src/noteburst/main.py @@ -13,6 +13,7 @@ from importlib.metadata import version from pathlib import Path +import structlog from fastapi import FastAPI from fastapi.openapi.utils import get_openapi from safir.dependencies.arq import arq_dependency @@ -20,6 +21,7 @@ from safir.fastapi import ClientRequestError, client_request_error_handler from safir.logging import configure_logging, configure_uvicorn_logging from safir.middleware.x_forwarded import XForwardedMiddleware +from safir.slack.webhook import SlackRouteErrorHandler from .config import config from .handlers.external import external_router @@ -36,6 +38,8 @@ ) configure_uvicorn_logging(config.log_level) +logger = structlog.get_logger(__name__) + @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: @@ -71,6 +75,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # Add middleware app.add_middleware(XForwardedMiddleware) +if config.slack_webhook_url: + SlackRouteErrorHandler.initialize( + str(config.slack_webhook_url), "Noteburst", logger + ) + app.exception_handler(ClientRequestError)(client_request_error_handler) From 617842d8515fd1f89973a8905459d60f7257af6d Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 8 Mar 2024 13:05:29 -0500 Subject: [PATCH 03/11] Raise NoteburstJobError to Slack for job retrieval For errors getting a job from Redis, when we otherwise know the job exists, we're now raising a NoteburstJobError, which will be sent to the Slack webhook. --- src/noteburst/exceptions.py | 17 ++++++++++++++++- src/noteburst/handlers/v1/handlers.py | 26 +++++++++++++++----------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/noteburst/exceptions.py b/src/noteburst/exceptions.py index b758ba1..10a788d 100644 --- a/src/noteburst/exceptions.py +++ b/src/noteburst/exceptions.py @@ -6,7 +6,7 @@ from fastapi import status from safir.fastapi import ClientRequestError -from safir.slack.blockkit import SlackException +from safir.slack.blockkit import SlackException, SlackMessage, SlackTextField __all__ = [ "TaskError", @@ -55,3 +55,18 @@ class NoteburstError(SlackException): This exception derives from SlackException so that uncaught internal exceptions are reported to Slack. """ + + +class NoteburstJobError(NoteburstError): + """Error related to a notebook execution job.""" + + def __init__(self, msg: str, *, user: str | None, job_id: str) -> None: + super().__init__(msg, user=user) + self.job_id = job_id + + def to_slack(self) -> SlackMessage: + message = super().to_slack() + message.fields.append( + SlackTextField(heading="Job ID", text=self.job_id) + ) + return message diff --git a/src/noteburst/handlers/v1/handlers.py b/src/noteburst/handlers/v1/handlers.py index a599fc0..8760e89 100644 --- a/src/noteburst/handlers/v1/handlers.py +++ b/src/noteburst/handlers/v1/handlers.py @@ -7,11 +7,14 @@ from fastapi import APIRouter, Depends, Query, Request, Response from safir.arq import ArqQueue, JobNotFound from safir.dependencies.arq import arq_dependency -from safir.dependencies.gafaelfawr import auth_logger_dependency +from safir.dependencies.gafaelfawr import ( + auth_dependency, + auth_logger_dependency, +) from safir.models import ErrorLocation, ErrorModel from safir.slack.webhook import SlackRouteErrorHandler -from noteburst.exceptions import JobNotFoundError +from noteburst.exceptions import JobNotFoundError, NoteburstJobError from .models import NotebookResponse, PostNotebookRequest @@ -102,6 +105,7 @@ async def get_nbexec_job( ), ), logger: Annotated[structlog.BoundLogger, Depends(auth_logger_dependency)], + user: Annotated[str, Depends(auth_dependency)], arq_queue: Annotated[ArqQueue, Depends(arq_dependency)], ) -> NotebookResponse: """Provides information about a notebook execution job, and the result @@ -132,12 +136,12 @@ async def get_nbexec_job( raise JobNotFoundError( "Job not found", location=ErrorLocation.path, field_path=["job_id"] ) from None - except Exception: - logger.exception( - "Error getting nbexec job metadata", + except Exception as e: + raise NoteburstJobError( + "Error getting job metadata", + user=user, job_id=job_id, - ) - raise + ) from e logger.debug( "Got nbexec job metadata", job_id=job_id, @@ -154,12 +158,12 @@ async def get_nbexec_job( location=ErrorLocation.path, field_path=["job_id"], ) from None - except Exception: - logger.exception( + except Exception as e: + raise NoteburstJobError( "Error getting nbexec job result", + user=user, job_id=job_id, - ) - raise + ) from e logger.debug( "Got nbexec job result", job_id=job_id, From ce958001c3ae6dcba88e05bf717e1543335d3f90 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 12:12:05 -0400 Subject: [PATCH 04/11] Post to Slack on worker start up --- src/noteburst/worker/main.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index 163da59..3436232 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -8,6 +8,8 @@ import structlog from arq import cron from safir.logging import configure_logging +from safir.slack.blockkit import SlackMessage, SlackTextField +from safir.slack.webhook import SlackWebhookClient from noteburst.config import WorkerConfig, WorkerKeepAliveSetting from noteburst.jupyterclient.jupyterlab import ( @@ -47,6 +49,9 @@ async def startup(ctx: dict[Any, Any]) -> None: http_client = httpx.AsyncClient() ctx["http_client"] = http_client + slack_client = SlackWebhookClient(config.slack_webhook_url) + ctx["slack"] = slack_client + jupyter_config = JupyterConfig( url_prefix=config.jupyterhub_path_prefix, image_selector=config.image_selector, @@ -85,7 +90,28 @@ async def startup(ctx: dict[Any, Any]) -> None: ctx["jupyter_client"] = jupyter_client ctx["logger"] = logger - logger.info("Start up complete") + logger.info( + "Noteburst worker startup complete.", + image_selector=config.image_selector, + image_reference=config.image_reference, + ) + + slack_client.post( + SlackMessage( + message="Noteburst worker started", + fields=[ + SlackTextField( + heading="Username", + text=identity.username, + ), + SlackTextField( + heading="Image Selector", + text=config.image_selector, + ), + SlackTextField(heading="Image", text=image_info.name), + ], + ) + ) async def shutdown(ctx: dict[Any, Any]) -> None: From c6eb4eb759f0ee81e8f4a825c0474ef0b7118a69 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 12:32:25 -0400 Subject: [PATCH 05/11] Refresh based on pre-commit/ruff style update --- src/noteburst/config.py | 12 +++++------ src/noteburst/worker/main.py | 41 +++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/noteburst/config.py b/src/noteburst/config.py index 6a3de17..d7a5fee 100644 --- a/src/noteburst/config.py +++ b/src/noteburst/config.py @@ -51,13 +51,13 @@ class Config(BaseSettings): name: Annotated[str, Field(alias="SAFIR_NAME")] = "Noteburst" - profile: Annotated[ - Profile, Field(alias="SAFIR_PROFILE") - ] = Profile.production + profile: Annotated[Profile, Field(alias="SAFIR_PROFILE")] = ( + Profile.production + ) - log_level: Annotated[ - LogLevel, Field(alias="SAFIR_LOG_LEVEL") - ] = LogLevel.INFO + log_level: Annotated[LogLevel, Field(alias="SAFIR_LOG_LEVEL")] = ( + LogLevel.INFO + ) logger_name: Annotated[ str, diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index 3436232..28635b7 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -49,8 +49,13 @@ async def startup(ctx: dict[Any, Any]) -> None: http_client = httpx.AsyncClient() ctx["http_client"] = http_client - slack_client = SlackWebhookClient(config.slack_webhook_url) - ctx["slack"] = slack_client + if config.slack_webhook_url: + slack_client = SlackWebhookClient( + str(config.slack_webhook_url), + "Noteburst worker", + logger=logger, + ) + ctx["slack"] = slack_client jupyter_config = JupyterConfig( url_prefix=config.jupyterhub_path_prefix, @@ -96,22 +101,24 @@ async def startup(ctx: dict[Any, Any]) -> None: image_reference=config.image_reference, ) - slack_client.post( - SlackMessage( - message="Noteburst worker started", - fields=[ - SlackTextField( - heading="Username", - text=identity.username, - ), - SlackTextField( - heading="Image Selector", - text=config.image_selector, - ), - SlackTextField(heading="Image", text=image_info.name), - ], + if "slack" in ctx: + slack_client = ctx["slack"] + await slack_client.post( + SlackMessage( + message="Noteburst worker started", + fields=[ + SlackTextField( + heading="Username", + text=identity.username, + ), + SlackTextField( + heading="Image Selector", + text=config.image_selector, + ), + SlackTextField(heading="Image", text=image_info.name), + ], + ) ) - ) async def shutdown(ctx: dict[Any, Any]) -> None: From 8b644ca056698ee1b132f2a6286f9a6df0f03ed0 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 13:49:28 -0400 Subject: [PATCH 06/11] Add slack message factory for worker This allows a worker to communicate its context when sending messages to slack (e.g. username, age). --- requirements/dev.txt | 129 +++++++++++++++++------------------ requirements/main.in | 1 + requirements/main.txt | 26 +++---- src/noteburst/worker/main.py | 33 +++++++-- 4 files changed, 108 insertions(+), 81 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 2df5a91..0e2d423 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -302,9 +302,9 @@ jsonschema-specifications==2023.12.1 \ --hash=sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc \ --hash=sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c # via jsonschema -latexcodec==2.0.1 \ - --hash=sha256:2aa2551c373261cefe2ad3a8953a6d6533e68238d180eb4bb91d7964adb3fe9a \ - --hash=sha256:c277a193638dc7683c4c30f6684e3db728a06efb0dc9cf346db8bd0aa6c5d271 +latexcodec==3.0.0 \ + --hash=sha256:6f3477ad5e61a0a99bd31a6a370c34e88733a6bad9c921a3ffcfacada12f41a7 \ + --hash=sha256:917dc5fe242762cc19d963e6548b42d63a118028cdd3361d62397e3b638b6bc5 # via pybtex linkify-it-py==2.0.3 \ --hash=sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048 \ @@ -388,34 +388,34 @@ mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -mypy==1.8.0 \ - --hash=sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6 \ - --hash=sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d \ - --hash=sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02 \ - --hash=sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d \ - --hash=sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3 \ - --hash=sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3 \ - --hash=sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3 \ - --hash=sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66 \ - --hash=sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259 \ - --hash=sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835 \ - --hash=sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd \ - --hash=sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d \ - --hash=sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8 \ - --hash=sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07 \ - --hash=sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b \ - --hash=sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e \ - --hash=sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6 \ - --hash=sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae \ - --hash=sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9 \ - --hash=sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d \ - --hash=sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a \ - --hash=sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592 \ - --hash=sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218 \ - --hash=sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817 \ - --hash=sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4 \ - --hash=sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410 \ - --hash=sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55 +mypy==1.9.0 \ + --hash=sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6 \ + --hash=sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913 \ + --hash=sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129 \ + --hash=sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc \ + --hash=sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974 \ + --hash=sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374 \ + --hash=sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150 \ + --hash=sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03 \ + --hash=sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9 \ + --hash=sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02 \ + --hash=sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89 \ + --hash=sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2 \ + --hash=sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d \ + --hash=sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3 \ + --hash=sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612 \ + --hash=sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e \ + --hash=sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3 \ + --hash=sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e \ + --hash=sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd \ + --hash=sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04 \ + --hash=sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed \ + --hash=sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185 \ + --hash=sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf \ + --hash=sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b \ + --hash=sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4 \ + --hash=sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f \ + --hash=sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6 # via -r requirements/dev.in mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ @@ -429,9 +429,9 @@ nodeenv==1.8.0 \ --hash=sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2 \ --hash=sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec # via pre-commit -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 # via # -c requirements/main.txt # pydata-sphinx-theme @@ -562,16 +562,16 @@ pygments==2.17.2 \ pylatexenc==2.10 \ --hash=sha256:3dd8fd84eb46dc30bee1e23eaab8d8fb5a7f507347b23e5f38ad9675c84f40d3 # via documenteer -pytest==8.0.2 \ - --hash=sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd \ - --hash=sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096 +pytest==8.1.1 \ + --hash=sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7 \ + --hash=sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044 # via # -r requirements/dev.in # pytest-asyncio # pytest-cov -pytest-asyncio==0.23.5 \ - --hash=sha256:3a048872a9c4ba14c3e90cc1aa20cbc2def7d01c7c8db3777ec281ba9c057675 \ - --hash=sha256:4e7093259ba018d58ede7d5315131d21923a60f8a6e9ee266ce1589685c89eac +pytest-asyncio==0.23.5.post1 \ + --hash=sha256:30f54d27774e79ac409778889880242b0403d09cabd65b727ce90fe92dd5d80e \ + --hash=sha256:b9a8806bea78c21276bc34321bbf234ba1b2ea5b30d9f0ce0f2dea45e4685813 # via -r requirements/dev.in pytest-cov==4.1.0 \ --hash=sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6 \ @@ -756,24 +756,24 @@ rpds-py==0.18.0 \ # via # jsonschema # referencing -ruff==0.3.0 \ - --hash=sha256:0886184ba2618d815067cf43e005388967b67ab9c80df52b32ec1152ab49f53a \ - --hash=sha256:128265876c1d703e5f5e5a4543bd8be47c73a9ba223fd3989d4aa87dd06f312f \ - --hash=sha256:19eacceb4c9406f6c41af806418a26fdb23120dfe53583df76d1401c92b7c14b \ - --hash=sha256:23dbb808e2f1d68eeadd5f655485e235c102ac6f12ad31505804edced2a5ae77 \ - --hash=sha256:2f7dbba46e2827dfcb0f0cc55fba8e96ba7c8700e0a866eb8cef7d1d66c25dcb \ - --hash=sha256:3ef655c51f41d5fa879f98e40c90072b567c666a7114fa2d9fe004dffba00932 \ - --hash=sha256:5da894a29ec018a8293d3d17c797e73b374773943e8369cfc50495573d396933 \ - --hash=sha256:755c22536d7f1889be25f2baf6fedd019d0c51d079e8417d4441159f3bcd30c2 \ - --hash=sha256:7deb528029bacf845bdbb3dbb2927d8ef9b4356a5e731b10eef171e3f0a85944 \ - --hash=sha256:9343690f95710f8cf251bee1013bf43030072b9f8d012fbed6ad702ef70d360a \ - --hash=sha256:a1f3ed501a42f60f4dedb7805fa8d4534e78b4e196f536bac926f805f0743d49 \ - --hash=sha256:b08b356d06a792e49a12074b62222f9d4ea2a11dca9da9f68163b28c71bf1dd4 \ - --hash=sha256:cc30a9053ff2f1ffb505a585797c23434d5f6c838bacfe206c0e6cf38c921a1e \ - --hash=sha256:d0d3d7ef3d4f06433d592e5f7d813314a34601e6c5be8481cccb7fa760aa243e \ - --hash=sha256:dd73fe7f4c28d317855da6a7bc4aa29a1500320818dd8f27df95f70a01b8171f \ - --hash=sha256:e1e0d4381ca88fb2b73ea0766008e703f33f460295de658f5467f6f229658c19 \ - --hash=sha256:e3a4a6d46aef0a84b74fcd201a4401ea9a6cd85614f6a9435f2d33dd8cefbf83 +ruff==0.3.2 \ + --hash=sha256:0ac06a3759c3ab9ef86bbeca665d31ad3aa9a4b1c17684aadb7e61c10baa0df4 \ + --hash=sha256:0c1bdd9920cab5707c26c8b3bf33a064a4ca7842d91a99ec0634fec68f9f4037 \ + --hash=sha256:1231eacd4510f73222940727ac927bc5d07667a86b0cbe822024dd00343e77e9 \ + --hash=sha256:2c6d613b19e9a8021be2ee1d0e27710208d1603b56f47203d0abbde906929a9b \ + --hash=sha256:5f65103b1d76e0d600cabd577b04179ff592064eaa451a70a81085930e907d0b \ + --hash=sha256:77f2612752e25f730da7421ca5e3147b213dca4f9a0f7e0b534e9562c5441f01 \ + --hash=sha256:967978ac2d4506255e2f52afe70dda023fc602b283e97685c8447d036863a302 \ + --hash=sha256:9966b964b2dd1107797be9ca7195002b874424d1d5472097701ae8f43eadef5d \ + --hash=sha256:9bd640a8f7dd07a0b6901fcebccedadeb1a705a50350fb86b4003b805c81385a \ + --hash=sha256:b74c3de9103bd35df2bb05d8b2899bf2dbe4efda6474ea9681280648ec4d237d \ + --hash=sha256:b83d17ff166aa0659d1e1deaf9f2f14cbe387293a906de09bc4860717eb2e2da \ + --hash=sha256:bb875c6cc87b3703aeda85f01c9aebdce3d217aeaca3c2e52e38077383f7268a \ + --hash=sha256:be75e468a6a86426430373d81c041b7605137a28f7014a72d2fc749e47f572aa \ + --hash=sha256:c8439338a6303585d27b66b4626cbde89bb3e50fa3cae86ce52c1db7449330a7 \ + --hash=sha256:de8b480d8379620cbb5ea466a9e53bb467d2fb07c7eca54a4aa8576483c35d36 \ + --hash=sha256:f380be9fc15a99765c9cf316b40b9da1f6ad2ab9639e551703e581a5e6da6745 \ + --hash=sha256:fa78ec9418eb1ca3db392811df3376b46471ae93792a81af2d1cbb0e5dcb5142 # via -r requirements/dev.in scriv==1.5.1 \ --hash=sha256:30ae9ff8d144f8e0cf394c4e1d379542f1b3823767642955b54ec40dc00b32b6 \ @@ -783,7 +783,6 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via - # latexcodec # pybtex # sphinxcontrib-redoc smmap==5.0.1 \ @@ -899,9 +898,9 @@ tomlkit==0.12.4 \ --hash=sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b \ --hash=sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3 # via documenteer -types-pyyaml==6.0.12.12 \ - --hash=sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062 \ - --hash=sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24 +types-pyyaml==6.0.12.20240311 \ + --hash=sha256:a9e0f0f88dc835739b0c1ca51ee90d04ca2a897a71af79de9aec5f38cb0a5342 \ + --hash=sha256:b845b06a1c7e54b8e5b4c683043de0d9caf205e7434b3edc678ff2411979b8f6 # via -r requirements/dev.in typing-extensions==4.10.0 \ --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \ @@ -921,9 +920,9 @@ urllib3==2.2.1 \ # via # documenteer # requests -uvicorn==0.27.1 \ - --hash=sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a \ - --hash=sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4 +uvicorn==0.28.0 \ + --hash=sha256:6623abbbe6176204a4226e67607b4d52cc60ff62cda0ff177613645cefa2ece1 \ + --hash=sha256:cab4473b5d1eaeb5a0f6375ac4bc85007ffc75c3cc1768816d9e5d589857b067 # via # -c requirements/main.txt # -r requirements/dev.in diff --git a/requirements/main.in b/requirements/main.in index 1cac71b..75fd586 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -19,3 +19,4 @@ pydantic_settings PyYAML httpx websockets +humanize diff --git a/requirements/main.txt b/requirements/main.txt index ed50546..d4a5cd1 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -30,9 +30,7 @@ arq==0.25.0 \ async-timeout==4.0.3 \ --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \ --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028 - # via - # aioredis - # redis + # via aioredis attrs==23.2.0 \ --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 @@ -321,15 +319,19 @@ httpx==0.27.0 \ # via # -r requirements/main.in # safir +humanize==4.9.0 \ + --hash=sha256:582a265c931c683a7e9b8ed9559089dea7edcf6cc95be39a3cbc2c5d5ac2bcfa \ + --hash=sha256:ce284a76d5b1377fd8836733b983bfb0b76f1aa1c090de2566fcf008d7f6ab16 + # via -r requirements/main.in idna==3.6 \ --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \ --hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f # via # anyio # httpx -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 # via gunicorn pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ @@ -495,9 +497,9 @@ pyyaml==6.0.1 \ # via # -r requirements/main.in # uvicorn -redis[hiredis]==5.0.2 \ - --hash=sha256:3f82cc80d350e93042c8e6e7a5d0596e4dd68715babffba79492733e1f367037 \ - --hash=sha256:4caa8e1fcb6f3c0ef28dba99535101d80934b7d4cd541bbb47f4a3826ee472d1 +redis[hiredis]==5.0.3 \ + --hash=sha256:4973bae7444c0fbed64a06b87446f79361cb7e4ec1538c022d696ed7a5015580 \ + --hash=sha256:5da9b8fe9e1254293756c16c008e8620b3d15fcc6dde6babde9541850e72a32d # via arq safir[arq]==5.2.1 \ --hash=sha256:1b61cc72881ddfb66e1f84b6c34ca7e062f27b5669b9d1d07377ebd117ce3ebf \ @@ -532,9 +534,9 @@ uritemplate==4.1.1 \ --hash=sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0 \ --hash=sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e # via gidgethub -uvicorn[standard]==0.27.1 \ - --hash=sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a \ - --hash=sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4 +uvicorn[standard]==0.28.0 \ + --hash=sha256:6623abbbe6176204a4226e67607b4d52cc60ff62cda0ff177613645cefa2ece1 \ + --hash=sha256:cab4473b5d1eaeb5a0f6375ac4bc85007ffc75c3cc1768816d9e5d589857b067 # via -r requirements/main.in uvloop==0.19.0 \ --hash=sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd \ diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index 28635b7..5a4ccf4 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -2,9 +2,11 @@ from __future__ import annotations +from datetime import UTC, datetime from typing import Any, ClassVar import httpx +import humanize import structlog from arq import cron from safir.logging import configure_logging @@ -103,9 +105,15 @@ async def startup(ctx: dict[Any, Any]) -> None: if "slack" in ctx: slack_client = ctx["slack"] - await slack_client.post( - SlackMessage( - message="Noteburst worker started", + + date_created = datetime.now(tz=UTC) + + def create_message(message: str) -> SlackMessage: + now = datetime.now(tz=UTC) + age = now - date_created + + return SlackMessage( + message=message, fields=[ SlackTextField( heading="Username", @@ -116,12 +124,21 @@ async def startup(ctx: dict[Any, Any]) -> None: text=config.image_selector, ), SlackTextField(heading="Image", text=image_info.name), + SlackTextField( + heading="Age", text=humanize.naturaldelta(age) + ), ], ) + + ctx["slack_message_factory"] = create_message + + # Make a start-up message + await slack_client.post( + ctx["slack_message_factory"]("Noteburst worker started") ) -async def shutdown(ctx: dict[Any, Any]) -> None: +async def shutdown(ctx: dict[Any, Any]) -> None: # noqa: PLR0912 """Clean up the worker context on shutdown.""" if "logger" in ctx: logger = ctx["logger"] @@ -171,6 +188,14 @@ async def shutdown(ctx: dict[Any, Any]) -> None: logger.info("Worker shutdown complete.") + if "slack" in ctx and "slack_message_factory" in ctx: + slack_client = ctx["slack"] + await slack_client.post( + ctx["slack_message_factory"]( + "Noteburst worker shut down complete." + ) + ) + # For info on ignoring the type checking here, see # https://github.com/samuelcolvin/arq/issues/249 From 7c220a1a74d71b11b0cb8efd12fd778d168fdcd3 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 14:17:31 -0400 Subject: [PATCH 07/11] Create Slack messages on nbexec failures --- src/noteburst/worker/functions/nbexec.py | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/noteburst/worker/functions/nbexec.py b/src/noteburst/worker/functions/nbexec.py index b4a38fd..f8b4fb3 100644 --- a/src/noteburst/worker/functions/nbexec.py +++ b/src/noteburst/worker/functions/nbexec.py @@ -9,6 +9,7 @@ from typing import Any, cast from arq import Retry +from safir.slack.blockkit import SlackCodeBlock, SlackTextField from noteburst.exceptions import NbexecTaskError from noteburst.jupyterclient.jupyterlab import JupyterClient, JupyterError @@ -64,11 +65,64 @@ async def nbexec( "Authentication error to Jupyter. Forcing worker shutdown", jupyter_status=e.status, ) + + if "slack" in ctx and "slack_message_factory" in ctx: + slack_client = ctx["slack"] + message = ctx["slack_message_factory"]( + "Noteburst worker shutting down due to Jupyter " + "authentication error during nbexec." + ) + message.blocks.append( + SlackCodeBlock(heading="Exception", code=str(e)) + ) + message.fields.append( + SlackTextField( + heading="Jupyter response", text=str(e.status) + ) + ) + message.fields.append( + SlackTextField( + heading="Job ID", text=ctx.get("job_id", "unknown") + ) + ) + message.fields.append( + SlackTextField( + heading="Attempt", text=ctx.get("job_try", "unknown") + ) + ) + await slack_client.post(message) + sys.exit("400 class error from Jupyter") elif enable_retry: logger.warning("nbexec triggering retry") raise Retry(defer=ctx["job_try"] * 5) from None else: + if "slack" in ctx and "slack_message_factory" in ctx: + slack_client = ctx["slack"] + message = ctx["slack_message_factory"]("Nbexec failed.") + message.blocks.append( + SlackCodeBlock(heading="Exception", code=str(e)) + ) + message.fields.append( + SlackTextField( + heading="Jupyter response", text=str(e.status) + ) + ) + message.fields.append( + SlackTextField( + heading="Job ID", text=ctx.get("job_id", "unknown") + ) + ) + message.fields.append( + SlackTextField( + heading="Attempt", text=ctx.get("job_try", "unknown") + ) + ) + message.blocks.append( + SlackCodeBlock(heading="Notebook", code=ipynb) + ) + await slack_client.post(message) + raise NbexecTaskError.from_exception(e) from e return execution_result.model_dump_json() From 88a217cd95f3834e90eaf5456cba334e7480c3ee Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 14:25:24 -0400 Subject: [PATCH 08/11] Update change log --- changelog.d/20240307_175948_jsick_DM_43226.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/changelog.d/20240307_175948_jsick_DM_43226.md b/changelog.d/20240307_175948_jsick_DM_43226.md index dfc4175..96774a1 100644 --- a/changelog.d/20240307_175948_jsick_DM_43226.md +++ b/changelog.d/20240307_175948_jsick_DM_43226.md @@ -1,17 +1,4 @@ - - -### Backwards-incompatible changes - -- - ### New features - Add formatted errors when a job is not found for the `GET /v1/notebooks/:job_id` endpoint. - -### Bug fixes - -- - -### Other changes - -- +- Errors and uncaught exceptions are now sent to Slack via a Slack webhook. The webhook URL is set via the `SLACK_WEBHOOK_URL` environment variable. From cb03538f988d6bdb49afbab47e78f2f045e2d470 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 16:07:16 -0400 Subject: [PATCH 09/11] Send Slack message earlier for nbexec error I think we weren't sending the message earlier because enable_retry is true --- src/noteburst/worker/functions/nbexec.py | 50 ++++++++++++------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/noteburst/worker/functions/nbexec.py b/src/noteburst/worker/functions/nbexec.py index f8b4fb3..c53c536 100644 --- a/src/noteburst/worker/functions/nbexec.py +++ b/src/noteburst/worker/functions/nbexec.py @@ -60,6 +60,30 @@ async def nbexec( logger.info("nbexec finished", error=execution_result.error) except JupyterError as e: logger.exception("nbexec error", jupyter_status=e.status) + if "slack" in ctx and "slack_message_factory" in ctx: + slack_client = ctx["slack"] + message = ctx["slack_message_factory"]("Nbexec failed.") + message.blocks.append( + SlackCodeBlock(heading="Exception", code=str(e)) + ) + message.fields.append( + SlackTextField(heading="Jupyter response", text=str(e.status)) + ) + message.fields.append( + SlackTextField( + heading="Job ID", text=ctx.get("job_id", "unknown") + ) + ) + message.fields.append( + SlackTextField( + heading="Attempt", text=ctx.get("job_try", "unknown") + ) + ) + message.blocks.append( + SlackCodeBlock(heading="Notebook", code=ipynb) + ) + await slack_client.post(message) + if e.status >= 400 and e.status < 500: logger.exception( "Authentication error to Jupyter. Forcing worker shutdown", @@ -97,32 +121,6 @@ async def nbexec( logger.warning("nbexec triggering retry") raise Retry(defer=ctx["job_try"] * 5) from None else: - if "slack" in ctx and "slack_message_factory" in ctx: - slack_client = ctx["slack"] - message = ctx["slack_message_factory"]("Nbexec failed.") - message.blocks.append( - SlackCodeBlock(heading="Exception", code=str(e)) - ) - message.fields.append( - SlackTextField( - heading="Jupyter response", text=str(e.status) - ) - ) - message.fields.append( - SlackTextField( - heading="Job ID", text=ctx.get("job_id", "unknown") - ) - ) - message.fields.append( - SlackTextField( - heading="Attempt", text=ctx.get("job_try", "unknown") - ) - ) - message.blocks.append( - SlackCodeBlock(heading="Notebook", code=ipynb) - ) - await slack_client.post(message) - raise NbexecTaskError.from_exception(e) from e return execution_result.model_dump_json() From 4985a4c9f0ee9fc0a112e861a3c7f357c3f23388 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Mon, 11 Mar 2024 16:29:52 -0400 Subject: [PATCH 10/11] Create a JupyterError timeout in execute_notebook Time-outs are generating Httpx errors, so we want to convert those into JupyterError to log to a Slack message. --- src/noteburst/jupyterclient/jupyterlab.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/noteburst/jupyterclient/jupyterlab.py b/src/noteburst/jupyterclient/jupyterlab.py index fa9ea83..1bbbfed 100644 --- a/src/noteburst/jupyterclient/jupyterlab.py +++ b/src/noteburst/jupyterclient/jupyterlab.py @@ -747,10 +747,22 @@ async def execute_notebook( Notebook execution extension. """ exec_url = self.url_for(f"user/{self.user.username}/rubin/execution") - r = await self.http_client.post( - exec_url, - content=json.dumps(notebook).encode("utf-8"), - ) + try: + r = await self.http_client.post( + exec_url, + content=json.dumps(notebook).encode("utf-8"), + ) + except httpx.HTTPError as e: + # This often occurs from timeouts, so we want to convert the + # generic HTTPError to a JupyterError. + raise JupyterError( + url=exec_url, + username=self.user.username, + status=500, + reason="Internal Server Error", + method="POST", + body=str(e), + ) from e if r.status_code != 200: raise JupyterError.from_response(self.user.username, r) self.logger.debug("Got response from /rubin/execution", text=r.text) From e81aa55821773ae57bc4aaa16c44afb71113054f Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Wed, 13 Mar 2024 13:29:38 -0400 Subject: [PATCH 11/11] Prepare change log for 0.9.0 release --- CHANGELOG.md | 23 ++++++++++++------- changelog.d/20240305_122219_jsick_DM_43173.md | 17 -------------- changelog.d/20240307_175948_jsick_DM_43226.md | 4 ---- 3 files changed, 15 insertions(+), 29 deletions(-) delete mode 100644 changelog.d/20240305_122219_jsick_DM_43173.md delete mode 100644 changelog.d/20240307_175948_jsick_DM_43226.md diff --git a/CHANGELOG.md b/CHANGELOG.md index be30307..0bf96a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,23 @@ - -## 0.8.0 (2024-01-04) + + +## 0.9.0 (2024-03-13) + +### New features + +- Add formatted errors when a job is not found for the `GET /v1/notebooks/:job_id` endpoint. -### Backwards-incompatible changes +- Errors and uncaught exceptions are now sent to Slack via a Slack webhook. The webhook URL is set via the `SLACK_WEBHOOK_URL` environment variable. + +### Other changes -- +- The code base now uses Ruff for linting and formatting, replacing black, isort, and flake8. This change is part of the ongoing effort to standardize SQuaRE code bases and improve the developer experience. + + + +## 0.8.0 (2024-01-04) ### New features @@ -17,10 +28,6 @@ - The user guide includes a new tutorial for using the Noteburst web API. -### Bug fixes - -- - ### Other changes - Update to Pydantic 2 diff --git a/changelog.d/20240305_122219_jsick_DM_43173.md b/changelog.d/20240305_122219_jsick_DM_43173.md deleted file mode 100644 index e9b2c6c..0000000 --- a/changelog.d/20240305_122219_jsick_DM_43173.md +++ /dev/null @@ -1,17 +0,0 @@ - - -### Backwards-incompatible changes - -- - -### New features - -- - -### Bug fixes - -- - -### Other changes - -- The code base now uses Ruff for linting and formatting, replacing black, isort, and flake8. This change is part of the ongoing effort to standardize SQuaRE code bases and improve the developer experience. diff --git a/changelog.d/20240307_175948_jsick_DM_43226.md b/changelog.d/20240307_175948_jsick_DM_43226.md deleted file mode 100644 index 96774a1..0000000 --- a/changelog.d/20240307_175948_jsick_DM_43226.md +++ /dev/null @@ -1,4 +0,0 @@ -### New features - -- Add formatted errors when a job is not found for the `GET /v1/notebooks/:job_id` endpoint. -- Errors and uncaught exceptions are now sent to Slack via a Slack webhook. The webhook URL is set via the `SLACK_WEBHOOK_URL` environment variable.