Merge pull request #61 from lsst-sqre/tickets/DM-40143

DM-40143: Adopt new notebook execution result with error field

.github/workflows/ci.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
         uses: pre-commit/[email protected]
@@ -40,7 +40,7 @@ jobs:
     strategy:
       matrix:
         python:
-          - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/dependencies.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Run neophile
         uses: lsst-sqre/run-neophile@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           mode: pr
           types: pre-commit
           app-id: ${{ secrets.NEOPHILE_APP_ID }}

.github/workflows/docs.yaml

@@ -23,15 +23,15 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install graphviz and ImageMagick
         run: sudo apt-get install graphviz imagemagick
 
       - name: Run tox
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           tox-envs: "docs"
 
       - name: Upload documentation

.github/workflows/periodic-ci.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         python:
-          - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v3
@@ -29,7 +29,7 @@ jobs:
       - name: Run neophile
         uses: lsst-sqre/run-neophile@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           mode: update
 
       - name: Run tests in tox

CHANGELOG.md

@@ -2,6 +2,35 @@
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-0.8.0'></a>
+## 0.8.0 (2024-01-04)
+
+### Backwards-incompatible changes
+
+-
+
+### New features
+
+- The response to `GET /notebooks/:job_id` now includes an `ipynb_error` field that contains structured information about any exception that occurred when executing the notebook. As well, if an exception occurred, the resultant notebook is still included in the response. That is, notebook failures are no longer considered failed jobs.
+
+- The `job_id` is now included in log messages when running the `nbexec` job under arq.
+
+- The user guide includes a new tutorial for using the Noteburst web API.
+
+### Bug fixes
+
+-
+
+### Other changes
+
+- Update to Pydantic 2
+- Adopt FastAPI's lifespan feature
+- Adopt scriv for changelog management
+
+- Update GitHub Actions workflows, including integrating Neophile for dependency updates.
+
+- Update to Python 3.12.
+
 ## 0.7.1 (2023-07-23)
 
 ### Bug fixes

Dockerfile

@@ -14,7 +14,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.11.4-slim-bullseye as base-image
+FROM python:3.12.1-slim-bullseye as base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2021 Association of Universities for Research in Astronomy, Inc. (AURA)
+Copyright (c) 2021-2024 Association of Universities for Research in Astronomy, Inc. (AURA)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -1,8 +1,26 @@
+# The dependencies need --allow-unsafe because kubernetes-asyncio and
+# (transitively) pre-commit depends on setuptools, which is normally not
+# allowed to appear in a hashed dependency file.
 .PHONY: update-deps
 update-deps:
 	pip install --upgrade pip-tools pip setuptools
-	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/main.txt requirements/main.in
-	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/dev.txt requirements/dev.in
+	pip-compile --upgrade --resolver=backtracking --build-isolation	\
+	    --allow-unsafe --generate-hashes				\
+	    --output-file requirements/main.txt requirements/main.in
+	pip-compile --upgrade --resolver=backtracking --build-isolation	\
+	    --allow-unsafe --generate-hashes				\
+	    --output-file requirements/dev.txt requirements/dev.in
+
+# Useful for testing against a Git version of Safir.
+.PHONY: update-deps-no-hashes
+update-deps-no-hashes:
+	pip install --upgrade pip-tools pip setuptools
+	pip-compile --upgrade --resolver=backtracking --build-isolation	\
+	    --allow-unsafe						\
+	    --output-file requirements/main.txt requirements/main.in
+	pip-compile --upgrade --resolver=backtracking --build-isolation	\
+	    --allow-unsafe						\
+	    --output-file requirements/dev.txt requirements/dev.in
 
 .PHONY: init
 init:
@@ -14,7 +32,3 @@ init:
 
 .PHONY: update
 update: update-deps init
-
-.PHONY: run
-run:
-	tox run -e=run

README.md

@@ -2,3 +2,10 @@
 
 Noteburst is a Rubin Science Platform service that coordinates running Jupyter Notebooks in a JupyterLab context.
 Noteburst can be used by CI and monitoring services, as well as for applications that need to compute and render Jupyter Notebooks programatically.
+[Times Square](https://times-square.lsst.io) uses Noteburst to render Jupyter Notebooks in the browser, for example.
+
+## Links
+
+- [User guide](https://noteburst.lsst.io)
+- [SQR-065](https://sqr-065.lsst.io/) (design document)
+- [Noteburst in Phalanx](https://phalanx.lsst.io/applications/noteburst/index.html)

docs/_rst_epilog.rst

@@ -2,3 +2,5 @@
 .. _pre-commit: https://pre-commit.com
 .. _pytest: https://docs.pytest.org/en/latest/
 .. _tox: https://tox.readthedocs.io/en/latest/
+.. _httpie: https://httpie.io/cli
+.. _`Rubin Science Platform environment`: 

docs/documenteer.toml

@@ -1,6 +1,6 @@
 [project]
 title = "Noteburst"
-copyright = "2021-2023 Association of Universities for Research in Astronomy, Inc. (AURA)"
+copyright = "2021-2024 Association of Universities for Research in Astronomy, Inc. (AURA)"
 
 [project.python]
 package = "noteburst"

docs/user-guide/basic-usage.rst

@@ -0,0 +1,136 @@
+########################################
+How to execute a notebook with Noteburst
+########################################
+
+This page shows how to execute a Jupyter Notebook programmatically on the Rubin Science Platform with Noteburst's web API.
+This approach is applicable to developers who want to integrate notebook execution into their own applications.
+Most users won't use this method directly, but instead use Notebook through an application like Times Square.
+
+Prerequisites
+=============
+
+HTTP client
+-----------
+
+To use the Noteburst web API, you need an HTTP client.
+For this tutorial, we'll use HTTPie_, a command-line HTTP client.
+You can also use curl or a Python library like Requests if you're more comfortable with that.
+
+Rubin Science Platform token
+----------------------------
+
+To use the Noteburst web API, you need a token from the appropriate Rubin Science Platform instance.
+This token generally needs the ``exec:notebook`` scope.
+
+For more information on getting a token, see `the Rubin Science Platform documentation <https://rsp.lsst.io/guides/auth/creating-user-tokens.html>`__.
+
+Once you get a token, it's convenient to store in an environment variable in your shell:
+
+.. prompt:: bash
+
+   export RSP_TOKEN="gt-..."
+
+Send a Notebook to Noteburst
+============================
+
+To execute a Jupyter Notebook (``.ipynb`` file), first send a ``POST`` request:
+
+.. prompt:: bash
+
+   http -A bearer -a $RSP_TOKEN post https://usdf-rsp.slac.stanford.edu/noteburst/v1/notebooks/ [email protected]
+
+Set the ``ipynb`` field to the contents of the notebook (HTTPie_\ ’s ``@`` syntax automatically loads the content from a file).
+Change the hostname as appropriate for your `Rubin Science Platform environment`_.
+
+The response contains a ``Location`` header with a unique URL for this execution request (you can also get the same URL from the ``self_url`` field in the JSON response body).
+You'll use this URL in the next step.
+
+.. code-block:: http
+   :caption: Example response headers
+   :emphasize-lines: 7
+
+   HTTP/1.1 202 Accepted
+   Connection: keep-alive
+   Content-Length: 226
+   Content-Type: application/json
+   Date: Thu, 04 Jan 2024 19:53:15 GMT
+   Strict-Transport-Security: max-age=31536000; includeSubDomains
+   location: https://data-dev.lsst.cloud/noteburst/v1/notebooks/821bd07de5e645b9b30a4e48a0a38b64
+
+.. code-block:: json
+   :caption: Example response body
+   :emphasize-lines: 5
+
+   {
+     "enqueue_time": "2024-01-04T19:53:15.045000Z",
+     "job_id": "821bd07de5e645b9b30a4e48a0a38b64",
+     "kernel_name": "LSST",
+     "self_url": "https://data-dev.lsst.cloud/noteburst/v1/notebooks/821bd07de5e645b9b30a4e48a0a38b64",
+     "status": "queued"
+   }
+
+Get the executed notebook
+=========================
+
+Now make another request, this time a ``GET``, to that ``Location`` URL to get the executed notebook:
+
+.. prompt:: bash
+
+   http -A bearer -a $RSP_TOKEN get https://data-dev.lsst.cloud/noteburst/v1/notebooks/821bd07de5e645b9b30a4e48a0a38b64
+
+The HTTP response body will look similar to the original response from the ``POST`` you made earlier.
+Look at the ``status`` field, though.
+If it's still ``queued``, the notebook hasn't been scheduled to execute yet.
+If the status is ``in_progress``, the notebook is being executed, but has not yet finished.
+You can periodically send more ``GET`` requests to check the status until it finally reads ``completed``.
+
+.. code-block:: json
+   :caption: Example response body
+
+   {
+     "enqueue_time": "2024-01-04T19:53:15.045000Z",
+     "finish_time": "2024-01-04T19:53:22.229000Z",
+     "ipynb": "{...}",
+     "job_id": "821bd07de5e645b9b30a4e48a0a38b64",
+     "kernel_name": "LSST",
+     "self_url": "https://data-dev.lsst.cloud/noteburst/v1/notebooks/821bd07de5e645b9b30a4e48a0a38b64",
+     "start_time": "2024-01-04T19:53:15.548000Z",
+     "status": "complete",
+     "success": true
+   }
+
+In the response, the executed notebook is in the ``ipynb`` field.
+
+You can use the other fields to get statistics and information about the notebook execution.
+For example, the difference between ``finish_time`` and ``start_time`` is the overall execution time.
+
+Notebooks that raise an exception
+=================================
+
+If the notebook raised an exception, the partially notebook is still returned (and the ``success`` field is still ``true``).
+However, now there will be an ``ipynb_error`` field with information about the exception:
+
+.. code-block:: json
+   :caption: Example response body with exception information
+   :emphasize-lines: 5,6,7,8
+
+   {
+     "enqueue_time": "2024-01-04T19:53:15.045000Z",
+     "finish_time": "2024-01-04T19:53:22.229000Z",
+     "ipynb": "{...}",
+     "ipynb_error": {
+       "message": "An error occurred while executing the following cell:\n------------------\nraise RuntimeError(\"This is an error.\")\n",
+       "name": "RuntimeError"
+     },
+     "job_id": "821bd07de5e645b9b30a4e48a0a38b64",
+     "kernel_name": "LSST",
+     "self_url": "https://data-dev.lsst.cloud/noteburst/v1/notebooks/821bd07de5e645b9b30a4e48a0a38b64",
+     "start_time": "2024-01-04T19:53:15.548000Z",
+     "status": "complete",
+     "success": true
+   }
+
+Further reading
+===============
+
+For more information about the Noteburst API, see the :doc:`API reference </api>`.

docs/user-guide/index.rst

@@ -8,3 +8,9 @@ User guide
 
    configuring-workers
    environment-variables
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Usage
+
+   basic-usage