diff --git a/.github/workflows/build-base-image.yml b/.github/workflows/build-base-image.yml index 691ee3934..f4f77af18 100644 --- a/.github/workflows/build-base-image.yml +++ b/.github/workflows/build-base-image.yml @@ -2,7 +2,7 @@ name: Build Base Image on: schedule: # Every Monday at 6:22am Eastern Time - - cron: '22 10 * * 1' + - cron: "22 10 * * 1" workflow_dispatch: # Allow us to manually trigger build @@ -46,11 +46,11 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/circ-baseimage + images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-baseimage # Generate tags for the image # We use the following tags: - # - The date in YYYYww format, which is the year and week number. 202052 is the last week of 2020. - # - The latest tag + # - The date in YYYYww format, which is the year and week number. 202052 is the last week of 2020. + # - The latest tag tags: | type=schedule,pattern={{date 'YYYYww'}} type=raw,value=latest diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 1d985105d..701c46cbf 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -158,7 +158,7 @@ jobs: id: baseimage-meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/circ-baseimage + images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-baseimage tags: | type=ref,event=branch type=sha @@ -169,7 +169,7 @@ jobs: id: baseimage-latest uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/circ-baseimage + images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-baseimage tags: | type=raw,value=latest @@ -192,7 +192,7 @@ jobs: type=registry,ref=${{ fromJSON(steps.baseimage-latest.outputs.json).tags[0] }} type=registry,ref=${{ fromJSON(steps.baseimage-meta.outputs.json).tags[0] }} cache-to: | - type=inline + type=inline platforms: linux/amd64, linux/arm64 tags: ${{ steps.baseimage-meta.outputs.tags }} labels: ${{ steps.baseimage-meta.outputs.labels }} @@ -213,7 +213,7 @@ jobs: elif [[ $tag_exists -eq 0 ]]; then tag="${{ fromJSON(steps.baseimage-latest.outputs.json).tags[0] }}" else - tag="ghcr.io/thepalaceproject/circ-baseimage:latest" + tag="ghcr.io/natlibfi/ekirjasto-circ-baseimage:latest" fi echo "Base image tag: $tag" echo tag="$tag" >> "$GITHUB_OUTPUT" @@ -358,7 +358,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ghcr.io/${{ github.repository_owner }}/circ-${{ matrix.image }} + images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-${{ matrix.image }} tags: | type=semver,pattern={{major}}.{{minor}},priority=10 type=semver,pattern={{version}},priority=20 diff --git a/README.md b/README.md index 63cc4f36d..ff9676637 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,23 @@ -# Palace Manager +# E-kirjasto Circulation Manager -[![Test & Build](https://github.com/ThePalaceProject/circulation/actions/workflows/test-build.yml/badge.svg)](https://github.com/ThePalaceProject/circulation/actions/workflows/test-build.yml) -[![codecov](https://codecov.io/github/thepalaceproject/circulation/branch/main/graph/badge.svg?token=T09QW6DLH6)](https://codecov.io/github/thepalaceproject/circulation) -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) -[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) -![Python: 3.8,3.9,3.10,3.11](https://img.shields.io/badge/Python-3.8%20|%203.9%20|%203.10%20|%203.11-blue) +This is the E-kirjasto fork of the [The Palace Project](https://thepalaceproject.org) Palace Manager (which is a fork of +[Library Simplified](http://www.librarysimplified.org/) Circulation Manager). -This is a [The Palace Project](https://thepalaceproject.org) maintained fork of the NYPL -[Library Simplified](http://www.librarysimplified.org/) Circulation Manager. + ## Git Branch Workflow -| Branch | Python Version | -| -------- | -------------- | -| main | Python 3 | -| python2 | Python 2 | - The default branch is `main` and that's the working branch that should be used when branching off for bug fixes or new features. -Python 2 stopped being supported after January 1st, 2020 but there is still a `python2` branch which can be used. As of -August 2021, development will be done in the `main` branch and the `python2` branch will not be updated unless -absolutely necessary. - ## Set Up ### Docker Compose @@ -77,7 +61,21 @@ Most distributions will offer Python packages. On Arch Linux, the following comm pacman -S python ``` -#### pyenv +You need to install dependencies: https://devguide.python.org/getting-started/setup-building/#build-dependencies + +Enable Source Packages: +Uncomment a deb-src in `/etc/apt/sources.list` e.g. `jammy main` + +Install build dependencies: + +```sh +$ sudo apt-get update +$ sudo apt-get build-dep python3 +$ sudo apt-get install pkg-config +$ sudo apt install libxmlsec1 libxmlsec1-dev +``` + +#### pyenv (Optional) [pyenv](https://github.com/pyenv/pyenv) pyenv lets you easily switch between multiple versions of Python. It can be [installed](https://github.com/pyenv/pyenv-installer) using the command `curl https://pyenv.run | bash`. You can then @@ -110,14 +108,14 @@ Poetry can be installed using the command `curl -sSL https://install.python-poet More information about installation options can be found in the [poetry documentation](https://python-poetry.org/docs/master/#installation). -### Opensearch +### OpenSearch -Palace now supports Opensearch: please use it instead of Elasticsearch. +Palace now supports OpenSearch: please use it instead of Elasticsearch. Elasticsearch is no longer supported. #### Docker -We recommend that you run Opensearch with docker using the following docker commands: +We recommend that you run OpenSearch with docker using the following docker commands: ```sh docker run --name opensearch -d --rm -p 9006:9200 -e "discovery.type=single-node" -e "plugins.security.disabled=true" "opensearchproject/opensearch:1" @@ -171,20 +169,20 @@ a storage service, you can set the following environment variables: public access to the files. - `PALACE_STORAGE_ANALYTICS_BUCKET`: Required if you want to use the storage service to store analytics data. - `PALACE_STORAGE_ACCESS_KEY`: The access key (optional). - - If this key is set it will be passed to boto3 when connecting to the storage service. - - If it is not set boto3 will attempt to find credentials as outlined in their - [documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). + - If this key is set it will be passed to boto3 when connecting to the storage service. + - If it is not set boto3 will attempt to find credentials as outlined in their + [documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). - `PALACE_STORAGE_SECRET_KEY`: The secret key (optional). - `PALACE_STORAGE_REGION`: The AWS region of the storage service (optional). - `PALACE_STORAGE_ENDPOINT_URL`: The endpoint of the storage service (optional). This is used if you are using a s3 compatible storage service like [minio](https://min.io/). - `PALACE_STORAGE_URL_TEMPLATE`: The url template to use when generating urls for files stored in the storage service (optional). - - The default value is `https://{bucket}.s3.{region}.amazonaws.com/{key}`. - - The following variables can be used in the template: - - `{bucket}`: The name of the bucket. - - `{key}`: The key of the file. - - `{region}`: The region of the storage service. + - The default value is `https://{bucket}.s3.{region}.amazonaws.com/{key}`. + - The following variables can be used in the template: + - `{bucket}`: The name of the bucket. + - `{key}`: The key of the file. + - `{region}`: The region of the storage service. #### Reporting @@ -209,9 +207,9 @@ the logging: - `PALACE_LOG_CLOUDWATCH_INTERVAL`: The interval in seconds to send logs to CloudWatch. Default is `60`. - `PALACE_LOG_CLOUDWATCH_CREATE_GROUP`: Whether to create the log group if it does not exist. Default is `true`. - `PALACE_LOG_CLOUDWATCH_ACCESS_KEY`: The access key to use when sending logs to CloudWatch. This is optional. - - If this key is set it will be passed to boto3 when connecting to CloudWatch. - - If it is not set boto3 will attempt to find credentials as outlined in their - [documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). + - If this key is set it will be passed to boto3 when connecting to CloudWatch. + - If it is not set boto3 will attempt to find credentials as outlined in their + [documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). - `PALACE_LOG_CLOUDWATCH_SECRET_KEY`: The secret key to use when sending logs to CloudWatch. This is optional. #### Patron `Basic Token` authentication @@ -219,6 +217,7 @@ the logging: Enables/disables patron "basic token" authentication through setting the designated environment variable to any (case-insensitive) value of "true"/"yes"/"on"/"1" or "false"/"no"/"off"/"0", respectively. If the value is the empty string or the variable is not present in the environment, it is disabled by default. + - `SIMPLIFIED_ENABLE_BASIC_TOKEN_AUTH` ```sh @@ -228,7 +227,8 @@ export SIMPLIFIED_ENABLE_BASIC_TOKEN_AUTH=true #### Firebase Cloud Messaging For Firebase Cloud Messaging (FCM) support (e.g., for notifications), `one` (and only one) of the following should be set: -- `SIMPLIFIED_FCM_CREDENTIALS_JSON` - the JSON-format Google Cloud Platform (GCP) service account key or + +- `SIMPLIFIED_FCM_CREDENTIALS_JSON` - the JSON-format Google Cloud Platform (GCP) service account key or - `SIMPLIFIED_FCM_CREDENTIALS_FILE` - the name of the file containing that key. ```sh @@ -258,6 +258,14 @@ Local analytics are enabled by default. S3 analytics can be enabled via the foll - PALACE_S3_ANALYTICS_ENABLED: A boolean value to disable or enable s3 analytics. The default is false. +## OpenSearch Analytics (E-Kirjasto, Finland) + +OpenSearch analytics can be enabled via the following environment variables: + +- PALACE_OPENSEARCH_ANALYTICS_ENABLED: A boolean value to disable or enable OpenSearch analytics. The default is false. +- PALACE_OPENSEARCH_ANALYTICS_URL: The url of your OpenSearch instance, eg. "http://localhost:9200" +- PALACE_OPENSEARCH_ANALYTICS_INDEX_PREFIX: The prefix of the event index name, eg. "circulation-events" + #### Email ### Email sending @@ -323,6 +331,7 @@ Run the application with: poetry run python app.py ``` +psear Check that there is now a web server listening on port `6500`: ```sh @@ -333,7 +342,7 @@ curl http://localhost:6500/ #### Access -By default, the application is configured to provide a built-in version of the [admin web interface](https://github.com/ThePalaceProject/circulation-admin). +By default, the application is configured to provide a built-in version of the [admin web interface](https://github.com/NatLibFi/ekirjasto-circulation-admin). The admin interface can be accessed by visiting the `/admin` endpoint: ```sh @@ -404,9 +413,9 @@ service has been configured. #### Configuring Search Navigate to `System Configuration → Search` and add a new search configuration. The required URL is -the URL of the [Opensearch instance configured earlier](#opensearch): +the URL of the [OpenSearch instance configured earlier](#opensearch): -![Opensearch](.github/readme/search.png) +![OpenSearch](.github/readme/search.png) #### Generating Search Indices @@ -534,13 +543,14 @@ the different lints that pre-commit runs. #### Built in Pre-commit ships with a [number of lints](https://pre-commit.com/hooks.html) out of the box, we are configured to use: + - `trailing-whitespace` - trims trailing whitespace. - `end-of-file-fixer` - ensures that a file is either empty, or ends with one newline. - `check-yaml` - checks yaml files for parseable syntax. - `check-json` - checks json files for parseable syntax. - `check-ast` - simply checks whether the files parse as valid python. - `check-shebang-scripts-are-executable` - ensures that (non-binary) files with a shebang are executable. -- `check-executables-have-shebangs` - ensures that (non-binary) executables have a shebang. +- `check-executables-have-shebangs` - ensures that (non-binary) executables have a shebang. - `check-merge-conflict` - checks for files that contain merge conflict strings. - `check-added-large-files` - prevents giant files from being committed. - `mixed-line-ending` - replaces or checks mixed line ending. @@ -593,7 +603,7 @@ with service dependencies running in docker containers. #### Python version | Factor | Python Version | -|--------|----------------| +| ------ | -------------- | | py38 | Python 3.8 | | py39 | Python 3.9 | | py310 | Python 3.10 | @@ -617,10 +627,10 @@ missing Python versions in your system for local testing. #### Module -| Factor | Module | -| ----------- | ----------------- | -| core | core tests | -| api | api tests | +| Factor | Module | +| ------ | ---------- | +| core | core tests | +| api | api tests | #### Docker @@ -703,7 +713,7 @@ enabled by setting environment variables while starting the application. - `PALACE_XRAY_NAME`: The name of the service shown in x-ray for these traces. - `PALACE_XRAY_ANNOTATE_`: Any environment variable starting with this prefix will be added to to the trace as an annotation. - - For example setting `PALACE_XRAY_ANNOTATE_KEY=value` will set the annotation `key=value` on all xray traces sent + - For example setting `PALACE_XRAY_ANNOTATE_KEY=value` will set the annotation `key=value` on all xray traces sent from the application. - `PALACE_XRAY_INCLUDE_BARCODE`: If this environment variable is set to `true` then the tracing code will try to include the patrons barcode in the user parameter of the trace, if a barcode is available. @@ -725,8 +735,9 @@ module under the hood to do the profiling. path specified in the environment variable. - The profile data will have the extension `.prof`. - The data can be accessed using the -[`pstats.Stats` class](https://docs.python.org/3/library/profile.html#the-stats-class). + [`pstats.Stats` class](https://docs.python.org/3/library/profile.html#the-stats-class). - Example code to print details of the gathered statistics: + ```python import os from pathlib import Path @@ -747,23 +758,25 @@ This profiler uses [PyInstrument](https://pyinstrument.readthedocs.io/en/latest/ - `PALACE_PYINSTRUMENT`: Profiling will the enabled if this variable is set. The saved profile data will be available at path specified in the environment variable. - - The profile data will have the extension `.pyisession`. - - The data can be accessed with the + + - The profile data will have the extension `.pyisession`. + - The data can be accessed with the [`pyinstrument.session.Session` class](https://pyinstrument.readthedocs.io/en/latest/reference.html#pyinstrument.session.Session). - - Example code to print details of the gathered statistics: - ```python - import os - from pathlib import Path - - from pyinstrument.renderers import HTMLRenderer - from pyinstrument.session import Session - - path = Path(os.environ.get("PALACE_PYINSTRUMENT")) - for file in path.glob("*.pyisession"): - session = Session.load(file) - renderer = HTMLRenderer() - renderer.open_in_browser(session) - ``` + - Example code to print details of the gathered statistics: + + ```python + import os + from pathlib import Path + + from pyinstrument.renderers import HTMLRenderer + from pyinstrument.session import Session + + path = Path(os.environ.get("PALACE_PYINSTRUMENT")) + for file in path.glob("*.pyisession"): + session = Session.load(file) + renderer = HTMLRenderer() + renderer.open_in_browser(session) + ``` ### Other Environment Variables diff --git a/api/admin/config.py b/api/admin/config.py index 957b0c339..b20b987a2 100644 --- a/api/admin/config.py +++ b/api/admin/config.py @@ -15,8 +15,8 @@ class OperationalMode(str, Enum): class Configuration(LoggerMixin): - APP_NAME = "Palace Collection Manager" - PACKAGE_NAME = "@thepalaceproject/circulation-admin" + APP_NAME = "E-kirjasto Collection Manager" + PACKAGE_NAME = "@natlibfi/ekirjasto-circulation-admin" PACKAGE_VERSION = "1.11.0" STATIC_ASSETS = { diff --git a/api/admin/controller/dashboard.py b/api/admin/controller/dashboard.py index fd53cb6e6..5774210f5 100644 --- a/api/admin/controller/dashboard.py +++ b/api/admin/controller/dashboard.py @@ -105,3 +105,36 @@ def get_date(field): date_end_label.strftime(date_format), library_short_name, ) + + # Finland + # Copied and modified from bulk_circulation_events + def circulation_loan_statistics_excel(self): + date_format = "%Y-%m-%d" + + def get_date(field): + today = date.today() + value = flask.request.args.get(field, None) + if not value: + return today + try: + return datetime.strptime(value, date_format).date() + except ValueError as e: + return today + + date_start = get_date("date") + date_end_label = get_date("dateEnd") + date_end = date_end_label + timedelta(days=1) + locations = flask.request.args.get("locations", None) + library = getattr(flask.request, "library", None) + library_short_name = library.short_name if library else None + + local_analytics_exporter = LocalAnalyticsExporter() + data = local_analytics_exporter.export_excel( + self._db, date_start, date_end, locations, library + ) + return ( + data, + date_start.strftime(date_format), + date_end_label.strftime(date_format), + library_short_name, + ) diff --git a/api/admin/routes.py b/api/admin/routes.py index d9de20fc3..9754dac2f 100644 --- a/api/admin/routes.py +++ b/api/admin/routes.py @@ -2,7 +2,7 @@ from functools import wraps import flask -from flask import Response, make_response, redirect, url_for +from flask import Response, make_response, redirect, url_for, request from flask_pydantic_spec import FileResponse as SpecFileResponse from flask_pydantic_spec import Request as SpecRequest from flask_pydantic_spec import Response as SpecResponse @@ -302,6 +302,34 @@ def bulk_circulation_events(): return response +# Finland +@library_route("/admin/circulation_loan_statistics_excel") +@returns_json_or_response_or_problem_detail +@allows_library +@requires_admin +def bulk_circulation_events_excel(): + """Returns an Excel file containing loan amounts and co-authors + for each work on a given timeframe.""" + ( + data, + date, + date_end, + library, + ) = app.manager.admin_dashboard_controller.circulation_loan_statistics_excel() + if isinstance(data, ProblemDetail): + return data + + response = Response(data) + response.headers[ + "Content-Type" + ] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + response.headers[ + "Content-Disposition" + ] = f"attachment; filename=lainaukset_{library}_{date}_{date_end}.xlsx" + + return response + + @library_route("/admin/circulation_events") @has_library @returns_json_or_response_or_problem_detail @@ -347,6 +375,37 @@ def get_quicksight_names(): return app.manager.admin_quicksight_controller.get_dashboard_names() +# Finland +@library_route("/admin/events/terms") +@returns_json_or_response_or_problem_detail +@has_library +@requires_admin +def events_query_terms(): + """Returns results from OpenSearch analytics events based on query parameters.""" + return app.manager.opensearch_analytics_search.events(params=request.args) + + +# Finland +@library_route("/admin/events/histogram") +@returns_json_or_response_or_problem_detail +@has_library +@requires_admin +def events_query_histogram(): + """Returns results from OpenSearch analytics based on query parameters + in histogram buckets for time series graphs""" + return app.manager.opensearch_analytics_search.events_histogram(params=request.args) + + +# Finland +@library_route("/admin/events/facets") +@returns_json_or_response_or_problem_detail +@has_library +@requires_admin +def events_get_facets(): + """Returns all the available facets from OpenSearch analytics events.""" + return app.manager.opensearch_analytics_search.get_facets() + + @app.route("/admin/libraries", methods=["GET", "POST"]) @returns_json_or_response_or_problem_detail @requires_admin diff --git a/api/admin/templates.py b/api/admin/templates.py index 712f2e61c..d2bfab477 100644 --- a/api/admin/templates.py +++ b/api/admin/templates.py @@ -4,6 +4,7 @@ {{ app_name }} + diff --git a/api/authenticator.py b/api/authenticator.py index 69a8ee508..bbad8c069 100644 --- a/api/authenticator.py +++ b/api/authenticator.py @@ -27,6 +27,7 @@ from api.custom_patron_catalog import CustomPatronCatalog from api.integration.registry.patron_auth import PatronAuthRegistry from api.problem_details import * +from api.ekirjasto_authentication import EkirjastoAuthenticationAPI # Finland from core.analytics import Analytics from core.integration.goals import Goals from core.integration.registry import IntegrationRegistry @@ -145,6 +146,11 @@ def saml_provider_lookup(self, *args, **kwargs): def decode_bearer_token(self, *args, **kwargs): return self.invoke_authenticator_method("decode_bearer_token", *args, **kwargs) + # Finland + @property + def ekirjasto_provider(self) -> EkirjastoAuthenticationAPI: + return self.invoke_authenticator_method("get_ekirjasto_provider") + class LibraryAuthenticator(LoggerMixin): """Use the registered AuthenticationProviders to turn incoming @@ -206,6 +212,10 @@ def from_config( BearerTokenSigner.bearer_token_signing_secret(_db) ) + # Finland + if authenticator.ekirjasto_provider: + authenticator.ekirjasto_provider.set_secrets(_db) + authenticator.assert_ready_for_token_signing() return authenticator @@ -216,6 +226,7 @@ def __init__( library: Library, basic_auth_provider: Optional[BasicAuthenticationProvider] = None, saml_providers: Optional[List[BaseSAMLAuthenticationProvider]] = None, + ekirjasto_provider: Optional[EkirjastoAuthenticationAPI] = None, # Finland bearer_token_signing_secret: Optional[str] = None, authentication_document_annotator: Optional[CustomPatronCatalog] = None, integration_registry: Optional[ @@ -253,6 +264,7 @@ def __init__( ) self.saml_providers_by_name = {} + self.ekirjasto_provider = ekirjasto_provider # Finland self.bearer_token_signing_secret = bearer_token_signing_secret self.initialization_exceptions: Dict[ Tuple[int | None, int | None], Exception @@ -276,6 +288,9 @@ def supports_patron_authentication(self) -> bool: """Does this library have any way of authenticating patrons at all?""" if self.basic_auth_provider or self.saml_providers_by_name: return True + # Finland + if self.ekirjasto_provider: + return True return False @property @@ -369,6 +384,8 @@ def register_provider( # the ability to run one. elif isinstance(provider, BaseSAMLAuthenticationProvider): self.register_saml_provider(provider) + elif isinstance(provider, EkirjastoAuthenticationAPI): # Finland + self.register_ekirjasto_provider(provider) else: raise CannotLoadConfiguration( f"Authentication provider {impl_cls.__name__} is neither a BasicAuthenticationProvider nor a " @@ -405,6 +422,19 @@ def register_saml_provider( ) self.saml_providers_by_name[provider.label()] = provider + # Finland + def register_ekirjasto_provider( + self, + provider: EkirjastoAuthenticationAPI, + ): + if self.ekirjasto_provider is not None and self.ekirjasto_provider != provider: + raise CannotLoadConfiguration("Two ekirjasto auth providers configured") + self.ekirjasto_provider = provider + + # Finland + def get_ekirjasto_provider(self) -> EkirjastoAuthenticationAPI: + return self.ekirjasto_provider + @property def providers(self) -> Iterable[AuthenticationProvider]: """An iterator over all registered AuthenticationProviders.""" @@ -412,6 +442,8 @@ def providers(self) -> Iterable[AuthenticationProvider]: yield self.access_token_authentication_provider if self.basic_auth_provider: yield self.basic_auth_provider + if self.ekirjasto_provider: # Finland + yield self.ekirjasto_provider yield from self.saml_providers_by_name.values() def _unique_basic_lookup_providers( @@ -457,7 +489,17 @@ def authenticated_patron( # BasicAuthenticationProvider. provider = self.basic_auth_provider provider_token = auth.parameters - elif auth.type.lower() == "bearer": + elif self.ekirjasto_provider and auth.type.lower() == "bearer": # Finland + # The patron wants to authenticate with the + # EkirjastoAuthenticationAPI. + if auth.token is None: + return INVALID_EKIRJASTO_DELEGATE_TOKEN + provider = self.ekirjasto_provider + # Get decoded payload from the delegate token. + provider_token = provider.validate_ekirjasto_delegate_token(auth.token) + if isinstance(provider_token, ProblemDetail): + return provider_token + elif self.saml_providers_by_name and auth.type.lower() == "bearer": # The patron wants to use an # SAMLAuthenticationProvider. Figure out which one. if auth.token is None: diff --git a/api/config.py b/api/config.py index 636fd6cc4..b8e43a916 100644 --- a/api/config.py +++ b/api/config.py @@ -49,6 +49,16 @@ class Configuration(CoreConfiguration): # used to sign bearer tokens. BEARER_TOKEN_SIGNING_SECRET = "bearer_token_signing_secret" + # Finland + # Name of the site-wide ConfigurationSetting containing the secret + # used to sign ekirjasto delegate tokens. + EKIRJASTO_TOKEN_SIGNING_SECRET = "ekirjasto_token_signing_secret" + + # Finland + # Name of the site-wide ConfigurationSetting containing the secret + # used to encrypt ekirjasto tokens. + EKIRJASTO_TOKEN_ENCRYPTING_SECRET = "ekirjasto_token_encrypting_secret" + # Maximum height and width for the saved logo image LOGO_MAX_DIMENSION = 135 diff --git a/api/controller.py b/api/controller.py index 4c854fa91..a19ff260b 100644 --- a/api/controller.py +++ b/api/controller.py @@ -30,6 +30,8 @@ from api.circulation import CirculationAPI from api.circulation_exceptions import * from api.config import CannotLoadConfiguration, Configuration +from api.ekirjasto_controller import EkirjastoController # Finland +from api.opensearch_analytics_search import OpenSearchAnalyticsSearch from api.custom_index import CustomIndexView from api.lanes import ( ContributorFacets, @@ -82,6 +84,8 @@ DeliveryMechanism, Hold, Identifier, + IntegrationConfiguration, + IntegrationLibraryConfiguration, Library, LicensePool, LicensePoolDeliveryMechanism, @@ -90,6 +94,7 @@ Representation, Session, get_one, + json_serializer, ) from core.model.devicetokens import ( DeviceToken, @@ -175,6 +180,7 @@ class CirculationManager: odl_notification_controller: ODLNotificationController static_files: StaticFileController playtime_entries: PlaytimeEntriesController + catalog_descriptions: CatalogDescriptionsController # Admin controllers admin_sign_in_controller: SignInController @@ -284,6 +290,9 @@ def load_settings(self): self.setup_external_search() + # Finland + self.setup_opensearch_analytics_search() + # Track the Lane configuration for each library by mapping its # short name to the top-level lane. new_top_level_lanes = {} @@ -353,6 +362,31 @@ def get_domain(url): max_len=1000, max_age_seconds=authentication_document_cache_time ) + # Finland + @property + def opensearch_analytics_search(self): + """Retrieve or create a connection to the OpenSearch + analytics interface. + + This is created lazily so that a failure to connect only + affects feeds that depend on the search engine, not the whole + circulation manager. + """ + if not self._opensearch_analytics_search: + self.setup_opensearch_analytics_search() + return self._opensearch_analytics_search + + # Finland + def setup_opensearch_analytics_search(self): + try: + self._opensearch_analytics_search = OpenSearchAnalyticsSearch() + self.opensearch_analytics_search_initialization_exception = None + except Exception as e: + self.log.error("Exception initializing search engine: %s", e) + self._opensearch_analytics_search = None + self.opensearch_analytics_search_initialization_exception = e + return self._opensearch_analytics_search + @property def external_search(self): """Retrieve or create a connection to the search interface. @@ -416,6 +450,7 @@ def setup_one_time_controllers(self): self.static_files = StaticFileController(self) self.patron_auth_token = PatronAuthTokenController(self) self.playtime_entries = PlaytimeEntriesController(self) + self.catalog_descriptions = CatalogDescriptionsController(self) def setup_configuration_dependent_controllers(self): """Set up all the controllers that depend on the @@ -425,6 +460,8 @@ def setup_configuration_dependent_controllers(self): configuration changes. """ self.saml_controller = SAMLController(self, self.auth) + # Finland + self.ekirjasto_controller = EkirjastoController(self, self.auth) def annotator(self, lane, facets=None, *args, **kwargs): """Create an appropriate OPDS annotator for the given lane. @@ -652,13 +689,27 @@ def load_licensepools(self, library, identifier_type, identifier): """ _db = Session.object_session(library) pools = ( - _db.query(LicensePool) - .join(LicensePool.collection) - .join(LicensePool.identifier) - .join(Collection.libraries) - .filter(Identifier.type == identifier_type) - .filter(Identifier.identifier == identifier) - .filter(Library.id == library.id) + _db.scalars( + select(LicensePool) + .join(Collection, LicensePool.collection_id == Collection.id) + .join(Identifier, LicensePool.identifier_id == Identifier.id) + .join( + IntegrationConfiguration, + Collection.integration_configuration_id + == IntegrationConfiguration.id, + ) + .join( + IntegrationLibraryConfiguration, + IntegrationConfiguration.id + == IntegrationLibraryConfiguration.parent_id, + ) + .where( + Identifier.type == identifier_type, + Identifier.identifier == identifier, + IntegrationLibraryConfiguration.library_id == library.id, + ) + ) + .unique() .all() ) if not pools: @@ -973,7 +1024,7 @@ def crawlable_collection_feed(self, collection_name): """Build or retrieve a crawlable acquisition feed for the requested collection. """ - collection = get_one(self._db, Collection, name=collection_name) + collection = Collection.by_name(self._db, collection_name) if not collection: return NO_SUCH_COLLECTION title = collection.name @@ -1386,7 +1437,7 @@ def borrow(self, identifier_type, identifier, mechanism_id=None): """ patron = flask.request.patron library = flask.request.library - + header = self.authorization_header() credential = self.manager.auth.get_credential_from_header(header) @@ -2392,6 +2443,111 @@ def image(self, filename): ) return self.static_file(directory, filename) +# Finland +class CatalogDescriptionsController(CirculationManagerController): + def get_catalogs(self, library_uuid=None): + catalogs = [] + libraries = [] + + + if library_uuid != None: + try: + libraries = [ + self._db.query(Library).filter(Library.uuid == library_uuid).one() + ] + except NoResultFound: + return LIBRARY_NOT_FOUND + else: + libraries = self._db.query(Library).order_by(Library.name).all() + + for library in libraries: + settings = library.settings_dict + images = [] + if library.logo: + images += [ + { + "rel": "http://opds-spec.org/image/thumbnail", + "href": library.logo.data_url, + "type": "image/png" + } + ] + + authentication_document_url = url_for( + "authentication_document", + library_short_name=library.short_name, + _external=True + ) + + catalog_url = url_for( + "acquisition_groups", + library_short_name=library.short_name, + _external=True + ) + + timenow = utc_now().strftime('%Y-%m-%dT%H:%M:%SZ') + + metadata = { + "id": "urn:uuid:" + library.uuid, + "title": library.name, + "short_name": library.short_name, + "modified": timenow, + "updated": timenow, + "isAutomatic": False + } + + if "library_description" in settings: + metadata["description"] = settings["library_description"] + + links = [ + { + "rel": "http://opds-spec.org/catalog", + "href": catalog_url, + "type": "application/atom+xml;profile=opds-catalog;kind=acquisition" + }, + { + "href": authentication_document_url, + "type": "application/vnd.opds.authentication.v1.0+json" + } + ] + + if "help_web" in settings: + links += [{ + "href": settings["help_web"], + "rel": "help" + }] + elif "help_email" in settings: + links += [{ + "href": "mailto:"+settings["help_email"], + "rel": "help" + }] + + catalogs += [ + { + "metadata": metadata, + "links": links, + "images": images + } + ] + + response_json = { + "metadata": { + "title": "Libraries" + }, + "catalogs": catalogs, + "links": [ + { + "rel": "self", + "href": url_for("client_libraries", _external=True), + "type": "application/opds+json" + } + ] + } + + return Response( + json_serializer(response_json), + status=200, + mimetype="application/json", + ) class PatronAuthTokenController(CirculationManagerController): def get_token(self): diff --git a/api/ekirjasto_authentication.py b/api/ekirjasto_authentication.py new file mode 100644 index 000000000..afc555f44 --- /dev/null +++ b/api/ekirjasto_authentication.py @@ -0,0 +1,687 @@ +from __future__ import annotations + +import datetime +import json +import jwt +import logging +import requests +import uuid + +from abc import ABC +from base64 import b64decode, b64encode +from cryptography.fernet import Fernet, InvalidToken +from enum import Enum +from flask import url_for +from flask_babel import lazy_gettext as _ +from typing import Any + +from sqlalchemy.orm import Session +from werkzeug.datastructures import Authorization + +from api.authentication.base import ( + AuthenticationProvider, + AuthProviderLibrarySettings, + AuthProviderSettings, + PatronData, +) + +from api.util.patron import PatronUtility +from .circulation_exceptions import ( + InternalServerError, + PatronNotFoundOnRemote, + RemoteInitiatedServerError, + RemotePatronCreationFailedException +) +from .config import Configuration +from core.analytics import Analytics +from core.integration.settings import ( + ConfigurationFormItem, + ConfigurationFormItemType, + FormField, +) +from core.model import ConfigurationSetting, Credential, DataSource, Patron, get_one +from core.util.datetime_helpers import from_timestamp, utc_now +from core.util.log import elapsed_time_logging +from core.util.problem_detail import ProblemDetail +from .problem_details import ( + UNSUPPORTED_AUTHENTICATION_MECHANISM, + EKIRJASTO_REMOTE_AUTHENTICATION_FAILED, + INVALID_EKIRJASTO_TOKEN, + INVALID_EKIRJASTO_DELEGATE_TOKEN +) + +class EkirjastoEnvironment(Enum): + FAKE = "http://localhost" + DEVELOPMENT = "https://e-kirjasto.loikka.dev" + PRODUCTION = "https://tunnus.e-kirjasto.fi" + + +class EkirjastoAuthAPISettings(AuthProviderSettings): + """Settings for the EkirjastoAuthenticationAPI.""" + + # API environment form field, choose between dev and prod. + ekirjasto_environment: EkirjastoEnvironment = FormField( + EkirjastoEnvironment.FAKE, + form=ConfigurationFormItem( + label=_("E-kirjasto API environment"), + description=_("Select what environment of E-kirjasto accounts should be used."), + type=ConfigurationFormItemType.SELECT, + options={ + EkirjastoEnvironment.FAKE: "Fake", + EkirjastoEnvironment.DEVELOPMENT: "Development", + EkirjastoEnvironment.PRODUCTION: "Production", + }, + required=True, + weight=10, + ), + ) + + delegate_expire_time: int = FormField( + 60*60*12, # 12 hours + form=ConfigurationFormItem( + label=_("Delegate token expire time in seconds"), + description=_("Expire time for a delegate token to authorize in behalf of a ekirjasto token. This should be less than the expire time for ekirjasto token, so it can be refreshed."), + required=True, + ), + ) + + +class EkirjastoAuthAPILibrarySettings(AuthProviderLibrarySettings): + ... + + +class EkirjastoAuthenticationAPI(AuthenticationProvider, ABC): + """Verify a token for E-kirjasto login, with a remote source of truth.""" + + def __init__( + self, + library_id: int, + integration_id: int, + settings: EkirjastoAuthAPISettings, + library_settings: EkirjastoAuthAPILibrarySettings, + analytics: Analytics | None = None, + ): + """Create a EkirjastoAuthenticationAPI.""" + super().__init__( + library_id, integration_id, settings, library_settings, analytics + ) + + self.ekirjasto_environment = settings.ekirjasto_environment + self.delegate_expire_timemestamp = settings.delegate_expire_time + + self.delegate_token_signing_secret = None + self.delegate_token_encrypting_secret = None + + self.analytics = analytics + + self.fake_ekirjasto_token = "4d2i2w3o1f6t3e1y0d46655q114q4d37200o3s6q5f1z2r4i1z0q1o5d3f695g1g" + + self._ekirjasto_api_url = self.ekirjasto_environment.value + if self.ekirjasto_environment == EkirjastoEnvironment.FAKE: + self._ekirjasto_api_url = EkirjastoEnvironment.DEVELOPMENT.value + + @property + def flow_type(self) -> str: + return "http://e-kirjasto.fi/authtype/ekirjasto" + + @classmethod + def label(cls) -> str: + return "E-kirjasto provider for circulation manager" + + @classmethod + def patron_delegate_id_credential_key(cls) -> str: + return "E-kirjasto patron uuid" + + @classmethod + def description(cls) -> str: + return ( + "Authenticate patrons with E-kirjasto accounts service." + ) + + @property + def identifies_individuals(self): + return True + + @classmethod + def settings_class(cls) -> type[EkirjastoAuthAPISettings]: + return EkirjastoAuthAPISettings + + @classmethod + def library_settings_class( + cls, + ) -> type[EkirjastoAuthAPILibrarySettings]: + return EkirjastoAuthAPILibrarySettings + + def _authentication_flow_document(self, _db: Session) -> dict[str, Any]: + """Create a Authentication Flow object for use in an Authentication for + OPDS document. + + This follows loosely the specification for OPDS authentication document (https://drafts.opds.io/authentication-for-opds-1.0.html#24-authentication-provider). + + Example: + { + "type": "http://e-kirjasto.fi/authtype/ekirjasto", + "description": "E-kirjasto", + "links": [ + { + "rel": "authenticate", + "href": "http://localhost:6500/ellibs-test/ekirjasto_authenticate?provider=E-kirjasto" + }, + { + "rel": "api", + "href": "https://e-kirjasto.loikka.dev" + }, + + ... + ] + } + """ + + flow_doc = { + "type": self.flow_type, + "description": self.label(), + "links": [ + { + "rel" : "authenticate", + "href": self._create_authenticate_url(_db) + }, + { + "rel" : "api", + "href": self._ekirjasto_api_url + }, + { + "rel" : "tunnistus_start", + "href": f'{self._ekirjasto_api_url}/v1/auth/tunnistus/start?locale=fi' + }, + { + "rel" : "tunnistus_finish", + "href": f'{self._ekirjasto_api_url}/v1/auth/tunnistus/finish' + }, + { + "rel" : "passkey_login_start", + "href": f'{self._ekirjasto_api_url}/v1/auth/passkey/login/start' + }, + { + "rel" : "passkey_login_finish", + "href": f'{self._ekirjasto_api_url}/v1/auth/passkey/login/finish' + }, + { + "rel" : "passkey_register_start", + "href": f'{self._ekirjasto_api_url}/v1/auth/passkey/register/start' + }, + { + "rel" : "passkey_register_finish", + "href": f'{self._ekirjasto_api_url}/v1/auth/passkey/register/finish' + }, + ] + } + + return flow_doc + + def _create_authenticate_url(self, db): + """Returns an authentication link used by clients to authenticate patrons + + :param db: Database session + :type db: sqlalchemy.orm.session.Session + + :return: URL for authentication using the chosen IdP + :rtype: string + """ + + library = self.library(db) + + return url_for( + "ekirjasto_authenticate", + _external=True, + library_short_name=library.short_name, + provider=self.label(), + ) + + def _run_self_tests(self, _db): + pass + + def _userinfo_to_patrondata(self, userinfo_json: dict) -> PatronData: + """ Convert user info JSON received from the ekirjasto API to PatronData. + + Example of userinfo_json + { + 'exp': 1703141144518, + 'family_name': 'Testi', + 'given_name': 'Testi', + 'name': 'Testi Testi', + 'role': 'customer', + 'sub': '1bf3c6ea-0502-45fc-a785-0113d8f78a51', + 'municipality': 'Helsinki', + 'verified': True, + 'passkeys': [] + } + """ + + def _get_key_or_none(userinfo_json, key): + if key in userinfo_json: + return userinfo_json[key] + return None + + patrondata = PatronData( + permanent_id=_get_key_or_none(userinfo_json, "sub"), + authorization_identifier=_get_key_or_none(userinfo_json, "sub"), # TODO: We don't know exactly what this should be. + external_type=_get_key_or_none(userinfo_json, "role"), + personal_name=_get_key_or_none(userinfo_json, "name"), + email_address=_get_key_or_none(userinfo_json, "email"), + username=_get_key_or_none(userinfo_json, "username"), + cached_neighborhood=_get_key_or_none(userinfo_json, "municipality"), + complete=True, + ) + + if patrondata.permanent_id == None: + # permanent_id is used to get the local Patron, we cannot proceed + # if it is missing. + message = "Value for permanent_id is missing in remote user info." + raise RemotePatronCreationFailedException(message, self.__class__.__name__) + + return patrondata + + def get_credential_from_header(self, auth: Authorization) -> str | None: + # We cannot extract the credential from the header, so we just return None. + # This is only needed for authentication providers where the external + # circulation API needs additional authentication. + return None + + def get_patron_delegate_id(self, _db: Session, patron: Patron) -> str: + """Find or randomly create an identifier to use when identifying + this patron from delegate token. + """ + def refresher_method(credential): + credential.credential = str(uuid.uuid4()) + + data_source = DataSource.lookup(_db, self.label(), autocreate=True) + if data_source == None: + raise InternalServerError("Ekirjasto authenticator failed to create DataSource for itself.") + + credential = Credential.lookup( + _db, + data_source, + self.patron_delegate_id_credential_key(), + patron, + refresher_method, + allow_persistent_token=True, + ) + return credential.credential + + def get_patron_with_delegate_id(self, _db: Session, patron_delegate_id: str) -> Patron | None: + """Find patron based on its delegate id. + """ + data_source = DataSource.lookup(_db, self.label()) + if data_source == None: + return None + + credential = Credential.lookup_by_token( + _db, + data_source, + self.patron_delegate_id_credential_key(), + patron_delegate_id, + allow_persistent_token=True, + ) + + if credential == None: + return None + + return credential.patron + + def set_secrets(self, _db): + self.delegate_token_signing_secret = ConfigurationSetting.sitewide_secret( + _db, Configuration.EKIRJASTO_TOKEN_SIGNING_SECRET + ) + + # Encrypting requires stronger secret than the sitewide_secret can provide. + secret = ConfigurationSetting.sitewide(_db, Configuration.EKIRJASTO_TOKEN_ENCRYPTING_SECRET) + if not secret.value: + secret.value = Fernet.generate_key().decode() + _db.commit() + self.delegate_token_encrypting_secret = secret.value.encode() + + def _check_secrets_or_throw(self): + if ( + self.delegate_token_signing_secret == None or len(self.delegate_token_signing_secret) == 0 + or self.delegate_token_encrypting_secret == None or len(self.delegate_token_encrypting_secret) == 0 + ): + raise InternalServerError("Ekirjasto authenticator not fully setup, secrets are missing.") + + def create_ekirjasto_delegate_token( + self, provider_token: str, patron_delegate_id: str, expires: int + ) -> str: + """ + Create a JSON Web Token fr patron with encrypted ekirjasto token in the payload. + + The patron will use this as the authentication toekn to authentiacte againsy circulation backend. + """ + self._check_secrets_or_throw() + + # Encrypt the ekirjasto token with a128cbc-hs256 algorithm. + fernet = Fernet(self.delegate_token_encrypting_secret) + encrypted_token = b64encode(fernet.encrypt(provider_token.encode())).decode("ascii") + + payload = dict( + token=encrypted_token, + iss=self.label(), + sub=patron_delegate_id, + iat=int(utc_now().timestamp()), + exp=expires, + ) + return jwt.encode(payload, self.delegate_token_signing_secret, algorithm="HS256") + + def decode_ekirjasto_delegate_token(self, delegate_token: str, validate_expire: bool = True, decrypt_ekirjasto_token: bool = False) -> dict: + """ + Validate and get payload of the JSON Web Token for circulation. + + return decoded payload + """ + self._check_secrets_or_throw() + + options = dict( + verify_signature=True, + require=["token", "iss", "sub", "iat", "exp"], + verify_iss=True, + verify_exp=validate_expire, + verify_iat=True, + ) + + decoded_payload = jwt.decode( + delegate_token, + self.delegate_token_signing_secret, + algorithms=["HS256"], + options=options, + issuer=self.label() + ) + + if decrypt_ekirjasto_token: + decoded_payload["token"] = self._decrypt_ekirjasto_token(decoded_payload["token"]) + + return decoded_payload + + def _decrypt_ekirjasto_token(self, token: str): + fernet = Fernet(self.delegate_token_encrypting_secret) + encrypted_token = b64decode(token.encode("ascii")) + return fernet.decrypt(encrypted_token).decode() + + def validate_ekirjasto_delegate_token(self, delegate_token: str, validate_expire: bool = True, decrypt_ekirjasto_token: bool = False) -> dict | ProblemDetail: + """ + Validate and get payload of the JSON Web Token for circulation. + + return decoded payload or ProblemDetail + """ + + try: + # Validate bearer token and get credential info. + decoded_payload = self.decode_ekirjasto_delegate_token(delegate_token, validate_expire, decrypt_ekirjasto_token) + except jwt.exceptions.InvalidTokenError as e: + return INVALID_EKIRJASTO_DELEGATE_TOKEN + except InvalidToken as e: + return INVALID_EKIRJASTO_DELEGATE_TOKEN + return decoded_payload + + def remote_refresh_token(self, token: str) -> (str, int): + """ Refresh ekirjasto token with ekirjasto API call. + + We assume that the token is valid, API call fails if not. + + :return: token and expire timestamp if refresh was succesfull or None | ProblemDetail otherwise. + """ + + if self.ekirjasto_environment == EkirjastoEnvironment.FAKE: + token = self.fake_ekirjasto_token + expires = utc_now() + datetime.timedelta(days=1) + return token, expires.timestamp() + + url = self._ekirjasto_api_url + "/v1/auth/refresh" + + try: + response = self.requests_post(url, token) + except requests.exceptions.ConnectionError as e: + raise RemoteInitiatedServerError(str(e), self.__class__.__name__) + + if response.status_code == 401: + # Do nothing if authentication fails, e.g. token expired. + return INVALID_EKIRJASTO_TOKEN, None + elif response.status_code != 200: + msg = "Got unexpected response code %d. Content: %s" % ( + response.status_code, + response.content, + ) + return EKIRJASTO_REMOTE_AUTHENTICATION_FAILED, None + else: + try: + content = response.json() + except requests.exceptions.JSONDecodeError as e: + raise RemoteInitiatedServerError(str(e), self.__class__.__name__) + + token = content["token"] + expires = content["exp"] + return token, expires + + def remote_patron_lookup( + self, ekirjasto_token: str | None + ) -> PatronData | ProblemDetail | None: + """Ask the remote for detailed information about patron related to the ekirjasto_token. + + If the patron is not found, or an error occurs communicating with the remote, + return None or a ProblemDetail. + + Otherwise, return a PatronData object with the complete property set to True. + """ + + if self.ekirjasto_environment == EkirjastoEnvironment.FAKE: + if ekirjasto_token == self.fake_ekirjasto_token: + # Fake authentication successful, return fake patron data. + return PatronData( + permanent_id='34637274574578', + authorization_identifier='test_34637274574578', + external_type='user', + personal_name='Fake User', + complete=True, + ) + else: + return None + + url = self._ekirjasto_api_url + "/v1/auth/userinfo" + + try: + response = self.requests_get(url, ekirjasto_token) + except requests.exceptions.ConnectionError as e: + raise RemoteInitiatedServerError(str(e), self.__class__.__name__) + + if response.status_code == 401: + # Do nothing if authentication fails, e.g. token expired. + return INVALID_EKIRJASTO_TOKEN + elif response.status_code != 200: + msg = "Got unexpected response code %d. Content: %s" % ( + response.status_code, + response.content, + ) + return EKIRJASTO_REMOTE_AUTHENTICATION_FAILED + else: + try: + content = response.json() + except requests.exceptions.JSONDecodeError as e: + raise RemoteInitiatedServerError(str(e), self.__class__.__name__) + + return self._userinfo_to_patrondata(content) + + return EKIRJASTO_REMOTE_AUTHENTICATION_FAILED + + def remote_authenticate( + self, ekirjasto_token: str | None + ) -> PatronData | ProblemDetail | None: + """Does the source of truth approve the ekirjasto_token? + + If the ekirjasto_token is valid, return a PatronData object. The PatronData object + has a `complete` field. + + If the ekirjasto_token is invalid, return None. + + If there is a problem communicating with the remote, return a ProblemDetail. + """ + + return self.remote_patron_lookup(ekirjasto_token) + + def authenticate_and_update_patron( + self, _db: Session, ekirjasto_token: str | None + ) -> Patron | PatronData | ProblemDetail | None: + """Turn an ekirjasto_token into a Patron object. + + :param ekirjasto_token: A token for e-kirjasto authorization. + + :return: A Patron if one can be authenticated; PatronData if + authenticated, but Patron not available; a ProblemDetail + if an error occurs; None if the credentials are missing or wrong. + """ + + # Check the ekirjasto token with the remote source of truth. + patrondata = self.remote_authenticate(ekirjasto_token) + + if not isinstance(patrondata, PatronData): + # Either an error occurred or the credentials did not correspond to any patron. + return patrondata + + # At this point we know there is _some_ authenticated patron, + # but it might not correspond to any Patron in our database. + # Try to look up the Patron object in our database. + patron = self.local_patron_lookup(_db, patrondata) + + if patron: + # Apply the remote information we have to the patron. + patrondata.apply(patron) + + return patron + + # No Patron found from the database, but we've got remote information (PatronData). + # Patron should be created through ekirjasto_authenticate. + return patrondata + + def local_patron_lookup( + self, _db: Session, patrondata: PatronData | None + ) -> Patron | None: + """Try to find a Patron object in the local database. + + :param patrondata: A PatronData object recently obtained from + the source of truth. This may make it possible to + identify the patron more precisely. Or it may be None, in + which case it's no help at all. + """ + patron = None + if patrondata and patrondata.permanent_id: + # Permanent ID is the most reliable way of identifying + # a patron, since this is supposed to be an internal + # ID that never changes. + lookup = dict( + external_identifier=patrondata.permanent_id, + library_id=self.library_id + ) + + patron = get_one(_db, Patron, **lookup) + + return patron + + def ekirjasto_authenticate( + self, _db: Session, ekirjasto_token: str + ) -> (Patron, bool): + """ Authenticate patron with remote ekirjasto API and if necessary, + create authenticated patron if not in database. + + :param ekirjasto_token: A token for e-kirjasto account endpoint. + """ + is_new = False + + with elapsed_time_logging( + log_method=self.logger().info, + message_prefix="authenticated_patron - ekirjasto_authenticate", + ): + patron = self.authenticate_and_update_patron(_db, ekirjasto_token) + + if isinstance(patron, PatronData): + # We didn't find the patron, but authentication to external truth was + # succesfull, so we create a new patron with the information we have. + patron, is_new = patron.get_or_create_patron( + _db, self.library_id, analytics=self.analytics + ) + patron.last_external_sync = utc_now() + + return patron, is_new + + def authenticated_patron( + self, _db: Session, authorization: dict | str + ) -> Patron | ProblemDetail | None: + """Go from a werkzeug.Authorization object to a Patron object. + + If the Patron needs to have their metadata updated, it happens + transparently at this point. + + :return: A Patron if one can be authenticated; a ProblemDetail + if an error occurs; None if the credentials are missing or wrong. + """ + # authorization is the decoded payload of the delegate token, including + # encrypted ekirjasto token. + if type(authorization) != dict: + return UNSUPPORTED_AUTHENTICATION_MECHANISM + + ekirjasto_token = None + delegate_patron = None + if "token" in authorization and "exp" in authorization and "sub" in authorization: + encrypted_ekirjasto_token = authorization["token"] + delegate_expired = from_timestamp(authorization["exp"]) < utc_now() + patron_delegate_id = authorization["sub"] + + if delegate_expired: + # Causes to return 401 error + return None + + delegate_patron = self.get_patron_with_delegate_id(_db, patron_delegate_id) + if delegate_patron == None: + # Causes to return 401 error + return None + + if PatronUtility.needs_external_sync(delegate_patron): + # We should sometimes try to update the patron from remote. + ekirjasto_token = self._decrypt_ekirjasto_token(encrypted_ekirjasto_token) + else: + # No need to update patron. + return delegate_patron + else: + return UNSUPPORTED_AUTHENTICATION_MECHANISM + + # If we come here, we have ekirjasto_token and we should try to update the patron. + with elapsed_time_logging( + log_method=self.logger().info, + message_prefix="authenticated_patron - authenticate", + ): + patron = self.authenticate_and_update_patron(_db, ekirjasto_token) + + if isinstance(patron, PatronData): + # Account not created, should first use ekirjasto_authenticate to + # create an account. Authenticated to remote, but not to circulation manager. + return None + if not isinstance(patron, Patron): + # Some issue with authentication. + return patron + if delegate_patron and patron.id != delegate_patron.id: + # This situation should never happen. + raise PatronNotFoundOnRemote(404, "Remote patron is conflicting with delegate patron.") + if patron.cached_neighborhood and not patron.neighborhood: + # Patron.neighborhood (which is not a model field) was not + # set, probably because we avoided an expensive metadata + # update. But we have a cached_neighborhood (which _is_ a + # model field) to use in situations like this. + patron.neighborhood = patron.cached_neighborhood + return patron + + def requests_get(self, url, ekirjasto_token=None): + headers = None + if ekirjasto_token: + headers = {'Authorization': f'Bearer {ekirjasto_token}'} + return requests.get(url, headers=headers) + + def requests_post(self, url, ekirjasto_token=None): + headers = None + if ekirjasto_token: + headers = {'Authorization': f'Bearer {ekirjasto_token}'} + return requests.post(url, headers=headers) \ No newline at end of file diff --git a/api/ekirjasto_controller.py b/api/ekirjasto_controller.py new file mode 100644 index 000000000..01c860842 --- /dev/null +++ b/api/ekirjasto_controller.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import datetime +import json +import jwt +import logging + +from flask import Response + +from core.model import Patron +from core.util.datetime_helpers import from_timestamp, utc_now +from core.util.problem_detail import ProblemDetail +from .problem_details import ( + EKIRJASTO_PROVIDER_NOT_CONFIGURED, + EKIRJASTO_REMOTE_AUTHENTICATION_FAILED, + INVALID_EKIRJASTO_DELEGATE_TOKEN +) + +class EkirjastoController(): + """Controller used for handing Ekirjasto authentication requests""" + + def __init__(self, circulation_manager, authenticator): + """Initializes a new instance of EkirjastoController class + + :param circulation_manager: Circulation Manager + :type circulation_manager: CirculationManager + + :param authenticator: Authenticator object used to route requests to the appropriate LibraryAuthenticator + :type authenticator: Authenticator + """ + self._circulation_manager = circulation_manager + self._authenticator = authenticator + + self._logger = logging.getLogger(__name__) + + def _get_delegate_expire_timestamp(self, ekirjasto_token_expires: int) -> int: + """ Get the expire time to use for delegate token, it is calculated based on + expire time of the ekirjasto token. + + :param ekirjasto_token_expires: Ekirjasto token expiration timestamp in milliseconds. + + :return: Timestamp for the delegate token expiration. + """ + + # Ekirjasto expire is in milliseconds but JWT uses seconds. + ekirjasto_token_expires = int(ekirjasto_token_expires/1000) + + delegate_token_expires = ( + utc_now() + datetime.timedelta( + seconds=self._authenticator.ekirjasto_provider.delegate_expire_timemestamp + ) + ).timestamp() + + # Use ekirjasto expire time at 70 % of the remaining duration, so we have some time to refresh it. + now_seconds = utc_now().timestamp() + ekirjasto_token_expires = (ekirjasto_token_expires - now_seconds) * 0.7 + now_seconds + + if ekirjasto_token_expires < delegate_token_expires: + return int(ekirjasto_token_expires) + + return int(delegate_token_expires) + + def get_tokens(self, authorization, validate_expire=False): + """ Extract possible delegate and ekirjasto tokens from the authorization header. + """ + if self.is_configured != True: + return EKIRJASTO_PROVIDER_NOT_CONFIGURED, None, None, None + + token = authorization.token + if token is None or len(token) == 0: + return EKIRJASTO_REMOTE_AUTHENTICATION_FAILED, None, None, None + + ekirjasto_token = None + delegate_token = None + delegate_sub = None + delegate_expired = True + try: + # We may attempt to refresh ekirjasto token in any case, so we don't validate + # delegate token expiration by default and we need the decrypted ekirjasto token. + delegate_payload = self._authenticator.ekirjasto_provider.validate_ekirjasto_delegate_token( + token, + validate_expire=validate_expire, + decrypt_ekirjasto_token=True + ) + if isinstance(delegate_payload, ProblemDetail): + # The ekirjasto_token might be ProblemDetail, indicating that the token + # is not valid. Still it might be ekirjasto_token (which is not JWT or + # at least not signed by us), so we can continue. + ekirjasto_token = token + else: + # Successful validation of a delegate token for circulation manager. + ekirjasto_token = delegate_payload["token"] + delegate_expired = from_timestamp(delegate_payload["exp"]) < utc_now() + delegate_sub = delegate_payload["sub"] + delegate_token = token + except jwt.exceptions.DecodeError as e: + # It might be just an ekirjasto_token, it will be used to authenticate + # with the remote ekirjasto API. We don't do anything further validation + # for it as it is valdiated with successful authentication. + ekirjasto_token = token + pass + + return delegate_token, ekirjasto_token, delegate_sub, delegate_expired + + @property + def is_configured(self): + if self._authenticator.ekirjasto_provider: + return True + return False + + def refresh_tokens_if_needed(self, authorization, _db, sync_patron): + """ Refresh delegate and ekirjasto tokens if delegate is expired. + """ + if self.is_configured != True: + return EKIRJASTO_PROVIDER_NOT_CONFIGURED, None, None + + ekirjasto_provider = self._authenticator.ekirjasto_provider + + delegate_token, ekirjasto_token, delegate_sub, delegate_expired = self.get_tokens(authorization) + if isinstance(delegate_token, ProblemDetail): + return delegate_token, None, None + + ekirjasto_token_expires = None + if delegate_expired: + ekirjasto_token, ekirjasto_token_expires = ekirjasto_provider.remote_refresh_token(ekirjasto_token) + + if isinstance(ekirjasto_token, ProblemDetail): + return ekirjasto_token, None, None + if ekirjasto_token == None or ekirjasto_token_expires == None: + return EKIRJASTO_REMOTE_AUTHENTICATION_FAILED, None, None + + is_patron_new = False + patron = None + if sync_patron: + # Synchronize or create patron + patron, is_patron_new = ekirjasto_provider.ekirjasto_authenticate(_db, ekirjasto_token) + if not isinstance(patron, Patron): + # Authentication was failed. + if patron == None: + return INVALID_EKIRJASTO_DELEGATE_TOKEN, None, None + # Return ProblemDetail + return patron, None, None + elif delegate_token != None: + patron = ekirjasto_provider.get_patron_with_delegate_id(_db, delegate_sub) + if patron == None: + # Causes to return 401 error + return INVALID_EKIRJASTO_DELEGATE_TOKEN, None, None + else: + return INVALID_EKIRJASTO_DELEGATE_TOKEN, None, None + + if ekirjasto_token_expires != None: + # We have new ekirjasto token. + # Create a delegate token which we can give to the patron. + delegate_expires = self._get_delegate_expire_timestamp(ekirjasto_token_expires) + delegate_token = ekirjasto_provider.create_ekirjasto_delegate_token( + ekirjasto_token, + ekirjasto_provider.get_patron_delegate_id(_db, patron), + delegate_expires + ) + + return delegate_token, ekirjasto_token, is_patron_new + + def authenticate(self, request, _db): + """ Authenticate patron with ekirjasto API and return delegate token for + circulation manager API access. + + New Patron is created to database if ekirjasto authentication was succesfull + and no patron for it was found. Token for ekirjasto API is stored for later usage. + """ + if self.is_configured != True: + return EKIRJASTO_PROVIDER_NOT_CONFIGURED + + delegate_token, ekirjasto_token, is_patron_new = self.refresh_tokens_if_needed( + request.authorization, + _db, + sync_patron=True + ) + if delegate_token == None or isinstance(delegate_token, ProblemDetail): + return delegate_token + + patron_info = None + patrondata = self._authenticator.ekirjasto_provider.remote_patron_lookup(ekirjasto_token) + if patrondata: + patron_info = json.dumps(patrondata.to_dict) + + response_json = json.dumps({"access_token": delegate_token, "patron_info": patron_info}) + response_code = 201 if is_patron_new else 200 + return Response(response_json, response_code, mimetype='application/json') \ No newline at end of file diff --git a/api/integration/registry/patron_auth.py b/api/integration/registry/patron_auth.py index 80cb1cd5f..cd7dd11a8 100644 --- a/api/integration/registry/patron_auth.py +++ b/api/integration/registry/patron_auth.py @@ -21,7 +21,7 @@ def __init__(self) -> None: from api.sirsidynix_authentication_provider import ( SirsiDynixHorizonAuthenticationProvider, ) - + self.register( SimpleAuthenticationProvider, canonical="api.simple_authentication" ) @@ -34,3 +34,7 @@ def __init__(self) -> None: SirsiDynixHorizonAuthenticationProvider, canonical="api.sirsidynix_authentication_provider", ) + + # Finland + from api.ekirjasto_authentication import EkirjastoAuthenticationAPI + self.register(EkirjastoAuthenticationAPI, canonical="api.ekirjasto") diff --git a/api/local_analytics_exporter.py b/api/local_analytics_exporter.py index d5a608d03..5d9c26e15 100644 --- a/api/local_analytics_exporter.py +++ b/api/local_analytics_exporter.py @@ -17,6 +17,12 @@ WorkGenre, ) +# Finland: +from core.model.contributor import Contribution, Contributor +from openpyxl import Workbook +from openpyxl.styles import Font, PatternFill, Border, Side +from tempfile import NamedTemporaryFile + class LocalAnalyticsExporter: """Export large numbers of analytics events in CSV format.""" @@ -55,6 +61,109 @@ def export(self, _db, start, end, locations=None, library=None): writer.writerows(results) return output.getvalue().decode("utf-8") + # Finland + def export_excel(self, _db, start, end, locations=None, library=None): + # Get the results from the database. + query = self.analytics_query_loan_statistics(start, end, locations, library) + results = _db.execute(query) + + # Prepare Excel workbook + workbook = Workbook() + sheet = workbook.active + + rows = [dict(row) for row in results] + + # Count how many contributor rows we need + max_contribs = max([0, *[len(row.get("contributors", [])) for row in rows]]) + + header = [ + "Tekijä (aakkostus)", + "Nimeke", + "Tunniste", + "Tunnisteen tyyppi", + "Kirjasto", + "Sijainti", + "Formaatti", + "Kategoria(t)", + "Kieli", + "Kustantaja/Julkaisija", + "Kaikki lainat", + ] + for i in range(max_contribs): + header.append(f"Tekijä {i+1}") + + sheet.append(header) + + for row in rows: + genres = row.get("genres") + categories = ", ".join(genres) if genres else "" + + contributors = row.get("contributors", []) + + sheet.append( + [ + # Tekijä (aakkostus) + row.get("sort_author", ""), + # Nimeke + row.get("sort_title", ""), + # Tunniste + row.get("identifier", ""), + # Tunnisteen tyyppi + row.get("identifier_type", ""), + # Kirjasto + row.get("library_name", ""), + # Sijainti + row.get("location", ""), + # Formaatti + row.get("medium", ""), + # Kategoria(t) + categories, + # Kieli + row.get("language", ""), + # Kustantaja/Julkaisija + row.get("publisher", ""), + # Kaikki lainat + row.get("count", ""), + # Tekijät (1-n rows) + *contributors, + ] + ) + + ### Adjust styles + column_width = 24 + + # Loop through all columns and set the width + for column in sheet.columns: + for cell in column: + sheet.column_dimensions[cell.column_letter].width = column_width + + # Define styles for the header row + header_style = Font(name="Calibri", bold=True, color="FFFFFF") + header_fill = PatternFill( + start_color="336699", end_color="336699", fill_type="solid" + ) + header_border = Border( + left=Side(border_style="thin", color="000000"), + right=Side(border_style="thin", color="000000"), + top=Side(border_style="thin", color="000000"), + bottom=Side(border_style="thin", color="000000"), + ) + + # Apply styles to the header row + for cell in sheet[1]: + cell.font = header_style + cell.fill = header_fill + cell.border = header_border + + # Make header row sticky + sheet.freeze_panes = "A2" + + with NamedTemporaryFile() as tmp: + workbook.save(tmp.name) + tmp.seek(0) + stream = tmp.read() + return stream + def analytics_query(self, start, end, locations=None, library=None): """Build a database query that fetches rows of analytics data. @@ -220,3 +329,183 @@ def analytics_query(self, start, end, locations=None, library=None): ] ).select_from(events_alias) return query + + # Finland + def analytics_query_loan_statistics(self, start, end, locations=None, library=None): + """Build a database query that fetches analytics data + for loan statistics Excel export. + + Heavily modified from analytics_query method. + + This method uses low-level SQLAlchemy code to do all + calculations and data conversations in the database. + + :return: A SQLAlchemy query + """ + + clauses = [] + + # Filter by date range + if start: + clauses += [CirculationEvent.start >= start] + if end: + clauses += [CirculationEvent.start < end] + + # Take only checkout events + clauses += [ + CirculationEvent.type.in_( + [CirculationEvent.CM_CHECKOUT, CirculationEvent.DISTRIBUTOR_CHECKOUT] + ) + ] + + if locations: + locations = locations.strip().split(",") + + clauses += [ + CirculationEvent.location.in_(locations), + ] + + if library: + clauses += [CirculationEvent.library == library] + + # Build the primary query. This is a query against the + # CirculationEvent table and a few other tables joined against + # it. This makes up the bulk of the data. + events_alias = ( + select( + [ + Identifier.identifier, + Identifier.type.label("identifier_type"), + Edition.sort_title, + Edition.sort_author, + Work.id.label("work_id"), + Edition.publisher, + Edition.language, + CirculationEvent.location, + Library.name.label("library_name"), + Edition.medium, + Edition.id.label("edition_id"), + func.count().label("count"), + ], + ) + .select_from( + join( + CirculationEvent, + LicensePool, + CirculationEvent.license_pool_id == LicensePool.id, + ) + .join(Identifier, LicensePool.identifier_id == Identifier.id) + .join(Work, Work.id == LicensePool.work_id) + .join(Edition, Work.presentation_edition_id == Edition.id) + .join(Collection, LicensePool.collection_id == Collection.id) + .outerjoin(Library, CirculationEvent.library_id == Library.id) + ) + .where(and_(*clauses)) + .group_by( + Work.id, + Identifier.identifier, + Identifier.type.label("identifier_type"), + Edition.sort_title, + Edition.sort_author, + Work.id.label("work_id"), + Edition.publisher, + Edition.language, + CirculationEvent.location, + Library.name.label("library_name"), + Edition.id.label("edition_id"), + Edition.medium, + ) + .order_by(Edition.sort_author.asc()) + .alias("events_alias") + ) + + edition_id_column = literal_column( + events_alias.name + "." + events_alias.c.edition_id.name + ) + + contributors_alias = ( + select( + [ + Contributor.sort_name, + Contributor.display_name, + Contributor.family_name, + Contributor.lc, + Contributor.viaf, + Contribution.role, + ] + ) + .where(Contribution.edition_id == edition_id_column) + .select_from( + join( + Contributor, + Contribution, + Contributor.id == Contribution.contributor_id, + ) + ) + .alias("contributors_alias") + ) + + # Combine contributor sort_name with role, eg. "sortname (role)" in a subquery + contributors_subquery = select( + [ + func.concat( + contributors_alias.c.sort_name, " (", contributors_alias.c.role, ")" + ).label("contributor_with_role") + ] + ).select_from(contributors_alias) + + contributors = select( + [ + func.array_agg(contributors_subquery.c.contributor_with_role).label( + "contributors_with_roles" + ) + ] + ).select_from(contributors_subquery) + + # A subquery can hook into the main query by referencing its + # 'work_id' field in its WHERE clause. + work_id_column = literal_column( + events_alias.name + "." + events_alias.c.work_id.name + ) + + # This subquery gets the names of a Work's genres as a single + # comma-separated string. + # + + # This Alias selects some number of rows, each containing one + # string column (Genre.name). Genres with higher affinities with + # this work go first. + genres_alias = ( + select([Genre.name.label("genre_name")]) + .select_from(join(WorkGenre, Genre, WorkGenre.genre_id == Genre.id)) + .where(WorkGenre.work_id == work_id_column) + .order_by(WorkGenre.affinity.desc(), Genre.name) + .alias("genres_subquery") + ) + + # Use array_agg() to consolidate the rows into one row -- this + # gives us a single value, an array of strings, for each + # Work. + genres = select([func.array_agg(genres_alias.c.genre_name)]).select_from( + genres_alias + ) + + # Build the main query out of the subqueries. + events = events_alias.c + query = select( + [ + events.identifier, + events.identifier_type, + events.sort_title, + events.sort_author, + events.publisher, + events.language, + genres.label("genres"), + contributors.label("contributors"), + events.location, + events.library_name, + events.medium, + events.count, + ] + ).select_from(events_alias) + return query diff --git a/api/opensearch_analytics_provider.py b/api/opensearch_analytics_provider.py new file mode 100644 index 000000000..eba83997b --- /dev/null +++ b/api/opensearch_analytics_provider.py @@ -0,0 +1,281 @@ +import datetime +from typing import Dict, Optional + +from opensearchpy import OpenSearch +from opensearch_dsl import Search +from flask_babel import lazy_gettext as _ + +from core.model.contributor import Contributor +from core.local_analytics_provider import LocalAnalyticsProvider +from core.model.library import Library +from core.model.licensing import LicensePool +from core.util.http import HTTP + + +class OpenSearchAnalyticsProvider(LocalAnalyticsProvider): + # Fields that get indexed as keyword for aggregating and building faceted search + KEYWORD_FIELDS = ( + "type", + "library_id", + "library_name", + "library_short_name", + "location", + "license_pool_id", + "publisher", + "genres", + "imprint", + "medium", + "collection", + "identifier_type", + "identifier", + "data_source", + "distributor", + "audience", + "author", + "series", + "language", + ) + + # Fields that get indexed as datetime for aggregations and date range searches + DATETIME_FIELDS = ( + "start", + "end", + "issued", + "availability_time", + ) + + # Fields that get indexed as numbers for aggregations and value range searches + NUMERIC_FIELDS = ( + "quality", + "rating", + "popularity", + "licenses_owned", + "licenses_available", + "licenses_reserved", + "patrons_in_hold_queue", + ) + + # Fields that get indexed as booleans + BOOLEAN_FIELDS = ( + "fiction", + "open_access", + ) + + def __init__( + self, + opensearch_analytics_url=None, + opensearch_analytics_index_prefix=None, + ): + self.url = opensearch_analytics_url + self.index_prefix = opensearch_analytics_index_prefix + + use_ssl = self.url.startswith("https://") + self.__client = OpenSearch(self.url, use_ssl=use_ssl, timeout=20, maxsize=25) + self.indices = self.__client.indices + self.index = self.__client.index + self.search = Search(using=self.__client, index=self.index_prefix) + + # Version v1 is hardcoded here. Implement external_search-type + # version system if needed in the future. + self.index_name = self.index_prefix + "-" + "v1" + self.setup_index(self.index_name) + + def setup_index(self, new_index=None, **index_settings): + """Create the event index with appropriate mapping.""" + + index_name = new_index + if self.indices.exists(index_name): + pass + + else: + properties = {} + for field in self.KEYWORD_FIELDS: + properties[field] = {"type": "keyword"} + for field in self.DATETIME_FIELDS: + properties[field] = {"type": "date"} + for field in self.BOOLEAN_FIELDS: + properties[field] = {"type": "boolean"} + for field in self.NUMERIC_FIELDS: + properties[field] = {"type": "float"} + + body = { + "mappings": {"properties": properties} + } # TODO: add settings if necessary + self.indices.create(index=index_name, body=body) + + # Copied from s3_analytics_provider.py (with minor edits) + @staticmethod + def _create_event_object( + library: Library, + license_pool: LicensePool, + event_type: str, + time: datetime.datetime, + old_value, + new_value, + neighborhood: Optional[str] = None, + ) -> Dict: + """Create a Python dict containing required information about the event. + + :param library: Library associated with the event + + :param license_pool: License pool associated with the event + + :param event_type: Type of the event + + :param time: Event's timestamp + + :param old_value: Old value of the metric changed by the event + + :param new_value: New value of the metric changed by the event + + :param neighborhood: Geographic location of the event + + :return: Python dict containing required information about the event + """ + start = time + if not start: + start = datetime.datetime.utcnow() + end = start + + if new_value is None or old_value is None: + delta = None + else: + delta = new_value - old_value + + data_source = license_pool.data_source if license_pool else None + identifier = license_pool.identifier if license_pool else None + collection = license_pool.collection if license_pool else None + work = license_pool.work if license_pool else None + edition = work.presentation_edition if work else None + if not edition and license_pool: + edition = license_pool.presentation_edition + + event = { + "type": event_type, + "start": start, + "end": end, + "library_id": library.id, + "library_name": library.name, + "library_short_name": library.short_name, + "old_value": old_value, + "new_value": new_value, + "delta": delta, + "location": neighborhood, + "license_pool_id": license_pool.id if license_pool else None, + "publisher": edition.publisher if edition else None, + "imprint": edition.imprint if edition else None, + "issued": edition.issued if edition else None, + "published": datetime.datetime.combine( + edition.published, datetime.datetime.min.time() + ) + if edition and edition.published + else None, + "medium": edition.medium if edition else None, + "collection": collection.name if collection else None, + "identifier_type": identifier.type if identifier else None, + "identifier": identifier.identifier if identifier else None, + "data_source": data_source.name if data_source else None, + "distributor": data_source.name if data_source else None, + "audience": work.audience if work else None, + "fiction": work.fiction if work else None, + "quality": work.quality if work else None, + "rating": work.rating if work else None, + "popularity": work.popularity if work else None, + "genres": [genre.name for genre in work.genres] if work else None, + "availability_time": license_pool.availability_time + if license_pool + else None, + "licenses_owned": license_pool.licenses_owned if license_pool else None, + "licenses_available": license_pool.licenses_available + if license_pool + else None, + "licenses_reserved": license_pool.licenses_reserved + if license_pool + else None, + "patrons_in_hold_queue": license_pool.patrons_in_hold_queue + if license_pool + else None, + "title": work.title if work else None, + "author": work.author if work else None, + "series": work.series if work else None, + "series_position": work.series_position if work else None, + "language": work.language if work else None, + "open_access": license_pool.open_access if license_pool else None, + "authors": [ + contribution.contributor.sort_name + for contribution in edition.contributions + if contribution.role == Contributor.AUTHOR_ROLE + ] + if edition + else None, + "contributions": [ + ": ".join( + contribution.contributor.role, + contribution.contributor.sort_name, + ) + for contribution in edition.contributions + if contribution.role != Contributor.AUTHOR_ROLE + ] + if edition + else None, + } + + return event + + def collect_event( + self, + library, + license_pool, + event_type, + time, + old_value=None, + new_value=None, + **kwargs, + ): + """Log the event using the appropriate for the specific provider's mechanism. + + :param db: Database session + :type db: sqlalchemy.orm.session.Session + + :param library: Library associated with the event + :type library: core.model.library.Library + + :param license_pool: License pool associated with the event + :type license_pool: core.model.licensing.LicensePool + + :param event_type: Type of the event + :type event_type: str + + :param time: Event's timestamp + :type time: datetime.datetime + + :param neighborhood: Geographic location of the event + :type neighborhood: str + + :param old_value: Old value of the metric changed by the event + :type old_value: Any + + :param new_value: New value of the metric changed by the event + :type new_value: Any + """ + + if not library and not license_pool: + raise ValueError("Either library or license_pool must be provided.") + + neighborhood = None + + # TODO: Check if we can use locations like in local_analytics + # if self.location_source == self.LOCATION_SOURCE_NEIGHBORHOOD: + # neighborhood = kwargs.pop("neighborhood", None) + + event = self._create_event_object( + library, license_pool, event_type, time, old_value, new_value, neighborhood + ) + + self.index( + index=self.index_name, + body=event, + ) + + def post(self, url, params): + HTTP.post_with_timeout(url, params) diff --git a/api/opensearch_analytics_search.py b/api/opensearch_analytics_search.py new file mode 100644 index 000000000..6f839fb7e --- /dev/null +++ b/api/opensearch_analytics_search.py @@ -0,0 +1,209 @@ +import os +import logging +import flask +from opensearchpy import OpenSearch +from opensearch_dsl import Search + +from api.opensearch_analytics_provider import OpenSearchAnalyticsProvider + + +class OpenSearchAnalyticsSearch: + TIME_INTERVALS = ("hour", "day", "month") + DEFAULT_TIME_INTERVAL = "day" + FACET_FIELDS = ( + "type", + "library_name", + "location", + "publisher", + "genres", + "imprint", + "medium", + "collection", + "data_source", + "distributor", + "audience", + "language", + ) + + def __init__(self): + self.log = logging.getLogger("OpenSearch analytics") + self.url = os.environ.get("PALACE_OPENSEARCH_ANALYTICS_URL", "") + index_prefix = os.environ.get("PALACE_OPENSEARCH_ANALYTICS_INDEX_PREFIX", "") + # Version v1 is hardcoded here. Implement external_search-type + # version system if needed in the future. + self.index_name = index_prefix + "-" + "v1" + + use_ssl = self.url.startswith("https://") + self.__client = OpenSearch(self.url, use_ssl=use_ssl, timeout=20, maxsize=25) + self.search = Search(using=self.__client, index=self.index_name) + + def events(self, params=None, pdebug=False): + """Run a search query on events. + + :return: An aggregated list of facet buckets + """ + + # Filter by library + library = getattr(flask.request, "library", None) + library_short_name = library.short_name if library else None + filters = ( + [{"match": {"library_short_name": library_short_name}}] + if library_short_name + else [] + ) + + # Add filter per provided keyword parameters + filters += [ + {"match": {key: value}} + for key, value in params.items() + if key in OpenSearchAnalyticsProvider.KEYWORD_FIELDS + ] + + # Use time range query if "from" and/or "to" parameters given + from_time = params.get("from") + to_time = params.get("to") + if from_time or to_time: + range = {} + if from_time: + range["gte"] = from_time + if to_time: + range["lte"] = to_time + filters.append({"range": {"start": range}}) + + # Add keyword aggregation buckets + aggs = {} + for field in OpenSearchAnalyticsProvider.KEYWORD_FIELDS: + aggs[field] = {"terms": {"field": field, "size": 100}} + + # Prepare and run the query + query = { + "size": 0, + "query": { + "bool": { + "filter": filters, + }, + }, + "aggs": aggs, + } + result = self.__client.search(index=self.index_name, body=query) + + # Simplify the result object for client + data = {} + for key, value in result["aggregations"].items(): + data[key] = value["buckets"] + + return {"data": data} + + def events_histogram(self, params=None, pdebug=False): + """Run a search query on events. + + :return: A nested aggregated list of event type buckets + inside date histogram buckets + """ + + # Filter by library + library = getattr(flask.request, "library", None) + library_short_name = library.short_name if library else None + filters = ( + [{"match": {"library_short_name": library_short_name}}] + if library_short_name + else [] + ) + + # Add filter per provided keyword parameters + filters += [ + {"match": {key: value}} + for key, value in params.items() + if key in OpenSearchAnalyticsProvider.KEYWORD_FIELDS + ] + + # Use time range query if "from" and/or "to" parameters given + from_param = params.get("from") + to_param = params.get("to") + if from_param or to_param: + range = {} + if from_param: + range["gte"] = from_param + if to_param: + range["lte"] = to_param + filters.append({"range": {"start": range}}) + + # Add time interval aggregation buckets + interval_param = params.get("interval") + interval = ( + interval_param + if interval_param in self.TIME_INTERVALS + else self.DEFAULT_TIME_INTERVAL + ) + aggs = {} + aggs["events_per_interval"] = { + "date_histogram": { + "field": "start", + "interval": interval, + "min_doc_count": 0, + "missing": 0, + "time_zone": "Europe/Helsinki", + "extended_bounds": { + "min": from_param if from_param else None, + "max": f"{to_param}T23:59" if to_param else None, + }, + }, + "aggs": {"type": {"terms": {"field": "type", "size": 100}}}, + } + + # Prepare and run the query + query = { + "size": 0, + "query": {"bool": {"must": filters}}, + "aggs": aggs, + } + result = self.__client.search(index=self.index_name, body=query) + + # Simplify the result object for client + data = { + "events_per_interval": { + "buckets": [ + { + "key": item["key"], + "key_as_string": item["key_as_string"], + "type": { + "buckets": item["type"]["buckets"], + }, + } + for item in result["aggregations"]["events_per_interval"]["buckets"] + ] + } + } + + return {"data": data} + + def get_facets(self, pdebug=False): + """Run a search query to get all the available facets. + + :return: An aggregated list of facet buckets + """ + + # Filter by library + library = getattr(flask.request, "library", None) + library_short_name = library.short_name if library else None + filters = ( + [{"match": {"library_short_name": library_short_name}}] + if library_short_name + else [] + ) + + # Add all term fields to aggregations + aggs = {} + for field in self.FACET_FIELDS: + aggs[field] = {"terms": {"field": field}} + + # Prepare and run the query (with 0 size) + query = {"size": 0, "query": {"bool": {"must": filters}}, "aggs": aggs} + result = self.__client.search(index=self.index_name, body=query) + + # Simplify the result object for client + data = {} + for key, value in result["aggregations"].items(): + data[key] = {"buckets": value.get("buckets", [])} + + return {"facets": data} diff --git a/api/problem_details.py b/api/problem_details.py index 1860cf4b4..ba15b94e3 100644 --- a/api/problem_details.py +++ b/api/problem_details.py @@ -250,6 +250,14 @@ detail=_("The specified SAML provider name isn't one of the known providers."), ) +# Finland +EKIRJASTO_PROVIDER_NOT_CONFIGURED = pd( + "http://librarysimplified.org/terms/problem/requested-provider-not-configured", + status_code=400, + title=_("Ekirjasto provider not configured."), + detail=_("Ekirjasto provider was not configured for the library"), +) + INVALID_SAML_BEARER_TOKEN = pd( "http://librarysimplified.org/terms/problem/credentials-invalid", status_code=401, @@ -257,6 +265,30 @@ detail=_("The provided SAML bearer token couldn't be verified."), ) +# Finland +INVALID_EKIRJASTO_DELEGATE_TOKEN = pd( + "http://librarysimplified.org/terms/problem/credentials-invalid", + status_code=401, + title=_("Invalid delegate token for ekirjasto authentication provider."), + detail=_("The provided delegate token couldn't be verified for ekirjasto authentication provider or it is expired."), +) + +# Finland +INVALID_EKIRJASTO_TOKEN = pd( + "http://librarysimplified.org/terms/problem/credentials-invalid", + status_code=401, + title=_("Invalid ekirjasto token for ekirjasto API."), + detail=_("The provided ekirjasto token couldn't be verified for ekirjasto API."), +) + +# Finland +EKIRJASTO_REMOTE_AUTHENTICATION_FAILED = pd( + "http://librarysimplified.org/terms/problem/credentials-invalid", + status_code=400, + title=_("Authentication with ekirjasto API failed."), + detail=_("Authentication with ekirjasto API failed, for unknown reason."), +) + UNSUPPORTED_AUTHENTICATION_MECHANISM = pd( "http://librarysimplified.org/terms/problem/unsupported-authentication-mechanism", status_code=400, diff --git a/api/routes.py b/api/routes.py index a1b61d99b..345cbe96d 100644 --- a/api/routes.py +++ b/api/routes.py @@ -555,7 +555,6 @@ def saml_authenticate(): flask.request.args, app.manager._db ) - # Redirect URI for SAML providers # NOTE: we cannot use @has_library decorator and append a library's name to saml_calback route # (e.g. https://cm.org/LIBRARY_NAME/saml_callback). @@ -572,6 +571,24 @@ def saml_callback(): request, app.manager._db ) +# Finland +# Authenticate with the ekirjasto token. +@library_route("/ekirjasto_authenticate", methods=["POST"]) +@has_library +@returns_problem_detail +def ekirjasto_authenticate(): + return app.manager.ekirjasto_controller.authenticate( + request, app.manager._db + ) + +# Finland +# Get descriptions for the library catalogs in the system. +# This is public route. +@app.route('/libraries', defaults={'library_uuid': None}, methods=["GET"]) +@app.route("/libraries/", methods=["GET"]) +@returns_problem_detail +def client_libraries(library_uuid): + return app.manager.catalog_descriptions.get_catalogs(library_uuid) # Loan notifications for ODL distributors, eg. Feedbooks @library_route("/odl_notify/", methods=["GET", "POST"]) diff --git a/build_docker_images.sh b/build_docker_images.sh new file mode 100755 index 000000000..a1fc6cc18 --- /dev/null +++ b/build_docker_images.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +if [ "$1" = "admin" ] +then + [ -d palace_circulation_admin_ui ] && rm -rf palace_circulation_admin_ui + git clone --branch=main ssh://git.lingsoft.fi/home/git/palace_circulation_admin_ui.git + docker compose --progress=plain build admin + retval=$? + if [ $retval -ne 0 ]; then + echo "Admin UI build failed" + exit $retval + fi + [ -d palace_circulation_admin_ui ] && rm -rf palace_circulation_admin_ui + docker create --name temp-admin "${PWD##*/}"-admin:latest + docker cp temp-admin:/mnt/tarball/admin_dist.tar.gz docker/admin_dist.tar.gz + docker rm temp-admin + docker image rm "${PWD##*/}"-admin:latest + elif [ "$#" -eq 1 ] + then + docker compose --progress=plain build $* + else + ./build_docker_images.sh admin + retval=$? + if [ $retval -ne 0 ]; then + echo "Admin UI build failed, exiting" + exit $retval + fi + docker compose --progress=plain build webapp scripts pg minio os + retval=$? + if [ $retval -ne 0 ]; then + echo "Build failed" + exit $retval + fi +fi diff --git a/core/analytics.py b/core/analytics.py index ab41fd20a..dc061e003 100644 --- a/core/analytics.py +++ b/core/analytics.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Optional +from api.opensearch_analytics_provider import OpenSearchAnalyticsProvider # Finland from api.s3_analytics_provider import S3AnalyticsProvider from core.local_analytics_provider import LocalAnalyticsProvider @@ -18,6 +19,9 @@ def __init__( self, s3_analytics_enabled: bool = False, s3_service: Optional[S3Service] = None, + opensearch_analytics_enabled: bool = False, + opensearch_analytics_url=Optional[str], + opensearch_analytics_index_prefix=Optional[str], ) -> None: self.providers = [LocalAnalyticsProvider()] @@ -29,6 +33,19 @@ def __init__( "S3 analytics is not configured: No analytics bucket was specified." ) + if opensearch_analytics_enabled: + if opensearch_analytics_url and opensearch_analytics_index_prefix: + self.providers.append( + OpenSearchAnalyticsProvider( + opensearch_analytics_url=opensearch_analytics_url, + opensearch_analytics_index_prefix=opensearch_analytics_index_prefix, + ) + ) + else: + self.log.info( + "OpenSearch analytics is not configured: Either analytics url or index prefix (or both) was not specified." + ) + def collect_event(self, library, license_pool, event_type, time=None, **kwargs): if not time: time = utc_now() diff --git a/core/service/analytics/configuration.py b/core/service/analytics/configuration.py index 9a4c3e3db..b741ec971 100644 --- a/core/service/analytics/configuration.py +++ b/core/service/analytics/configuration.py @@ -3,3 +3,6 @@ class AnalyticsConfiguration(ServiceConfiguration): s3_analytics_enabled: bool = False + opensearch_analytics_enabled: bool = False + opensearch_analytics_url: str = "" + opensearch_analytics_index_prefix: str = "" diff --git a/core/service/analytics/container.py b/core/service/analytics/container.py index 0ee8d7a4d..b7d88603a 100644 --- a/core/service/analytics/container.py +++ b/core/service/analytics/container.py @@ -12,4 +12,7 @@ class AnalyticsContainer(DeclarativeContainer): Analytics, s3_analytics_enabled=config.s3_analytics_enabled, s3_service=storage.analytics, + opensearch_analytics_enabled=config.opensearch_analytics_enabled, + opensearch_analytics_url=config.opensearch_analytics_url, + opensearch_analytics_index_prefix=config.opensearch_analytics_index_prefix, ) diff --git a/docker-compose.yml b/docker-compose.yml index 258847373..11c30e56f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,3 +56,8 @@ services: environment: discovery.type: single-node DISABLE_SECURITY_PLUGIN: true + + admin: + build: + dockerfile: docker/Dockerfile.admin + target: admin diff --git a/docker/Dockerfile b/docker/Dockerfile index 3acdd271b..4a47975a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,7 +6,7 @@ # The only difference are what services are enabled. ############################################################################### -ARG BASE_IMAGE=ghcr.io/thepalaceproject/circ-baseimage:latest +ARG BASE_IMAGE=ghcr.io/natlibfi/ekirjasto-circ-baseimage:latest FROM ${BASE_IMAGE} AS common @@ -29,6 +29,7 @@ COPY --chown=simplified:simplified . /var/www/circulation FROM common AS exec +ENV TZ=Europe/Helsinki ENV SIMPLIFIED_SCRIPT_NAME "" VOLUME /var/log @@ -44,7 +45,7 @@ CMD ["/sbin/my_init", "--skip-runit", "--quiet", "--", \ FROM common AS scripts # Set the local timezone and setup cron -ENV TZ=US/Eastern +ENV TZ=Europe/Helsinki RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ touch /var/log/cron.log @@ -61,16 +62,24 @@ CMD ["/sbin/my_init"] ############################################################################### FROM common AS webapp +ENV TZ=Europe/Helsinki # Setup nginx COPY docker/services/nginx /etc/nginx/ +COPY docker/admin_dist.tar.gz /mnt/admin_dist.tar.gz # Setup uwsgi COPY docker/services/uwsgi /etc/ RUN mkdir -p /var/log/uwsgi && \ chown -RHh simplified:simplified /var/log/uwsgi && \ mkdir /var/run/uwsgi && \ - chown simplified:simplified /var/run/uwsgi + chown simplified:simplified /var/run/uwsgi && \ + cd /var/www/circulation/api/admin && \ + tar xf /mnt/admin_dist.tar.gz && \ + chown -R simplified:simplified dist && \ + rm /mnt/admin_dist.tar.gz + +COPY --chown=simplified:simplified docker/config.py_admin /var/www/circulation/api/admin/config.py # Setup runit COPY docker/runit /etc/service/ @@ -80,3 +89,4 @@ WORKDIR /home/simplified/circulation EXPOSE 80 CMD ["/sbin/my_init"] + diff --git a/docker/Dockerfile.admin b/docker/Dockerfile.admin new file mode 100755 index 000000000..a1359b9cc --- /dev/null +++ b/docker/Dockerfile.admin @@ -0,0 +1,83 @@ +FROM ubuntu:22.04 as admin + +LABEL maintainer="Joe Random " + +ARG BUILDENV=devz + +#RUN if [ "${BUILDENV}" != "lfapi" -a "${BUILDENV}" != "lsstest" -a "${BUILDENV}" != "lfapi2" -a "${BUILDENV}" != "ltstest" -a "${BUILDENV}" != "prod" ]; \ +# then exit 1 ; \ +# fi + +RUN sed -i 's/http:\/\/archive\.ubuntu\.com\/ubuntu/http:\/\/mirrors.nic.funet.fi\/ubuntu\//' \ + /etc/apt/sources.list + +ENV DEBIAN_FRONTEND noninteractive +ENV TERM linux + +#ENV TINI_VERSION v0.19.0 +#ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini + +RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ + apt-utils \ + apt-transport-https \ + ca-certificates \ + curl \ + debconf-utils \ + gnupg \ + gzip \ + net-tools \ + software-properties-common \ + tar \ + telnet \ + tzdata \ + wget && \ + mkdir -p /etc/apt/keyrings && \ + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | \ + tee /etc/apt/sources.list.d/nodesource.list && \ + echo Europe/Helsinki >/etc/timezone && \ + ln -sf /usr/share/zoneinfo/Europe/Helsinki /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata && \ + apt-get update && apt-get install -y --allow-unauthenticated --no-install-recommends \ + less \ + nodejs && \ + npm update -g npm && \ + rm -f /etc/cron.daily/* && \ + rm -rf /var/lib/apt/lists/* && \ + groupadd -g 1000 node && \ + useradd -u 1000 -r -g node -d /home/node -s /bin/bash -c "Node image user" node && \ + apt-get remove -y \ + apt-transport-https \ + apt-utils \ + debconf-utils \ + gnupg \ + software-properties-common \ + wget && \ + apt-get autoremove -y + +COPY palace_circulation_admin_ui /opt/circulation/admin + +RUN cd /opt/circulation/admin && \ + npm install && \ + npm run test-finland && \ + retval=$? && \ + if [ $retval -ne 0 ]; then \ + echo "Admin UI test failed" && \ + exit $retval ; \ + fi && \ + mkdir -p /mnt/tarball && \ + tar czvf /mnt/tarball/admin_dist.tar.gz dist + +# Specify the port number the container should expose +#EXPOSE 8080 + +# Set up working dir +#WORKDIR /opt/circulation/admin + +# Entrypoint is run prior to CMD +#ENTRYPOINT ["/tini", "--"] + +# Actual command to start, can be overridden to start e.g. ASR API +#CMD ["npm", "run", "dev-server", "--", "--env=backend=webapp:6500"] + + diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 3fb803bb9..7409c1c25 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -1,4 +1,7 @@ FROM opensearchproject/opensearch:1 as opensearch # Install ICU Analysis Plugin +USER root +RUN ln -sf /usr/share/zoneinfo/Europe/Helsinki /etc/localtime +USER opensearch RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu diff --git a/docker/README.md b/docker/README.md index e009866e7..225c0e5b4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -20,7 +20,7 @@ Once the webapp Docker image is built, we can run it in a container with the fol $ docker run --name webapp -d \ --p 80:80 \ -e SIMPLIFIED_PRODUCTION_DATABASE='postgresql://[username]:[password]@[host]:[port]/[database_name]' \ - ghcr.io/thepalaceproject/circ-webapp:main + ghcr.io/natlibfi/ekirjasto-circ-webapp:main ``` If the database and OpenSearch(OS) are running in containers, use the --link option to let the webapp docker container @@ -32,7 +32,7 @@ docker run \ --name circ \ -e SIMPLIFIED_PRODUCTION_DATABASE='postgresql://[username]:[password]@[host]:[port]/[database_name]' \ -d -p 6500:80 \ -ghcr.io/thepalaceproject/circ-webapp:main +ghcr.io/natlibfi/ekirjasto-circ-webapp:main ``` Navigate to `http://localhost/admin` in your browser to visit the web admin for the Circulation Manager. In the admin, @@ -49,7 +49,7 @@ Once the scripts Docker image is built, we can run it in a container with the fo $ docker run --name scripts -d \ -e TZ='YOUR_TIMEZONE_STRING' \ -e SIMPLIFIED_PRODUCTION_DATABASE='postgresql://[username]:[password]@[host]:[port]/[database_name]' \ - ghcr.io/thepalaceproject/circ-scripts:main + ghcr.io/natlibfi/ekirjasto-circ-scripts:main ``` Using `docker exec -it scripts /bin/bash` in your console, navigate to `/var/log/simplified` in the container. After @@ -58,7 +58,7 @@ Using `docker exec -it scripts /bin/bash` in your console, navigate to `/var/log ### circ-exec This image builds containers that will run a single script and stop. It's useful in conjunction with a tool like Amazon - ECS Scheduled Tasks, where you can run script containers on a cron-style schedule. +ECS Scheduled Tasks, where you can run script containers on a cron-style schedule. Unlike the `circ-scripts` image, which runs constantly and executes every possible maintenance script--whether or not your configuration requires it--`circ-exec` offers more nuanced control of your Library Simplified Circulation Manager @@ -74,7 +74,7 @@ external log aggregator if you wish to capture logs from the job. $ docker run --name search_index_refresh -it \ -e SIMPLIFIED_SCRIPT_NAME='refresh_materialized_views' \ -e SIMPLIFIED_PRODUCTION_DATABASE='postgresql://[username]:[password]@[host]:[port]/[database_name]' \ - ghcr.io/thepalaceproject/circ-exec:main + ghcr.io/natlibfi/ekirjasto-circ-exec:main ``` ## Environment Variables @@ -84,35 +84,35 @@ Environment variables can be set with the `-e VARIABLE_KEY='variable_value'` opt ### `SIMPLIFIED_PRODUCTION_DATABASE` -*Required.* The URL of the production PostgreSQL database for the application. +_Required._ The URL of the production PostgreSQL database for the application. ### `SIMPLIFIED_TEST_DATABASE` -*Optional.* The URL of a PostgreSQL database for tests. This optional variable allows unit tests to be run in the +_Optional._ The URL of a PostgreSQL database for tests. This optional variable allows unit tests to be run in the container. ### `TZ` -*Optional. Applies to `circ-scripts` only.* The time zone that cron should use to run scheduled scripts--usually the +_Optional. Applies to `circ-scripts` only._ The time zone that cron should use to run scheduled scripts--usually the time zone of the library or libraries on the circulation manager instance. This value should be selected according to - [Debian-system time zone options](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). - This value allows scripts to be run at ideal times. +[Debian-system time zone options](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). +This value allows scripts to be run at ideal times. ### `UWSGI_PROCESSES` -*Optional.* The number of processes to use when running uWSGI. This value can be updated in `docker-compose.yml` or +_Optional._ The number of processes to use when running uWSGI. This value can be updated in `docker-compose.yml` or added directly in `Dockerfile` under webapp stage. Defaults to 6. ### `UWSGI_THREADS` -*Optional.* The number of threads to use when running uWSGI. This value can be updated in `docker-compose.yml` or added +_Optional._ The number of threads to use when running uWSGI. This value can be updated in `docker-compose.yml` or added directly in `Dockerfile` under webapp stage. Defaults to 2. ## Building new images If you plan to work with stable versions of the Circulation Manager, we strongly recommend using the latest stable versions of circ-webapp and circ-scripts -[published to the GitHub Container Registry](https://github.com/orgs/ThePalaceProject/packages?repo_name=circulation). +[published to the GitHub Container Registry](https://github.com/orgs/NatLibFi/packages?repo_name=circulation). However, there may come a time in development when you want to build Docker containers for a particular version of the Circulation Manager. If so, please use the instructions below. diff --git a/docker/config.py_admin b/docker/config.py_admin new file mode 100755 index 000000000..623e5f7c8 --- /dev/null +++ b/docker/config.py_admin @@ -0,0 +1,145 @@ +import os +from enum import Enum +from typing import Optional +from urllib.parse import urljoin + + +class OperationalMode(str, Enum): + production = "production" + development = "development" + + +class Configuration: + APP_NAME = "E-kirjasto Collection Manager" + PACKAGE_NAME = "@natlibfi/ekirjasto-circulation-admin" + PACKAGE_VERSION = "1.7.0" + + STATIC_ASSETS = { + "admin_js": "circulation-admin.js", + "admin_css": "circulation-admin.css", + "admin_logo": "PalaceCollectionManagerLogo.svg", + } + + # For proper operation, `package_url` MUST end with a slash ('/') and + # `asset_rel_url` MUST NOT begin with one. + # (Finland: Modified to serve static files from admin-ui build in production) + PACKAGE_TEMPLATES = { + OperationalMode.production: { + "package_url": "/admin/", + "asset_rel_url": "static/{filename}", + }, + OperationalMode.development: { + "package_url": "/admin/", + "asset_rel_url": "static/{filename}", + }, + } + + DEVELOPMENT_MODE_PACKAGE_TEMPLATE = "node_modules/{name}" + STATIC_ASSETS_REL_PATH = "dist" + + ADMIN_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) + + # Environment variables that contain admin client package information. + ENV_ADMIN_UI_PACKAGE_NAME = "TPP_CIRCULATION_ADMIN_PACKAGE_NAME" + ENV_ADMIN_UI_PACKAGE_VERSION = "TPP_CIRCULATION_ADMIN_PACKAGE_VERSION" + + @classmethod + def operational_mode(cls) -> OperationalMode: + return ( + OperationalMode.development + if os.path.isdir(cls.package_development_directory()) + else OperationalMode.production + ) + + @classmethod + def package_name(cls) -> str: + """Get the effective package name. + + :return: A package name. + :rtype: str + """ + return os.environ.get(cls.ENV_ADMIN_UI_PACKAGE_NAME) or cls.PACKAGE_NAME + + @classmethod + def package_version(cls) -> str: + """Get the effective package version. + + :return Package verison. + """ + return os.environ.get(cls.ENV_ADMIN_UI_PACKAGE_VERSION) or cls.PACKAGE_VERSION + + @classmethod + def lookup_asset_url( + cls, key: str, *, _operational_mode: Optional[OperationalMode] = None + ) -> str: + """Get the URL for the asset_type. + + :param key: The key used to lookup an asset's filename. If the key is + not found in the asset list, then the key itself is used as the asset. + :type key: str + :param _operational_mode: Provided for testing purposes. The operational + mode is normally determined by local state + :type _operational_mode: OperationalMode + :return: A URL string. + :rtype: str + """ + operational_mode = _operational_mode or cls.operational_mode() + filename = cls.STATIC_ASSETS.get(key, key) + return urljoin( + cls.package_url(_operational_mode=operational_mode), + cls.PACKAGE_TEMPLATES[operational_mode]["asset_rel_url"].format( + filename=filename + ), + ) + + @classmethod + def package_url(cls, *, _operational_mode: Optional[OperationalMode] = None) -> str: + """Compute the URL for the admin UI package. + + :param _operational_mode: For testing. The operational mode is + normally determined by local state. + :type _operational_mode: OperationalMode + :return: String representation of the URL/path for either the asset + of the given type or, if no type is specified, the base path + of the package. + :rtype: str + """ + operational_mode = _operational_mode or cls.operational_mode() + template = cls.PACKAGE_TEMPLATES[operational_mode]["package_url"] + url = template.format(name=cls.package_name(), version=cls.package_version()) + if not url.endswith("/"): + url += "/" + return url + + @classmethod + def package_development_directory(cls, *, _base_dir: Optional[str] = None) -> str: + """Absolute path for the admin UI package when in development mode. + + :param _base_dir: For testing purposes. Not used in normal operation. + :type _base_dir: str + :returns: String containing absolute path to the admin UI package. + :rtype: str + """ + base_dir = _base_dir or cls.ADMIN_DIRECTORY + return os.path.join( + base_dir, + cls.DEVELOPMENT_MODE_PACKAGE_TEMPLATE.format(name=cls.package_name()), + ) + + @classmethod + def static_files_directory(cls, *, _base_dir: Optional[str] = None) -> str: + """Absolute path for the admin UI static files. + (Finland: Modified to serve static files from admin-ui build in production) + + :param _base_dir: For testing purposes. Not used in normal operation. + :type _base_dir: str + :returns: String containing absolute path to the admin UI package. + :rtype: str + """ + operational_mode = cls.operational_mode() + package_dir = ( + cls.package_development_directory(_base_dir=_base_dir) + if operational_mode == OperationalMode.development + else cls.ADMIN_DIRECTORY + ) + return os.path.join(package_dir, cls.STATIC_ASSETS_REL_PATH) diff --git a/docker/services/cron/cron.d/circulation b/docker/services/cron/cron.d/circulation index 7a8a40f6e..744a67097 100644 --- a/docker/services/cron/cron.d/circulation +++ b/docker/services/cron/cron.d/circulation @@ -115,3 +115,7 @@ HOME=/var/www/circulation 0 8,20 * * * root core/bin/run playtime_summation >> /var/log/cron.log 2>&1 # On the 2nd of every month 0 4 2 * * root core/bin/run playtime_reporting >> /var/log/cron.log 2>&1 + +# Update the status of the different Authentication Providers +# +*/5 * * * * root core/bin/run update_integration_statuses >> /var/log/cron.log 2>&1 diff --git a/poetry.lock b/poetry.lock index f3c2e2acc..a46dcc669 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "alembic" @@ -1062,6 +1062,17 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "et-xmlfile" +version = "1.1.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, + {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, +] + [[package]] name = "exceptiongroup" version = "1.0.4" @@ -1282,11 +1293,11 @@ files = [ google-auth = ">=2.14.1,<3.0dev" googleapis-common-protos = ">=1.56.2,<2.0dev" grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -2530,6 +2541,20 @@ files = [ {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, ] +[[package]] +name = "openpyxl" +version = "3.1.2" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = false +python-versions = ">=3.6" +files = [ + {file = "openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"}, + {file = "openpyxl-3.1.2.tar.gz", hash = "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184"}, +] + +[package.dependencies] +et-xmlfile = "*" + [[package]] name = "opensearch-dsl" version = "1.0.0" @@ -2778,8 +2803,6 @@ files = [ {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, @@ -2822,7 +2845,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"}, {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"}, @@ -2831,8 +2853,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"}, {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"}, - {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"}, {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"}, @@ -3412,7 +3432,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -3420,15 +3439,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -3445,7 +3457,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -3453,7 +3464,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -3951,7 +3961,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} mypy = {version = ">=0.910", optional = true, markers = "python_version >= \"3\" and extra == \"mypy\""} sqlalchemy2-stubs = {version = "*", optional = true, markers = "extra == \"mypy\""} @@ -4414,16 +4424,6 @@ files = [ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, - {file = "wrapt-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55"}, - {file = "wrapt-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9"}, - {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335"}, - {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9"}, - {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8"}, - {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf"}, - {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a"}, - {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be"}, - {file = "wrapt-1.14.1-cp311-cp311-win32.whl", hash = "sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204"}, - {file = "wrapt-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, @@ -4512,4 +4512,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.8,<4" -content-hash = "c96cd038c5187aba1670c495d8dc0a03aae508ae60d76362782143168ce6a652" +content-hash = "0fdab9c3480492547bd6804f15d137ba5ee6a9ced8eb21052b8df03fc8ea582c" diff --git a/pyproject.toml b/pyproject.toml index ffc7598b6..b0f235d1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,13 +164,13 @@ module = [ ] [tool.poetry] -authors = ["The Palace Project "] -description = "The Palace Project Manager Application" -homepage = "https://thepalaceproject.org" +authors = ["E-kirjasto"] +description = "E-kirjasto Circulation Manager Application" +homepage = "https://kansalliskirjasto.fi" license = "Apache-2.0" -name = "Palace Manager" +name = "E-kirjasto Circulation Manager" readme = "README.md" -repository = "https://github.com/ThePalaceProject/circulation" +repository = "https://github.com/NatLibFi/ekirjasto-circulation" version = "0" # Version number is managed with tags in git [tool.poetry.dependencies] @@ -200,6 +200,7 @@ money = "1.3.0" multipledispatch = "^1.0" nameparser = "^1.1" # nameparser is for author name manipulations nltk = "3.8.1" # nltk is a textblob dependency. +openpyxl = "3.1.2" # Finland opensearch-dsl = "~1.0" opensearch-py = "~1.1" palace-webpub-manifest-parser = "^3.1" diff --git a/resources/images/favicon.ico b/resources/images/favicon.ico new file mode 100644 index 000000000..c6922a128 Binary files /dev/null and b/resources/images/favicon.ico differ diff --git a/tests/api/admin/controller/test_patron_auth.py b/tests/api/admin/controller/test_patron_auth.py index b1925c04b..38a382ad9 100644 --- a/tests/api/admin/controller/test_patron_auth.py +++ b/tests/api/admin/controller/test_patron_auth.py @@ -105,7 +105,7 @@ def test_patron_auth_services_get_with_no_services( assert response.get("patron_auth_services") == [] protocols = response.get("protocols") assert isinstance(protocols, list) - assert 7 == len(protocols) + assert 8 == len(protocols) assert SimpleAuthenticationProvider.__module__ == protocols[0].get("name") assert "settings" in protocols[0] assert "library_settings" in protocols[0] diff --git a/tests/api/admin/test_config.py b/tests/api/admin/test_config.py index ec3b93395..e336cfc73 100644 --- a/tests/api/admin/test_config.py +++ b/tests/api/admin/test_config.py @@ -90,7 +90,7 @@ def test_resolve_package_version(self, caplog): None, None, OperationalMode.production, - "https://cdn.jsdelivr.net/npm/@thepalaceproject/circulation-admin@", + "https://cdn.jsdelivr.net/npm/@natlibfi/ekirjasto-circulation-admin@", ], [ "@some-scope/some-package", @@ -132,12 +132,12 @@ def test_package_url( [ None, None, - "/my-base-dir/node_modules/@thepalaceproject/circulation-admin", + "/my-base-dir/node_modules/@natlibfi/circulation-admin", ], [ None, "1.0.0", - "/my-base-dir/node_modules/@thepalaceproject/circulation-admin", + "/my-base-dir/node_modules/@natlibfi/circulation-admin", ], ["some-package", "1.0.0", "/my-base-dir/node_modules/some-package"], ], diff --git a/tests/api/admin/test_routes.py b/tests/api/admin/test_routes.py index 9cbd5561c..717cda2b4 100644 --- a/tests/api/admin/test_routes.py +++ b/tests/api/admin/test_routes.py @@ -871,7 +871,7 @@ def test_static_file(self, fixture: AdminRouteFixture): root_path = Path(__file__).parent.parent.parent.parent local_path = ( root_path - / "api/admin/node_modules/@thepalaceproject/circulation-admin/dist" + / "api/admin/node_modules/@natlibfi/ekirjasto-circulation-admin/dist" ) url = "/admin/static/circulation-admin.js" diff --git a/tests/finland/__init__.py b/tests/finland/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/finland/conftest.py b/tests/finland/conftest.py new file mode 100644 index 000000000..7efef71ed --- /dev/null +++ b/tests/finland/conftest.py @@ -0,0 +1,10 @@ +from pytest import register_assert_rewrite + +register_assert_rewrite("tests.fixtures.database") +register_assert_rewrite("tests.fixtures.files") +register_assert_rewrite("tests.fixtures.vendor_id") + +pytest_plugins = [ + "tests.fixtures.api_controller", + "tests.fixtures.database", +] diff --git a/tests/finland/test_ekirjasto.py b/tests/finland/test_ekirjasto.py new file mode 100644 index 000000000..971b69117 --- /dev/null +++ b/tests/finland/test_ekirjasto.py @@ -0,0 +1,681 @@ +import datetime +import os +import time +import urllib.parse +import uuid +from base64 import b64decode, b64encode +from cryptography.fernet import Fernet, InvalidToken +from flask import url_for +from functools import partial +from typing import Callable + +import jwt +import pytest +import requests + +from api.authentication.base import PatronData +from api.circulation_exceptions import ( + InternalServerError, + PatronNotFoundOnRemote, + RemoteInitiatedServerError, + RemotePatronCreationFailedException +) +from api.ekirjasto_authentication import ( + EkirjastoAuthenticationAPI, + EkirjastoAuthAPISettings, + EkirjastoAuthAPILibrarySettings, + EkirjastoEnvironment +) +from api.ekirjasto_controller import EkirjastoController +from api.util.patron import PatronUtility +from core.model import Credential, DataSource, Patron +from core.util.datetime_helpers import from_timestamp, utc_now +from core.util.problem_detail import ProblemDetail +from tests.core.mock import MockRequestsResponse +from tests.fixtures.api_controller import ControllerFixture +from tests.fixtures.database import DatabaseTransactionFixture + +class MockEkirjastoRemoteAPI: + + def __init__(self): + self.test_users = { + # TODO: currently there is no difference with (in our perspective) verified and unverified, but this might change. + 'unverified': { + 'sub': '0CA5A0D5-ABA2-4104-AEFB-E37A30B66E23', + 'name': 'Unverified User', + 'family_name': 'User', + 'given_name': 'Unverified', + 'email': 'unverified.user@example.com', + 'username': 'unverified.user', + 'role': 'customer', + 'municipality': 'Helsinki', + # When True, this will be replaced with real expire timestamp in init. + 'exp': True, + 'verified': False, + 'passkeys': [] + }, + 'verified': { + 'sub': '6B0DDBEE-572A-4B94-8619-500CAD9747D6', + 'name': 'Verified User', + 'family_name': 'User', + 'given_name': 'Verified', + 'email': 'verified.user@example.com', + 'username': 'verified.user', + 'role': 'customer', + 'municipality': 'Helsinki', + 'exp': True, + 'verified': True, + 'passkeys': [] + } + } + self.access_tokens = {} + for user_id, user in self.test_users.items(): + if user['exp'] == True: + self._refresh_token_for_user_id(user_id) + + def _generate_auth_token_info(self, exp_duration_seconds: int = 600): + return { + 'token': b64encode(uuid.uuid4().bytes).decode("ascii"), + # Ekirjasto API returns expire timestamp in milliseconds + 'exp': int(((utc_now() + datetime.timedelta(seconds=exp_duration_seconds)).timestamp()) * 1000) + } + + def _refresh_token_for_user_id(self, user_id): + # Remove (invalidate) old tokens for the user. + self.access_tokens = {key:val for key, val in self.access_tokens.items() if val['user_id'] != user_id} + + # Create new valid token, with meta info. + token_info = self._generate_auth_token_info() + self.access_tokens[token_info['token']] = { + 'user_id': user_id, + **token_info + } + + # Update token expiration in user info. + self.test_users[user_id]['exp'] = token_info['exp'] + + return token_info + + def get_test_access_token_for_user(self, user_id): + # Normally this is got after using some login method with Ekirjasto. + for token, token_info in self.access_tokens.items(): + if token_info["user_id"] == user_id: + return token_info["token"], token_info["exp"] + assert None, f"Token info for user '{user_id}' was not found." + + def _check_authentication(self, access_token): + token_info = None + if access_token in self.access_tokens.keys(): + token_info = self.access_tokens[access_token] + + user_id = None + if token_info != None and float(token_info['exp']/1000) > utc_now().timestamp(): + # Token is not expired. + user_id = token_info['user_id'] + + if user_id != None and not user_id in self.test_users: + user_id = None + + return user_id + + def api_userinfo(self, access_token): + user_id = self._check_authentication(access_token) + + if user_id != None: + return MockRequestsResponse(200, content=self.test_users[user_id]) + + return MockRequestsResponse(401) + + def api_refresh(self, access_token): + user_id = self._check_authentication(access_token) + + if user_id != None: + token_info = self._refresh_token_for_user_id(user_id) + return MockRequestsResponse(200, content=token_info) + + return MockRequestsResponse(401) + +class MockEkirjastoAuthenticationAPI(EkirjastoAuthenticationAPI): + + def __init__( + self, + library_id, + integration_id, + settings, + library_settings, + analytics=None, + bad_connection=False, + failure_status_code=None, + ): + super().__init__(library_id, integration_id, settings, library_settings, None) + + self.bad_connection = bad_connection + self.failure_status_code = failure_status_code + + self.mock_api = MockEkirjastoRemoteAPI() + + def _create_authenticate_url(self, db): + return url_for( + "ekirjasto_authenticate", + _external=True, + library_short_name="test-library", + provider=self.label(), + ) + + def requests_get(self, url, ekirjasto_token=None): + if self.bad_connection: + raise requests.exceptions.ConnectionError(str("Connection error"), self.__class__.__name__) + elif self.failure_status_code: + # Simulate a server returning an unexpected error code. + return MockRequestsResponse( + self.failure_status_code, "Error %s" % self.failure_status_code + ) + if "userinfo" in url: + return self.mock_api.api_userinfo(ekirjasto_token) + + assert None, f"Mockup for GET {url} not created" + + def requests_post(self, url, ekirjasto_token=None): + if self.bad_connection: + raise requests.exceptions.ConnectionError(str("Connection error"), self.__class__.__name__) + elif self.failure_status_code: + # Simulate a server returning an unexpected error code. + return MockRequestsResponse( + self.failure_status_code, "Error %s" % self.failure_status_code + ) + if "refresh" in url: + return self.mock_api.api_refresh(ekirjasto_token) + + assert None, f"Mockup for POST {url} not created" + + +@pytest.fixture +def mock_library_id() -> int: + return 20 + + +@pytest.fixture +def mock_integration_id() -> int: + return 20 + + +@pytest.fixture +def create_settings() -> Callable[..., EkirjastoAuthAPISettings]: + return partial( + EkirjastoAuthAPISettings, + ekirjasto_environment=EkirjastoEnvironment.DEVELOPMENT + ) + + +@pytest.fixture +def create_provider( + mock_library_id: int, + mock_integration_id: int, + create_settings: Callable[..., EkirjastoAuthAPISettings], +) -> Callable[..., MockEkirjastoAuthenticationAPI]: + return partial( + MockEkirjastoAuthenticationAPI, + library_id=mock_library_id, + integration_id=mock_integration_id, + settings=create_settings(), + library_settings=EkirjastoAuthAPILibrarySettings() + ) + +class TestEkirjastoAuthentication: + + def test_authentication_flow_document( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + # We're about to call url_for, so we must create an + # application context. + provider = create_provider() + controller_fixture.app.config["SERVER_NAME"] = "localhost" + + with controller_fixture.app.test_request_context("/"): + doc = provider._authentication_flow_document(None) + assert provider.label() == doc["description"] + assert provider.flow_type == doc["type"] + + assert doc["links"][0]["href"] == "http://localhost/test-library/ekirjasto_authenticate?provider=E-kirjasto+provider+for+circulation+manager" + + def test_from_config( + self, + create_settings: Callable[..., EkirjastoAuthAPISettings], + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + settings = create_settings( + ekirjasto_environment=EkirjastoEnvironment.FAKE, + delegate_expire_time=537457, + ) + provider = create_provider(settings=settings) + + # Verify that the configuration details were stored properly. + assert EkirjastoEnvironment.FAKE == provider.ekirjasto_environment + assert 537457 == provider.delegate_expire_timemestamp + + def test_persistent_patron_delegate_id( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + provider = create_provider() + + db_patron = controller_fixture.db.patron() + assert isinstance(db_patron, Patron) + + delegate_id = provider.get_patron_delegate_id(controller_fixture.db.session, db_patron) + delegate_id2 = provider.get_patron_delegate_id(controller_fixture.db.session, db_patron) + + assert delegate_id == delegate_id2 + + data_source = DataSource.lookup(controller_fixture.db.session, provider.label()) + assert isinstance(data_source, DataSource) + + credential = Credential.lookup_by_token( + controller_fixture.db.session, + data_source, + provider.patron_delegate_id_credential_key(), + delegate_id, + allow_persistent_token=True + ) + + assert credential.credential == delegate_id + + # Must be persistent credential + assert credential.expires == None + + delegate_patron = provider.get_patron_with_delegate_id(controller_fixture.db.session, delegate_id) + + assert isinstance(delegate_patron, Patron) + assert db_patron.id == delegate_patron.id + + def test_secrets( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + provider = create_provider() + + # Secrets are not set, so this will fail. + pytest.raises(InternalServerError, provider._check_secrets_or_throw) + assert provider.delegate_token_signing_secret == None + assert provider.delegate_token_encrypting_secret == None + + provider.set_secrets(controller_fixture.db.session) + + provider._check_secrets_or_throw() + + assert provider.delegate_token_signing_secret != None + assert provider.delegate_token_encrypting_secret != None + + # Screts should be strong enough. + assert len(provider.delegate_token_signing_secret) > 30 + assert len(provider.delegate_token_encrypting_secret) > 30 + + def test_create_delegate_token( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + provider = create_provider() + + decrypted_provider_token = "test_token" + patron_delegate_id = "test_delegate_id" + expires_at = int(utc_now().timestamp()) + 500 + + # Secrets are not set, so this will fail. + pytest.raises(InternalServerError, provider.create_ekirjasto_delegate_token, decrypted_provider_token, patron_delegate_id, expires_at) + + provider.set_secrets(controller_fixture.db.session) + delegate_token = provider.create_ekirjasto_delegate_token(decrypted_provider_token, patron_delegate_id, expires_at) + + # Is valid, otherwise throws exception. + decoded_payload = provider.decode_ekirjasto_delegate_token(delegate_token, decrypt_ekirjasto_token=False) + decoded_payload_decrypted = provider.decode_ekirjasto_delegate_token(delegate_token, decrypt_ekirjasto_token=True) + + # Double validate payload + required_options = ["token", "iss", "sub", "iat", "exp"] + payload_options = decoded_payload.keys() + for option in required_options: + assert option in payload_options + + assert decoded_payload["token"] != decrypted_provider_token + assert decoded_payload_decrypted["token"] == decrypted_provider_token + assert decoded_payload_decrypted["iss"] == provider.label() + assert decoded_payload["iss"] == provider.label() + assert decoded_payload_decrypted["sub"] == patron_delegate_id + assert decoded_payload["sub"] == patron_delegate_id + + timestamp_now = utc_now().timestamp() + assert decoded_payload_decrypted["iat"] < timestamp_now + assert decoded_payload["iat"] < timestamp_now + + @pytest.mark.parametrize("wrong_signing_secret,wrong_encrypting_secret", [ + (True, True), + (True, False), + (False, True), + ]) + def test_wrong_secret_delegate_token( + self, + wrong_signing_secret, + wrong_encrypting_secret, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + provider = create_provider() + + decrypted_provider_token = "test_token" + patron_delegate_id = "test_delegate_id" + expires_at = int(utc_now().timestamp()) + 500 + + provider.set_secrets(controller_fixture.db.session) + delegate_token = provider.create_ekirjasto_delegate_token(decrypted_provider_token, patron_delegate_id, expires_at) + + # Change secrets + if wrong_signing_secret: + provider.delegate_token_signing_secret = Fernet.generate_key().decode() + if wrong_encrypting_secret: + provider.delegate_token_encrypting_secret = Fernet.generate_key() + + # Try to decode with wrong secrets, should throw exception. + if wrong_signing_secret: + # Signing validate error due to wrong key. + pytest.raises(jwt.exceptions.InvalidTokenError, provider.decode_ekirjasto_delegate_token, delegate_token, decrypt_ekirjasto_token=True) + else: + # Signing is ok, decrypt error due to wrong key. + pytest.raises(InvalidToken, provider.decode_ekirjasto_delegate_token, delegate_token, decrypt_ekirjasto_token=True) + + # Also this method should fail. + decoded_payload = provider.validate_ekirjasto_delegate_token(delegate_token, decrypt_ekirjasto_token=True) + assert isinstance(decoded_payload, ProblemDetail), "Validation must not work with wrong secrets." + + # Set secrets back to the original secrets from database. + provider.set_secrets(controller_fixture.db.session) + + # Verify that it now works + decoded_payload = provider.decode_ekirjasto_delegate_token(delegate_token, decrypt_ekirjasto_token=True) + validated_payload = provider.validate_ekirjasto_delegate_token(delegate_token, decrypt_ekirjasto_token=True) + + assert decoded_payload == validated_payload + + def test_expired_delegate_token( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + provider = create_provider() + + decrypted_provider_token = "test_token" + patron_delegate_id = "test_delegate_id" + # Expire time 1 second in history. + expires_at = int((utc_now()-datetime.timedelta(seconds=1)).timestamp()) + + provider.set_secrets(controller_fixture.db.session) + delegate_token = provider.create_ekirjasto_delegate_token(decrypted_provider_token, patron_delegate_id, expires_at) + + # This fails with InvalidTokenError as expected + pytest.raises(jwt.exceptions.InvalidTokenError, provider.decode_ekirjasto_delegate_token, delegate_token) + + # This will not fail. + decoded_payload = provider.decode_ekirjasto_delegate_token(delegate_token, validate_expire=False) + assert decoded_payload != None + + def test_refresh_token_remote_success( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider() + user_id = "verified" + + first_token, first_token_exp = provider.mock_api.get_test_access_token_for_user(user_id) + + token = first_token + expires = first_token_exp + previous_token = first_token + previous_expires = first_token_exp + # Test that refresh works multiple times in row. Running this multiple + # times mostly ensures that the mock API works properly. + for i in range(3): + previous_token = token + previous_expires = expires + token, expires = provider.remote_refresh_token(token) + + assert isinstance(token, str) + assert isinstance(expires, int) + + assert token != first_token + assert expires >= first_token_exp + assert token != previous_token + assert expires >= previous_expires + + # Verify the refresh happened correctly in the mock API. + assert provider.mock_api.get_test_access_token_for_user(user_id)[0] == token + assert provider.mock_api._check_authentication(token) == user_id + + def test_refresh_token_remote_invalidated( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider() + user_id = "verified" + first_token, first_token_exp = provider.mock_api.get_test_access_token_for_user(user_id) + + # This works properly + token, expires = provider.remote_refresh_token(first_token) + assert isinstance(token, str) + assert isinstance(expires, int) + + # This fails, because we use old token. + token, expires = provider.remote_refresh_token(first_token) + + assert isinstance(token, ProblemDetail) + assert token.status_code == 401 + assert expires == None + + def test_refresh_token_remote_bad_status_code( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider(failure_status_code=502) + user_id = "verified" + first_token, first_token_exp = provider.mock_api.get_test_access_token_for_user(user_id) + + token, expires = provider.remote_refresh_token(first_token) + + assert isinstance(token, ProblemDetail) + assert token.status_code == 400 + assert expires == None + + def test_refresh_token_remote_bad_connection( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider(bad_connection=True) + user_id = "verified" + first_token, first_token_exp = provider.mock_api.get_test_access_token_for_user(user_id) + + pytest.raises(RemoteInitiatedServerError, provider.remote_refresh_token, first_token) + + + def test_patron_lookup_remote_success( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider() + user_id = "verified" + user = provider.mock_api.test_users[user_id] + token, expires = provider.mock_api.get_test_access_token_for_user(user_id) + + patrondata = provider.remote_patron_lookup(token) + + assert isinstance(patrondata, PatronData) + + assert patrondata.permanent_id == user['sub'] + + def test_patron_lookup_remote_invalidated( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider() + user_id = "verified" + first_token, first_token_exp = provider.mock_api.get_test_access_token_for_user(user_id) + + # Invalidate first_token + token, expires = provider.remote_refresh_token(first_token) + assert isinstance(token, str) + assert isinstance(expires, int) + + # This fails, because we use old (invalid) token. + patrondata = provider.remote_patron_lookup(first_token) + + assert isinstance(patrondata, ProblemDetail) + assert patrondata.status_code == 401 + + def test_patron_lookup_remote_status_code( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider(failure_status_code=502) + user_id = "verified" + token, _ = provider.mock_api.get_test_access_token_for_user(user_id) + + patrondata = provider.remote_patron_lookup(token) + + assert isinstance(patrondata, ProblemDetail) + assert patrondata.status_code == 400 + + def test_patron_lookup_remote_connection( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + ): + provider = create_provider(bad_connection=True) + user_id = "verified" + token, _ = provider.mock_api.get_test_access_token_for_user(user_id) + + pytest.raises(RemoteInitiatedServerError, provider.remote_patron_lookup, token) + + def test_update_patron_from_remote( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + default_library = controller_fixture.db.default_library() + provider = create_provider(library_id=default_library.id) + user_id = "verified" + + # Update patrons role on remote API. + first_role = "first_user_role" + provider.mock_api.test_users[user_id]['role'] = first_role + + token, expires = provider.mock_api.get_test_access_token_for_user(user_id) + + # Create new patron, because we don't have one in database yet. + patron, is_new = provider.ekirjasto_authenticate( + controller_fixture.db.session, token + ) + + assert is_new == True + assert isinstance(patron, Patron) + assert patron.external_type == first_role + + # Update patrons role on remote API. + new_role = "new_user_role" + provider.mock_api.test_users[user_id]['role'] = new_role + + # Now we get updated patron with new role. + updated_patron = provider.authenticate_and_update_patron(controller_fixture.db.session, token) + + assert isinstance(updated_patron, Patron) + assert updated_patron.external_type == new_role + assert updated_patron.id == patron.id + + def test_authenticated_patron_delegate_token_expired( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + default_library = controller_fixture.db.default_library() + provider = create_provider(library_id=default_library.id) + user_id = "verified" + + decrypted_provider_token = "test_token" + patron_delegate_id = "test_delegate_id" + # Expire time 1 second in history. + expires_at = int((utc_now()-datetime.timedelta(seconds=1)).timestamp()) + + provider.set_secrets(controller_fixture.db.session) + ekirjasto_token, _ = provider.mock_api.get_test_access_token_for_user(user_id) + delegate_token = provider.create_ekirjasto_delegate_token(ekirjasto_token, patron_delegate_id, expires_at) + assert isinstance(delegate_token, str) + + # Our delegate_token is now expired. + decoded_payload = provider.validate_ekirjasto_delegate_token(delegate_token, validate_expire=False) + patron = provider.authenticated_patron(controller_fixture.db.session, decoded_payload) + + assert patron is None + + def test_authenticated_patron_ekirjasto_token_invald( + self, + create_provider: Callable[..., MockEkirjastoAuthenticationAPI], + controller_fixture: ControllerFixture, + ): + default_library = controller_fixture.db.default_library() + provider = create_provider(library_id=default_library.id) + + user_id = "verified" + # Expire time far in future. + expires_at = int((utc_now()+datetime.timedelta(seconds=600)).timestamp()) + + provider.set_secrets(controller_fixture.db.session) + + # Get valid ekirjasto_token. + ekirjasto_token, _ = provider.mock_api.get_test_access_token_for_user(user_id) + + # Create new patron, because we don't have one in database yet. + patron, is_new = provider.ekirjasto_authenticate( + controller_fixture.db.session, ekirjasto_token + ) + assert is_new == True + assert isinstance(patron, Patron) + patron_delegate_id = provider.get_patron_delegate_id(controller_fixture.db.session, patron) + + # Delegate token with the ekirjasto token. + delegate_token = provider.create_ekirjasto_delegate_token(ekirjasto_token, patron_delegate_id, expires_at) + assert isinstance(delegate_token, str) + + # Invalidate ekirjasto_token. + valid_ekirjasto_token = provider.mock_api._refresh_token_for_user_id(user_id)['token'] + + # Our delegate_token is not expired. + decoded_payload = provider.validate_ekirjasto_delegate_token(delegate_token, validate_expire=True) + assert isinstance(decoded_payload, dict) + + # Patron is synced, because it was just created. + assert PatronUtility.needs_external_sync(patron) == False + + # This works because delegate token is valid, even though ekirjsto token is not. + patron = provider.authenticated_patron(controller_fixture.db.session, decoded_payload) + assert isinstance(patron, Patron) + + # Change that patron is synced far in history and needs new remote sync. + patron.last_external_sync = utc_now() - Patron.MAX_SYNC_TIME - datetime.timedelta(minutes=1) + assert PatronUtility.needs_external_sync(patron) == True + + # This fails because remote sync is needed, but ekirjasto token is invalid. + result = provider.authenticated_patron(controller_fixture.db.session, decoded_payload) + assert isinstance(result, ProblemDetail) + assert result.status_code == 401 + + # Delegate token with the valid ekirjasto token. + delegate_token = provider.create_ekirjasto_delegate_token(valid_ekirjasto_token, patron_delegate_id, expires_at) + assert isinstance(delegate_token, str) + decoded_payload = provider.validate_ekirjasto_delegate_token(delegate_token, validate_expire=True) + assert isinstance(decoded_payload, dict) + + # Now we can sync the patron. + patron = provider.authenticated_patron(controller_fixture.db.session, decoded_payload) + assert isinstance(patron, Patron) + assert PatronUtility.needs_external_sync(patron) == False + \ No newline at end of file diff --git a/tests/finland/test_loan_excel_export.py b/tests/finland/test_loan_excel_export.py new file mode 100644 index 000000000..731d9cea1 --- /dev/null +++ b/tests/finland/test_loan_excel_export.py @@ -0,0 +1,86 @@ +import io +import unittest +from unittest.mock import MagicMock, patch +from openpyxl import load_workbook +from api.local_analytics_exporter import LocalAnalyticsExporter + +LOAN_DB_FIXTURE = [ + { + "identifier": "https://standardebooks.org/ebooks/charles-dickens/bleak-house", + "identifier_type": "URI", + "sort_title": "Bleak House", + "sort_author": "Dickens, Charles", + "publisher": "Standard Ebooks", + "language": "eng", + "genres": None, + "contributors": ["Dickens, Charles (Author)"], + "location": None, + "library_name": "Open Access Library", + "medium": "Book", + "count": 5, + }, + { + "identifier": "https://standardebooks.org/ebooks/george-eliot/silas-marner", + "identifier_type": "URI", + "sort_title": "Silas Marner", + "sort_author": "Eliot, George", + "publisher": "Standard Ebooks", + "language": "eng", + "genres": ["Literary Fiction"], + "contributors": ["Eliot, George (Author)", "Fake contributor (Author)"], + "location": None, + "library_name": "Open Access Library", + "medium": "Book", + "count": 8, + }, +] + + +class TestExcelExport(unittest.TestCase): + def test_export_excel(self): + # Mock the database connection and its execute method + mock_db = MagicMock() + mock_db.execute.return_value = LOAN_DB_FIXTURE + + exporter = LocalAnalyticsExporter() + + # Patch the database connection and run the export method + with patch.object(exporter, "analytics_query_loan_statistics") as mock_query: + mock_query.return_value = "Mock SQL query" + stream = exporter.export_excel(mock_db, "2023-01-01", "2023-12-31") + + # Load the stream into an openpyxl workbook + bytes_in = io.BytesIO(stream) + wb = load_workbook(bytes_in) + sheet = wb.active + + # Validate the header row content + expected_headers = ( + "Tekijä (aakkostus)", + "Nimeke", + "Tunniste", + "Tunnisteen tyyppi", + "Kirjasto", + "Sijainti", + "Formaatti", + "Kategoria(t)", + "Kieli", + "Kustantaja/Julkaisija", + "Kaikki lainat", + "Tekijä 1", + "Tekijä 2", + ) + header_row = list(sheet.iter_rows(min_row=1, max_row=1, values_only=True))[0] + self.assertEqual(header_row, expected_headers, "Header row mismatch") + + # Validate the number of rows in the worksheet + expected_row_count = len(LOAN_DB_FIXTURE) + actual_row_count = sheet.max_row - 1 # Subtracting the header row + self.assertEqual(actual_row_count, expected_row_count, "Row count mismatch") + + # Validate database query interaction + mock_db.execute.assert_called_once_with("Mock SQL query") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/finland/test_opensearch_analytics_provider.py b/tests/finland/test_opensearch_analytics_provider.py new file mode 100644 index 000000000..8ba89c59b --- /dev/null +++ b/tests/finland/test_opensearch_analytics_provider.py @@ -0,0 +1,20 @@ +from core.analytics import Analytics +from core.local_analytics_provider import LocalAnalyticsProvider +from api.opensearch_analytics_provider import OpenSearchAnalyticsProvider + +# The test set is based on core/test_analytics.py + +MOCK_PROTOCOL = "../core/mock_analytics_provider" + + +class TestOpenSearchAnalytics: + def test_init_opensource_analytics(self): + analytics = Analytics( + opensearch_analytics_enabled=True, + opensearch_analytics_index_prefix="circulation-events", + opensearch_analytics_url="http://localhost:9200", + ) + + assert len(analytics.providers) == 2 + assert type(analytics.providers[0]) == LocalAnalyticsProvider + assert type(analytics.providers[1]) == OpenSearchAnalyticsProvider diff --git a/tests/finland/test_opensearch_analytics_search.py b/tests/finland/test_opensearch_analytics_search.py new file mode 100644 index 000000000..7384fa415 --- /dev/null +++ b/tests/finland/test_opensearch_analytics_search.py @@ -0,0 +1,202 @@ +import flask +from flask import Flask + +import unittest +from unittest.mock import patch + +from api.opensearch_analytics_search import OpenSearchAnalyticsSearch + + +class MockLibrary: + def short_name(): + return "testlib" + + +class TestOpenSearchAnalyticsSearch(unittest.TestCase): + # Patch environmental variables + @patch.dict( + "os.environ", + { + "PALACE_OPENSEARCH_ANALYTICS_URL": "http://localhost:9300", + "PALACE_OPENSEARCH_ANALYTICS_INDEX_PREFIX": "circulation-events", + }, + ) + # Patch OpenSearch client in OpeSearchAnalyticsSearch + @patch("api.opensearch_analytics_search.OpenSearch") + def test_events(self, mock_opensearch): + mock_response = { + "aggregations": { + "field_1": {"buckets": [{"key": "value1"}]}, + "field_2": {"buckets": [{"key": "value2"}]}, + } + } + # Mock the response for the search method of the mocked OpenSearch client + mock_opensearch.return_value.search.return_value = mock_response + + # Create a Flask app context for the test + app = Flask(__name__) + with app.test_request_context(): + # Set up the Flask request context variables + setattr(flask.request, "library", MockLibrary()) + + search_instance = OpenSearchAnalyticsSearch() + + test_params = { + "from": "2023-01-01", + "to": "2023-01-31", + } + result = search_instance.events(params=test_params) + + expected_result = { + "data": {"field_1": [{"key": "value1"}], "field_2": [{"key": "value2"}]} + } + self.assertEqual(result, expected_result) + + # Patch environmental variables + @patch.dict( + "os.environ", + { + "PALACE_OPENSEARCH_ANALYTICS_URL": "http://localhost:9300", + "PALACE_OPENSEARCH_ANALYTICS_INDEX_PREFIX": "circulation-events", + }, + ) + # Patch OpenSearch client in OpeSearchAnalyticsSearch + @patch("api.opensearch_analytics_search.OpenSearch") + def test_histogram(self, mock_opensearch): + # Prepare mock data for OpenSearch response + mock_response = { + "aggregations": { + "events_per_interval": { + "buckets": [ + { + "key": 1696107600000, + "key_as_string": "2023-10-01T00:00:00.000+03:00", + "type": { + "buckets": [ + { + "doc_count": 24, + "key": "circulation_manager_check_out", + }, + { + "doc_count": 17, + "key": "circulation_manager_check_in", + }, + ] + }, + } + ] + } + } + } + # Create a MagicMock for the search method of the mocked OpenSearch client + mock_opensearch.return_value.search.return_value = mock_response + # Create a Flask app context for the test + app = Flask(__name__) + with app.test_request_context(): + # Set up the Flask request context variables + setattr(flask.request, "library", MockLibrary()) + + # Initialize the OpenSearchAnalyticsSearch class + search_instance = OpenSearchAnalyticsSearch() + + # Call the events method with the test parameters + test_params = { + "from": "2023-01-01", + "to": "2023-01-31", + "interval": "month", + } + result = search_instance.events_histogram(params=test_params) + + expected_result = { + "data": { + "events_per_interval": { + "buckets": [ + { + "key": 1696107600000, + "key_as_string": "2023-10-01T00:00:00.000+03:00", + "type": { + "buckets": [ + { + "doc_count": 24, + "key": "circulation_manager_check_out", + }, + { + "doc_count": 17, + "key": "circulation_manager_check_in", + }, + ] + }, + } + ] + } + } + } + self.assertEqual(result, expected_result) + + # Patch environmental variables + @patch.dict( + "os.environ", + { + "PALACE_OPENSEARCH_ANALYTICS_URL": "http://localhost:9300", + "PALACE_OPENSEARCH_ANALYTICS_INDEX_PREFIX": "circulation-events", + }, + ) + # Patch OpenSearch client in OpeSearchAnalyticsSearch + @patch("api.opensearch_analytics_search.OpenSearch") + def test_facets(self, mock_opensearch): + # Prepare mock data for OpenSearch response + mock_response = { + "aggregations": { + "audience": { + "buckets": [ + {"doc_count": 72, "key": "Adult"}, + {"doc_count": 4, "key": "Young Adult"}, + ] + }, + "collection": { + "buckets": [ + { + "doc_count": 76, + "key": "Library Simplified Content Server Crawlable", + } + ] + }, + } + } + # Create a MagicMock for the search method of the mocked OpenSearch client + mock_opensearch.return_value.search.return_value = mock_response + # Create a Flask app context for the test + app = Flask(__name__) + with app.test_request_context(): + # Set up the Flask request context variables + setattr(flask.request, "library", MockLibrary()) + + # Initialize the OpenSearchAnalyticsSearch class + search_instance = OpenSearchAnalyticsSearch() + + # Call the events method with the test parameters + result = search_instance.get_facets() + + expected_result = { + "facets": { + "audience": { + "buckets": [ + {"doc_count": 72, "key": "Adult"}, + {"doc_count": 4, "key": "Young Adult"}, + ] + }, + "collection": { + "buckets": [ + { + "doc_count": 76, + "key": "Library Simplified Content Server Crawlable", + } + ] + }, + } + } + self.assertEqual(result, expected_result) + + +if __name__ == "__main__": + unittest.main()