Merge pull request #1541 from alan-turing-institute/1496-add-database

Pulumi: add database support

data_safe_haven/commands/deploy.py

@@ -10,7 +10,7 @@
     validate_ip_address,
     validate_timezone,
 )
-from data_safe_haven.utility import SoftwarePackageCategory
+from data_safe_haven.utility import DatabaseSystem, SoftwarePackageCategory
 
 from .deploy_shm import deploy_shm
 from .deploy_sre import deploy_sre
@@ -109,16 +109,12 @@ def sre(
             callback=lambda vms: [validate_ip_address(vm) for vm in vms],
         ),
     ] = None,
-    workspace_skus: Annotated[
-        Optional[list[str]],  # noqa: UP007
+    databases: Annotated[
+        Optional[list[DatabaseSystem]],  # noqa: UP007
         typer.Option(
-            "--workspace-sku",
-            "-w",
-            help=(
-                "A virtual machine SKU to make available to your users as a workspace."
-                " [*may be specified several times*]"
-            ),
-            callback=lambda ips: [validate_azure_vm_sku(ip) for ip in ips],
+            "--database",
+            "-b",
+            help="Make a database of this system available to users of this SRE.",
         ),
     ] = None,
     software_packages: Annotated[
@@ -138,14 +134,27 @@ def sre(
             callback=lambda ips: [validate_ip_address(ip) for ip in ips],
         ),
     ] = None,
+    workspace_skus: Annotated[
+        Optional[list[str]],  # noqa: UP007
+        typer.Option(
+            "--workspace-sku",
+            "-w",
+            help=(
+                "A virtual machine SKU to make available to your users as a workspace."
+                " [*may be specified several times*]"
+            ),
+            callback=lambda ips: [validate_azure_vm_sku(ip) for ip in ips],
+        ),
+    ] = None,
 ) -> None:
     """Deploy a Secure Research Environment"""
     deploy_sre(
         name,
         allow_copy=allow_copy,
         allow_paste=allow_paste,
         data_provider_ip_addresses=data_provider_ip_addresses,
-        workspace_skus=workspace_skus,
+        databases=databases,
         software_packages=software_packages,
         user_ip_addresses=user_ip_addresses,
+        workspace_skus=workspace_skus,
     )

data_safe_haven/commands/deploy_sre.py

@@ -7,7 +7,7 @@
 from data_safe_haven.functions import alphanumeric, password
 from data_safe_haven.provisioning import SREProvisioningManager
 from data_safe_haven.pulumi import PulumiSHMStack, PulumiSREStack
-from data_safe_haven.utility import SoftwarePackageCategory
+from data_safe_haven.utility import DatabaseSystem, SoftwarePackageCategory
 
 
 def deploy_sre(
@@ -16,6 +16,7 @@ def deploy_sre(
     allow_copy: bool | None = None,
     allow_paste: bool | None = None,
     data_provider_ip_addresses: list[str] | None = None,
+    databases: list[DatabaseSystem] | None = None,
     workspace_skus: list[str] | None = None,
     software_packages: SoftwarePackageCategory | None = None,
     user_ip_addresses: list[str] | None = None,
@@ -32,6 +33,7 @@ def deploy_sre(
             allow_copy=allow_copy,
             allow_paste=allow_paste,
             data_provider_ip_addresses=data_provider_ip_addresses,
+            databases=databases,
             workspace_skus=workspace_skus,
             software_packages=software_packages,
             user_ip_addresses=user_ip_addresses,
@@ -133,6 +135,7 @@ def deploy_sre(
         )
         # Add necessary secrets
         stack.copy_secret("password-domain-ldap-searcher", shm_stack)
+        stack.add_secret("password-database-service-admin", password(20), replace=False)
         stack.add_secret("password-gitea-database-admin", password(20), replace=False)
         stack.add_secret(
             "password-hedgedoc-database-admin", password(20), replace=False

data_safe_haven/config/config.py

@@ -30,7 +30,11 @@
     validate_timezone,
     validate_type,
 )
-from data_safe_haven.utility import LoggingSingleton, SoftwarePackageCategory
+from data_safe_haven.utility import (
+    DatabaseSystem,
+    LoggingSingleton,
+    SoftwarePackageCategory,
+)
 
 from .backend_settings import BackendSettings
 
@@ -214,6 +218,7 @@ def update(
                 f"[bold]Pasting text into the SRE[/] will be [green]{'allowed' if self.allow_paste else 'forbidden'}[/]."
             )
 
+    databases: list[DatabaseSystem] = field(default_factory=list)
     data_provider_ip_addresses: list[str] = field(default_factory=list)
     index: int = 0
     remote_desktop: ConfigSubsectionRemoteDesktopOpts = field(
@@ -227,6 +232,7 @@ def update(
         "data_provider_ip_addresses": partial(
             validate_list, validator=validate_ip_address
         ),
+        "databases": lambda pkg: isinstance(pkg, DatabaseSystem),
         "index": lambda idx: isinstance(idx, int) and idx >= 0,
         "remote_desktop": lambda dsktop: dsktop.validate(),
         "workspace_skus": partial(validate_list, validator=validate_azure_vm_sku),
@@ -242,6 +248,7 @@ def update(
         allow_copy: bool | None = None,
         allow_paste: bool | None = None,
         data_provider_ip_addresses: list[str] | None = None,
+        databases: list[DatabaseSystem] | None = None,
         workspace_skus: list[str] | None = None,
         software_packages: SoftwarePackageCategory | None = None,
         user_ip_addresses: list[str] | None = None,
@@ -251,6 +258,7 @@ def update(
         Args:
             allow_copy: Allow/deny copying text out of the SRE
             allow_paste: Allow/deny pasting text into the SRE
+            databases: List of database systems to deploy
             data_provider_ip_addresses: List of IP addresses belonging to data providers
             workspace_skus: List of VM SKUs for workspaces
             software_packages: Whether to allow packages from external repositories
@@ -263,6 +271,14 @@ def update(
         logger.info(
             f"[bold]IP addresses used by data providers[/] will be [green]{self.data_provider_ip_addresses}[/]."
         )
+        # Set which databases to deploy
+        if databases:
+            self.databases = sorted(set(databases))
+            if len(self.databases) != len(databases):
+                logger.warning("Discarding duplicate values for 'database'.")
+        logger.info(
+            f"[bold]Databases available to users[/] will be [green]{[database.value for database in self.databases]}[/]."
+        )
         # Pass allow_copy and allow_paste to remote desktop
         self.remote_desktop.update(allow_copy=allow_copy, allow_paste=allow_paste)
         # Set research desktop SKUs
@@ -273,7 +289,7 @@ def update(
         if software_packages:
             self.software_packages = software_packages
         logger.info(
-            f"[bold]Software packages[/] from [green]{self.software_packages}[/] sources will be installable."
+            f"[bold]Software packages[/] from [green]{self.software_packages.value}[/] sources will be installable."
         )
         # Set user IP addresses
         if user_ip_addresses:

data_safe_haven/pulumi/common/enums.py

@@ -17,7 +17,8 @@ class NetworkingPriorities(int, Enum):
     INTERNAL_SHM_UPDATE_SERVERS = 1400
     INTERNAL_SRE_PRIVATE_DATA = 1500
     INTERNAL_SRE_REMOTE_DESKTOP = 1600
-    INTERNAL_SRE_USER_SERVICES = 1700
+    INTERNAL_SRE_USER_SERVICES_CONTAINERS = 1700
+    INTERNAL_SRE_USER_SERVICES_DATABASES = 1800
     INTERNAL_DSH_VIRTUAL_NETWORK = 1999
     # Authorised external IPs: 2000-2999
     AUTHORISED_EXTERNAL_ADMIN_IPS = 2000

data_safe_haven/pulumi/components/sre_data.py

@@ -66,8 +66,8 @@ def __init__(
         self.networking_resource_group_name = Output.from_input(
             networking_resource_group
         ).apply(get_name_from_rg)
-        self.password_workspace_admin = self.get_secret(
-            pulumi_opts, "password-workspace-admin"
+        self.password_database_service_admin = self.get_secret(
+            pulumi_opts, "password-database-service-admin"
         )
         self.password_gitea_database_admin = self.get_secret(
             pulumi_opts, "password-gitea-database-admin"
@@ -79,6 +79,9 @@ def __init__(
         self.password_user_database_admin = self.get_secret(
             pulumi_opts, "password-user-database-admin"
         )
+        self.password_workspace_admin = self.get_secret(
+            pulumi_opts, "password-workspace-admin"
+        )
         self.private_dns_zone_base_id = self.get_secret(
             pulumi_opts, "shm-networking-private_dns_zone_base_id"
         )
@@ -212,10 +215,12 @@ def __init__(
         )
 
         # Define SSL certificate for this FQDN
-        certificate = SSLCertificate(
-            f"{self._name}_ssl_certificate",
+        sre_fqdn_certificate = SSLCertificate(
+            f"{self._name}_kvc_https_certificate",
             SSLCertificateProps(
-                certificate_secret_name="ssl-certificate-sre-remote-desktop",
+                certificate_secret_name=Output.from_input(props.sre_fqdn).apply(
+                    lambda s: replace_separators(s, "-")
+                ),
                 domain_name=props.sre_fqdn,
                 admin_email_address=props.admin_email_address,
                 key_vault_name=key_vault.name,
@@ -233,12 +238,12 @@ def __init__(
 
         # Deploy key vault secrets
         keyvault.Secret(
-            f"{self._name}_kvs_password_workspace_admin",
+            f"{self._name}_kvs_password_database_service_admin",
             properties=keyvault.SecretPropertiesArgs(
-                value=props.password_workspace_admin
+                value=props.password_database_service_admin
             ),
             resource_group_name=resource_group.name,
-            secret_name="password-workspace-admin",
+            secret_name="password-database-service-admin",
             vault_name=key_vault.name,
             opts=ResourceOptions.merge(child_opts, ResourceOptions(parent=key_vault)),
         )
@@ -280,6 +285,16 @@ def __init__(
             vault_name=key_vault.name,
             opts=ResourceOptions.merge(child_opts, ResourceOptions(parent=key_vault)),
         )
+        keyvault.Secret(
+            f"{self._name}_kvs_password_workspace_admin",
+            properties=keyvault.SecretPropertiesArgs(
+                value=props.password_workspace_admin
+            ),
+            resource_group_name=resource_group.name,
+            secret_name="password-workspace-admin",
+            vault_name=key_vault.name,
+            opts=ResourceOptions(parent=key_vault),
+        )
 
         # Deploy state storage account
         storage_account_state = storage.StorageAccount(
@@ -575,16 +590,18 @@ def __init__(
         )
 
         # Register outputs
+        self.sre_fqdn_certificate_secret_id = sre_fqdn_certificate.secret_id
         self.storage_account_userdata_name = storage_account_userdata.name
         self.storage_account_securedata_name = storage_account_securedata.name
         self.storage_account_state_key = Output.secret(
             storage_account_state_keys.keys[0].value
         )
         self.storage_account_state_name = storage_account_state.name
-        self.certificate_secret_id = certificate.secret_id
         self.managed_identity = identity_key_vault_reader
         self.password_nexus_admin = Output.secret(props.password_nexus_admin)
-        self.password_workspace_admin = Output.secret(props.password_workspace_admin)
+        self.password_database_service_admin = Output.secret(
+            props.password_database_service_admin
+        )
         self.password_gitea_database_admin = Output.secret(
             props.password_gitea_database_admin
         )
@@ -594,4 +611,5 @@ def __init__(
         self.password_user_database_admin = Output.secret(
             props.password_user_database_admin
         )
+        self.password_workspace_admin = Output.secret(props.password_workspace_admin)
         self.resource_group_name = resource_group.name