Merge branch 'alan-turing-institute:develop' into inbound-docs

alan-turing-institute · Aug 3, 2023 · 6457e49 · 6457e49
2 parents 03821ee + b877b1d
commit 6457e49
Show file tree

Hide file tree

Showing 16 changed files with 48 additions and 22 deletions.
diff --git a/.github/scripts/update_rstudio.py b/.github/scripts/update_rstudio.py
@@ -7,7 +7,7 @@
 root = html.fromstring(remote_page.content)
 short_links = [link for link in root.xpath("//a[contains(text(), '.deb')]/@href") if "debian" not in link]
 
-for ubuntu_version in ["bionic", "jammy"]:
+for ubuntu_version in ["focal", "jammy"]:
     short_link = [link for link in short_links if ubuntu_version in link][0]
     remote_content = requests.get(short_link, allow_redirects=True)
     sha256 = hashlib.sha256(remote_content.content).hexdigest()

diff --git a/.github/workflows/build_allow_lists.yaml b/.github/workflows/build_allow_lists.yaml
@@ -7,6 +7,7 @@ on:  # yamllint disable-line rule:truthy
     branches: [develop]
   schedule:
     - cron: "0 0 */6 * *"  # run every six days in order to keep the cache fresh
+  workflow_dispatch:  # allow this workflow to be manually triggered
 
 # checkout needs 'contents:read'
 # pull request needs 'pull-requests:write' and 'contents:write'

diff --git a/.github/workflows/update_package_versions.yaml b/.github/workflows/update_package_versions.yaml
@@ -7,6 +7,7 @@ on:  # yamllint disable-line rule:truthy
     branches: [develop]
   schedule:
     - cron: "0 0 */7 * *"  # run once per week
+  workflow_dispatch:  # allow this workflow to be manually triggered
 
 # checkout needs 'contents:read'
 # pull request needs 'pull-requests:write' and 'contents:write'
@@ -19,10 +20,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
 
@@ -54,7 +55,7 @@ jobs:
       - name: Create pull request
         if: ${{ ! env.ACT }}
         id: pull-request
-        uses: peter-evans/create-pull-request@dcd5fd746d53dd8de555c0f10bca6c35628be47a  # This commit corresponds to tag 3.12.0
+        uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38  # This commit corresponds to tag 5.0.2
         with:
           commit-message: Update SRD package versions
           committer: GitHub Actions <[email protected]>

diff --git a/.lychee.toml b/.lychee.toml
@@ -92,8 +92,9 @@ glob_ignore_case = false
 exclude = [
   'github\.com',  # Requires authentication (403)
   'turing\.ac\.uk',  # DDOS protection
-  'ipaddressguide\.com', # 403
-  'opensource\.org' # 403
+  'ipaddressguide\.com',  # 403
+  'opensource\.org',  # 403
+  'portal\.azure\.com',  # 403
 ]
 
 # Exclude these filesystem paths from getting checked.

diff --git a/deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1 b/deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1
@@ -32,18 +32,21 @@ function Test-PackageExistence {
         if ($Repository -eq "pypi") {
             # The best PyPI results come from the package JSON files
             $response = Invoke-RestMethod -Uri "https://pypi.org/${Repository}/${Package}/json" -MaximumRetryCount 4 -RetryIntervalSec 1 -ErrorAction Stop
+            if ($response -is [String]) { $response = $response | ConvertFrom-Json -AsHashtable }
             $versions = $response.releases | Get-Member -MemberType NoteProperty | ForEach-Object { $_.Name }
             $name = $response.info.name
         } elseif ($Repository -eq "cran") {
             # Use the RStudio package manager for CRAN packages
             $response = Invoke-RestMethod -Uri "https://packagemanager.rstudio.com/__api__/repos/${RepositoryId}/packages?name=${Package}&case_insensitive=true" -MaximumRetryCount 4 -RetryIntervalSec 1 -ErrorAction Stop
+            if ($response -is [String]) { $response = $response | ConvertFrom-Json -AsHashtable }
             $name = $response.name
             $response = Invoke-RestMethod -Uri "https://packagemanager.rstudio.com/__api__/repos/${RepositoryId}/packages/${name}" -MaximumRetryCount 4 -RetryIntervalSec 1 -ErrorAction Stop
             $versions = @($response.version) + ($response.archived | ForEach-Object { $_.version })
         } else {
             # For other repositories we use libraries.io
             # As we are rate-limited to 60 requests per minute this request can fail. If it does, we retry every few seconds for 1 minute
             $response = Invoke-RestMethod -Uri "https://libraries.io/api/${Repository}/${Package}?api_key=${ApiKey}" -MaximumRetryCount 16 -RetryIntervalSec 4 -ErrorAction Stop
+            if ($response -is [String]) { $response = $response | ConvertFrom-Json -AsHashtable }
             $versions = $response.versions | ForEach-Object { $_.number }
             $name = $response.Name
         }

diff --git a/deployment/safe_haven_management_environment/network_rules/shm-firewall-rules.json b/deployment/safe_haven_management_environment/network_rules/shm-firewall-rules.json
@@ -192,8 +192,15 @@
                         "targetFqdns": [
                             "*-sb.servicebus.windows.net",
                             "passwordreset.microsoftonline.com",
+                            "ssprdedicatedsbprodeus2-1.servicebus.windows.net",
+                            "ssprdedicatedsbprodfra-1.servicebus.windows.net",
+                            "ssprdedicatedsbprodncu-2.servicebus.windows.net",
                             "ssprdedicatedsbprodncu.servicebus.windows.net",
-                            "ssprdedicatedsbprodscu.servicebus.windows.net"
+                            "ssprdedicatedsbprodneu.servicebus.windows.net",
+                            "ssprdedicatedsbprodscu-2.servicebus.windows.net",
+                            "ssprdedicatedsbprodscu.servicebus.windows.net",
+                            "ssprdedicatedsbprodsea-1.servicebus.windows.net",
+                            "ssprdedicatedsbprodweu.servicebus.windows.net"
                         ],
                         "sourceAddresses": [
                             "{{network.vnet.subnets.identity.cidr}}"

diff --git a/...oyment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml b/...oyment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml
@@ -122,7 +122,7 @@ write_files:
   - path: "/opt/build/rstudio.debinfo"
     permissions: "0400"
     content: |
-      {{deb-rstudio-bionic.version}}
+      {{deb-rstudio-focal.version}}
 
   - path: "/opt/monitoring/analyse_build.py"
     permissions: "0755"

diff --git a/...oyment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml b/...oyment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml
@@ -122,7 +122,7 @@ write_files:
   - path: "/opt/build/rstudio.debinfo"
     permissions: "0400"
     content: |
-      {{deb-rstudio-bionic.version}}
+      {{deb-rstudio-focal.version}}
 
   - path: "/opt/monitoring/analyse_build.py"
     permissions: "0755"

diff --git a/deployment/secure_research_desktop/packages/deb-azuredatastudio.version b/deployment/secure_research_desktop/packages/deb-azuredatastudio.version
@@ -1,4 +1,4 @@
-hash: f413c7e1315516dc37c2a7d0c31cece0663556266dcb3a81aa9c402620a5c236
-version: 1.44.1
+hash: 5c037e28402b05e32b8371428544d2ffb0c1e7ffbed59421e64f067bbf883a76
+version: 1.45.0
 debfile: azuredatastudio-linux-|VERSION|.deb
-remote: https://sqlopsbuilds.azureedge.net/stable/8f53a316fa00a98264f1ab119641cd540b5af25c/|DEBFILE|
+remote: https://sqlopsbuilds.azureedge.net/stable/8eff468100f02157f2219335f7a566d14451ca49/|DEBFILE|
diff --git a/deployment/secure_research_desktop/packages/deb-rstudio-bionic.version b/deployment/secure_research_desktop/packages/deb-rstudio-bionic.version
diff --git a/deployment/secure_research_desktop/packages/deb-rstudio-focal.version b/deployment/secure_research_desktop/packages/deb-rstudio-focal.version
@@ -0,0 +1,4 @@
+hash: 49e24a6956f9f12ffeded493f571cd39f3e6c89411fc60d3bb228661793320da
+version: 2023.06.1-524
+debfile: rstudio-|VERSION|-amd64.deb
+remote: https://download1.rstudio.org/electron/focal/amd64/|DEBFILE|
diff --git a/deployment/secure_research_desktop/packages/deb-rstudio-jammy.version b/deployment/secure_research_desktop/packages/deb-rstudio-jammy.version
@@ -1,4 +1,4 @@
-hash: 464038393a380321afb96793f8775852ce75f29340bd5d49a0c7ccbe77e411c0
-version: 2023.03.1-446
+hash: c030ec8338f1c76b3ae27997ec4411a0af43b2367dedb3d48e95c319b5425698
+version: 2023.06.1-524
 debfile: rstudio-|VERSION|-amd64.deb
 remote: https://download1.rstudio.org/electron/jammy/amd64/|DEBFILE|
diff --git a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1
@@ -29,7 +29,7 @@ $allowedFqdns = @($firewallRules.applicationRuleCollections | ForEach-Object { $
                 @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name })
 # List all unique FQDNs
 $allowedFqdns = $allowedFqdns |
-                Where-Object { $_ -notlike "g*.servicebus.windows.net" } | # Remove AzureADConnect password reset endpoints
+                Where-Object { $_ -notlike "*-sb.servicebus.windows.net" } | # Remove AzureADConnect password reset endpoints
                 Where-Object { $_ -notlike "pksproddatastore*.blob.core.windows.net" } | # Remove AzureAD operations endpoints
                 Sort-Object -Unique
 Add-LogMessage -Level Info "Restricted networks will be allowed to run DNS lookup on the following $($allowedFqdns.Count) FQDNs:"
@@ -81,7 +81,8 @@ Add-LogMessage -Level Info "Looking for SRD with IP address '$vmIpAddress'..."
 if (-not $vmIpAddress) {
     Add-LogMessage -Level Fatal "No SRD found with IP address '$vmIpAddress'. Cannot run test to confirm external DNS resolution."
 } else {
-    $vmName = @(Get-AzNetworkInterface | Where-Object { $_.IpConfigurations.PrivateIpAddress -eq $vmIpAddress } | ForEach-Object { $_.VirtualMachine.Id.Split("/")[-1] })[0]
+    # Match on IP address within approriate SRE resource group
+    $vmName = @(Get-AzNetworkInterface -ResourceGroupName $config.sre.srd.rg | Where-Object { $_.IpConfigurations.PrivateIpAddress -eq $vmIpAddress } | ForEach-Object { $_.VirtualMachine.Id.Split("/")[-1] })[0]
     Add-LogMessage -Level Info "Testing external DNS resolution on VM '$vmName'..."
     $params = @{
         SHM_DOMAIN_FQDN   = $config.shm.domain.fqdn

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -14,7 +14,7 @@ babel==2.12.1
     #   sphinx
 beautifulsoup4==4.12.2
     # via pydata-sphinx-theme
-certifi==2023.5.7
+certifi==2023.7.22
     # via requests
 charset-normalizer==3.1.0
     # via requests

diff --git a/environment_configs/package_lists/allowlist-full-python-pypi-tier3.list b/environment_configs/package_lists/allowlist-full-python-pypi-tier3.list
@@ -1,5 +1,4 @@
 absl-py
-aenum
 aero-calc
 aesara
 affine
@@ -14,6 +13,7 @@ alabaster
 altair
 annotated-types
 annoy
+ansi2html
 ansimarkup
 anyio
 apispec
@@ -212,17 +212,22 @@ graphlib-backport
 graphviz
 grpcio
 gym
+h11
+h2
 h5netcdf
 h5py
 HeapDict
 hijri-converter
 holidays
 hotelling
+hpack
 html5lib
 htmlmin
+httpcore
 httplib2
 httpstan
 huggingface-hub
+hyperframe
 hyperlink
 hyperopt
 idna
@@ -538,6 +543,7 @@ ruamel.yaml
 s3fs
 s3transfer
 sacremoses
+safetensors
 safety
 scandir
 scikit-image

diff --git a/environment_configs/package_lists/allowlist-full-r-cran-tier3.list b/environment_configs/package_lists/allowlist-full-r-cran-tier3.list
@@ -213,6 +213,7 @@ httpuv
 httr
 httr2
 hunspell
+ICS
 ids
 igraph
 infer
@@ -295,6 +296,7 @@ mgcv
 mime
 miniUI
 minqa
+mitools
 mlapi
 mlbench
 MLmetrics
@@ -335,6 +337,7 @@ patchwork
 pbdMPI
 pbdZMQ
 pbkrtest
+pcaPP
 permute
 phosphoricons
 pillar
@@ -440,12 +443,14 @@ RMySQL
 RNetCDF
 rngtools
 robustbase
+robustX
 ROCR
 roxygen2
 rpart
 RPostgres
 RPostgreSQL
 rprojroot
+rrcov
 rsample
 rsconnect
 rsparse
@@ -503,6 +508,7 @@ stopwords
 stringi
 stringr
 styler
+survey
 survival
 svglite
 svgPanZoom