Merge pull request #1519 from alan-turing-institute/develop

Update release 4.0.4

.github/pull_request_template.md

@@ -8,10 +8,10 @@ Replace the empty checkboxes [ ] below with checked ones [x] accordingly.
 -->
 
 - [ ] You have given your pull request a meaningful title (_e.g._ `Enable foobar integration` rather than `515 foobar`).
-- [ ] You are targeting the **develop branch**.
-- [ ] Your branch is up-to-date with the **develop branch** (you probably started your branch from `develop` but it may have changed since then).
-- [ ] If-and-only-if your changes are not yet ready to merge, you have marked this pull request as a **draft** pull request and added '[WIP]' to the title.
-- [ ] If-and-only-if you have changed any Powershell code, you have run the code formatter. You can do this with `./tests/AutoFormat_Powershell.ps1 -TargetPath <path to file or directory>`.
+- [ ] You are targeting the appropriate branch. If you're not certain which one this is, it should be **`develop`**.
+- [ ] Your branch is up-to-date with the **target branch** (it probably was when you started, but it may have changed since then).
+- [ ] You have marked this pull request as a **draft** and added `'[WIP]'` to the title if needed (if you're not yet ready to merge).
+- [ ] You have formatted your code using appropriate automated tools (for example `./tests/AutoFormat_Powershell.ps1 -TargetPath <path to file or directory>` for Powershell).
 
 ### :arrow_heading_up: Summary
 

.github/scripts/update_rstudio.py

@@ -7,7 +7,7 @@
 root = html.fromstring(remote_page.content)
 short_links = [link for link in root.xpath("//a[contains(text(), '.deb')]/@href") if "debian" not in link]
 
-for ubuntu_version in ["bionic", "jammy"]:
+for ubuntu_version in ["focal", "jammy"]:
     short_link = [link for link in short_links if ubuntu_version in link][0]
     remote_content = requests.get(short_link, allow_redirects=True)
     sha256 = hashlib.sha256(remote_content.content).hexdigest()

.github/workflows/build_allow_lists.yaml

@@ -7,6 +7,7 @@ on:  # yamllint disable-line rule:truthy
     branches: [develop]
   schedule:
     - cron: "0 0 */6 * *"  # run every six days in order to keep the cache fresh
+  workflow_dispatch:  # allow this workflow to be manually triggered
 
 # checkout needs 'contents:read'
 # pull request needs 'pull-requests:write' and 'contents:write'

.github/workflows/documentation.yaml

@@ -53,5 +53,5 @@ jobs:
       - name: Link Checker
         uses: lycheeverse/[email protected]
         with:
-          args: --config='./.lychee.toml' --verbose --no-progress './docs/build/html/**/*.html'
+          args: --config='./.lychee.toml' --no-progress './docs/build/html/**/*.html'
           fail: true  # fail on broken links

.github/workflows/test_code.yaml

@@ -31,7 +31,8 @@ jobs:
       - name: Install requirements
         shell: bash
         run: npm install -g markdown-link-check
-      - name: Test Markdown for dead links
-        shell: bash
-        run: find . -name "*.md" | xargs -n 1 markdown-link-check -p -c .markdownlinkcheck.json
-        continue-on-error: true
+      - name: Link Checker
+        uses: lycheeverse/[email protected]
+        with:
+          args: --config='./.lychee.toml' --no-progress --offline '**/*.md' --exclude-path './docs'
+          fail: true  # fail on broken links

.github/workflows/update_package_versions.yaml

@@ -7,6 +7,7 @@ on:  # yamllint disable-line rule:truthy
     branches: [develop]
   schedule:
     - cron: "0 0 */7 * *"  # run once per week
+  workflow_dispatch:  # allow this workflow to be manually triggered
 
 # checkout needs 'contents:read'
 # pull request needs 'pull-requests:write' and 'contents:write'
@@ -19,10 +20,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
 
@@ -54,7 +55,7 @@ jobs:
       - name: Create pull request
         if: ${{ ! env.ACT }}
         id: pull-request
-        uses: peter-evans/create-pull-request@dcd5fd746d53dd8de555c0f10bca6c35628be47a  # This commit corresponds to tag 3.12.0
+        uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38  # This commit corresponds to tag 5.0.2
         with:
           commit-message: Update SRD package versions
           committer: GitHub Actions <[email protected]>

.gitignore

@@ -28,8 +28,9 @@ docs/*.pdf
 # ... except the tiers flowchart
 !docs/tiersflowchart.pdf
 
-# Sphinx output
-docs/_output
+# Sphinx output and venv
+docs/build
+docs/venv
 
 # Files produced during testing
 .mustache_config.json

.lychee.toml

@@ -2,7 +2,7 @@
 
 # Verbose program output
 # Accepts log level: "error", "warn", "info", "debug", "trace"
-verbose = "info"
+verbose = "error"
 
 # Don't show interactive progress bar while checking links.
 no_progress = false
@@ -92,8 +92,9 @@ glob_ignore_case = false
 exclude = [
   'github\.com',  # Requires authentication (403)
   'turing\.ac\.uk',  # DDOS protection
-  'ipaddressguide\.com', # 403
-  'opensource\.org' # 403
+  'ipaddressguide\.com',  # 403
+  'opensource\.org',  # 403
+  'portal\.azure\.com',  # 403
 ]
 
 # Exclude these filesystem paths from getting checked.

.readthedocs.yaml

@@ -12,8 +12,9 @@ build:
 sphinx:
   configuration: docs/source/conf.py
 
-# formats:
-#    - pdf
+formats:
+  - htmlzip
+  - pdf
 
 python:
   install:

CODEOWNERS

@@ -11,3 +11,4 @@
 # main and other branches that may be deployed from.
 # Note: /dir/ applies to directory and all subdirectories
 /deployment/      @martintoreilly @jemrobinson @JimMadge
+/docs/            @martintoreilly @jemrobinson @JimMadge @craddm @edwardchalstrey1

CONTRIBUTING.md

@@ -34,7 +34,7 @@ For the Data Safe Haven project, we follow a DevOps development philosophy.
 This means different things to different people, but the definition that we're using is [best summed up here](https://www.guru99.com/agile-vs-devops.html).
 
 <p align="center">
-   <img src="docs/contributing/devops_gap.png" width="80%" title="Taken from https://www.guru99.com/agile-vs-devops.html">
+   <img src="docs/source/contributing/devops_gap.png" width="80%" title="Taken from https://www.guru99.com/agile-vs-devops.html">
 </p>
 
 This is a software development method which focuses on:
@@ -87,7 +87,7 @@ In order to contribute via GitHub you'll need to set up a free account and sign
 We use the [Gitflow Workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow).
 
 <p align="center">
-   <img src="docs/contributing/gitflow.svg" width="80%" title="Taken from https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow">
+   <img src="docs/source/contributing/gitflow.svg" width="80%" title="Taken from https://www.atlassian.com/git/tutorials/comparing-workflows/gitflow-workflow">
 </p>
 
 This means that:
@@ -172,7 +172,7 @@ For example, `@KirstieJane` will send a notification to `Kirstie Whitaker` so sh
 Alternatively (and this is encouraged) you can use the issue to keep track of where you're up to with the task and add information about next steps and barriers.
 
 <p align="center">
-   <img src="docs/contributing/example-conversation-in-issue.png" width="80%" title="example-conversation-in-issue">
+   <img src="docs/source/contributing/example-conversation-in-issue.png" width="80%" title="example-conversation-in-issue">
 </p>
 
 ### Working in a private repository
@@ -226,21 +226,7 @@ The docs, including for older releases, are available [here](https://data-safe-h
 
 You should follow the same instructions as above to [make a change with a pull request](#making-a-change-with-a-pull-request) when editing the documentation.
 
-To preview your changes, you can build the docs locally.
-The documentation build dependencies are listed in [a requirements file](docs/build/requirements.txt) that can be installed using `pip`:
-
-```{shell}
-pip install -r docs/build/requirements.txt
-```
-
-Check out your branch, navigate to the `docs` folder and `make` them:
-
-```{shell}
-cd data-safe-haven/docs
-make html
-```
-
-This will add the contents to a folder called `_output` inside `docs`. Open the index html from a browser and you should be able to navigate the docs and view your changes.
+To preview your changes, you can build the docs locally. See [docs/README.md](docs/README.md).
 
 ### Who's involved in the project
 

README.md

@@ -1,4 +1,4 @@
-![Data Safe Haven cartoon by Scriberia for The Alan Turing Institute](docs/_static/scriberia_diagram.jpg)
+![Data Safe Haven cartoon by Scriberia for The Alan Turing Institute](docs/source/_static/scriberia_diagram.jpg)
 
 # :eyes: What is the Turing Data Safe Haven?
 
@@ -12,7 +12,7 @@ It was developed as part of the Alan Turing Institute's [Data Safe Havens in the
 [![Latest version](https://img.shields.io/github/v/release/alan-turing-institute/data-safe-haven?style=flat&label=Latest&color=%234B78E6)](https://github.com/alan-turing-institute/data-safe-haven/releases)
 [![Slack](https://img.shields.io/badge/Join%20us!-yellow?style=flat&logo=slack&logoColor=white&labelColor=4A154B&label=Slack)](https://join.slack.com/t/turingdatasafehaven/signup)
 ![Licence](https://img.shields.io/github/license/alan-turing-institute/data-safe-haven)
-[![Citation](https://img.shields.io/badge/citation-cite%20this%20project-informational)](https://github.com/alan-turing-institute/data-safe-haven/blob/badges/CITATION.cff)
+[![Citation](https://img.shields.io/badge/citation-cite%20this%20project-informational)](https://github.com/alan-turing-institute/data-safe-haven/blob/develop/CITATION.cff)
 
 ## :family: Community & support
 

deployment/CheckRequirements.ps1

@@ -16,7 +16,7 @@ $ModuleVersionRequired = @{
     "Az.DataProtection"                            = @("ge", "0.4.0")
     "Az.Dns"                                       = @("ge", "1.1.2")
     "Az.KeyVault"                                  = @("ge", "4.9.1")
-    "Az.Monitor"                                   = @("ge", "3.0.1")
+    "Az.Monitor"                                   = @("ge", "4.2.0")
     "Az.MonitoringSolutions"                       = @("ge", "0.1.0")
     "Az.Network"                                   = @("ge", "5.3.0")
     "Az.OperationalInsights"                       = @("ge", "3.1.0")

deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1

@@ -32,18 +32,21 @@ function Test-PackageExistence {
         if ($Repository -eq "pypi") {
             # The best PyPI results come from the package JSON files
             $response = Invoke-RestMethod -Uri "https://pypi.org/${Repository}/${Package}/json" -MaximumRetryCount 4 -RetryIntervalSec 1 -ErrorAction Stop
+            if ($response -is [String]) { $response = $response | ConvertFrom-Json -AsHashtable }
             $versions = $response.releases | Get-Member -MemberType NoteProperty | ForEach-Object { $_.Name }
             $name = $response.info.name
         } elseif ($Repository -eq "cran") {
             # Use the RStudio package manager for CRAN packages
             $response = Invoke-RestMethod -Uri "https://packagemanager.rstudio.com/__api__/repos/${RepositoryId}/packages?name=${Package}&case_insensitive=true" -MaximumRetryCount 4 -RetryIntervalSec 1 -ErrorAction Stop
+            if ($response -is [String]) { $response = $response | ConvertFrom-Json -AsHashtable }
             $name = $response.name
             $response = Invoke-RestMethod -Uri "https://packagemanager.rstudio.com/__api__/repos/${RepositoryId}/packages/${name}" -MaximumRetryCount 4 -RetryIntervalSec 1 -ErrorAction Stop
             $versions = @($response.version) + ($response.archived | ForEach-Object { $_.version })
         } else {
             # For other repositories we use libraries.io
             # As we are rate-limited to 60 requests per minute this request can fail. If it does, we retry every few seconds for 1 minute
             $response = Invoke-RestMethod -Uri "https://libraries.io/api/${Repository}/${Package}?api_key=${ApiKey}" -MaximumRetryCount 16 -RetryIntervalSec 4 -ErrorAction Stop
+            if ($response -is [String]) { $response = $response | ConvertFrom-Json -AsHashtable }
             $versions = $response.versions | ForEach-Object { $_.number }
             $name = $response.Name
         }

deployment/common/AzureMonitor.psm1

@@ -70,7 +70,7 @@ function Set-LogAnalyticsDiagnostics {
         [string]$WorkspaceId
     )
     Add-LogMessage -Level Info "Enable logging for $ResourceName to log analytics workspace"
-    $null = Set-AzDiagnosticSetting -ResourceId $ResourceId -WorkspaceId $WorkspaceId -Enabled $true
+    $null = New-AzDiagnosticSetting -Name "LogToWorkspace" -ResourceId $ResourceId -WorkspaceId $WorkspaceId
     if ($?) {
         Add-LogMessage -Level Success "Enabled logging for $ResourceName to log analytics workspace"
     } else {

deployment/safe_haven_management_environment/desired_state_configuration/dc1Artifacts/CreateUsers.ps1

@@ -26,7 +26,7 @@ Import-Csv $userFilePath | ForEach-Object {
         Path                 = "$userOuPath"
         Enabled              = $True
         AccountPassword      = (ConvertTo-SecureString $Password -AsPlainText -Force)
-        PasswordNeverExpires = $False
+        PasswordNeverExpires = $True
         Mobile               = $_.Mobile
         Email                = $_.SecondaryEmail
         Country              = "GB"