From fc6947adc11243ab2c29e601306155bdd451eb34 Mon Sep 17 00:00:00 2001
From: Paul Taylor <178183+trxcllnt@users.noreply.github.com>
Date: Thu, 19 Sep 2024 11:00:50 -0700
Subject: [PATCH 1/2] Use `gh` `nv-gha-aws` CLI extension to generate
 credentials (#392)

* install
[`nv-gha-aws`](https://github.com/nv-gha-runners/gh-nv-gha-aws) CLI
extension
* use `nv-gha-aws` to generate credentials if `AWS_ROLE_ARN` is set
---
 .devcontainer/rapids.Dockerfile               |   3 +-
 .../actions/build-and-test-feature/action.yml |   2 +
 .github/workflows/build-and-test-feature.yml  |   1 +
 features/src/utils/devcontainer-feature.json  |   2 +-
 features/src/utils/install.sh                 |  43 +++++-
 .../opt/devcontainer/bin/creds/s3/generate.sh |  23 +++
 .../devcontainer/bin/creds/s3/gh/generate.sh  |  68 +++++++++
 .../opt/devcontainer/bin/creds/s3/init.sh     |  58 ++++++++
 .../opt/devcontainer/bin/creds/s3/persist.sh  | 109 ++++++++++++++
 .../devcontainer/bin/creds/s3/propagate.sh    |  42 ++++++
 .../opt/devcontainer/bin/creds/s3/schedule.sh |  64 +++++++++
 .../opt/devcontainer/bin/creds/s3/test.sh     |  44 ++++++
 .../bin/creds/s3/vault/generate.sh            | 135 ++++++++++++++++++
 .../devcontainer/bin/creds/s3/vault/github.sh |  39 +++++
 .../devcontainer/bin/post-attach-command.sh   |   2 +-
 .../bin/vault/s3/creds/generate.sh            |  20 +--
 .../bin/vault/s3/creds/schedule.sh            |  10 +-
 .../opt/devcontainer/bin/vault/s3/init.sh     |   2 +-
 features/test/utils/scenarios.json            |   3 +
 features/test/utils/ubuntu18.04.sh            |  62 ++++++++
 features/test/utils/ubuntu20.04.sh            |  62 ++++++++
 features/test/utils/ubuntu22.04.sh            |  62 ++++++++
 22 files changed, 832 insertions(+), 24 deletions(-)
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/init.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/schedule.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/test.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh
 create mode 100755 features/src/utils/opt/devcontainer/bin/creds/s3/vault/github.sh

diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile
index b2e8c0c8..74cfe238 100644
--- a/.devcontainer/rapids.Dockerfile
+++ b/.devcontainer/rapids.Dockerfile
@@ -47,7 +47,8 @@ ENV PYTHONDONTWRITEBYTECODE="1"
 
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
-ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
+ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
+
 ENV HISTFILE="/home/coder/.cache/._bash_history"
 
 ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"
diff --git a/.github/actions/build-and-test-feature/action.yml b/.github/actions/build-and-test-feature/action.yml
index 4c02bcd8..30562ee8 100644
--- a/.github/actions/build-and-test-feature/action.yml
+++ b/.github/actions/build-and-test-feature/action.yml
@@ -6,6 +6,7 @@ inputs:
   args: {type: string, required: true}
   gh_token: {type: string, defaut: '', required: false}
   vault_host: {type: string, defaut: '', required: false}
+  aws_role_arn: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
 
@@ -30,5 +31,6 @@ runs:
         VAULT_S3_TTL: "900" # 15 minutes
         gh_token: "${{ inputs.gh_token }}"
         vault_host: "${{ inputs.vault_host }}"
+        aws_role_arn: "${{ inputs.aws_role_arn }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"
diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml
index cedc9bdf..a0d514f8 100644
--- a/.github/workflows/build-and-test-feature.yml
+++ b/.github/workflows/build-and-test-feature.yml
@@ -38,5 +38,6 @@ jobs:
           args: "${{ inputs.args }}"
           gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}"
           vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
+          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"
diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json
index 7fb87a98..9bcbfffc 100644
--- a/features/src/utils/devcontainer-feature.json
+++ b/features/src/utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "24.10.3",
+  "version": "24.10.4",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"
diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh
index 56894de9..11a79001 100644
--- a/features/src/utils/install.sh
+++ b/features/src/utils/install.sh
@@ -67,9 +67,9 @@ chmod u+s "$(realpath -m "$(which cron)")";
 
 # shellcheck disable=SC2174
 mkdir -m 0775 -p /var/log/devcontainer-utils;
-touch /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-chmod 0664 /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-chgrp crontab /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
+touch /var/log/devcontainer-utils/creds-s3.log;
+chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
 
 # Install Devcontainer utility scripts to /opt/devcontainer
 cp -ar ./opt/devcontainer /opt/;
@@ -79,6 +79,15 @@ declare -a commands_and_sources=(
     "parse-args                         parse-args.sh"
     "parse-args-from-docstring          parse-args-from-docstring.sh"
     "bash-completion.tmpl               bash/completion.tmpl.sh"
+    "creds-s3-init                      creds/s3/init.sh"
+    "creds-s3-generate                  creds/s3/generate.sh"
+    "creds-s3-persist                   creds/s3/persist.sh"
+    "creds-s3-propagate                 creds/s3/propagate.sh"
+    "creds-s3-schedule                  creds/s3/schedule.sh"
+    "creds-s3-test                      creds/s3/test.sh"
+    "creds-s3-gh-generate               creds/s3/gh/generate.sh"
+    "creds-s3-vault-generate            creds/s3/vault/generate.sh"
+    "creds-s3-vault-github              creds/s3/vault/github.sh"
     "generate-bash-completion           bash/generate-bash-completion.sh"
     "shell-is-interactive               shell-is-interactive.sh"
     "post-create-command                post-create-command.sh"
@@ -107,10 +116,10 @@ declare -a commands_and_sources=(
 
 # Install alternatives
 for entry in "${commands_and_sources[@]}"; do
-    declare -a pair=(${entry});
+    declare -a pair="(${entry})";
     declare cmd="devcontainer-utils-${pair[0]}";
     declare src="/opt/devcontainer/bin/${pair[1]}";
-    update-alternatives --install /usr/bin/${cmd} ${cmd} ${src} 0;
+    update-alternatives --install "/usr/bin/${cmd}" "${cmd}" "${src}" 0;
 done
 
 declare -a commands="($(for pair in "${commands_and_sources[@]}"; do cut -d' ' -f1 <<< "${pair}"; done))";
@@ -182,6 +191,30 @@ find_non_root_user;
 
 if test -n "${USERNAME-}"; then
     USERHOME="$(bash -c "echo ~${USERNAME-}")";
+
+    if type gh >/dev/null 2>&1; then
+        mkdir -p -m 0755                                         \
+            "$USERHOME/.local"                                   \
+            "$USERHOME/.local/share"                             \
+            "$USERHOME/.local/share/gh"                          \
+            "$USERHOME/.local/share/gh/extensions"               \
+            "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws" \
+            ;
+        NV_GHA_AWS_VERSION=latest
+        find_version_from_git_tags NV_GHA_AWS_VERSION https://github.com/nv-gha-runners/gh-nv-gha-aws;
+        wget --no-hsts -q -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \
+            "https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')";
+        chmod 0755 "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws";
+        cat <<EOF >"$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/manifest.yml"
+owner: nv-gha-runners
+name: gh-nv-gha-aws
+host: github.com
+tag: v${NV_GHA_AWS_VERSION}
+ispinned: false
+path: $USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws
+EOF
+    fi
+
     # Add user to the crontab group
     usermod -aG crontab "${USERNAME}";
     # Allow user to edit the crontab
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh
new file mode 100755
index 00000000..7069b4c9
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh
@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+_creds_s3_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-generate';
+
+    if test -z "${SCCACHE_BUCKET:-}"; then
+        exit 1;
+    fi
+
+    if test -n "${AWS_ROLE_ARN:-}" && gh nv-gha-aws --help >/dev/null 2>&1; then
+        # shellcheck disable=SC1091
+        devcontainer-utils-creds-s3-gh-generate;
+    elif test -n "${VAULT_HOST}"; then
+        # shellcheck disable=SC1091
+        devcontainer-utils-creds-s3-vault-generate;
+    fi
+}
+
+_creds_s3_generate "$@" <&0;
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
new file mode 100755
index 00000000..8c2c45d2
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
@@ -0,0 +1,68 @@
+#! /usr/bin/env bash
+
+_creds_github_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate';
+
+    if test -z "${AWS_ROLE_ARN:-}" \
+    || test -z "${SCCACHE_BUCKET:-}" \
+    || ! gh nv-gha-aws --help >/dev/null 2>&1; then
+        exit 1;
+    fi
+
+    # Remove existing credentials in case vault declines to issue new ones.
+    rm -rf ~/.aws/{stamp,config,credentials};
+
+    SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
+
+    devcontainer-utils-creds-s3-persist - <<< \
+        --bucket="${SCCACHE_BUCKET:-}"        \
+        --region="${SCCACHE_REGION:-}"        ;
+
+    # Initialize the GitHub CLI with the appropriate user scopes
+    # shellcheck disable=SC1091
+    . devcontainer-utils-init-github-cli;
+
+    # Check whether the user is in one of the allowed GitHub orgs
+    local allowed_orgs="${AWS_GITHUB_ORGS:-${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}}";
+    allowed_orgs="${allowed_orgs// /|}";
+    allowed_orgs="${allowed_orgs//;/|}";
+    allowed_orgs="${allowed_orgs//,/|}";
+
+    local -ra user_orgs="($(                         \
+        gh api user/orgs --jq '.[].login'            \
+            -H "Accept: application/vnd.github+json" \
+        | grep --color=never -iE "(${allowed_orgs})" \
+    ))";
+
+    if test "${#user_orgs[@]}" -eq 0; then
+        exit 1;
+    fi
+
+    local org;
+    local generated_at;
+    local -a nv_gha_aws_args=(
+        --profile default
+        --output creds-file
+        --role-arn "${AWS_ROLE_ARN}"
+        --aud "${AWS_AUDIENCE:-sts.amazonaws.com}"
+        --duration "${AWS_S3_TTL:-${VAULT_S3_TTL:-43200}}"
+        --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}"
+    );
+
+    for org in "${user_orgs[@]}"; do
+        generated_at="$(date '+%s')";
+        if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then
+            if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then
+                echo "${generated_at}" > ~/.aws/stamp;
+                return 0;
+            fi
+        fi
+    done
+
+}
+
+_creds_github_generate "$@" <&0;
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/init.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/init.sh
new file mode 100755
index 00000000..b0d33356
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/init.sh
@@ -0,0 +1,58 @@
+#! /usr/bin/env bash
+
+_s3_cred() {
+    sed -n "s/$1=//p" ~/.aws/credentials 2>/dev/null;
+}
+
+_s3_creds_init() {
+    local -
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-init';
+
+    if type sccache >/dev/null 2>&1; then
+        if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then
+            if grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}"     \
+            && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then
+                if ! grep -qE "^$" <<< "${VAULT_HOST:-${AWS_ROLE_ARN:-}}"; then
+                    # Generate S3 creds if they don't exist (or are expired)
+                    if devcontainer-utils-creds-s3-test \
+                    || devcontainer-utils-creds-s3-generate; then
+                        # Persist creds in ~/.aws dir
+                        devcontainer-utils-creds-s3-persist - <<< "                     \
+                            --bucket '${SCCACHE_BUCKET:-}'                              \
+                            --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}'       \
+                            --aws-access-key-id '$(_s3_cred aws_access_key_id)'         \
+                            --aws-session-token '$(_s3_cred aws_session_token)'         \
+                            --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
+                        ";
+                        # Install a crontab to refresh the credentials
+                        devcontainer-utils-creds-s3-schedule;
+                    else
+                        devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
+                    fi
+                elif devcontainer-utils-creds-s3-test; then
+                    devcontainer-utils-creds-s3-persist - <<< "                     \
+                        --bucket '${SCCACHE_BUCKET:-}'                              \
+                        --region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}'       \
+                        --aws-access-key-id '$(_s3_cred aws_access_key_id)'         \
+                        --aws-session-token '$(_s3_cred aws_session_token)'         \
+                        --aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
+                    ";
+                else
+                    # bucket is inaccessible
+                    devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
+                fi
+            elif ! devcontainer-utils-creds-s3-propagate; then
+                # bucket is inaccessible
+                devcontainer-utils-creds-s3-persist <<< "--no-bucket --no-region";
+            fi
+        fi
+    fi
+}
+
+_s3_creds_init "$@";
+
+# shellcheck disable=SC1090
+. /etc/profile.d/*-devcontainer-utils.sh;
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
new file mode 100755
index 00000000..b89992b7
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+# Usage:
+#  devcontainer-utils-creds-s3-persist [OPTION]...
+#
+# Set, unset, or reset the S3 bucket, region, and credentials in the environment.
+#
+# Boolean options:
+#  -h,--help                      print this text
+#  --no-bucket                    Unset the $SCCACHE_BUCKET environment variable for all shells.
+#                                 (default: false)
+#  --no-region                    Unset the $SCCACHE_REGION environment variable for all shells.
+#                                 (default: false)
+#
+# Options that require values:
+#  --stamp <stamp>                Timestamp when the S3 credentials were generated.
+#                                 (default: none)
+#  --bucket <bucket>              Set the $SCCACHE_BUCKET environment variable for all shells to <bucket> and persist in ~/.aws/config.
+#                                 (default: none)
+#  --region <region>              Set the $SCCACHE_REGION environment variable for all shells to <region> and persist in ~/.aws/config.
+#                                 (default: none)
+#  --aws-access-key-id <id>       Set the $AWS_ACCESS_KEY_ID environment variable for all shells to <id> and persist in ~/.aws/credentials.
+#                                 (default: none)
+#  --aws-session-token <token>    Set the $AWS_SESSION_TOKEN environment variable for all shells to <token> and persist in ~/.aws/credentials.
+#                                 (default: none)
+#  --aws-secret-access-key <key>  Set the $AWS_SECRET_ACCESS_KEY environment variable for all shells to <key> and persist in ~/.aws/credentials.
+#                                 (default: none)
+
+# shellcheck disable=SC1091
+. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh";
+
+_creds_s3_persist() {
+    local -;
+    set -euo pipefail;
+
+    eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)";
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-persist';
+
+    # Reset envvars
+    reset_envvar "SCCACHE_BUCKET";
+    reset_envvar "SCCACHE_REGION";
+    reset_envvar "AWS_ACCESS_KEY_ID";
+    reset_envvar "AWS_SESSION_TOKEN";
+    reset_envvar "AWS_SECRET_ACCESS_KEY";
+
+    mkdir -p ~/.aws;
+    rm -f ~/.aws/{config,credentials};
+
+    if test -n "${stamp:-}"; then
+        echo "${stamp:-}" > ~/.aws/stamp;
+    fi
+
+    if ! grep -qE "^$" <<< "${no_bucket-}"; then
+        unset_envvar "SCCACHE_BUCKET";
+    elif ! grep -qE "^$" <<< "${bucket:-}"; then
+        export_envvar "SCCACHE_BUCKET" "${bucket}";
+        cat <<________EOF >> ~/.aws/config
+bucket=${bucket:-}
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${no_region-}"; then
+        unset_envvar "SCCACHE_REGION";
+    elif ! grep -qE "^$" <<< "${region:-}"; then
+        export_envvar "SCCACHE_REGION" "${region}";
+        cat <<________EOF >> ~/.aws/config
+region=${region:-}
+________EOF
+    fi
+
+    if test -f ~/.aws/config; then
+        cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,}
+[default]
+$(cat ~/.aws/config)
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
+aws_access_key_id=${aws_access_key_id}
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
+aws_secret_access_key=${aws_secret_access_key}
+________EOF
+    fi
+
+    if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
+aws_session_token=${aws_session_token}
+________EOF
+    fi
+
+    if test -f ~/.aws/credentials; then
+        cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,}
+[default]
+$(cat ~/.aws/credentials)
+________EOF
+        chmod 0600 ~/.aws/credentials;
+    fi
+}
+
+if [ "$(basename "${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}")" = devcontainer-utils-creds-s3-persist ]; then
+    _creds_s3_persist "$@" <&0;
+fi
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh
new file mode 100755
index 00000000..7a4139da
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/propagate.sh
@@ -0,0 +1,42 @@
+#! /usr/bin/env bash
+
+_creds_s3_propagate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-propagate';
+
+    if ! type sccache >/dev/null 2>&1; then
+        return;
+    fi
+
+    local num_restarts="0";
+
+    if test -n "$(pgrep sccache || echo)"; then
+        sccache --stop-server >/dev/null 2>&1 || true;
+    fi
+
+    while true; do
+
+        if sccache --start-server >/dev/null 2>&1; then
+            if [ "${num_restarts}" -gt "0" ]; then echo "Success!"; fi
+            exit 0;
+        fi
+
+        if [ "${num_restarts}" -ge "20" ]; then
+            if [ "${num_restarts}" -gt "0" ]; then echo "Skipping."; fi
+            exit 1;
+        fi
+
+        num_restarts="$((num_restarts + 1))";
+
+        if [ "${num_restarts}" -eq "1" ]; then
+            echo -n "Waiting for AWS S3 credentials to propagate... ";
+        fi
+
+        sleep 1;
+    done
+}
+
+_creds_s3_propagate "$@";
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/schedule.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/schedule.sh
new file mode 100755
index 00000000..54ea6a90
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/schedule.sh
@@ -0,0 +1,64 @@
+#! /usr/bin/env bash
+
+_creds_s3_schedule() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-schedule';
+
+    local -r now="$(date '+%s')";
+    local ttl="${VAULT_S3_TTL:-"43200"}";
+    ttl="${ttl%s}";
+
+    local -r stamp="$(cat ~/.aws/stamp 2>/dev/null || echo "${now}")";
+    local ttime="$((ttl - (now - stamp)))";
+    ttime="$((ttime < ttl ? ttime : ttl))";
+    ttime="$((( (ttime + 59) / 60) * 60))";
+
+    # Regenerate creds if within `min(ttl / 5, 2hrs)` of keys expiring
+    local refresh_window="$(((ttl / 5) > 7200 ? 7200 : ttl / 5))";
+
+    if test "${ttime}" -le "${refresh_window}"; then
+        if devcontainer-utils-creds-s3-generate; then
+            devcontainer-utils-creds-s3-schedule;
+        fi
+    else
+        ttime="$((now + ttime - refresh_window))";
+
+        crontab -u "$(whoami)" -r 2>/dev/null || true;
+
+        sudo mkdir -m 0775 -p /var/log/devcontainer-utils;
+        sudo touch /var/log/devcontainer-utils/creds-s3.log;
+        sudo chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+        sudo chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
+
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
+$(date --date="@${now}")
+Scheduling cron to regerate S3 creds $(date -u --date="@$((ttime - now))" '+%T') from now.
+________EOF
+
+        cat <<________EOF | crontab -u "$(whoami)" -
+SHELL=/bin/bash
+BASH_ENV="${BASH_ENV:-}"
+VAULT_HOST="${VAULT_HOST:-}"
+GITHUB_USER="${GITHUB_USER:-}"
+AWS_S3_TTL="${AWS_S3_TTL:-}"
+AWS_IDP_URL="${AWS_IDP_URL:-}"
+AWS_ROLE_ARN="${AWS_ROLE_ARN:-}"
+AWS_AUDIENCE="${AWS_AUDIENCE:-}"
+AWS_GITHUB_ORGS="${AWS_GITHUB_ORGS:-}"
+VAULT_S3_TTL="${VAULT_S3_TTL:-}"
+SCCACHE_BUCKET="${SCCACHE_BUCKET:-}"
+SCCACHE_REGION="${SCCACHE_REGION:-}"
+VAULT_GITHUB_ORGS="${VAULT_GITHUB_ORGS:-}"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-}"
+$(date --date="@${ttime}" '+%M %H %d %m %w') \
+devcontainer-utils-creds-s3-schedule
+________EOF
+
+        sudo /etc/init.d/cron restart >>/var/log/devcontainer-utils/creds-s3.log 2>&1;
+    fi
+}
+
+_creds_s3_schedule "$@";
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh
new file mode 100755
index 00000000..00e0a90b
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/test.sh
@@ -0,0 +1,44 @@
+#! /usr/bin/env bash
+
+# Test sccache AWS S3 credentials are valid
+
+_creds_s3_test() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-test';
+
+    if ! type sccache >/dev/null 2>&1; then exit 1; fi
+
+    if test -f ~/.aws/stamp; then
+        local -r now="$(date '+%s')";
+        local -r stamp="$(cat ~/.aws/stamp)";
+        local ttl="${VAULT_S3_TTL:-"43200"}";
+        if [ $((now - stamp)) -ge "${ttl%s}" ]; then
+            exit 1;
+        fi
+    fi
+
+    local bucket="${SCCACHE_BUCKET:-"$(sed -n 's/bucket=//p' ~/.aws/config 2>/dev/null)"}";
+    if [ -z "${bucket:-}" ]; then exit 1; fi
+
+    local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(sed -n 's/region=//p' ~/.aws/config 2>/dev/null)"}"}";
+    local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)"}";
+    local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}";
+    local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}";
+
+    if test -n "$(pgrep sccache || echo)"; then
+        sccache --stop-server >/dev/null 2>&1 || true;
+    fi
+
+    SCCACHE_BUCKET="${bucket:-}" \
+    SCCACHE_REGION="${region:-}" \
+    AWS_ACCESS_KEY_ID="${aws_access_key_id:-}" \
+    AWS_SESSION_TOKEN="${aws_session_token:-}" \
+    AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \
+    sccache --start-server >/dev/null 2>&1;
+    sccache --show-stats | grep -qE 'Cache location \s+ s3';
+}
+
+_creds_s3_test "$@";
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh
new file mode 100755
index 00000000..76323e61
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh
@@ -0,0 +1,135 @@
+#! /usr/bin/env bash
+
+# Attempt to retrieve temporary AWS credentials from a vault instance using GitHub OAuth.
+
+_creds_vault_generate() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate';
+
+    if test -z "${VAULT_HOST:-}" \
+    || test -z "${SCCACHE_BUCKET:-}"; then
+        exit 1;
+    fi
+
+    SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
+
+    # Remove existing credentials in case vault declines to issue new ones.
+    rm -rf ~/.aws/{stamp,config,credentials};
+
+    devcontainer-utils-creds-s3-persist - <<< \
+        --bucket="${SCCACHE_BUCKET:-}"        \
+        --region="${SCCACHE_REGION:-}"        ;
+
+    # Initialize the GitHub CLI with the appropriate user scopes
+    # shellcheck disable=SC1091
+    . devcontainer-utils-init-github-cli;
+
+    if test -z "${GITHUB_USER:-}"; then
+        exit 1;
+    fi
+
+    # Check whether the user is in one of the allowed GitHub orgs
+    local allowed_orgs="${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}";
+    allowed_orgs="${allowed_orgs// /|}";
+    allowed_orgs="${allowed_orgs//;/|}";
+    allowed_orgs="${allowed_orgs//,/|}";
+
+    local -r user_orgs="$(                           \
+        gh api user/orgs --jq '.[].login'            \
+            -H "Accept: application/vnd.github+json" \
+        | grep --color=never -iE "(${allowed_orgs})" \
+    )";
+
+    if test "${#user_orgs}" -eq 0; then
+        exit 1;
+    fi
+
+    sudo mkdir -m 0775 -p /var/log/devcontainer-utils;
+    sudo touch /var/log/devcontainer-utils/creds-s3.log;
+    sudo chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+    sudo chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
+
+    cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
+$(date)
+Attempting to use your GitHub account to authenticate with vault at:
+'${VAULT_HOST}'.
+____EOF
+
+    local vault_token="null";
+
+    # Attempt to authenticate with GitHub
+    eval "$(devcontainer-utils-creds-s3-vault-github "${VAULT_HOST}" "${user_orgs}")";
+
+    if [ "${vault_token:-null}" = "null" ]; then
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
+Your GitHub user was not recognized by vault. Skipping.
+________EOF
+        exit 1;
+    fi
+
+    cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
+Successfully authenticated with vault!
+____EOF
+
+    local -r generated_at="$(date '+%s')";
+    local ttl="${VAULT_S3_TTL:-"43200"}";
+    local uri="${VAULT_S3_URI:-"v1/aws/creds/devs"}";
+
+    if grep -qE '^[0-9]$' <<< "${ttl}"; then
+        ttl+="s";
+    fi
+
+    # Generate temporary AWS creds
+    local -r aws_creds="$(                      \
+        curl -s                                 \
+            -X GET                              \
+            -H "X-Vault-Token: ${vault_token}"  \
+            -H "Content-Type: application/json" \
+            "${VAULT_HOST}/${uri}?ttl=${ttl}"   \
+        | jq -r '.data'                         \
+    )";
+
+    local -r aws_access_key_id="$(jq -r '.access_key' <<< "${aws_creds}" || echo)";
+    local -r aws_secret_access_key="$(jq -r '.secret_key' <<< "${aws_creds}" || echo)";
+
+    if grep -qE "^null$" <<< "${aws_access_key_id:-null}"; then
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
+Failed to retrieve AWS S3 credentials. Skipping.
+________EOF
+        exit 1;
+    fi
+
+    if grep -qE "^null$" <<< "${aws_secret_access_key:-null}"; then
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
+Failed to retrieve AWS S3 credentials. Skipping.
+________EOF
+        exit 1;
+    fi
+
+    cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
+Successfully generated temporary AWS S3 credentials!
+____EOF
+
+    # Block until the new temporary AWS S3 credentials propagate
+    if  SCCACHE_BUCKET="${SCCACHE_BUCKET:-}" \
+        SCCACHE_REGION="${SCCACHE_REGION:-}" \
+        AWS_ACCESS_KEY_ID="${aws_access_key_id:-}"         \
+        AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \
+        devcontainer-utils-creds-s3-propagate | tee -a /var/log/devcontainer-utils/creds-s3.log; then
+        # Store creds in ~/.aws dir
+        devcontainer-utils-creds-s3-persist - <<<               \
+            --stamp="${generated_at:-}"                         \
+            --bucket="${SCCACHE_BUCKET:-}"                      \
+            --region="${SCCACHE_REGION:-}"                      \
+            --aws-access-key-id="${aws_access_key_id:-}"        \
+            --aws-secret-access-key="${aws_secret_access_key:-}";
+    fi
+}
+
+_creds_vault_generate "$@";
+
+# shellcheck disable=SC1090
+. /etc/profile.d/*-devcontainer-utils.sh;
diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/vault/github.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/github.sh
new file mode 100755
index 00000000..2425cf93
--- /dev/null
+++ b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/github.sh
@@ -0,0 +1,39 @@
+#! /usr/bin/env bash
+
+_creds_vault_auth_github() {
+    local -;
+    set -euo pipefail;
+
+    # shellcheck disable=SC1091
+    . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-auth-github';
+
+    local -r VAULT_HOST="$1";
+    local -r user_orgs=("${@:2}");
+    local -r gh_token="$(gh auth token)";
+    local vault_token=null;
+    local o;
+    local org;
+
+    for o in "${user_orgs[@]}"; do
+        for org in $(echo -e "${o}\n${o,,}\n${o^^}" | sort -su); do
+            vault_token="$(                                     \
+                curl -s                                         \
+                    -X POST                                     \
+                    -H "Content-Type: application/json"         \
+                    -d "{\"token\": \"${gh_token}\"}"           \
+                    "${VAULT_HOST}/v1/auth/github-${org}/login" \
+              | jq -r '.auth.client_token'                      \
+            )";
+            if test "${vault_token:-null}" != null; then
+                break;
+            fi
+        done
+        if test "${vault_token:-null}" != null; then
+            break;
+        fi
+    done
+
+    echo "vault_token='$vault_token'";
+}
+
+_creds_vault_auth_github "$@";
diff --git a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
index 67d1caef..695e0d9e 100755
--- a/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
+++ b/features/src/utils/opt/devcontainer/bin/post-attach-command.sh
@@ -4,5 +4,5 @@ if test -z "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:-}"; then
     # shellcheck disable=SC1091
     . devcontainer-utils-init-git-interactive;
     # shellcheck disable=SC1091
-    . devcontainer-utils-vault-s3-init;
+    . devcontainer-utils-creds-s3-init;
 fi
diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh
index 5f6f8185..711962b5 100755
--- a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh
+++ b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/generate.sh
@@ -48,11 +48,11 @@ generate_s3_creds() {
     fi
 
     sudo mkdir -m 0775 -p /var/log/devcontainer-utils;
-    sudo touch /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-    sudo chmod 0664 /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-    sudo chgrp crontab /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
+    sudo touch /var/log/devcontainer-utils/creds-s3.log;
+    sudo chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+    sudo chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
 
-    cat <<____EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log
+    cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
 $(date)
 Attempting to use your GitHub account to authenticate with vault at:
 '${VAULT_HOST}'.
@@ -64,13 +64,13 @@ ____EOF
     eval "$(devcontainer-utils-vault-auth-github "${VAULT_HOST}" "${user_orgs}")";
 
     if [ "${vault_token:-null}" = "null" ]; then
-        cat <<________EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log >&2
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Your GitHub user was not recognized by vault. Skipping.
 ________EOF
         exit 1;
     fi
 
-    cat <<____EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log
+    cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
 Successfully authenticated with vault!
 ____EOF
 
@@ -96,20 +96,20 @@ ____EOF
     local -r aws_secret_access_key="$(jq -r '.secret_key' <<< "${aws_creds}" || echo)";
 
     if grep -qE "^null$" <<< "${aws_access_key_id:-null}"; then
-        cat <<________EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log >&2
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Failed to retrieve AWS S3 credentials. Skipping.
 ________EOF
         exit 1;
     fi
 
     if grep -qE "^null$" <<< "${aws_secret_access_key:-null}"; then
-        cat <<________EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log >&2
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Failed to retrieve AWS S3 credentials. Skipping.
 ________EOF
         exit 1;
     fi
 
-    cat <<____EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log
+    cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
 Successfully generated temporary AWS S3 credentials!
 ____EOF
 
@@ -118,7 +118,7 @@ ____EOF
         SCCACHE_REGION="${SCCACHE_REGION:-}" \
         AWS_ACCESS_KEY_ID="${aws_access_key_id:-}"         \
         AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \
-        devcontainer-utils-vault-s3-creds-propagate | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log; then
+        devcontainer-utils-vault-s3-creds-propagate | tee -a /var/log/devcontainer-utils/creds-s3.log; then
         # Store creds in ~/.aws dir
         devcontainer-utils-vault-s3-creds-persist - <<<         \
             --stamp="${generated_at:-}"                         \
diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/schedule.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/schedule.sh
index 3e1f98b2..9630445b 100755
--- a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/schedule.sh
+++ b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/schedule.sh
@@ -29,11 +29,11 @@ schedule_s3_creds_refresh() {
         crontab -u "$(whoami)" -r 2>/dev/null || true;
 
         sudo mkdir -m 0775 -p /var/log/devcontainer-utils;
-        sudo touch /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-        sudo chmod 0664 /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
-        sudo chgrp crontab /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
+        sudo touch /var/log/devcontainer-utils/creds-s3.log;
+        sudo chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
+        sudo chgrp crontab /var/log/devcontainer-utils/creds-s3.log;
 
-        cat <<________EOF | tee -a /var/log/devcontainer-utils/vault-s3-creds-refresh.log
+        cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
 $(date --date="@${now}")
 Scheduling cron to regerate S3 creds $(date -u --date="@$((ttime - now))" '+%T') from now.
 ________EOF
@@ -52,7 +52,7 @@ $(date --date="@${ttime}" '+%M %H %d %m %w') \
 devcontainer-utils-vault-s3-creds-schedule
 ________EOF
 
-        sudo /etc/init.d/cron restart >>/var/log/devcontainer-utils/vault-s3-creds-refresh.log 2>&1;
+        sudo /etc/init.d/cron restart >>/var/log/devcontainer-utils/creds-s3.log 2>&1;
     fi
 }
 
diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh
index adbfb478..6db085c3 100755
--- a/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh
+++ b/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh
@@ -44,7 +44,7 @@ init_vault_s3_creds() {
                 fi
             elif ! devcontainer-utils-vault-s3-creds-propagate; then
                 # bucket is inaccessible
-                devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
+                devcontainer-utils-vault-s3-creds-persist <<< "--no-bucket --no-region";
             fi
         fi
     fi
diff --git a/features/test/utils/scenarios.json b/features/test/utils/scenarios.json
index 28b56a45..3d8f4349 100644
--- a/features/test/utils/scenarios.json
+++ b/features/test/utils/scenarios.json
@@ -5,6 +5,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
       "rw_sccache_region": "${localEnv:rw_sccache_region}"
@@ -49,6 +50,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
       "rw_sccache_region": "${localEnv:rw_sccache_region}"
@@ -93,6 +95,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
       "rw_sccache_region": "${localEnv:rw_sccache_region}"
diff --git a/features/test/utils/ubuntu18.04.sh b/features/test/utils/ubuntu18.04.sh
index d83f684d..ee7e21b3 100644
--- a/features/test/utils/ubuntu18.04.sh
+++ b/features/test/utils/ubuntu18.04.sh
@@ -24,6 +24,7 @@ utils_profile_script="$(find /etc/profile.d/ -type f -name '*-devcontainer-utils
 reset_state() {
     export GH_TOKEN=;
     export VAULT_HOST=;
+    export AWS_ROLE_ARN=;
     export SCCACHE_BUCKET=;
     export SCCACHE_REGION=;
     export AWS_ACCESS_KEY_ID=;
@@ -32,6 +33,7 @@ reset_state() {
 
     export -n GH_TOKEN;
     export -n VAULT_HOST;
+    export -n AWS_ROLE_ARN;
     export -n SCCACHE_BUCKET;
     export -n SCCACHE_REGION;
     export -n AWS_ACCESS_KEY_ID;
@@ -40,6 +42,7 @@ reset_state() {
 
     unset GH_TOKEN;
     unset VAULT_HOST;
+    unset AWS_ROLE_ARN;
     unset SCCACHE_BUCKET;
     unset SCCACHE_REGION;
     unset AWS_ACCESS_KEY_ID;
@@ -143,6 +146,7 @@ if test -n "${gh_token:-}" \
     check "no creds with GH_TOKEN, VAULT_HOST, and SCCACHE_BUCKET should generate credentials" no_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_generate_credentials;
 
     existing_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_reuse_credentials() {
+        rm -rf /tmp/.aws;
         mv ~/.aws /tmp/;
         reset_state;
         cp -ar /tmp/.aws ~/;
@@ -183,6 +187,64 @@ if test -n "${gh_token:-}" \
     check "bad stored creds with GH_TOKEN, VAULT_HOST, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_regenerate_credentials;
 fi
 
+if test -n "${gh_token:-}" \
+&& test -n "${aws_role_arn:-}" \
+&& test -n "${rw_sccache_bucket:-}"; then
+
+    no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
+        reset_state;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "no creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should generate credentials" no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials;
+
+    existing_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_reuse_credentials() {
+        rm -rf /tmp/.aws;
+        mv ~/.aws /tmp/;
+        reset_state;
+        cp -ar /tmp/.aws ~/;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "existing creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should reuse credentials" existing_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_reuse_credentials;
+
+    should_use_provided_AWS_envvar_credentials() {
+        reset_state;
+        export SCCACHE_BUCKET="${rw_sccache_bucket}";
+        export SCCACHE_REGION="${rw_sccache_region}";
+        export AWS_ACCESS_KEY_ID="$(sed -n 's/aws_access_key_id=//p' /tmp/.aws/credentials 2>/dev/null)";
+        export AWS_SESSION_TOKEN="$(sed -n 's/aws_session_token=//p' /tmp/.aws/credentials 2>/dev/null)";
+        export AWS_SECRET_ACCESS_KEY="$(sed -n 's/aws_secret_access_key=//p' /tmp/.aws/credentials 2>/dev/null)";
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "should use provided AWS envvar credentials" should_use_provided_AWS_envvar_credentials;
+
+    bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials() {
+        reset_state;
+        write_bad_creds;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "bad stored creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials;
+fi
+
 # Report result
 # If any of the checks above exited with a non-zero exit code, the test will fail.
 reportResults
diff --git a/features/test/utils/ubuntu20.04.sh b/features/test/utils/ubuntu20.04.sh
index d83f684d..ee7e21b3 100644
--- a/features/test/utils/ubuntu20.04.sh
+++ b/features/test/utils/ubuntu20.04.sh
@@ -24,6 +24,7 @@ utils_profile_script="$(find /etc/profile.d/ -type f -name '*-devcontainer-utils
 reset_state() {
     export GH_TOKEN=;
     export VAULT_HOST=;
+    export AWS_ROLE_ARN=;
     export SCCACHE_BUCKET=;
     export SCCACHE_REGION=;
     export AWS_ACCESS_KEY_ID=;
@@ -32,6 +33,7 @@ reset_state() {
 
     export -n GH_TOKEN;
     export -n VAULT_HOST;
+    export -n AWS_ROLE_ARN;
     export -n SCCACHE_BUCKET;
     export -n SCCACHE_REGION;
     export -n AWS_ACCESS_KEY_ID;
@@ -40,6 +42,7 @@ reset_state() {
 
     unset GH_TOKEN;
     unset VAULT_HOST;
+    unset AWS_ROLE_ARN;
     unset SCCACHE_BUCKET;
     unset SCCACHE_REGION;
     unset AWS_ACCESS_KEY_ID;
@@ -143,6 +146,7 @@ if test -n "${gh_token:-}" \
     check "no creds with GH_TOKEN, VAULT_HOST, and SCCACHE_BUCKET should generate credentials" no_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_generate_credentials;
 
     existing_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_reuse_credentials() {
+        rm -rf /tmp/.aws;
         mv ~/.aws /tmp/;
         reset_state;
         cp -ar /tmp/.aws ~/;
@@ -183,6 +187,64 @@ if test -n "${gh_token:-}" \
     check "bad stored creds with GH_TOKEN, VAULT_HOST, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_regenerate_credentials;
 fi
 
+if test -n "${gh_token:-}" \
+&& test -n "${aws_role_arn:-}" \
+&& test -n "${rw_sccache_bucket:-}"; then
+
+    no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
+        reset_state;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "no creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should generate credentials" no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials;
+
+    existing_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_reuse_credentials() {
+        rm -rf /tmp/.aws;
+        mv ~/.aws /tmp/;
+        reset_state;
+        cp -ar /tmp/.aws ~/;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "existing creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should reuse credentials" existing_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_reuse_credentials;
+
+    should_use_provided_AWS_envvar_credentials() {
+        reset_state;
+        export SCCACHE_BUCKET="${rw_sccache_bucket}";
+        export SCCACHE_REGION="${rw_sccache_region}";
+        export AWS_ACCESS_KEY_ID="$(sed -n 's/aws_access_key_id=//p' /tmp/.aws/credentials 2>/dev/null)";
+        export AWS_SESSION_TOKEN="$(sed -n 's/aws_session_token=//p' /tmp/.aws/credentials 2>/dev/null)";
+        export AWS_SECRET_ACCESS_KEY="$(sed -n 's/aws_secret_access_key=//p' /tmp/.aws/credentials 2>/dev/null)";
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "should use provided AWS envvar credentials" should_use_provided_AWS_envvar_credentials;
+
+    bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials() {
+        reset_state;
+        write_bad_creds;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "bad stored creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials;
+fi
+
 # Report result
 # If any of the checks above exited with a non-zero exit code, the test will fail.
 reportResults
diff --git a/features/test/utils/ubuntu22.04.sh b/features/test/utils/ubuntu22.04.sh
index d83f684d..ee7e21b3 100644
--- a/features/test/utils/ubuntu22.04.sh
+++ b/features/test/utils/ubuntu22.04.sh
@@ -24,6 +24,7 @@ utils_profile_script="$(find /etc/profile.d/ -type f -name '*-devcontainer-utils
 reset_state() {
     export GH_TOKEN=;
     export VAULT_HOST=;
+    export AWS_ROLE_ARN=;
     export SCCACHE_BUCKET=;
     export SCCACHE_REGION=;
     export AWS_ACCESS_KEY_ID=;
@@ -32,6 +33,7 @@ reset_state() {
 
     export -n GH_TOKEN;
     export -n VAULT_HOST;
+    export -n AWS_ROLE_ARN;
     export -n SCCACHE_BUCKET;
     export -n SCCACHE_REGION;
     export -n AWS_ACCESS_KEY_ID;
@@ -40,6 +42,7 @@ reset_state() {
 
     unset GH_TOKEN;
     unset VAULT_HOST;
+    unset AWS_ROLE_ARN;
     unset SCCACHE_BUCKET;
     unset SCCACHE_REGION;
     unset AWS_ACCESS_KEY_ID;
@@ -143,6 +146,7 @@ if test -n "${gh_token:-}" \
     check "no creds with GH_TOKEN, VAULT_HOST, and SCCACHE_BUCKET should generate credentials" no_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_generate_credentials;
 
     existing_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_reuse_credentials() {
+        rm -rf /tmp/.aws;
         mv ~/.aws /tmp/;
         reset_state;
         cp -ar /tmp/.aws ~/;
@@ -183,6 +187,64 @@ if test -n "${gh_token:-}" \
     check "bad stored creds with GH_TOKEN, VAULT_HOST, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_VAULT_HOST_and_SCCACHE_BUCKET_should_regenerate_credentials;
 fi
 
+if test -n "${gh_token:-}" \
+&& test -n "${aws_role_arn:-}" \
+&& test -n "${rw_sccache_bucket:-}"; then
+
+    no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials() {
+        reset_state;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "no creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should generate credentials" no_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_generate_credentials;
+
+    existing_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_reuse_credentials() {
+        rm -rf /tmp/.aws;
+        mv ~/.aws /tmp/;
+        reset_state;
+        cp -ar /tmp/.aws ~/;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "existing creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should reuse credentials" existing_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_reuse_credentials;
+
+    should_use_provided_AWS_envvar_credentials() {
+        reset_state;
+        export SCCACHE_BUCKET="${rw_sccache_bucket}";
+        export SCCACHE_REGION="${rw_sccache_region}";
+        export AWS_ACCESS_KEY_ID="$(sed -n 's/aws_access_key_id=//p' /tmp/.aws/credentials 2>/dev/null)";
+        export AWS_SESSION_TOKEN="$(sed -n 's/aws_session_token=//p' /tmp/.aws/credentials 2>/dev/null)";
+        export AWS_SECRET_ACCESS_KEY="$(sed -n 's/aws_secret_access_key=//p' /tmp/.aws/credentials 2>/dev/null)";
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "should use provided AWS envvar credentials" should_use_provided_AWS_envvar_credentials;
+
+    bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials() {
+        reset_state;
+        write_bad_creds;
+        GH_TOKEN="${gh_token}" \
+        AWS_ROLE_ARN="${aws_role_arn}" \
+        SCCACHE_BUCKET="${rw_sccache_bucket}" \
+        SCCACHE_REGION="${rw_sccache_region}" \
+        devcontainer-utils-post-attach-command;
+        expect_s3_cache_is_used;
+    }
+
+    check "bad stored creds with GH_TOKEN, AWS_ROLE_ARN, and SCCACHE_BUCKET should regenerate credentials" bad_stored_creds_with_GH_TOKEN_AWS_ROLE_ARN_and_SCCACHE_BUCKET_should_regenerate_credentials;
+fi
+
 # Report result
 # If any of the checks above exited with a non-zero exit code, the test will fail.
 reportResults

From 88c449fe9bef3a147f5b8ca72b46d88dfedfe725 Mon Sep 17 00:00:00 2001
From: Christopher Harris <charris@nvidia.com>
Date: Thu, 19 Sep 2024 14:17:48 -0500
Subject: [PATCH 2/2] Delay submodule initiation until after branch checkout
 (#393)

This fixes an issue where a submodule branch is not included in the
history of the default branch upon first clone (where the clone
recursively fetches the submodule) when the user passes the
`--single-branch` argument to `clone-*`.

---------

Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com>
---
 features/src/rapids-build-utils/devcontainer-feature.json       | 2 +-
 .../opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh          | 1 -
 features/src/utils/devcontainer-feature.json                    | 2 +-
 features/src/utils/opt/devcontainer/bin/git/repo/clone.sh       | 2 +-
 4 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json
index fe3190f2..b6792894 100644
--- a/features/src/rapids-build-utils/devcontainer-feature.json
+++ b/features/src/rapids-build-utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.10.9",
+  "version": "24.10.10",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh
index 0ba647d4..d3dd0f45 100755
--- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh
+++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/repo.clone.tmpl.sh
@@ -51,7 +51,6 @@ clone_${NAME}() {
         --branch "${branch}"                    \
         --ssh-url "${ssh_url}"                  \
         --https-url "${https_url}"              \
-        --recurse-submodules                    \
         -j ${n_jobs:-$(nproc --all --ignore=1)} \
         -c checkout.defaultRemote=upstream      \
         "${OPTS[@]}"                            \
diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json
index 9bcbfffc..e46c6150 100644
--- a/features/src/utils/devcontainer-feature.json
+++ b/features/src/utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "24.10.4",
+  "version": "24.10.5",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"
diff --git a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh
index ea045224..43ca2aee 100755
--- a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh
+++ b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh
@@ -108,7 +108,7 @@ clone_git_repo() {
     fi
 
     # shellcheck disable=SC2086
-    git -C "${directory}" submodule update --recursive "${qj[@]}";
+    git -C "${directory}" submodule update --init --recursive "${qj[@]}";
 }
 
 clone_git_repo "$@" <&0;