Skip to content

Commit

Permalink
Use gh nv-gha-aws CLI extension to generate credentials (#392)
Browse files Browse the repository at this point in the history
* install
[`nv-gha-aws`](https://github.com/nv-gha-runners/gh-nv-gha-aws) CLI
extension
* use `nv-gha-aws` to generate credentials if `AWS_ROLE_ARN` is set
  • Loading branch information
trxcllnt authored Sep 19, 2024
1 parent 3952e76 commit fc6947a
Show file tree
Hide file tree
Showing 22 changed files with 832 additions and 24 deletions.
3 changes: 2 additions & 1 deletion .devcontainer/rapids.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ ENV PYTHONDONTWRITEBYTECODE="1"

ENV SCCACHE_REGION="us-east-2"
ENV SCCACHE_BUCKET="rapids-sccache-devs"
ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"

ENV HISTFILE="/home/coder/.cache/._bash_history"

ENV LIBCUDF_KERNEL_CACHE_PATH="/home/coder/cudf/cpp/build/${PYTHON_PACKAGE_MANAGER}/cuda-${CUDA_VERSION}/latest/jitify_cache"
2 changes: 2 additions & 0 deletions .github/actions/build-and-test-feature/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ inputs:
args: {type: string, required: true}
gh_token: {type: string, defaut: '', required: false}
vault_host: {type: string, defaut: '', required: false}
aws_role_arn: {type: string, defaut: '', required: false}
rw_sccache_bucket: {type: string, defaut: '', required: false}
rw_sccache_region: {type: string, defaut: '', required: false}

Expand All @@ -30,5 +31,6 @@ runs:
VAULT_S3_TTL: "900" # 15 minutes
gh_token: "${{ inputs.gh_token }}"
vault_host: "${{ inputs.vault_host }}"
aws_role_arn: "${{ inputs.aws_role_arn }}"
rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
rw_sccache_region: "${{ inputs.rw_sccache_region }}"
1 change: 1 addition & 0 deletions .github/workflows/build-and-test-feature.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,6 @@ jobs:
args: "${{ inputs.args }}"
gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}"
vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
rw_sccache_region: "${{ vars.AWS_REGION }}"
2 changes: 1 addition & 1 deletion features/src/utils/devcontainer-feature.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "devcontainer-utils",
"id": "utils",
"version": "24.10.3",
"version": "24.10.4",
"description": "A feature to install RAPIDS devcontainer utility scripts",
"containerEnv": {
"BASH_ENV": "/etc/bash.bash_env"
Expand Down
43 changes: 38 additions & 5 deletions features/src/utils/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ chmod u+s "$(realpath -m "$(which cron)")";

# shellcheck disable=SC2174
mkdir -m 0775 -p /var/log/devcontainer-utils;
touch /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
chmod 0664 /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
chgrp crontab /var/log/devcontainer-utils/vault-s3-creds-refresh.log;
touch /var/log/devcontainer-utils/creds-s3.log;
chmod 0664 /var/log/devcontainer-utils/creds-s3.log;
chgrp crontab /var/log/devcontainer-utils/creds-s3.log;

# Install Devcontainer utility scripts to /opt/devcontainer
cp -ar ./opt/devcontainer /opt/;
Expand All @@ -79,6 +79,15 @@ declare -a commands_and_sources=(
"parse-args parse-args.sh"
"parse-args-from-docstring parse-args-from-docstring.sh"
"bash-completion.tmpl bash/completion.tmpl.sh"
"creds-s3-init creds/s3/init.sh"
"creds-s3-generate creds/s3/generate.sh"
"creds-s3-persist creds/s3/persist.sh"
"creds-s3-propagate creds/s3/propagate.sh"
"creds-s3-schedule creds/s3/schedule.sh"
"creds-s3-test creds/s3/test.sh"
"creds-s3-gh-generate creds/s3/gh/generate.sh"
"creds-s3-vault-generate creds/s3/vault/generate.sh"
"creds-s3-vault-github creds/s3/vault/github.sh"
"generate-bash-completion bash/generate-bash-completion.sh"
"shell-is-interactive shell-is-interactive.sh"
"post-create-command post-create-command.sh"
Expand Down Expand Up @@ -107,10 +116,10 @@ declare -a commands_and_sources=(

# Install alternatives
for entry in "${commands_and_sources[@]}"; do
declare -a pair=(${entry});
declare -a pair="(${entry})";
declare cmd="devcontainer-utils-${pair[0]}";
declare src="/opt/devcontainer/bin/${pair[1]}";
update-alternatives --install /usr/bin/${cmd} ${cmd} ${src} 0;
update-alternatives --install "/usr/bin/${cmd}" "${cmd}" "${src}" 0;
done

declare -a commands="($(for pair in "${commands_and_sources[@]}"; do cut -d' ' -f1 <<< "${pair}"; done))";
Expand Down Expand Up @@ -182,6 +191,30 @@ find_non_root_user;

if test -n "${USERNAME-}"; then
USERHOME="$(bash -c "echo ~${USERNAME-}")";

if type gh >/dev/null 2>&1; then
mkdir -p -m 0755 \
"$USERHOME/.local" \
"$USERHOME/.local/share" \
"$USERHOME/.local/share/gh" \
"$USERHOME/.local/share/gh/extensions" \
"$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws" \
;
NV_GHA_AWS_VERSION=latest
find_version_from_git_tags NV_GHA_AWS_VERSION https://github.com/nv-gha-runners/gh-nv-gha-aws;
wget --no-hsts -q -O "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws" \
"https://github.com/nv-gha-runners/gh-nv-gha-aws/releases/download/v${NV_GHA_AWS_VERSION}/gh-nv-gha-aws_v${NV_GHA_AWS_VERSION}_linux-$(dpkg --print-architecture | awk -F'-' '{print $NF}')";
chmod 0755 "$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws";
cat <<EOF >"$USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/manifest.yml"
owner: nv-gha-runners
name: gh-nv-gha-aws
host: github.com
tag: v${NV_GHA_AWS_VERSION}
ispinned: false
path: $USERHOME/.local/share/gh/extensions/gh-nv-gha-aws/gh-nv-gha-aws
EOF
fi

# Add user to the crontab group
usermod -aG crontab "${USERNAME}";
# Allow user to edit the crontab
Expand Down
23 changes: 23 additions & 0 deletions features/src/utils/opt/devcontainer/bin/creds/s3/generate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#! /usr/bin/env bash

_creds_s3_generate() {
local -;
set -euo pipefail;

# shellcheck disable=SC1091
. devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-generate';

if test -z "${SCCACHE_BUCKET:-}"; then
exit 1;
fi

if test -n "${AWS_ROLE_ARN:-}" && gh nv-gha-aws --help >/dev/null 2>&1; then
# shellcheck disable=SC1091
devcontainer-utils-creds-s3-gh-generate;
elif test -n "${VAULT_HOST}"; then
# shellcheck disable=SC1091
devcontainer-utils-creds-s3-vault-generate;
fi
}

_creds_s3_generate "$@" <&0;
68 changes: 68 additions & 0 deletions features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#! /usr/bin/env bash

_creds_github_generate() {
local -;
set -euo pipefail;

# shellcheck disable=SC1091
. devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-vault creds-s3-vault-generate';

if test -z "${AWS_ROLE_ARN:-}" \
|| test -z "${SCCACHE_BUCKET:-}" \
|| ! gh nv-gha-aws --help >/dev/null 2>&1; then
exit 1;
fi

# Remove existing credentials in case vault declines to issue new ones.
rm -rf ~/.aws/{stamp,config,credentials};

SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";

devcontainer-utils-creds-s3-persist - <<< \
--bucket="${SCCACHE_BUCKET:-}" \
--region="${SCCACHE_REGION:-}" ;
# Initialize the GitHub CLI with the appropriate user scopes
# shellcheck disable=SC1091
. devcontainer-utils-init-github-cli;
# Check whether the user is in one of the allowed GitHub orgs
local allowed_orgs="${AWS_GITHUB_ORGS:-${VAULT_GITHUB_ORGS:-nvidia nv-morpheus nv-legate rapids}}";
allowed_orgs="${allowed_orgs// /|}";
allowed_orgs="${allowed_orgs//;/|}";
allowed_orgs="${allowed_orgs//,/|}";
local -ra user_orgs="($( \
gh api user/orgs --jq '.[].login' \
-H "Accept: application/vnd.github+json" \
| grep --color=never -iE "(${allowed_orgs})" \
))";
if test "${#user_orgs[@]}" -eq 0; then
exit 1;
fi
local org;
local generated_at;
local -a nv_gha_aws_args=(
--profile default
--output creds-file
--role-arn "${AWS_ROLE_ARN}"
--aud "${AWS_AUDIENCE:-sts.amazonaws.com}"
--duration "${AWS_S3_TTL:-${VAULT_S3_TTL:-43200}}"
--idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}"
);
for org in "${user_orgs[@]}"; do
generated_at="$(date '+%s')";
if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then
if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then
echo "${generated_at}" > ~/.aws/stamp;
return 0;
fi
fi
done
}
_creds_github_generate "$@" <&0;
58 changes: 58 additions & 0 deletions features/src/utils/opt/devcontainer/bin/creds/s3/init.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#! /usr/bin/env bash

_s3_cred() {
sed -n "s/$1=//p" ~/.aws/credentials 2>/dev/null;
}

_s3_creds_init() {
local -
set -euo pipefail;

# shellcheck disable=SC1091
. devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-init';

if type sccache >/dev/null 2>&1; then
if ! grep -qE "^$" <<< "${SCCACHE_BUCKET:-}"; then
if grep -qE "^$" <<< "${AWS_ACCESS_KEY_ID:-}" \
&& grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then
if ! grep -qE "^$" <<< "${VAULT_HOST:-${AWS_ROLE_ARN:-}}"; then
# Generate S3 creds if they don't exist (or are expired)
if devcontainer-utils-creds-s3-test \
|| devcontainer-utils-creds-s3-generate; then
# Persist creds in ~/.aws dir
devcontainer-utils-creds-s3-persist - <<< " \
--bucket '${SCCACHE_BUCKET:-}' \
--region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}' \
--aws-access-key-id '$(_s3_cred aws_access_key_id)' \
--aws-session-token '$(_s3_cred aws_session_token)' \
--aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
";
# Install a crontab to refresh the credentials
devcontainer-utils-creds-s3-schedule;
else
devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
fi
elif devcontainer-utils-creds-s3-test; then
devcontainer-utils-creds-s3-persist - <<< " \
--bucket '${SCCACHE_BUCKET:-}' \
--region '${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}' \
--aws-access-key-id '$(_s3_cred aws_access_key_id)' \
--aws-session-token '$(_s3_cred aws_session_token)' \
--aws-secret-access-key '$(_s3_cred aws_secret_access_key)' \
";
else
# bucket is inaccessible
devcontainer-utils-creds-s3-persist - <<< "--no-bucket --no-region";
fi
elif ! devcontainer-utils-creds-s3-propagate; then
# bucket is inaccessible
devcontainer-utils-creds-s3-persist <<< "--no-bucket --no-region";
fi
fi
fi
}

_s3_creds_init "$@";

# shellcheck disable=SC1090
. /etc/profile.d/*-devcontainer-utils.sh;
109 changes: 109 additions & 0 deletions features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/usr/bin/env bash

# Usage:
# devcontainer-utils-creds-s3-persist [OPTION]...
#
# Set, unset, or reset the S3 bucket, region, and credentials in the environment.
#
# Boolean options:
# -h,--help print this text
# --no-bucket Unset the $SCCACHE_BUCKET environment variable for all shells.
# (default: false)
# --no-region Unset the $SCCACHE_REGION environment variable for all shells.
# (default: false)
#
# Options that require values:
# --stamp <stamp> Timestamp when the S3 credentials were generated.
# (default: none)
# --bucket <bucket> Set the $SCCACHE_BUCKET environment variable for all shells to <bucket> and persist in ~/.aws/config.
# (default: none)
# --region <region> Set the $SCCACHE_REGION environment variable for all shells to <region> and persist in ~/.aws/config.
# (default: none)
# --aws-access-key-id <id> Set the $AWS_ACCESS_KEY_ID environment variable for all shells to <id> and persist in ~/.aws/credentials.
# (default: none)
# --aws-session-token <token> Set the $AWS_SESSION_TOKEN environment variable for all shells to <token> and persist in ~/.aws/credentials.
# (default: none)
# --aws-secret-access-key <key> Set the $AWS_SECRET_ACCESS_KEY environment variable for all shells to <key> and persist in ~/.aws/credentials.
# (default: none)

# shellcheck disable=SC1091
. "$(dirname "$(realpath -m "${BASH_SOURCE[0]}")")/../../update-envvars.sh";

_creds_s3_persist() {
local -;
set -euo pipefail;

eval "$(devcontainer-utils-parse-args "$0" "$@" <&0)";

# shellcheck disable=SC1091
. devcontainer-utils-debug-output 'devcontainer_utils_debug' 'creds-s3 creds-s3-persist';

# Reset envvars
reset_envvar "SCCACHE_BUCKET";
reset_envvar "SCCACHE_REGION";
reset_envvar "AWS_ACCESS_KEY_ID";
reset_envvar "AWS_SESSION_TOKEN";
reset_envvar "AWS_SECRET_ACCESS_KEY";

mkdir -p ~/.aws;
rm -f ~/.aws/{config,credentials};

if test -n "${stamp:-}"; then
echo "${stamp:-}" > ~/.aws/stamp;
fi

if ! grep -qE "^$" <<< "${no_bucket-}"; then
unset_envvar "SCCACHE_BUCKET";
elif ! grep -qE "^$" <<< "${bucket:-}"; then
export_envvar "SCCACHE_BUCKET" "${bucket}";
cat <<________EOF >> ~/.aws/config
bucket=${bucket:-}
________EOF
fi

if ! grep -qE "^$" <<< "${no_region-}"; then
unset_envvar "SCCACHE_REGION";
elif ! grep -qE "^$" <<< "${region:-}"; then
export_envvar "SCCACHE_REGION" "${region}";
cat <<________EOF >> ~/.aws/config
region=${region:-}
________EOF
fi

if test -f ~/.aws/config; then
cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,}
[default]
$(cat ~/.aws/config)
________EOF
fi

if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
cat <<________EOF >> ~/.aws/credentials
aws_access_key_id=${aws_access_key_id}
________EOF
fi

if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
cat <<________EOF >> ~/.aws/credentials
aws_secret_access_key=${aws_secret_access_key}
________EOF
fi

if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
cat <<________EOF >> ~/.aws/credentials
aws_session_token=${aws_session_token}
________EOF
fi

if test -f ~/.aws/credentials; then
cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,}
[default]
$(cat ~/.aws/credentials)
________EOF
chmod 0600 ~/.aws/credentials;
fi
}

if [ "$(basename "${BASH_SOURCE[${#BASH_SOURCE[@]}-1]}")" = devcontainer-utils-creds-s3-persist ]; then
_creds_s3_persist "$@" <&0;
fi
Loading

0 comments on commit fc6947a

Please sign in to comment.