Merge pull request #1119 from nextstrain/use-pathogen-repo-build

Use pathogen-repo-build in GH Action workflows
nextstrain · Jun 27, 2024 · 0336d0e · 0336d0e
2 parents eb488d6 + c30b752
commit 0336d0e
Show file tree

Hide file tree

Showing 5 changed files with 59 additions and 226 deletions.
diff --git a/.github/workflows/rebuild-100k.yml b/.github/workflows/rebuild-100k.yml
@@ -9,80 +9,49 @@ on:
 
 jobs:
   gisaid:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
-
-    - name: Launch GISAID build
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
       run: |
         set -x
 
         declare -a config
         config+=(slack_token=$SLACK_TOKEN)
 
         nextstrain build \
-          --aws-batch \
           --detach \
           --cpus 16 \
           --memory 31GiB \
           . \
             upload \
             --configfile nextstrain_profiles/100k/config-gisaid.yaml \
             --config "${config[@]}" \
-            --set-threads tree=8 \
-        |& tee build-launch-gisaid.log
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-
-    - name: Launch open build
+            --set-threads tree=8
+      artifact-name: gisaid-build-output
+
+  open:
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
       run: |
         set -x
 
         declare -a config
         config+=(slack_token=$SLACK_TOKEN)
 
         nextstrain build \
-          --aws-batch \
           --detach \
           --cpus 16 \
           --memory 31GiB \
           . \
             upload \
             --configfile nextstrain_profiles/100k/config-open.yaml \
             --config "${config[@]}" \
-            --set-threads tree=8 \
-        |& tee build-launch-open.log
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-
-
-    - name: Build info
-      run: |
-        echo "--> 100k samples for GISAID + Open data rebuilding (using separate AWS jobs)"
-        echo
-        echo "--> When completed, the following files will be updated:"
-        echo "s3://nextstrain-data/files/ncov/open/100k/metadata.tsv.xz"
-        echo "s3://nextstrain-data/files/ncov/open/100k/sequences.fasta.xz"
-        echo "s3://nextstrain-ncov-private/100k/metadata.tsv.xz"
-        echo "s3://nextstrain-ncov-private/100k/sequences.fasta.xz"
-        echo
-        echo "--> You can attach to the GISAID AWS job via:"
-        tail -n1 build-launch-gisaid.log
-        echo
-        echo "--> You can attach to the Open AWS job via:"
-        tail -n1 build-launch-open.log
-        echo
-        JOBID=$( tail -n1 build-launch-gisaid.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
-        echo "--> View the GISAID job in the AWS console via"
-        echo "    https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
-        echo
-        JOBID=$( tail -n1 build-launch-open.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
-        echo "--> View the Open job in the AWS console via"
-        echo "    https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
-        echo
+            --set-threads tree=8
+      artifact-name: open-build-output
diff --git a/.github/workflows/rebuild-country.yml b/.github/workflows/rebuild-country.yml
@@ -17,19 +17,17 @@ on:
         description: 'Specific container image to use for build (will override the default of "nextstrain build")'
         required: false
 
-env:
-  TRIAL_NAME: ${{ github.event.inputs.trial_name }}
-  NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
-
 jobs:
-  gisaid:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
-
-    - name: Launch build
+  nextstrain-country:
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
+      env: |
+        TRIAL_NAME: ${{ github.event.inputs.trial_name }}
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
       run: |
         set -x
 
@@ -46,7 +44,6 @@ jobs:
         fi
 
         nextstrain build \
-          --aws-batch \
           --detach \
           --cpus 72 \
           --memory 140GiB \
@@ -55,35 +52,4 @@ jobs:
             upload \
             --config "${config[@]}" \
             --profile nextstrain_profiles/nextstrain-country \
-            --set-threads tree=8 \
-        |& tee build-launch.log
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-
-    - name: Build info
-      run: |
-        if [[ "$TRIAL_NAME" ]]; then
-          echo "--> Trial name is: $TRIAL_NAME"
-          echo
-          echo "--> When completed, the following will be available:"
-          echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/global/all-time"
-        else
-          echo "--> GISAID phylogenetic analysis rebuilding on AWS"
-          echo
-          echo "--> When completed, the following will be updated:"
-          echo "build files: s3://nextstrain-ncov-private/REGION_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/ncov/gisaid/global/all-time"
-        fi
-        echo
-        echo "--> You can attach to this AWS job via:"
-        tail -n1 build-launch.log
-        echo
-        JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
-        echo "--> View this job in the AWS console via"
-        echo "    https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
-        echo
+            --set-threads tree=8
diff --git a/.github/workflows/rebuild-gisaid-21L.yml b/.github/workflows/rebuild-gisaid-21L.yml
@@ -17,19 +17,17 @@ on:
         description: 'Specific container image to use for build (will override the default of "nextstrain build")'
         required: false
 
-env:
-  TRIAL_NAME: ${{ github.event.inputs.trial_name }}
-  NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
-
 jobs:
   gisaid-21L:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
-
-    - name: Launch build
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
+      env: |
+        TRIAL_NAME: ${{ github.event.inputs.trial_name }}
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
       run: |
         set -x
 
@@ -46,7 +44,6 @@ jobs:
         fi
 
         nextstrain build \
-          --aws-batch \
           --detach \
           --cpus 72 \
           --memory 140GiB \
@@ -55,35 +52,4 @@ jobs:
             upload \
             --config "${config[@]}" \
             --profile nextstrain_profiles/nextstrain-gisaid-21L \
-            --set-threads tree=8 \
-        |& tee build-launch.log
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-
-    - name: Build info
-      run: |
-        if [[ "$TRIAL_NAME" ]]; then
-          echo "--> Trial name is: $TRIAL_NAME"
-          echo
-          echo "--> When completed, the following will be available:"
-          echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/21L/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/staging/ncov/gisaid/21L/trial/$TRIAL_NAME/global/all-time"
-        else
-          echo "--> GISAID 21L phylogenetic analysis rebuilding on AWS"
-          echo
-          echo "--> When completed, the following will be updated:"
-          echo "build files: s3://nextstrain-ncov-private/REGION_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/21L/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/ncov/gisaid/21L/global/all-time"
-        fi
-        echo
-        echo "--> You can attach to this AWS job via:"
-        tail -n1 build-launch.log
-        echo
-        JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
-        echo "--> View this job in the AWS console via"
-        echo "    https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
-        echo
+            --set-threads tree=8
diff --git a/.github/workflows/rebuild-gisaid.yml b/.github/workflows/rebuild-gisaid.yml
@@ -17,19 +17,17 @@ on:
         description: 'Specific container image to use for build (will override the default of "nextstrain build")'
         required: false
 
-env:
-  TRIAL_NAME: ${{ github.event.inputs.trial_name }}
-  NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
-
 jobs:
   gisaid:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
-
-    - name: Launch build
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
+      env: |
+        TRIAL_NAME: ${{ github.event.inputs.trial_name }}
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
       run: |
         set -x
 
@@ -46,7 +44,6 @@ jobs:
         fi
 
         nextstrain build \
-          --aws-batch \
           --detach \
           --cpus 72 \
           --memory 140GiB \
@@ -55,35 +52,4 @@ jobs:
             upload \
             --config "${config[@]}" \
             --profile nextstrain_profiles/nextstrain-gisaid \
-            --set-threads tree=8 \
-        |& tee build-launch.log
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-
-    - name: Build info
-      run: |
-        if [[ "$TRIAL_NAME" ]]; then
-          echo "--> Trial name is: $TRIAL_NAME"
-          echo
-          echo "--> When completed, the following will be available:"
-          echo "build files: s3://nextstrain-ncov-private/trial/$TRIAL_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/staging/ncov/gisaid/trial/$TRIAL_NAME/global/all-time"
-        else
-          echo "--> GISAID phylogenetic analysis rebuilding on AWS"
-          echo
-          echo "--> When completed, the following will be updated:"
-          echo "build files: s3://nextstrain-ncov-private/REGION_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/ncov/gisaid/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/ncov/gisaid/global/all-time"
-        fi
-        echo
-        echo "--> You can attach to this AWS job via:"
-        tail -n1 build-launch.log
-        echo
-        JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
-        echo "--> View this job in the AWS console via"
-        echo "    https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
-        echo
+            --set-threads tree=8
diff --git a/.github/workflows/rebuild-open.yml b/.github/workflows/rebuild-open.yml
@@ -18,19 +18,17 @@ on:
         description: 'Specific container image to use for build (will override the default of "nextstrain build")'
         required: false
 
-env:
-  TRIAL_NAME: ${{ github.event.inputs.trial_name }}
-  NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
-
 jobs:
   open:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: nextstrain/.github/actions/setup-nextstrain-cli@master
-
-    - name: Launch build
+    permissions:
+      id-token: write
+    uses: nextstrain/.github/.github/workflows/pathogen-repo-build.yaml@master
+    secrets: inherit
+    with:
+      runtime: aws-batch
+      env: |
+        TRIAL_NAME: ${{ github.event.inputs.trial_name }}
+        NEXTSTRAIN_DOCKER_IMAGE: ${{ github.event.inputs.image }}
       run: |
         set -x
 
@@ -47,7 +45,6 @@ jobs:
         fi
 
         nextstrain build \
-          --aws-batch \
           --detach \
           --cpus 72 \
           --memory 140GiB \
@@ -56,35 +53,4 @@ jobs:
             upload \
             --config "${config[@]}" \
             --profile nextstrain_profiles/nextstrain-open \
-            --set-threads tree=8 \
-        |& tee build-launch.log
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
-
-    - name: Build info
-      run: |
-        if [[ "$TRIAL_NAME" ]]; then
-          echo "--> Trial name is: $TRIAL_NAME"
-          echo
-          echo "--> When completed, the following will be available:"
-          echo "build files: s3://nextstrain-staging/files/ncov/open/trial/$TRIAL_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/staging/ncov/open/trial/$TRIAL_NAME/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/staging/ncov/open/trial/$TRIAL_NAME/global/all-time"
-        else
-          echo "--> Open (GenBank) phylogenetic analysis rebuilding on AWS"
-          echo
-          echo "--> When completed, the following will be updated:"
-          echo "build files: s3://nextstrain-data/files/ncov/open/REGION_NAME/"
-          echo "nextstrain URLs: https://nextstrain.org/ncov/open/REGION_NAME/TIME_SPAN"
-          echo "e.g. https://nextstrain.org/ncov/open/global/all-time"
-        fi
-        echo
-        echo "--> You can attach to this AWS job via:"
-        tail -n1 build-launch.log
-        echo
-        JOBID=$( tail -n1 build-launch.log | sed -E 's/.+attach ([-a-f0-9]+).+/\1/' )
-        echo "--> View this job in the AWS console via"
-        echo "    https://console.aws.amazon.com/batch/home?region=us-east-1#jobs/detail/${JOBID}"
-        echo
+            --set-threads tree=8