Action inputs to dispatch n-runs of a single test in CI

.github/workflows/run-tests.yml

@@ -13,6 +13,54 @@ on:
       commit:
         description: "Commit SHA"
         required: true
+      run_single_functional_test:
+        description: "Run a single functional test"
+        type: boolean
+        default: false
+      run_single_unit_test:
+        description: "Run a single unit test (INSTEAD of functional test)"
+        type: boolean
+        default: false
+      unit_test_directory:
+        description: "[Unit Test Only] Directory to run unit tests in"
+        type: string
+        default: "./temporal"
+      n_runs:
+        description: "Number of times to repeat the single test per database type"
+        type: number
+        default: 1
+      test_name:
+        description: "Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
+        type: string
+      timeout_minutes:
+        description: "Test timeout in minutes"
+        type: number
+        default: 120
+      test_runner:
+        description: "Which runner to use. Choose higher RAM if your n_runs is high."
+        type: choice
+        default: "16GB RAM (ubuntu-20.04)"
+        options:
+          - "16GB RAM (ubuntu-20.04)"
+          - "64GB RAM (ubuntu-22.04)"
+      cassandra:
+        description: "cassandra"
+        type: boolean
+        default: false
+      sqlite:
+        description: "sqlite"
+        type: boolean
+        default: false
+      #      mysql8:
+      #        description: "mysql8"
+      #        type: boolean
+      #        default: false
+#      postgres12:
+#        description: "postgres12"
+#        type: boolean
+#        default: false
+          # if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi
+        # if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -29,6 +77,52 @@ env:
   BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
 
 jobs:
+  set-up-single-test:
+    name: Set up single test
+    runs-on: ubuntu-20.04
+    outputs:
+      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      total_shards: ${{ steps.generate_output.outputs.shards }}
+      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
+      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      runs_on: ${{ steps.generate_output.outputs.runs_on }}
+      dbs: ${{ steps.generate_output.outputs.dbs }}
+    steps:
+      - id: generate_output
+        run: |
+          shards=3
+          timeout=30
+          runs_on='["ubuntu-20.04"]'
+          if [[ "${{ inputs.run_single_functional_test }}" == "true" || "${{ inputs.run_single_unit_test }}" == "true" ]]; then
+            shards=1
+            timeout=${{ inputs.timeout_minutes }}
+            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
+              runs_on='[ "ubuntu-latest-16-cores" ]'
+            fi 
+            dbs_arr=()
+            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi
+            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi
+            if [[ ${#dbs_arr[@]} == 0 ]]; then
+              dbs='["sqlite"]'
+            else
+              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
+            fi
+          fi
+          {
+            echo "shard_indices=[ $(seq -s, 0 $((shards-1))) ]"
+            echo "shards=$shards"
+            echo "github_timeout=$((timeout+5))"
+            echo "test_timeout=${timeout}m"
+            echo "single_test_args=$single_test_args"
+            echo "runs_on=$runs_on"
+            echo "dbs=$dbs" 
+          } >> "$GITHUB_OUTPUT"
+      - id: cat_output
+        run: |
+          cat "$GITHUB_OUTPUT"
+
   misc-checks:
     name: Misc checks
     strategy:
@@ -60,13 +154,20 @@ jobs:
       - run: make build-tests
 
   unit-test:
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
-    needs: misc-checks
+    needs: [misc-checks, set-up-single-test]
     strategy:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}"
+      SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
+      UNIT_TEST_DIR: ${{ inputs.unit_test_directory }}
+      TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
+      RUN_SINGLE_UNIT_TEST: ${{ inputs.run_single_unit_test }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -78,22 +179,31 @@ jobs:
           go-version-file: 'go.mod'
           check-latest: true
 
-      - name: Run unit test
+      - name: Run unit tests
+        if: ${{ !cancelled() && !inputs.run_single_unit_test }}
         timeout-minutes: 15
         run: make unit-test-coverage
 
+      - name: Run single unit test
+        if: ${{ !cancelled() && inputs.run_single_unit_test }}
+        timeout-minutes: 15
+        run: UNIT_TEST_DIRS=$UNIT_TEST_DIR make unit-test-coverage
+
       - name: Upload test results
         if: ${{ !cancelled() }}
         run: make upload-test-results
 
   integration-test:
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Integration test
     needs: misc-checks
     strategy:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -123,21 +233,27 @@ jobs:
         if: ${{ !cancelled() }}
         run: make upload-test-results
 
+      - name: Tear down docker compose
+        if: ${{ always() }}
+        run: |
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
+
   functional-test:
+    if: ${{ inputs.run_single_unit_test != true }}
     name: Functional test
-    needs: misc-checks
+    needs: [misc-checks, set-up-single-test]
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [ubuntu-20.04]
+        runs-on: ${{ fromJson(needs.set-up-single-test.outputs.runs_on) }}
+        shard_index: ${{ fromJson(needs.set-up-single-test.outputs.shard_indices) }}
         name:
           - cass_es
           - cass_es8
           - sqlite
           - mysql8
           - postgres12
           - postgres12_pgx
-        shard_index: [0, 1, 2]
         include:
           - name: cass_es
             persistence_type: nosql
@@ -167,38 +283,52 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: 3
+      TEST_TOTAL_SHARDS: ${{ needs.set-up-single-test.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
+      TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
+      - name: Print functional test
+        run: echo "${{ needs.set-up-single-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
+      - uses: ScribeMD/[email protected]
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
       - uses: actions/checkout@v4
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ env.COMMIT }}
 
       - uses: actions/setup-go@v5
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           go-version-file: 'go.mod'
           check-latest: true
 
       - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' }}
+        if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         uses: hoverkraft-tech/[email protected]
         with:
           compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
           services: "${{ join(matrix.containers, '\n') }}"
           down-flags: -v
 
       - name: Run functional test
-        timeout-minutes: 30 # make sure this is larger than the test timeout in the Makefile
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         run: make upload-test-results
 
   functional-test-xdc:
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Functional test xdc
     needs: misc-checks
     strategy:
@@ -232,6 +362,7 @@ jobs:
     env:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -260,6 +391,7 @@ jobs:
         run: make upload-test-results
 
   functional-test-ndc:
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Functional test ndc
     needs: misc-checks
     strategy:
@@ -324,6 +456,7 @@ jobs:
         run: make functional-test-ndc-coverage
 
   test-status:
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Test Status
     needs:
       - unit-test
@@ -332,7 +465,6 @@ jobs:
       - functional-test-xdc
       - functional-test-ndc
     runs-on: ubuntu-20.04
-    if: always()
     env:
       RESULTS: ${{ toJSON(needs.*.result) }}
     steps:

Makefile

@@ -64,7 +64,7 @@ define NEWLINE
 
 endef
 
-TEST_TIMEOUT := 25m
+TEST_TIMEOUT ?= 25m
 
 PROTO_ROOT := proto
 PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto")
@@ -88,7 +88,7 @@ FUNCTIONAL_TEST_NDC_ROOT      := ./tests/ndc
 DB_INTEGRATION_TEST_ROOT      := ./common/persistence/tests
 DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests
 INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest ./internal/temporalite
-UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% ./internal/temporalite%,$(TEST_DIRS))
+UNIT_TEST_DIRS ?= $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% ./internal/temporalite%,$(TEST_DIRS))
 
 # github.com/urfave/cli/[email protected]             - needs to accept comma in values before unlocking https://github.com/urfave/cli/pull/1241.
 PINNED_DEPENDENCIES := \
@@ -377,8 +377,8 @@ prepare-coverage-test: $(GOTESTSUM) $(TEST_OUTPUT_ROOT)
 
 unit-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run unit tests with coverage..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) -coverprofile=$(NEW_COVER_PROFILE)
+	$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+		$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) $(SINGLE_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE)
 
 integration-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run integration tests with coverage..."
@@ -391,8 +391,10 @@ pre-build-functional-test-coverage: prepare-coverage-test
 
 functional-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
+	$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) \
+		 -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) \
+		 $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
 
 functional-test-xdc-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..."

develop/upload-test-results.sh

@@ -21,5 +21,6 @@ for file in *.junit.xml; do
     -F "run_env[branch]=${GITHUB_REF}" \
     -F "run_env[commit_sha]=${GITHUB_SHA}" \
     -F "run_env[url]=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+    -F "run_env[message]=${BUILDKITE_MESSAGE}" \
     https://analytics-api.buildkite.com/v1/uploads
-done
+done