From f5301c61702449ce257c957f5da490727f87e23b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Wed, 10 Jul 2024 12:59:46 -0700
Subject: [PATCH 01/76] create Run Test N-times gh action

---
 .github/workflows/run-test-n-times.yml | 141 +++++++++++++++++++++++++
 Makefile                               |   5 +
 2 files changed, 146 insertions(+)
 create mode 100644 .github/workflows/run-test-n-times.yml

diff --git a/.github/workflows/run-test-n-times.yml b/.github/workflows/run-test-n-times.yml
new file mode 100644
index 00000000000..f9c4fd301db
--- /dev/null
+++ b/.github/workflows/run-test-n-times.yml
@@ -0,0 +1,141 @@
+name: Run Test N-times
+
+on:
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: "Commit SHA"
+        required: true
+      n_runs:
+        description: "Number of times to repeat the test"
+        required: true
+      test_name:
+        description: "Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
+        required: true
+
+env:
+  COMMIT: ${{ github.event.inputs.commit || github.sha }}
+  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
+  TEMPORAL_VERSION_CHECK_DISABLED: 1
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
+
+jobs:
+  misc-checks:
+    name: Misc checks
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ env.COMMIT }}
+          submodules: true
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - uses: arduino/setup-protoc@v3
+
+      - run: make ci-build-misc
+
+      - run: make build-tests
+
+  cache-docker-images:
+    name: Cache Docker images
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: ScribeMD/docker-cache@0.3.7
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ env.COMMIT }}
+
+      - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
+
+  functional-test:
+    name: Functional test
+    needs: [misc-checks, cache-docker-images]
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+        name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
+        shard_index: [0, 1, 2]
+        include:
+          - name: cass_es
+            persistence_type: nosql
+            persistence_driver: cassandra
+            containers: [cassandra, elasticsearch]
+          - name: cass_es8
+            persistence_type: nosql
+            persistence_driver: cassandra
+            containers: [cassandra, elasticsearch8]
+          - name: sqlite
+            persistence_type: sql
+            persistence_driver: sqlite
+            containers: []
+          - name: mysql8
+            persistence_type: sql
+            persistence_driver: mysql8
+            containers: [mysql]
+          - name: postgres12
+            persistence_type: sql
+            persistence_driver: postgres12
+            containers: [postgresql]
+          - name: postgres12_pgx
+            persistence_type: sql
+            persistence_driver: postgres12_pgx
+            containers: [postgresql]
+    runs-on: ${{ matrix.runs-on }}
+    env:
+      TEST_TOTAL_SHARDS: 3
+      TEST_SHARD_INDEX: ${{ matrix.shard_index }}
+      PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
+      PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+    steps:
+      - uses: ScribeMD/docker-cache@0.3.7
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ env.COMMIT }}
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - name: Start containerized dependencies
+        if: ${{ toJson(matrix.containers) != '[]' }}
+        run: |
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
+
+      - name: Run functional test N times
+        timeout-minutes: 30 # make sure this is larger than the test timeout in the Makefile
+        run: make functional-test-n-times
+        env:
+          TEST_NAME: ${{ inputs.test_name }}
+          N_TEST_RUNS: ${{ inputs.n_runs }}
+
+      - name: Upload test results
+        if: ${{ !cancelled() }}
+        run: make upload-test-results
+
+      - name: Tear down docker compose
+        if: ${{ always() }}
+        run: |
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 249c18fdb6d..b50ec2d7035 100644
--- a/Makefile
+++ b/Makefile
@@ -414,6 +414,11 @@ coverage-report: $(SUMMARY_COVER_PROFILE)
 	@printf $(COLOR) "Generate HTML report from $(SUMMARY_COVER_PROFILE) to $(SUMMARY_COVER_PROFILE).html..."
 	@go tool cover -html=$(SUMMARY_COVER_PROFILE) -o $(SUMMARY_COVER_PROFILE).html
 
+functional-test-n-times: prepare-coverage-test
+	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_TEST_RUNS) times with $(PERSISTENCE_DRIVER) driver..."
+	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) -test.run $(TEST_NAME) -count $(N_TEST_RUNS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER)
+
 upload-test-results:
 	@(cd $(TEST_OUTPUT_ROOT) && sh $(ROOT)/develop/upload-test-results.sh)
 

From 377d713472fcdc3c61929199f39289b72a35d3f3 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Wed, 10 Jul 2024 13:07:25 -0700
Subject: [PATCH 02/76] change name of step

---
 .github/workflows/run-test-n-times.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-test-n-times.yml b/.github/workflows/run-test-n-times.yml
index f9c4fd301db..b82fca6c85f 100644
--- a/.github/workflows/run-test-n-times.yml
+++ b/.github/workflows/run-test-n-times.yml
@@ -64,8 +64,8 @@ jobs:
 
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
-  functional-test:
-    name: Functional test
+  functional-test-n-times:
+    name: Functional test n-times
     needs: [misc-checks, cache-docker-images]
     strategy:
       fail-fast: false

From c05a1a15cffe9d6aae34bf0bd3981ae477a46886 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Wed, 10 Jul 2024 23:58:31 -0700
Subject: [PATCH 03/76] make existing test workflow allow single test run

---
 .github/workflows/run-tests.yml | 34 ++++++++++++++++++++++++++++++---
 Makefile                        | 12 ++++++------
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b0fd72d44f9..199c0fc2df5 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -13,6 +13,18 @@ on:
       commit:
         description: "Commit SHA"
         required: true
+      run_single_functional_test:
+        description: "Whether to run a single test. If so, the rest of the input fields are required."
+        type: boolean
+        default: false
+      n_runs:
+        description: "Number of times to repeat the single test"
+        type: number
+        default: 1
+      test_name:
+        description: "Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
+        type: string
+
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -50,6 +62,17 @@ jobs:
 
       - run: make build-tests
 
+      - name: build functional test matrix
+        id: json
+        run: |
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then shard_index='{ "shard_index": [0] }'; else shard_index='{ "shard_index": [0, 1, 2] }'; fi
+          echo "shard_index=$shard_index" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then total_shards=1; else total_shards=3; fi
+          echo "total_shards=$total_shards" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"; fi
+          echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
+
+
   cache-docker-images:
     name: Cache Docker images
     strategy:
@@ -70,6 +93,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   unit-test:
+    if: ${{ inputs.run_single_functional_test }} == 'false'
     name: Unit test
     needs: misc-checks
     strategy:
@@ -97,6 +121,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
+    if: ${{ inputs.run_single_functional_test }} == 'false'
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -144,7 +169,7 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: [0, 1, 2]
+        shard_index: ${{ fromJson(needs.misc-checks.outputs.shard_index) }}
         include:
           - name: cass_es
             persistence_type: nosql
@@ -172,10 +197,11 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: 3
+      TEST_TOTAL_SHARDS: ${{ needs.misc-checks.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      SINGLE_TEST_ARGS: ${{ needs.misc-checks.outputs.single_test_args }}
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:
@@ -210,6 +236,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
+    if: ${{ inputs.run_single_functional_test }} == 'false'
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -277,6 +304,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
+    if: ${{ inputs.run_single_functional_test }} == 'false'
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -339,6 +367,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
+    if: ${{ inputs.run_single_functional_test }} == 'false'
     name: Test Status
     needs:
       - unit-test
@@ -347,7 +376,6 @@ jobs:
       - functional-test-xdc
       - functional-test-ndc
     runs-on: ubuntu-20.04
-    if: always()
     env:
       RESULTS: ${{ toJSON(needs.*.result) }}
     steps:
diff --git a/Makefile b/Makefile
index b50ec2d7035..c19a36634e2 100644
--- a/Makefile
+++ b/Makefile
@@ -385,7 +385,12 @@ pre-build-functional-test-coverage: prepare-coverage-test
 functional-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..."
 	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
+		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
+
+functional-test-n-times: prepare-coverage-test
+	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_TEST_RUNS) times with $(PERSISTENCE_DRIVER) driver..."
+	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) -test.run $(TEST_NAME) -count $(N_TEST_RUNS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER)
 
 functional-test-xdc-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..."
@@ -414,11 +419,6 @@ coverage-report: $(SUMMARY_COVER_PROFILE)
 	@printf $(COLOR) "Generate HTML report from $(SUMMARY_COVER_PROFILE) to $(SUMMARY_COVER_PROFILE).html..."
 	@go tool cover -html=$(SUMMARY_COVER_PROFILE) -o $(SUMMARY_COVER_PROFILE).html
 
-functional-test-n-times: prepare-coverage-test
-	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_TEST_RUNS) times with $(PERSISTENCE_DRIVER) driver..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) -test.run $(TEST_NAME) -count $(N_TEST_RUNS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER)
-
 upload-test-results:
 	@(cd $(TEST_OUTPUT_ROOT) && sh $(ROOT)/develop/upload-test-results.sh)
 

From 6fb13fdd38dda84295a6345f437ff70f10370f4c Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Thu, 11 Jul 2024 00:15:16 -0700
Subject: [PATCH 04/76] fix false check syntax

---
 .github/workflows/run-tests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 199c0fc2df5..3b7f4cb4a1b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -93,7 +93,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test }} == 'false'
+    if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -121,7 +121,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test }} == 'false'
+    if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -236,7 +236,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test }} == 'false'
+    if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -304,7 +304,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test }} == 'false'
+    if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -367,7 +367,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test }} == 'false'
+    if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Test Status
     needs:
       - unit-test

From 82acc5c25fdacd98901c5ed585afd2ade2590ee8 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Thu, 11 Jul 2024 00:28:43 -0700
Subject: [PATCH 05/76] don't run unit tests in single functional test mode

---
 .github/workflows/run-tests.yml | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 3b7f4cb4a1b..7e2f14f83b9 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -38,6 +38,7 @@ env:
 
 jobs:
   misc-checks:
+    if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -161,15 +162,33 @@ jobs:
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
+  set-up-functional-test:
+    name: Set up functional test
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - name: build functional test matrix
+        id: json
+        run: |
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then shard_index='{ "shard_index": [0] }'; else shard_index='{ "shard_index": [0, 1, 2] }'; fi
+          echo "shard_index=$shard_index" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then total_shards=1; else total_shards=3; fi
+          echo "total_shards=$total_shards" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"; fi
+          echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
+
   functional-test:
     name: Functional test
-    needs: [misc-checks, cache-docker-images]
+    needs: [misc-checks, cache-docker-images, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: ${{ fromJson(needs.misc-checks.outputs.shard_index) }}
+        shard_index: ${{ fromJson(steps.set-up-functional-test.outputs.shard_index) }}
         include:
           - name: cass_es
             persistence_type: nosql
@@ -197,11 +216,11 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: ${{ needs.misc-checks.outputs.total_shards }}
+      TEST_TOTAL_SHARDS: ${{ steps.set-up-functional-test.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      SINGLE_TEST_ARGS: ${{ needs.misc-checks.outputs.single_test_args }}
+      SINGLE_TEST_ARGS: ${{ steps.set-up-functional-test.outputs.single_test_args }}
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:

From 56b97cf6b1955bbb69c141dd0a4668cfa2fa71ae Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Thu, 11 Jul 2024 00:33:05 -0700
Subject: [PATCH 06/76] fix needs

---
 .github/workflows/run-tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 7e2f14f83b9..2196aee83f7 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -188,7 +188,7 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: ${{ fromJson(steps.set-up-functional-test.outputs.shard_index) }}
+        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
         include:
           - name: cass_es
             persistence_type: nosql
@@ -216,11 +216,11 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: ${{ steps.set-up-functional-test.outputs.total_shards }}
+      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      SINGLE_TEST_ARGS: ${{ steps.set-up-functional-test.outputs.single_test_args }}
+      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:

From 320ed3dcde3cd0be77bb7ca6a893082e1637d92c Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Thu, 11 Jul 2024 00:38:17 -0700
Subject: [PATCH 07/76] hardcode single test mode

---
 .github/workflows/run-tests.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 2196aee83f7..ebe546e83ec 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -188,7 +188,8 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+        #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+        shard_index: [0]
         include:
           - name: cass_es
             persistence_type: nosql
@@ -216,11 +217,13 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
+      #TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
+      TEST_TOTAL_SHARDS: 1
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      #SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      SINGLE_TEST_ARGS: -test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:

From 94a6f2c9ec290af95b0ca4a206a901328ccab41a Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Thu, 11 Jul 2024 00:45:55 -0700
Subject: [PATCH 08/76] run functional-test always()

---
 .github/workflows/run-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index ebe546e83ec..cbb8ddf5449 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -181,6 +181,7 @@ jobs:
           echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
 
   functional-test:
+    if: always()
     name: Functional test
     needs: [misc-checks, cache-docker-images, set-up-functional-test]
     strategy:

From 774403dc7a46803aa21d4a7d102713d6f1c6d228 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 12 Jul 2024 11:02:22 -0700
Subject: [PATCH 09/76] print what the makefile runs

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c19a36634e2..7fa3f9b92a2 100644
--- a/Makefile
+++ b/Makefile
@@ -384,11 +384,11 @@ pre-build-functional-test-coverage: prepare-coverage-test
 
 functional-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+	$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
 		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
 
 functional-test-n-times: prepare-coverage-test
-	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_TEST_RUNS) times with $(PERSISTENCE_DRIVER) driver..."
+	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_RUNS) times with $(PERSISTENCE_DRIVER) driver..."
 	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
 		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) -test.run $(TEST_NAME) -count $(N_TEST_RUNS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER)
 

From cd5160b330b2c63b282af62e890644584a006a52 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 12 Jul 2024 11:02:55 -0700
Subject: [PATCH 10/76] temporary code change to test flaky test

---
 service/history/worker_versioning_util.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/service/history/worker_versioning_util.go b/service/history/worker_versioning_util.go
index 7307da28766..827b7fe843f 100644
--- a/service/history/worker_versioning_util.go
+++ b/service/history/worker_versioning_util.go
@@ -26,6 +26,7 @@ package history
 
 import (
 	"context"
+	"go.temporal.io/server/common"
 
 	"go.temporal.io/api/serviceerror"
 	persistencespb "go.temporal.io/server/api/persistence/v1"
@@ -196,7 +197,7 @@ func initializeWorkflowAssignedBuildId(
 		return err
 	}
 
-	if mutableState.HasCompletedAnyWorkflowTask() {
+	if mutableState.HasCompletedAnyWorkflowTask() || workflowTask.StartedEventID != common.EmptyEventID {
 		// workflow has already completed a wft. buildId is stale and useless.
 		// workflow's assigned build ID should be already updated via RecordWorkflowTaskStarted
 		return nil

From f055929f056497a4fe4bb1f81723889cc0c2f12b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 12 Jul 2024 12:05:13 -0700
Subject: [PATCH 11/76] add db options

---
 .github/workflows/run-tests.yml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index cbb8ddf5449..d28a9bb2694 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -24,7 +24,17 @@ on:
       test_name:
         description: "Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
         type: string
-
+      test_db:
+        description: "Which database to run your test against"
+        default: "all"
+        type: choice
+        options:
+          - "cass_es"
+          - "cass_es8"
+          - "sqlite"
+          - "mysql8"
+          - "postgres12"
+          - "postgres12_pgx"
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -188,7 +198,8 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
+        #name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
+        name: [sqlite]
         #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
         shard_index: [0]
         include:

From 451f80c2d68e07523d9ab2b95ad9e08d4a433162 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 12 Jul 2024 17:10:12 -0700
Subject: [PATCH 12/76] hard code sqlite and postgres12

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index d28a9bb2694..6ce4c842886 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -199,7 +199,7 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         #name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        name: [sqlite]
+        name: [sqlite, postgres12]
         #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
         shard_index: [0]
         include:

From ceeaca42b304e99e905a0c3576ea9387e19c19b3 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 12 Jul 2024 17:15:12 -0700
Subject: [PATCH 13/76] delete comment below runs-on

---
 .github/workflows/run-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 6ce4c842886..4f6ebdd6683 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -198,8 +198,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        #name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        name: [sqlite, postgres12]
+        name: [sqlite]
         #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
         shard_index: [0]
         include:

From 6ddda0b806a4cdf448b250fd2806e43fded335f8 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 12 Jul 2024 18:25:31 -0700
Subject: [PATCH 14/76] fix db issue?

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 4f6ebdd6683..7b9b71ae24f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -198,7 +198,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name: [sqlite]
+        name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
         #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
         shard_index: [0]
         include:

From fc7ead3e4d7da37d5b82be66c1fb54790dafaf07 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 11:50:48 -0700
Subject: [PATCH 15/76] removed duplicate matrix step, updated options

---
 .github/workflows/run-tests.yml | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 7b9b71ae24f..891e0527b7e 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -18,15 +18,15 @@ on:
         type: boolean
         default: false
       n_runs:
-        description: "Number of times to repeat the single test"
+        description: "[Single Test Only] Number of times to repeat the single test"
         type: number
         default: 1
       test_name:
-        description: "Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
+        description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
         type: string
       test_db:
-        description: "Which database to run your test against"
-        default: "all"
+        description: "[Single Test Only] Which database to run your test against"
+        default: "sqlite"
         type: choice
         options:
           - "cass_es"
@@ -73,16 +73,6 @@ jobs:
 
       - run: make build-tests
 
-      - name: build functional test matrix
-        id: json
-        run: |
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then shard_index='{ "shard_index": [0] }'; else shard_index='{ "shard_index": [0, 1, 2] }'; fi
-          echo "shard_index=$shard_index" >> "$GITHUB_OUTPUT"
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then total_shards=1; else total_shards=3; fi
-          echo "total_shards=$total_shards" >> "$GITHUB_OUTPUT"
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"; fi
-          echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
-
 
   cache-docker-images:
     name: Cache Docker images

From ce29c6514a35dde530d20755daad3592b3b2bad9 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 15:36:38 -0700
Subject: [PATCH 16/76] define setup job outputs

---
 .github/workflows/run-tests.yml | 51 ++++++++++++++++++++++++---------
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 891e0527b7e..94de1b3bca8 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -75,6 +75,7 @@ jobs:
 
 
   cache-docker-images:
+    if: always()
     name: Cache Docker images
     strategy:
       fail-fast: false
@@ -162,21 +163,47 @@ jobs:
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
+#jobs:
+  job1:
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      output1: ${{ steps.step1.outputs.test }}
+      output2: ${{ steps.step2.outputs.test }}
+    steps:
+      - id: step1
+        run: echo "test=hello" >> "$GITHUB_OUTPUT"
+      - id: step2
+        run: echo "test=world" >> "$GITHUB_OUTPUT"
+  job2:
+    runs-on: ubuntu-latest
+    needs: job1
+    steps:
+      - env:
+          OUTPUT1: ${{needs.job1.outputs.output1}}
+          OUTPUT2: ${{needs.job1.outputs.output2}}
+        run: echo "$OUTPUT1 $OUTPUT2"
+
+
   set-up-functional-test:
+    if: always()
     name: Set up functional test
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-    runs-on: ${{ matrix.runs-on }}
+    runs-on: ubuntu-20.04
+    outputs:
+      shard_index: ${{ steps.shard_matrix_step.outputs.shard_index }}
+      total_shards: ${{ steps.total_shards_step.outputs.total_shards }}
+      single_test_args: ${{ steps.test_args_step.outputs.single_test_args }}
     steps:
-      - name: build functional test matrix
-        id: json
+      - id: shard_matrix_step
         run: |
           if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then shard_index='{ "shard_index": [0] }'; else shard_index='{ "shard_index": [0, 1, 2] }'; fi
           echo "shard_index=$shard_index" >> "$GITHUB_OUTPUT"
+      - id: total_shards_step
+        run: |  
           if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then total_shards=1; else total_shards=3; fi
           echo "total_shards=$total_shards" >> "$GITHUB_OUTPUT"
+      - id: test_args_step
+        run: |  
           if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"; fi
           echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
 
@@ -189,8 +216,7 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
-        shard_index: [0]
+        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
         include:
           - name: cass_es
             persistence_type: nosql
@@ -218,13 +244,12 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      #TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
-      TEST_TOTAL_SHARDS: 1
+      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      #SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
-      SINGLE_TEST_ARGS: -test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}
+      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      #SINGLE_TEST_ARGS: -test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:

From 8fbe6df9cbe5f8e4d20b6398e91f9d5c38eadafb Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 15:59:30 -0700
Subject: [PATCH 17/76] configurable timeout

---
 .github/workflows/run-tests.yml | 39 ++++++++++++++-------------------
 1 file changed, 16 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 94de1b3bca8..02ac9e1db3f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -24,6 +24,10 @@ on:
       test_name:
         description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
         type: string
+      timeout_minutes:
+        description: "[Single Test Only] When should the Github Action time out?"
+        type: number
+        default: 120
       test_db:
         description: "[Single Test Only] Which database to run your test against"
         default: "sqlite"
@@ -163,27 +167,6 @@ jobs:
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
-#jobs:
-  job1:
-    runs-on: ubuntu-latest
-    # Map a step output to a job output
-    outputs:
-      output1: ${{ steps.step1.outputs.test }}
-      output2: ${{ steps.step2.outputs.test }}
-    steps:
-      - id: step1
-        run: echo "test=hello" >> "$GITHUB_OUTPUT"
-      - id: step2
-        run: echo "test=world" >> "$GITHUB_OUTPUT"
-  job2:
-    runs-on: ubuntu-latest
-    needs: job1
-    steps:
-      - env:
-          OUTPUT1: ${{needs.job1.outputs.output1}}
-          OUTPUT2: ${{needs.job1.outputs.output2}}
-        run: echo "$OUTPUT1 $OUTPUT2"
-
 
   set-up-functional-test:
     if: always()
@@ -193,6 +176,7 @@ jobs:
       shard_index: ${{ steps.shard_matrix_step.outputs.shard_index }}
       total_shards: ${{ steps.total_shards_step.outputs.total_shards }}
       single_test_args: ${{ steps.test_args_step.outputs.single_test_args }}
+      functional_test_timeout: ${{ steps.test_args_step.outputs.timeout }}
     steps:
       - id: shard_matrix_step
         run: |
@@ -206,6 +190,10 @@ jobs:
         run: |  
           if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"; fi
           echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
+      - id: timeout_step
+        run: |
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then timeout="${{ inputs.timeout_minutes }}"; else timeout="30"; fi
+          echo "timeout=$timeout" >> "$GITHUB_OUTPUT" 
 
   functional-test:
     if: always()
@@ -216,7 +204,8 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+        #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+        shard_index: [0] # todo
         include:
           - name: cass_es
             persistence_type: nosql
@@ -249,8 +238,12 @@ jobs:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      FUNCTIONAL_TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.functional_test_timeout }}
       #SINGLE_TEST_ARGS: -test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}
     steps:
+      - name: Print things
+        run: echo "$TEST_TOTAL_SHARDS" "$SINGLE_TEST_ARGS" "$FUNCTIONAL_TEST_TIMEOUT"
+
       - uses: ScribeMD/docker-cache@0.3.7
         with:
           key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
@@ -271,7 +264,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
-        timeout-minutes: 30 # make sure this is larger than the test timeout in the Makefile
+        timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results

From bc768897f0090f3084bbca1771170da989f0ce8c Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:05:39 -0700
Subject: [PATCH 18/76] test string timeout

---
 .github/workflows/run-tests.yml | 104 +++++++++++++++++---------------
 1 file changed, 54 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 02ac9e1db3f..d23228c9ed7 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -51,6 +51,27 @@ env:
   BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
 
 jobs:
+
+  cache-docker-images:
+    if: ${{ inputs.run_single_functional_test == 'false' }} # todo: remove
+    name: Cache Docker images
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: ScribeMD/docker-cache@0.3.7
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ env.COMMIT }}
+
+      - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
+
   misc-checks:
     if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Misc checks
@@ -77,27 +98,6 @@ jobs:
 
       - run: make build-tests
 
-
-  cache-docker-images:
-    if: always()
-    name: Cache Docker images
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-    runs-on: ${{ matrix.runs-on }}
-    steps:
-      - uses: ScribeMD/docker-cache@0.3.7
-        with:
-          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          ref: ${{ env.COMMIT }}
-
-      - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
-
   unit-test:
     if: ${{ inputs.run_single_functional_test == 'false' }}
     name: Unit test
@@ -198,7 +198,8 @@ jobs:
   functional-test:
     if: always()
     name: Functional test
-    needs: [misc-checks, cache-docker-images, set-up-functional-test]
+    #needs: [misc-checks, cache-docker-images, set-up-functional-test]
+    needs: [misc-checks, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:
@@ -239,42 +240,45 @@ jobs:
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
       FUNCTIONAL_TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.functional_test_timeout }}
-      #SINGLE_TEST_ARGS: -test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}
     steps:
       - name: Print things
-        run: echo "$TEST_TOTAL_SHARDS" "$SINGLE_TEST_ARGS" "$FUNCTIONAL_TEST_TIMEOUT"
-
-      - uses: ScribeMD/docker-cache@0.3.7
-        with:
-          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          ref: ${{ env.COMMIT }}
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: 'go.mod'
-          check-latest: true
-
-      - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' }}
         run: |
-          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
+          echo "$TEST_TOTAL_SHARDS"
+          echo "$SINGLE_TEST_ARGS"
+          echo "$FUNCTIONAL_TEST_TIMEOUT"
+          echo ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+
+  #    - uses: ScribeMD/docker-cache@0.3.7
+  #      with:
+  #        key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
+  #    - uses: actions/checkout@v4
+  #      with:
+  #        token: ${{ secrets.GITHUB_TOKEN }}
+  #        ref: ${{ env.COMMIT }}
+
+  #    - uses: actions/setup-go@v5
+  #      with:
+  #        go-version-file: 'go.mod'
+  #        check-latest: true
+
+  #    - name: Start containerized dependencies
+  #      if: ${{ toJson(matrix.containers) != '[]' }}
+  #      run: |
+  #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
-        timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
+        timeout-minutes: ${{ env.FUNCTIONAL_TEST_TIMEOUT }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
-      - name: Upload test results
-        if: ${{ !cancelled() }}
-        run: make upload-test-results
+  #    - name: Upload test results
+  #      if: ${{ !cancelled() }}
+  #      run: make upload-test-results
 
-      - name: Tear down docker compose
-        if: ${{ always() }}
-        run: |
-          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
+  #    - name: Tear down docker compose
+  #      if: ${{ always() }}
+  #      run: |
+  #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
     if: ${{ inputs.run_single_functional_test == 'false' }}

From 69a0a3777be3af40eac46f28351bd57941b159a0 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:06:50 -0700
Subject: [PATCH 19/76] fix timeout pipe

---
 .github/workflows/run-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index d23228c9ed7..c108e695317 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -176,7 +176,7 @@ jobs:
       shard_index: ${{ steps.shard_matrix_step.outputs.shard_index }}
       total_shards: ${{ steps.total_shards_step.outputs.total_shards }}
       single_test_args: ${{ steps.test_args_step.outputs.single_test_args }}
-      functional_test_timeout: ${{ steps.test_args_step.outputs.timeout }}
+      functional_test_timeout: ${{ steps.timeout_step.outputs.timeout }}
     steps:
       - id: shard_matrix_step
         run: |
@@ -268,7 +268,7 @@ jobs:
   #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
-        timeout-minutes: ${{ env.FUNCTIONAL_TEST_TIMEOUT }} # make sure this is larger than the test timeout in the Makefile
+        timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
   #    - name: Upload test results

From ee8e47642aeff67eeac26ec7abcb7f60aca5a1c1 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:10:51 -0700
Subject: [PATCH 20/76] try dynamic shard matrix

---
 .github/workflows/run-tests.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index c108e695317..e727f9d6844 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -205,8 +205,8 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        #shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
-        shard_index: [0] # todo
+        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+        #shard_index: [0] # todo
         include:
           - name: cass_es
             persistence_type: nosql
@@ -244,9 +244,12 @@ jobs:
       - name: Print things
         run: |
           echo "$TEST_TOTAL_SHARDS"
+          echo "$TEST_SHARD_INDEX"
+          echo "$PERSISTENCE_TYPE"
+          echo "$PERSISTENCE_DRIVER"
           echo "$SINGLE_TEST_ARGS"
           echo "$FUNCTIONAL_TEST_TIMEOUT"
-          echo ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+          
 
   #    - uses: ScribeMD/docker-cache@0.3.7
   #      with:
@@ -267,9 +270,9 @@ jobs:
   #      run: |
   #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
-      - name: Run functional test
-        timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
-        run: make functional-test-coverage
+  #    - name: Run functional test
+  #      timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
+  #      run: make functional-test-coverage
 
   #    - name: Upload test results
   #      if: ${{ !cancelled() }}

From be78354f30833dbc281be4022cdb0eec48919a2d Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:17:07 -0700
Subject: [PATCH 21/76] try other dynamic matrix format

---
 .github/workflows/run-tests.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index e727f9d6844..0e1cd2a1640 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -173,15 +173,17 @@ jobs:
     name: Set up functional test
     runs-on: ubuntu-20.04
     outputs:
-      shard_index: ${{ steps.shard_matrix_step.outputs.shard_index }}
+      shard_indices: ${{ steps.shard_matrix_step.outputs.shard_indices }}
       total_shards: ${{ steps.total_shards_step.outputs.total_shards }}
       single_test_args: ${{ steps.test_args_step.outputs.single_test_args }}
       functional_test_timeout: ${{ steps.timeout_step.outputs.timeout }}
     steps:
       - id: shard_matrix_step
         run: |
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then shard_index='{ "shard_index": [0] }'; else shard_index='{ "shard_index": [0, 1, 2] }'; fi
-          echo "shard_index=$shard_index" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
+            then echo 'shard_indices=[0]' >> "$GITHUB_OUTPUT";
+            else echo 'shard_indices=[0, 1, 2]' >> "$GITHUB_OUTPUT";
+          fi
       - id: total_shards_step
         run: |  
           if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then total_shards=1; else total_shards=3; fi
@@ -205,7 +207,7 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
         name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_index) }}
+        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
         #shard_index: [0] # todo
         include:
           - name: cass_es

From 0a9d4730fea4a3951f0678ad31fbc8e9150098f7 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:25:00 -0700
Subject: [PATCH 22/76] add db matrix

---
 .github/workflows/run-tests.yml | 31 +++++++++++++++++++++----------
 1 file changed, 21 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 0e1cd2a1640..88c6c0c3b40 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -173,29 +173,41 @@ jobs:
     name: Set up functional test
     runs-on: ubuntu-20.04
     outputs:
-      shard_indices: ${{ steps.shard_matrix_step.outputs.shard_indices }}
+      shard_indices: ${{ steps.shard_indices_step.outputs.shard_indices }}
+      db_names: ${{ steps.db_names_step.outputs.db_names }}
       total_shards: ${{ steps.total_shards_step.outputs.total_shards }}
       single_test_args: ${{ steps.test_args_step.outputs.single_test_args }}
       functional_test_timeout: ${{ steps.timeout_step.outputs.timeout }}
     steps:
-      - id: shard_matrix_step
+      - id: shard_indices_step
         run: |
           if [ ${{ inputs.run_single_functional_test }} = 'true' ];
             then echo 'shard_indices=[0]' >> "$GITHUB_OUTPUT";
             else echo 'shard_indices=[0, 1, 2]' >> "$GITHUB_OUTPUT";
           fi
+      - id: db_names_step
+        run: |
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
+            then echo 'db_names=[${{ inputs.test_db }}]' >> "$GITHUB_OUTPUT";
+            else echo 'db_names=[cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]' >> "$GITHUB_OUTPUT";
+          fi
       - id: total_shards_step
         run: |  
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then total_shards=1; else total_shards=3; fi
-          echo "total_shards=$total_shards" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
+            then echo 'total_shards=1' >> "$GITHUB_OUTPUT";
+            else echo 'total_shards=3' >> "$GITHUB_OUTPUT";
+          fi
       - id: test_args_step
         run: |  
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"; fi
-          echo "single_test_args=$single_test_args" >> "$GITHUB_OUTPUT"
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
+            then echo 'single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"' >> $GITHUB_OUTPUT";
+          fi
       - id: timeout_step
         run: |
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ]; then timeout="${{ inputs.timeout_minutes }}"; else timeout="30"; fi
-          echo "timeout=$timeout" >> "$GITHUB_OUTPUT" 
+          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
+            then echo 'timeout="${{ inputs.timeout_minutes }}"' >> $GITHUB_OUTPUT;
+            else echo 'timeout="30"' >> $GITHUB_OUTPUT;
+          fi
 
   functional-test:
     if: always()
@@ -206,9 +218,8 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
+        name: ${{ fromJson(needs.set-up-functional-test.outputs.db_names) }}
         shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
-        #shard_index: [0] # todo
         include:
           - name: cass_es
             persistence_type: nosql

From f6b52eaef9ea662dca1b337d8826b9a70bbf404e Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:31:04 -0700
Subject: [PATCH 23/76] fix misuse of single quotes

---
 .github/workflows/run-tests.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 88c6c0c3b40..7bb24cbb295 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -181,32 +181,32 @@ jobs:
     steps:
       - id: shard_indices_step
         run: |
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
-            then echo 'shard_indices=[0]' >> "$GITHUB_OUTPUT";
-            else echo 'shard_indices=[0, 1, 2]' >> "$GITHUB_OUTPUT";
+          if [ ${{ inputs.run_single_functional_test }} = "true" ];
+            then echo "shard_indices=[0]" >> "$GITHUB_OUTPUT";
+            else echo "shard_indices=[0, 1, 2]" >> "$GITHUB_OUTPUT";
           fi
       - id: db_names_step
         run: |
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
-            then echo 'db_names=[${{ inputs.test_db }}]' >> "$GITHUB_OUTPUT";
-            else echo 'db_names=[cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]' >> "$GITHUB_OUTPUT";
+          if [ ${{ inputs.run_single_functional_test }} = "true" ];
+            then echo "db_names=[${{ inputs.test_db }}]" >> "$GITHUB_OUTPUT";
+            else echo "db_names=[cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]" >> "$GITHUB_OUTPUT";
           fi
       - id: total_shards_step
         run: |  
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
-            then echo 'total_shards=1' >> "$GITHUB_OUTPUT";
-            else echo 'total_shards=3' >> "$GITHUB_OUTPUT";
+          if [ ${{ inputs.run_single_functional_test }} = "true" ];
+            then echo "total_shards=1" >> "$GITHUB_OUTPUT";
+            else echo "total_shards=3" >> "$GITHUB_OUTPUT";
           fi
       - id: test_args_step
         run: |  
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
-            then echo 'single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"' >> $GITHUB_OUTPUT";
+          if [ ${{ inputs.run_single_functional_test }} = "true" ];
+            then echo "single_test_args=\"-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}\" >> $GITHUB_OUTPUT";
           fi
       - id: timeout_step
         run: |
-          if [ ${{ inputs.run_single_functional_test }} = 'true' ];
-            then echo 'timeout="${{ inputs.timeout_minutes }}"' >> $GITHUB_OUTPUT;
-            else echo 'timeout="30"' >> $GITHUB_OUTPUT;
+          if [ ${{ inputs.run_single_functional_test }} = "true" ];
+            then echo "timeout=${{ inputs.timeout_minutes }}" >> $GITHUB_OUTPUT;
+            else echo "timeout=30" >> $GITHUB_OUTPUT;
           fi
 
   functional-test:

From 5647d443007b601d01d9789dd20d9dad36b961b0 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:33:19 -0700
Subject: [PATCH 24/76] hard-code sqlite for single test mode

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 7bb24cbb295..8cfce0ca2d1 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -188,7 +188,7 @@ jobs:
       - id: db_names_step
         run: |
           if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "db_names=[${{ inputs.test_db }}]" >> "$GITHUB_OUTPUT";
+            then echo "db_names=[sqlite]" >> "$GITHUB_OUTPUT";
             else echo "db_names=[cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]" >> "$GITHUB_OUTPUT";
           fi
       - id: total_shards_step

From dd629bc5e612540f96961bbda49e7efda4d2f4a9 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 17:35:40 -0700
Subject: [PATCH 25/76] quote the db names

---
 .github/workflows/run-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 8cfce0ca2d1..32f6e361c4e 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -188,8 +188,8 @@ jobs:
       - id: db_names_step
         run: |
           if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "db_names=[sqlite]" >> "$GITHUB_OUTPUT";
-            else echo "db_names=[cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]" >> "$GITHUB_OUTPUT";
+            then echo "db_names=[\"sqlite\"]" >> "$GITHUB_OUTPUT";
+            else echo "db_names=[\"cass_es\", \"cass_es8\", \"sqlite\", \"mysql8\", \"postgres12\", \"postgres12_pgx\"]" >> "$GITHUB_OUTPUT";
           fi
       - id: total_shards_step
         run: |  

From 9a78b44cc3f6ade8a16ea4132a800164f4ec562b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 18:32:14 -0700
Subject: [PATCH 26/76] remove comment above runs-on and dynamically config
 test_db

---
 .github/workflows/run-tests.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 32f6e361c4e..c14aaceac04 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -188,7 +188,7 @@ jobs:
       - id: db_names_step
         run: |
           if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "db_names=[\"sqlite\"]" >> "$GITHUB_OUTPUT";
+            then echo "db_names=[\"${{ inputs.test_db }}\"]" >> "$GITHUB_OUTPUT";
             else echo "db_names=[\"cass_es\", \"cass_es8\", \"sqlite\", \"mysql8\", \"postgres12\", \"postgres12_pgx\"]" >> "$GITHUB_OUTPUT";
           fi
       - id: total_shards_step
@@ -212,8 +212,7 @@ jobs:
   functional-test:
     if: always()
     name: Functional test
-    #needs: [misc-checks, cache-docker-images, set-up-functional-test]
-    needs: [misc-checks, set-up-functional-test]
+    needs: [misc-checks, cache-docker-images, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:

From 7292977d91ee817ea1cb8fa5dd8baa784845a52c Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 18:44:56 -0700
Subject: [PATCH 27/76] change job if condition so steps run when needed

---
 .github/workflows/run-tests.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index c14aaceac04..ecc78fd7a95 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -53,7 +53,7 @@ env:
 jobs:
 
   cache-docker-images:
-    if: ${{ inputs.run_single_functional_test == 'false' }} # todo: remove
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Cache Docker images
     strategy:
       fail-fast: false
@@ -73,7 +73,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   misc-checks:
-    if: ${{ inputs.run_single_functional_test == 'false' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -99,7 +99,7 @@ jobs:
       - run: make build-tests
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test == 'false' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -127,7 +127,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test == 'false' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -296,7 +296,7 @@ jobs:
   #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test == 'false' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -364,7 +364,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test == 'false' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -427,7 +427,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test == 'false' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Test Status
     needs:
       - unit-test

From 189b6a999ee9a628fd48c2b5fc26fa983181eea6 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:12:47 -0700
Subject: [PATCH 28/76] fix lint and clean code

---
 .github/workflows/run-tests.yml | 53 +++++++++++++--------------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 603978aeeca..4b30e7010f1 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -182,41 +182,30 @@ jobs:
     name: Set up functional test
     runs-on: ubuntu-20.04
     outputs:
-      shard_indices: ${{ steps.shard_indices_step.outputs.shard_indices }}
-      db_names: ${{ steps.db_names_step.outputs.db_names }}
-      total_shards: ${{ steps.total_shards_step.outputs.total_shards }}
-      single_test_args: ${{ steps.test_args_step.outputs.single_test_args }}
-      functional_test_timeout: ${{ steps.timeout_step.outputs.timeout }}
+      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      db_names: ${{ steps.generate_output.outputs.db_names }}
+      total_shards: ${{ steps.generate_output.outputs.shards }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      functional_test_timeout: ${{ steps.generate_output.outputs.timeout }}
     steps:
-      - id: shard_indices_step
+      - id: generate_output
         run: |
-          if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "shard_indices=[0]" >> "$GITHUB_OUTPUT";
-            else echo "shard_indices=[0, 1, 2]" >> "$GITHUB_OUTPUT";
-          fi
-      - id: db_names_step
-        run: |
-          if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "db_names=[\"${{ inputs.test_db }}\"]" >> "$GITHUB_OUTPUT";
-            else echo "db_names=[\"cass_es\", \"sqlite\", \"mysql8\", \"postgres12\", \"postgres12_pgx\"]" >> "$GITHUB_OUTPUT";
-          fi
-      - id: total_shards_step
-        run: |  
-          if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "total_shards=1" >> "$GITHUB_OUTPUT";
-            else echo "total_shards=3" >> "$GITHUB_OUTPUT";
-          fi
-      - id: test_args_step
-        run: |  
-          if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "single_test_args=\"-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}\" >> $GITHUB_OUTPUT";
-          fi
-      - id: timeout_step
-        run: |
-          if [ ${{ inputs.run_single_functional_test }} = "true" ];
-            then echo "timeout=${{ inputs.timeout_minutes }}" >> $GITHUB_OUTPUT;
-            else echo "timeout=30" >> $GITHUB_OUTPUT;
+          shards=3
+          db_names="['cass_es', 'sqlite', 'mysql8', 'postgres12', 'postgres12_pgx']"
+          timeout=30
+          if [[ ${{ inputs.run_single_functional_test }} = true ]]; then
+            shards=1
+            db_names="['sqlite']"
+            timeout=30
+            single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
           fi
+          {
+            echo "shard_indices={\"shard_indices\":[$(seq -s, 0 $((shards-1)))]}"
+            echo "shards=$shards"
+            echo "db_names=$db_names"
+            echo "timeout=$timeout"
+            echo "single_test_args=$single_test_args" 
+          } >> "$GITHUB_OUTPUT"
 
   functional-test:
     if: always()

From 241cf332b0e47fa9f0b5e8be7e68ac6d78ca8416 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:26:08 -0700
Subject: [PATCH 29/76] remove db configuration

---
 .github/workflows/run-tests.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 4b30e7010f1..8816604f7d8 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -195,7 +195,6 @@ jobs:
           timeout=30
           if [[ ${{ inputs.run_single_functional_test }} = true ]]; then
             shards=1
-            db_names="['sqlite']"
             timeout=30
             single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
           fi

From 9c1f858cfbb90e4d7883e8ca265652432a06ed83 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:30:42 -0700
Subject: [PATCH 30/76] change if condition syntax and pipe in timeout

---
 .github/workflows/run-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 8816604f7d8..f73227a2256 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -193,9 +193,9 @@ jobs:
           shards=3
           db_names="['cass_es', 'sqlite', 'mysql8', 'postgres12', 'postgres12_pgx']"
           timeout=30
-          if [[ ${{ inputs.run_single_functional_test }} = true ]]; then
+          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
-            timeout=30
+            timeout=${{ inputs.timeout_minutes }}
             single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
           fi
           {

From f4f8752d9847ecf979647b51cbc8540ddbbe7b5b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:41:01 -0700
Subject: [PATCH 31/76] give up on configuring test db

---
 .github/workflows/run-tests.yml | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index f73227a2256..d29d6e7c14f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -28,17 +28,6 @@ on:
         description: "[Single Test Only] When should the Github Action time out?"
         type: number
         default: 120
-      test_db:
-        description: "[Single Test Only] Which database to run your test against"
-        default: "sqlite"
-        type: choice
-        options:
-          - "cass_es"
-          - "cass_es8"
-          - "sqlite"
-          - "mysql8"
-          - "postgres12"
-          - "postgres12_pgx"
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -183,15 +172,13 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
-      db_names: ${{ steps.generate_output.outputs.db_names }}
       total_shards: ${{ steps.generate_output.outputs.shards }}
-      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
       functional_test_timeout: ${{ steps.generate_output.outputs.timeout }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
     steps:
       - id: generate_output
         run: |
           shards=3
-          db_names="['cass_es', 'sqlite', 'mysql8', 'postgres12', 'postgres12_pgx']"
           timeout=30
           if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
@@ -214,7 +201,13 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name: ${{ fromJson(needs.set-up-functional-test.outputs.db_names) }}
+        name:
+          - cass_es
+          # - cass_es8
+          - sqlite
+          - mysql8
+          - postgres12
+          - postgres12_pgx
         shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
         include:
           - name: cass_es

From b29a900f3578bf9772e09cba48ab6389f02d4251 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:41:35 -0700
Subject: [PATCH 32/76] specify # of times per db

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index d29d6e7c14f..450d41bd3ad 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -18,7 +18,7 @@ on:
         type: boolean
         default: false
       n_runs:
-        description: "[Single Test Only] Number of times to repeat the single test"
+        description: "[Single Test Only] Number of times to repeat the single test per database"
         type: number
         default: 1
       test_name:

From 2eacb72b9c3765fb4a4d0b90c5673d72df62dece Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:48:30 -0700
Subject: [PATCH 33/76] Revert "specify # of times per db"

This reverts commit b29a900f3578bf9772e09cba48ab6389f02d4251.
---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 450d41bd3ad..d29d6e7c14f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -18,7 +18,7 @@ on:
         type: boolean
         default: false
       n_runs:
-        description: "[Single Test Only] Number of times to repeat the single test per database"
+        description: "[Single Test Only] Number of times to repeat the single test"
         type: number
         default: 1
       test_name:

From 8d05095d3de6321e873078d0728c2daafac5168e Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:48:32 -0700
Subject: [PATCH 34/76] Revert "give up on configuring test db"

This reverts commit f4f8752d9847ecf979647b51cbc8540ddbbe7b5b.
---
 .github/workflows/run-tests.yml | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index d29d6e7c14f..f73227a2256 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -28,6 +28,17 @@ on:
         description: "[Single Test Only] When should the Github Action time out?"
         type: number
         default: 120
+      test_db:
+        description: "[Single Test Only] Which database to run your test against"
+        default: "sqlite"
+        type: choice
+        options:
+          - "cass_es"
+          - "cass_es8"
+          - "sqlite"
+          - "mysql8"
+          - "postgres12"
+          - "postgres12_pgx"
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -172,13 +183,15 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      db_names: ${{ steps.generate_output.outputs.db_names }}
       total_shards: ${{ steps.generate_output.outputs.shards }}
-      functional_test_timeout: ${{ steps.generate_output.outputs.timeout }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      functional_test_timeout: ${{ steps.generate_output.outputs.timeout }}
     steps:
       - id: generate_output
         run: |
           shards=3
+          db_names="['cass_es', 'sqlite', 'mysql8', 'postgres12', 'postgres12_pgx']"
           timeout=30
           if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
@@ -201,13 +214,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name:
-          - cass_es
-          # - cass_es8
-          - sqlite
-          - mysql8
-          - postgres12
-          - postgres12_pgx
+        name: ${{ fromJson(needs.set-up-functional-test.outputs.db_names) }}
         shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
         include:
           - name: cass_es

From 537294b449aa6faa0bc962e5f3e39e71bbc135ae Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:54:17 -0700
Subject: [PATCH 35/76] try db config again with new json formatting

---
 .github/workflows/run-tests.yml | 19 +++++++++----------
 Makefile                        |  4 +++-
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index f73227a2256..21499b94a6e 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -57,7 +57,7 @@ env:
 jobs:
 
   cache-docker-images:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Cache Docker images
     strategy:
       fail-fast: false
@@ -77,7 +77,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   misc-checks:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -108,7 +108,7 @@ jobs:
       - run: make build-tests
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -136,7 +136,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -178,7 +178,6 @@ jobs:
 
 
   set-up-functional-test:
-    if: always()
     name: Set up functional test
     runs-on: ubuntu-20.04
     outputs:
@@ -191,10 +190,11 @@ jobs:
       - id: generate_output
         run: |
           shards=3
-          db_names="['cass_es', 'sqlite', 'mysql8', 'postgres12', 'postgres12_pgx']"
+          db_names='["cass_es", "sqlite", "mysql8", "postgres12", "postgres12_pgx"]'
           timeout=30
           if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
+            db_names='["cass_es", "sqlite"]'
             timeout=${{ inputs.timeout_minutes }}
             single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
           fi
@@ -207,7 +207,6 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
   functional-test:
-    if: always()
     name: Functional test
     needs: [misc-checks, cache-docker-images, set-up-functional-test]
     strategy:
@@ -293,7 +292,7 @@ jobs:
   #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -361,7 +360,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -429,7 +428,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Test Status
     needs:
       - unit-test
diff --git a/Makefile b/Makefile
index 2cb2b5dcab5..7b3feb3e054 100644
--- a/Makefile
+++ b/Makefile
@@ -387,7 +387,9 @@ pre-build-functional-test-coverage: prepare-coverage-test
 functional-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..."
 	$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
+		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) $(SINGLE_TEST_ARGS) $(TEST_TAG_FLAG) \
+		 -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) \
+		 $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
 
 functional-test-n-times: prepare-coverage-test
 	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_RUNS) times with $(PERSISTENCE_DRIVER) driver..."

From e9d8f11d464f932da69a6dd2814670847b7a2cbf Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 19:58:11 -0700
Subject: [PATCH 36/76] go back tp unquoted != true

---
 .github/workflows/run-tests.yml | 52 ++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 21499b94a6e..617da159b03 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -56,28 +56,8 @@ env:
 
 jobs:
 
-  cache-docker-images:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
-    name: Cache Docker images
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-    runs-on: ${{ matrix.runs-on }}
-    steps:
-      - uses: ScribeMD/docker-cache@0.3.7
-        with:
-          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          ref: ${{ env.COMMIT }}
-
-      - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
-
   misc-checks:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -107,8 +87,28 @@ jobs:
 
       - run: make build-tests
 
+  cache-docker-images:
+    if: ${{ inputs.run_single_functional_test != true }}
+    name: Cache Docker images
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+      - uses: ScribeMD/docker-cache@0.3.7
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ env.COMMIT }}
+
+      - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
+
   unit-test:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -136,7 +136,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -292,7 +292,7 @@ jobs:
   #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -360,7 +360,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -428,7 +428,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Test Status
     needs:
       - unit-test

From c33c0a45e1241b29adb3b686bb37709912d285df Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 20:33:09 -0700
Subject: [PATCH 37/76] run functional-test always

---
 .github/workflows/run-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 617da159b03..b0ad76e672b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -207,6 +207,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
   functional-test:
+    if: always()
     name: Functional test
     needs: [misc-checks, cache-docker-images, set-up-functional-test]
     strategy:

From 56e19ef6c68bbbf606976e5768472465bc5d84c3 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 21:54:13 -0700
Subject: [PATCH 38/76] give up on one-db-at-a-time

---
 .github/workflows/run-tests.yml | 40 ++++++++++-----------------------
 1 file changed, 12 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b0ad76e672b..18a33739860 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -18,27 +18,16 @@ on:
         type: boolean
         default: false
       n_runs:
-        description: "[Single Test Only] Number of times to repeat the single test"
+        description: "[Single Test Only] Number of times to repeat the single test per database type"
         type: number
         default: 1
       test_name:
         description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
         type: string
       timeout_minutes:
-        description: "[Single Test Only] When should the Github Action time out?"
+        description: "[Single Test Only] Github Action timeout in minutes"
         type: number
         default: 120
-      test_db:
-        description: "[Single Test Only] Which database to run your test against"
-        default: "sqlite"
-        type: choice
-        options:
-          - "cass_es"
-          - "cass_es8"
-          - "sqlite"
-          - "mysql8"
-          - "postgres12"
-          - "postgres12_pgx"
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -88,7 +77,6 @@ jobs:
       - run: make build-tests
 
   cache-docker-images:
-    if: ${{ inputs.run_single_functional_test != true }}
     name: Cache Docker images
     strategy:
       fail-fast: false
@@ -182,7 +170,6 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
-      db_names: ${{ steps.generate_output.outputs.db_names }}
       total_shards: ${{ steps.generate_output.outputs.shards }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
       functional_test_timeout: ${{ steps.generate_output.outputs.timeout }}
@@ -190,32 +177,34 @@ jobs:
       - id: generate_output
         run: |
           shards=3
-          db_names='["cass_es", "sqlite", "mysql8", "postgres12", "postgres12_pgx"]'
           timeout=30
           if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
-            db_names='["cass_es", "sqlite"]'
             timeout=${{ inputs.timeout_minutes }}
             single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
           fi
           {
-            echo "shard_indices={\"shard_indices\":[$(seq -s, 0 $((shards-1)))]}"
+            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
             echo "shards=$shards"
-            echo "db_names=$db_names"
             echo "timeout=$timeout"
             echo "single_test_args=$single_test_args" 
           } >> "$GITHUB_OUTPUT"
 
   functional-test:
-    if: always()
     name: Functional test
-    needs: [misc-checks, cache-docker-images, set-up-functional-test]
+    needs: [cache-docker-images, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name: ${{ fromJson(needs.set-up-functional-test.outputs.db_names) }}
         shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
+        name:
+          - cass_es
+          # - cass_es8
+          - sqlite
+          - mysql8
+          - postgres12
+          - postgres12_pgx
         include:
           - name: cass_es
             persistence_type: nosql
@@ -252,12 +241,7 @@ jobs:
     steps:
       - name: Print things
         run: |
-          echo "$TEST_TOTAL_SHARDS"
-          echo "$TEST_SHARD_INDEX"
-          echo "$PERSISTENCE_TYPE"
-          echo "$PERSISTENCE_DRIVER"
-          echo "$SINGLE_TEST_ARGS"
-          echo "$FUNCTIONAL_TEST_TIMEOUT"
+          echo "${TEST_TOTAL_SHARDS} | ${TEST_SHARD_INDEX} | ${PERSISTENCE_TYPE} | ${PERSISTENCE_DRIVER} | ${SINGLE_TEST_ARGS} | ${FUNCTIONAL_TEST_TIMEOUT}"
           
 
   #    - uses: ScribeMD/docker-cache@0.3.7

From 8278452be5b077b2a0290b978f33eb7f58e72dde Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 22:01:17 -0700
Subject: [PATCH 39/76] uncomment real functional test work

---
 .github/workflows/run-tests.yml | 65 +++++++++++++++------------------
 1 file changed, 30 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 18a33739860..b20604c99cd 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -239,42 +239,37 @@ jobs:
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
       FUNCTIONAL_TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.functional_test_timeout }}
     steps:
-      - name: Print things
+      - uses: ScribeMD/docker-cache@0.3.7
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ env.COMMIT }}
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - name: Start containerized dependencies
+        if: ${{ toJson(matrix.containers) != '[]' }}
         run: |
-          echo "${TEST_TOTAL_SHARDS} | ${TEST_SHARD_INDEX} | ${PERSISTENCE_TYPE} | ${PERSISTENCE_DRIVER} | ${SINGLE_TEST_ARGS} | ${FUNCTIONAL_TEST_TIMEOUT}"
-          
-
-  #    - uses: ScribeMD/docker-cache@0.3.7
-  #      with:
-  #        key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
-
-  #    - uses: actions/checkout@v4
-  #      with:
-  #        token: ${{ secrets.GITHUB_TOKEN }}
-  #        ref: ${{ env.COMMIT }}
-
-  #    - uses: actions/setup-go@v5
-  #      with:
-  #        go-version-file: 'go.mod'
-  #        check-latest: true
-
-  #    - name: Start containerized dependencies
-  #      if: ${{ toJson(matrix.containers) != '[]' }}
-  #      run: |
-  #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
-
-  #    - name: Run functional test
-  #      timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
-  #      run: make functional-test-coverage
-
-  #    - name: Upload test results
-  #      if: ${{ !cancelled() }}
-  #      run: make upload-test-results
-
-  #    - name: Tear down docker compose
-  #      if: ${{ always() }}
-  #      run: |
-  #        docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
+
+      - name: Run functional test
+        timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
+        run: make functional-test-coverage
+
+      - name: Upload test results
+        if: ${{ !cancelled() }}
+        run: make upload-test-results
+
+      - name: Tear down docker compose
+        if: ${{ always() }}
+        run: |
+          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
     if: ${{ inputs.run_single_functional_test != true }}

From 1ae8df090ccebdf8d3342a499bfc342c2702575a Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 15 Jul 2024 22:06:29 -0700
Subject: [PATCH 40/76] clean up extra files

---
 .github/workflows/run-test-n-times.yml    | 141 ----------------------
 service/history/worker_versioning_util.go |   3 +-
 2 files changed, 1 insertion(+), 143 deletions(-)
 delete mode 100644 .github/workflows/run-test-n-times.yml

diff --git a/.github/workflows/run-test-n-times.yml b/.github/workflows/run-test-n-times.yml
deleted file mode 100644
index b82fca6c85f..00000000000
--- a/.github/workflows/run-test-n-times.yml
+++ /dev/null
@@ -1,141 +0,0 @@
-name: Run Test N-times
-
-on:
-  workflow_dispatch:
-    inputs:
-      commit:
-        description: "Commit SHA"
-        required: true
-      n_runs:
-        description: "Number of times to repeat the test"
-        required: true
-      test_name:
-        description: "Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
-        required: true
-
-env:
-  COMMIT: ${{ github.event.inputs.commit || github.sha }}
-  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
-  TEMPORAL_VERSION_CHECK_DISABLED: 1
-  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
-
-jobs:
-  misc-checks:
-    name: Misc checks
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-    runs-on: ${{ matrix.runs-on }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          ref: ${{ env.COMMIT }}
-          submodules: true
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: 'go.mod'
-          check-latest: true
-
-      - uses: arduino/setup-protoc@v3
-
-      - run: make ci-build-misc
-
-      - run: make build-tests
-
-  cache-docker-images:
-    name: Cache Docker images
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-    runs-on: ${{ matrix.runs-on }}
-    steps:
-      - uses: ScribeMD/docker-cache@0.3.7
-        with:
-          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          ref: ${{ env.COMMIT }}
-
-      - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
-
-  functional-test-n-times:
-    name: Functional test n-times
-    needs: [misc-checks, cache-docker-images]
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-        name: [cass_es, cass_es8, sqlite, mysql8, postgres12, postgres12_pgx]
-        shard_index: [0, 1, 2]
-        include:
-          - name: cass_es
-            persistence_type: nosql
-            persistence_driver: cassandra
-            containers: [cassandra, elasticsearch]
-          - name: cass_es8
-            persistence_type: nosql
-            persistence_driver: cassandra
-            containers: [cassandra, elasticsearch8]
-          - name: sqlite
-            persistence_type: sql
-            persistence_driver: sqlite
-            containers: []
-          - name: mysql8
-            persistence_type: sql
-            persistence_driver: mysql8
-            containers: [mysql]
-          - name: postgres12
-            persistence_type: sql
-            persistence_driver: postgres12
-            containers: [postgresql]
-          - name: postgres12_pgx
-            persistence_type: sql
-            persistence_driver: postgres12_pgx
-            containers: [postgresql]
-    runs-on: ${{ matrix.runs-on }}
-    env:
-      TEST_TOTAL_SHARDS: 3
-      TEST_SHARD_INDEX: ${{ matrix.shard_index }}
-      PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
-      PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-    steps:
-      - uses: ScribeMD/docker-cache@0.3.7
-        with:
-          key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
-
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          ref: ${{ env.COMMIT }}
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: 'go.mod'
-          check-latest: true
-
-      - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' }}
-        run: |
-          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
-
-      - name: Run functional test N times
-        timeout-minutes: 30 # make sure this is larger than the test timeout in the Makefile
-        run: make functional-test-n-times
-        env:
-          TEST_NAME: ${{ inputs.test_name }}
-          N_TEST_RUNS: ${{ inputs.n_runs }}
-
-      - name: Upload test results
-        if: ${{ !cancelled() }}
-        run: make upload-test-results
-
-      - name: Tear down docker compose
-        if: ${{ always() }}
-        run: |
-          docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
\ No newline at end of file
diff --git a/service/history/worker_versioning_util.go b/service/history/worker_versioning_util.go
index 827b7fe843f..7307da28766 100644
--- a/service/history/worker_versioning_util.go
+++ b/service/history/worker_versioning_util.go
@@ -26,7 +26,6 @@ package history
 
 import (
 	"context"
-	"go.temporal.io/server/common"
 
 	"go.temporal.io/api/serviceerror"
 	persistencespb "go.temporal.io/server/api/persistence/v1"
@@ -197,7 +196,7 @@ func initializeWorkflowAssignedBuildId(
 		return err
 	}
 
-	if mutableState.HasCompletedAnyWorkflowTask() || workflowTask.StartedEventID != common.EmptyEventID {
+	if mutableState.HasCompletedAnyWorkflowTask() {
 		// workflow has already completed a wft. buildId is stale and useless.
 		// workflow's assigned build ID should be already updated via RecordWorkflowTaskStarted
 		return nil

From 2a95b20d7024dcaacb62ba2f92662cf916940235 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 08:50:22 -0700
Subject: [PATCH 41/76] pass timeout to makefile

---
 .github/workflows/run-tests.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b20604c99cd..dd795bfb07d 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -25,7 +25,7 @@ on:
         description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
         type: string
       timeout_minutes:
-        description: "[Single Test Only] Github Action timeout in minutes"
+        description: "[Single Test Only] test timeout in minutes"
         type: number
         default: 120
 
@@ -237,7 +237,8 @@ jobs:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
-      FUNCTIONAL_TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.functional_test_timeout }}
+      TEST_TIMEOUT_MINUTES: ${{ needs.set-up-functional-test.outputs.functional_test_timeout }}
+      TEST_TIMEOUT: "${{ env.TEST_TIMEOUT_MINUTES }}m"
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:
@@ -259,7 +260,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
-        timeout-minutes: ${{ fromJSON(env.FUNCTIONAL_TEST_TIMEOUT) }} # make sure this is larger than the test timeout in the Makefile
+        timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT_MINUTES) + 5 }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results

From 3ef72612119c685f3b8ed2491c1bb632038d8324 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 09:02:55 -0700
Subject: [PATCH 42/76] pass timeouts correctly

---
 .github/workflows/run-tests.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index dd795bfb07d..db85c156a4b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -171,8 +171,9 @@ jobs:
     outputs:
       shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
       total_shards: ${{ steps.generate_output.outputs.shards }}
+      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
+      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
-      functional_test_timeout: ${{ steps.generate_output.outputs.timeout }}
     steps:
       - id: generate_output
         run: |
@@ -186,7 +187,8 @@ jobs:
           {
             echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
             echo "shards=$shards"
-            echo "timeout=$timeout"
+            echo "github_timeout=$((timeout+5))"
+            echo "test_timeout=${timeout}m"
             echo "single_test_args=$single_test_args" 
           } >> "$GITHUB_OUTPUT"
 
@@ -237,8 +239,7 @@ jobs:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
-      TEST_TIMEOUT_MINUTES: ${{ needs.set-up-functional-test.outputs.functional_test_timeout }}
-      TEST_TIMEOUT: "${{ env.TEST_TIMEOUT_MINUTES }}m"
+      TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:
@@ -260,7 +261,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
-        timeout-minutes: ${{ fromJSON(env.TEST_TIMEOUT_MINUTES) + 5 }} # make sure this is larger than the test timeout in the Makefile
+        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results

From 945dc2715b77d989be192a484ea36565353dce95 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 10:12:01 -0700
Subject: [PATCH 43/76] use TEST_TIMEOUT variable in makefile

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 7b3feb3e054..55d4abc75cd 100644
--- a/Makefile
+++ b/Makefile
@@ -64,7 +64,7 @@ define NEWLINE
 
 endef
 
-TEST_TIMEOUT := 25m
+TEST_TIMEOUT ?= 25m
 
 PROTO_ROOT := proto
 PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto")

From 5470ba712b085a991fefa5944e81c3a5d75075f0 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 10:41:38 -0700
Subject: [PATCH 44/76] remove unused make target, try == false syntax for if:

---
 .github/workflows/run-tests.yml | 12 ++++++------
 Makefile                        |  5 -----
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index db85c156a4b..b248304769e 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -46,7 +46,7 @@ env:
 jobs:
 
   misc-checks:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test == false }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -96,7 +96,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test == false }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -124,7 +124,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test == false }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -274,7 +274,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test == false }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -342,7 +342,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test == false }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -410,7 +410,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test == false }}
     name: Test Status
     needs:
       - unit-test
diff --git a/Makefile b/Makefile
index 55d4abc75cd..5d56f8da163 100644
--- a/Makefile
+++ b/Makefile
@@ -391,11 +391,6 @@ functional-test-coverage: prepare-coverage-test
 		 -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) \
 		 $(FUNCTIONAL_TEST_COVERPKG) -coverprofile=$(NEW_COVER_PROFILE)
 
-functional-test-n-times: prepare-coverage-test
-	@printf $(COLOR) "Run functional test $(TEST_NAME) $(N_RUNS) times with $(PERSISTENCE_DRIVER) driver..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(FUNCTIONAL_TEST_ROOT) -shuffle on -timeout=$(TEST_TIMEOUT) $(TEST_ARGS) -test.run $(TEST_NAME) -count $(N_TEST_RUNS) $(TEST_TAG_FLAG) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER)
-
 functional-test-xdc-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..."
 	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \

From 55962e82082a77403adeda1ef29944ba4f80cdde Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 10:51:01 -0700
Subject: [PATCH 45/76] go back to != true

---
 .github/workflows/run-tests.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b248304769e..db85c156a4b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -46,7 +46,7 @@ env:
 jobs:
 
   misc-checks:
-    if: ${{ inputs.run_single_functional_test == false }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -96,7 +96,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test == false }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -124,7 +124,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test == false }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -274,7 +274,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test == false }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -342,7 +342,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test == false }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -410,7 +410,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test == false }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Test Status
     needs:
       - unit-test

From f9f153348593802a1c3cf195433a25b7fa4dd150 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 10:57:44 -0700
Subject: [PATCH 46/76] try != 'true'

---
 .github/workflows/run-tests.yml  | 12 ++---
 .github/workflows/run-tests2.yml | 86 ++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/run-tests2.yml

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index db85c156a4b..d287be84acc 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -46,7 +46,7 @@ env:
 jobs:
 
   misc-checks:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -96,7 +96,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -124,7 +124,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -274,7 +274,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -342,7 +342,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -410,7 +410,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Test Status
     needs:
       - unit-test
diff --git a/.github/workflows/run-tests2.yml b/.github/workflows/run-tests2.yml
new file mode 100644
index 00000000000..4f07c3cf5f0
--- /dev/null
+++ b/.github/workflows/run-tests2.yml
@@ -0,0 +1,86 @@
+name: All Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - release/**
+      - cloud/**
+
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: "Commit SHA"
+        required: true
+      run_single_functional_test:
+        description: "Whether to run a single test. If so, the rest of the input fields are required."
+        type: boolean
+        default: false
+      n_runs:
+        description: "[Single Test Only] Number of times to repeat the single test per database type"
+        type: number
+        default: 1
+      test_name:
+        description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
+        type: string
+      timeout_minutes:
+        description: "[Single Test Only] test timeout in minutes"
+        type: number
+        default: 120
+
+concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
+  group: run-tests-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  # For workflow_dispatch: use the given commit.
+  # For pull_request: use the head of the PR branch (not the merge branch which is the default!)
+  # For push: use the pushed commit.
+  COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }}
+  PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
+  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
+  TEMPORAL_VERSION_CHECK_DISABLED: 1
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
+
+jobs:
+
+  misc-checks:
+    if: ${{ inputs.run_single_functional_test != true }}
+    name: Misc checks
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: [ubuntu-20.04]
+    runs-on: ${{ matrix.runs-on }}
+    steps:
+
+      - run: echo "misc checks"
+
+
+  set-up-functional-test:
+    name: Set up functional test
+    runs-on: ubuntu-20.04
+    outputs:
+      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      total_shards: ${{ steps.generate_output.outputs.shards }}
+      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
+      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+    steps:
+      - id: generate_output
+        run: |
+          shards=3
+          timeout=30
+          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
+            shards=1
+            timeout=${{ inputs.timeout_minutes }}
+            single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+          fi
+          {
+            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
+            echo "shards=$shards"
+            echo "github_timeout=$((timeout+5))"
+            echo "test_timeout=${timeout}m"
+            echo "single_test_args=$single_test_args" 
+          } >> "$GITHUB_OUTPUT"

From 532c0a9dc7d35ec3a4f289ad83eb3385b34e1925 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 11:30:59 -0700
Subject: [PATCH 47/76] try != true again

---
 .github/workflows/run-tests.yml | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index d287be84acc..d90c858fcbb 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -46,7 +46,6 @@ env:
 jobs:
 
   misc-checks:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
     name: Misc checks
     strategy:
       fail-fast: false
@@ -96,7 +95,7 @@ jobs:
       - run: docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} pull
 
   unit-test:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
     needs: misc-checks
     strategy:
@@ -124,7 +123,7 @@ jobs:
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Integration test
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -194,7 +193,7 @@ jobs:
 
   functional-test:
     name: Functional test
-    needs: [cache-docker-images, set-up-functional-test]
+    needs: [misc-checks, cache-docker-images, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:
@@ -274,7 +273,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test xdc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -342,7 +341,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Functional test ndc
     needs: [misc-checks, cache-docker-images]
     strategy:
@@ -410,7 +409,7 @@ jobs:
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
   test-status:
-    if: ${{ inputs.run_single_functional_test != 'true' }}
+    if: ${{ inputs.run_single_functional_test != true }}
     name: Test Status
     needs:
       - unit-test

From 07ce206f55a3b1e4e224583983e35b4b039435db Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 12:22:31 -0700
Subject: [PATCH 48/76] remove extra file

---
 .github/workflows/run-tests2.yml | 86 --------------------------------
 1 file changed, 86 deletions(-)
 delete mode 100644 .github/workflows/run-tests2.yml

diff --git a/.github/workflows/run-tests2.yml b/.github/workflows/run-tests2.yml
deleted file mode 100644
index 4f07c3cf5f0..00000000000
--- a/.github/workflows/run-tests2.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-name: All Tests
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-      - release/**
-      - cloud/**
-
-  workflow_dispatch:
-    inputs:
-      commit:
-        description: "Commit SHA"
-        required: true
-      run_single_functional_test:
-        description: "Whether to run a single test. If so, the rest of the input fields are required."
-        type: boolean
-        default: false
-      n_runs:
-        description: "[Single Test Only] Number of times to repeat the single test per database type"
-        type: number
-        default: 1
-      test_name:
-        description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
-        type: string
-      timeout_minutes:
-        description: "[Single Test Only] test timeout in minutes"
-        type: number
-        default: 120
-
-concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
-  group: run-tests-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # For workflow_dispatch: use the given commit.
-  # For pull_request: use the head of the PR branch (not the merge branch which is the default!)
-  # For push: use the pushed commit.
-  COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }}
-  PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
-  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
-  TEMPORAL_VERSION_CHECK_DISABLED: 1
-  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
-
-jobs:
-
-  misc-checks:
-    if: ${{ inputs.run_single_functional_test != true }}
-    name: Misc checks
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: [ubuntu-20.04]
-    runs-on: ${{ matrix.runs-on }}
-    steps:
-
-      - run: echo "misc checks"
-
-
-  set-up-functional-test:
-    name: Set up functional test
-    runs-on: ubuntu-20.04
-    outputs:
-      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
-      total_shards: ${{ steps.generate_output.outputs.shards }}
-      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
-      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
-      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
-    steps:
-      - id: generate_output
-        run: |
-          shards=3
-          timeout=30
-          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
-            shards=1
-            timeout=${{ inputs.timeout_minutes }}
-            single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
-          fi
-          {
-            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
-            echo "shards=$shards"
-            echo "github_timeout=$((timeout+5))"
-            echo "test_timeout=${timeout}m"
-            echo "single_test_args=$single_test_args" 
-          } >> "$GITHUB_OUTPUT"

From 3d226a4ea3f64c232cc3e46cec862a994771bb2f Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 13:04:59 -0700
Subject: [PATCH 49/76] improve test flag consistency

---
 .github/workflows/run-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index d90c858fcbb..b9a58df42bd 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -22,7 +22,7 @@ on:
         type: number
         default: 1
       test_name:
-        description: "[Single Test Only] Name of the test suite to run, include -testify.m flag here if desired (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite -testify.m=TestUpdateWorkflow')"
+        description: "[Single Test Only] Name of the test suite to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
         type: string
       timeout_minutes:
         description: "[Single Test Only] test timeout in minutes"
@@ -181,7 +181,7 @@ jobs:
           if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
             timeout=${{ inputs.timeout_minutes }}
-            single_test_args="-test.run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
           fi
           {
             echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"

From 78242914cf9cb9eb125949df6fa5c3127e837ec5 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 13:05:36 -0700
Subject: [PATCH 50/76] change wording

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b9a58df42bd..897d00574ae 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -22,7 +22,7 @@ on:
         type: number
         default: 1
       test_name:
-        description: "[Single Test Only] Name of the test suite to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
+        description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
         type: string
       timeout_minutes:
         description: "[Single Test Only] test timeout in minutes"

From 3acb316a2b2fbe9f70e59eb182b265e8146a2cd7 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 13:28:26 -0700
Subject: [PATCH 51/76] fix TestWorkflowTaskRedirectInRetryFirstTask flake

---
 service/history/worker_versioning_util.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/service/history/worker_versioning_util.go b/service/history/worker_versioning_util.go
index 7307da28766..12223401295 100644
--- a/service/history/worker_versioning_util.go
+++ b/service/history/worker_versioning_util.go
@@ -30,6 +30,7 @@ import (
 	"go.temporal.io/api/serviceerror"
 	persistencespb "go.temporal.io/server/api/persistence/v1"
 	taskqueuespb "go.temporal.io/server/api/taskqueue/v1"
+	"go.temporal.io/server/common"
 	"go.temporal.io/server/common/log"
 	"go.temporal.io/server/common/log/tag"
 	"go.temporal.io/server/common/metrics"
@@ -196,8 +197,8 @@ func initializeWorkflowAssignedBuildId(
 		return err
 	}
 
-	if mutableState.HasCompletedAnyWorkflowTask() {
-		// workflow has already completed a wft. buildId is stale and useless.
+	if mutableState.HasCompletedAnyWorkflowTask() || workflowTask.StartedEventID != common.EmptyEventID {
+		// workflow task is running or already completed. buildId is potentially stale and useless.
 		// workflow's assigned build ID should be already updated via RecordWorkflowTaskStarted
 		return nil
 	}

From e525fd45cd450926a2509dfc809665dc2bbd2028 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 14:30:12 -0700
Subject: [PATCH 52/76] add buildkite message

---
 .github/workflows/run-tests.yml | 6 ++++++
 develop/upload-test-results.sh  | 1 +
 2 files changed, 7 insertions(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 897d00574ae..762475d7b60 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -103,6 +103,8 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -131,6 +133,8 @@ jobs:
       matrix:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}"
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:
@@ -239,6 +243,7 @@ jobs:
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
       TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:
@@ -307,6 +312,7 @@ jobs:
     env:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
         with:
diff --git a/develop/upload-test-results.sh b/develop/upload-test-results.sh
index 47eec2b3f40..6204952f2cf 100644
--- a/develop/upload-test-results.sh
+++ b/develop/upload-test-results.sh
@@ -21,5 +21,6 @@ for file in *.junit.xml; do
     -F "run_env[branch]=${GITHUB_REF}" \
     -F "run_env[commit_sha]=${GITHUB_SHA}" \
     -F "run_env[url]=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+    -F "run_env[message]=${BUILDKITE_MESSAGE}" \
     https://analytics-api.buildkite.com/v1/uploads
 done
\ No newline at end of file

From c74c23d2f6bc6943a662a8e9f8e915929143df23 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 15:13:55 -0700
Subject: [PATCH 53/76] try configurable runner size

---
 .github/workflows/run-tests.yml | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 762475d7b60..70ec9728746 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -25,9 +25,17 @@ on:
         description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
         type: string
       timeout_minutes:
-        description: "[Single Test Only] test timeout in minutes"
+        description: "[Single Test Only] Test timeout in minutes"
         type: number
         default: 120
+      test_runner:
+        description: "[Single Test Only] Which runner to use. Choose higher RAM if your n_runs is high."
+        type: choice
+        default: "16GB RAM (ubuntu-20.04)"
+        options:
+          - "16GB RAM (ubuntu-20.04)"
+          - "64GB RAM (ubuntu-22.04)"
+
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -44,7 +52,6 @@ env:
   BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
 
 jobs:
-
   misc-checks:
     name: Misc checks
     strategy:
@@ -177,22 +184,28 @@ jobs:
       github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
       test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      runs_on: ${{ steps.generate_output.outputs.runs_on }}
     steps:
       - id: generate_output
         run: |
           shards=3
           timeout=30
+          runs_on='["ubuntu-20.04"]'
           if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
             shards=1
             timeout=${{ inputs.timeout_minutes }}
             single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
+              runs_on='["ubuntu-latest-16-cores"]'
+            fi
           fi
           {
             echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
             echo "shards=$shards"
             echo "github_timeout=$((timeout+5))"
             echo "test_timeout=${timeout}m"
-            echo "single_test_args=$single_test_args" 
+            echo "single_test_args=$single_test_args"
+            echo "runs_on=$runs_on" 
           } >> "$GITHUB_OUTPUT"
 
   functional-test:
@@ -201,7 +214,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [ubuntu-20.04]
+        runs-on: ${{ fromJson(needs.set-up-functional-test.outputs.runs_on) }}
         shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
         name:
           - cass_es

From fe5c7969295049e5ca5cf703ba3436004ad87d8d Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 15:48:28 -0700
Subject: [PATCH 54/76] add configurable db via if

---
 .github/workflows/run-tests.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 70ec9728746..f13d2cde12b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -35,6 +35,17 @@ on:
         options:
           - "16GB RAM (ubuntu-20.04)"
           - "64GB RAM (ubuntu-22.04)"
+      db:
+        description: "[Single Test Only] The DB to use for your tests"
+        type: choice
+        default: "sqlite"
+        options:
+          - cass_es
+          # - cass_es8
+          - sqlite
+          - mysql8
+          - postgres12
+          - postgres12_pgx
 
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
@@ -273,16 +284,17 @@ jobs:
           check-latest: true
 
       - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' }}
+        if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER)) }}
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
         timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER)) }}
         run: make upload-test-results
 
       - name: Tear down docker compose

From b4ff73967531036af43311463efc554ffde3fbdd Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 15:59:17 -0700
Subject: [PATCH 55/76] run job set up conditionally based on db as well

---
 .github/workflows/run-tests.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index f13d2cde12b..ece1538e34b 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -270,15 +270,18 @@ jobs:
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
         with:
           key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
 
       - uses: actions/checkout@v4
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ env.COMMIT }}
 
       - uses: actions/setup-go@v5
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
         with:
           go-version-file: 'go.mod'
           check-latest: true

From 3e826a3c48410671f70cf69c18ddbab6291be704 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 16:21:41 -0700
Subject: [PATCH 56/76] only tear down docker compose if we made it

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index ece1538e34b..31a87e1a9b3 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -301,7 +301,7 @@ jobs:
         run: make upload-test-results
 
       - name: Tear down docker compose
-        if: ${{ always() }}
+        if: ${{ (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER)) && always() }}
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 

From a082423f80b1bbf30e16d713929a11f7cf8dc5fd Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 16:42:05 -0700
Subject: [PATCH 57/76] revert test flake fix

---
 service/history/worker_versioning_util.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/service/history/worker_versioning_util.go b/service/history/worker_versioning_util.go
index 12223401295..7307da28766 100644
--- a/service/history/worker_versioning_util.go
+++ b/service/history/worker_versioning_util.go
@@ -30,7 +30,6 @@ import (
 	"go.temporal.io/api/serviceerror"
 	persistencespb "go.temporal.io/server/api/persistence/v1"
 	taskqueuespb "go.temporal.io/server/api/taskqueue/v1"
-	"go.temporal.io/server/common"
 	"go.temporal.io/server/common/log"
 	"go.temporal.io/server/common/log/tag"
 	"go.temporal.io/server/common/metrics"
@@ -197,8 +196,8 @@ func initializeWorkflowAssignedBuildId(
 		return err
 	}
 
-	if mutableState.HasCompletedAnyWorkflowTask() || workflowTask.StartedEventID != common.EmptyEventID {
-		// workflow task is running or already completed. buildId is potentially stale and useless.
+	if mutableState.HasCompletedAnyWorkflowTask() {
+		// workflow has already completed a wft. buildId is stale and useless.
 		// workflow's assigned build ID should be already updated via RecordWorkflowTaskStarted
 		return nil
 	}

From a31d78bccbb22fdab9b2dd176ba70e62edac14b6 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 16 Jul 2024 18:46:37 -0700
Subject: [PATCH 58/76] fix cassandra db input

---
 .github/workflows/run-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 31a87e1a9b3..c20f80c2e50 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -40,8 +40,7 @@ on:
         type: choice
         default: "sqlite"
         options:
-          - cass_es
-          # - cass_es8
+          - cassandra
           - sqlite
           - mysql8
           - postgres12

From f6f6873b476dfef27d16432a5e88fb49eb3dca9d Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 19:48:16 -0700
Subject: [PATCH 59/76] multi-select test db

---
 .github/workflows/run-tests.yml  |  69 ++++++++++----
 .github/workflows/run-tests2.yml | 153 +++++++++++++++++++++++++++++++
 2 files changed, 204 insertions(+), 18 deletions(-)
 create mode 100644 .github/workflows/run-tests2.yml

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index c20f80c2e50..ec4d274c7f3 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -17,6 +17,26 @@ on:
         description: "Whether to run a single test. If so, the rest of the input fields are required."
         type: boolean
         default: false
+      cassandra:
+        description: "cassandra"
+        type: boolean
+        default: false
+      sqlite:
+        description: "sqlite"
+        type: boolean
+        default: false
+      mysql8:
+        description: "mysql8"
+        type: boolean
+        default: false
+      postgres12:
+        description: "postgres12"
+        type: boolean
+        default: false
+      postgres12_pgx:
+        description: "postgres12_pgx"
+        type: boolean
+        default: false
       n_runs:
         description: "[Single Test Only] Number of times to repeat the single test per database type"
         type: number
@@ -35,16 +55,16 @@ on:
         options:
           - "16GB RAM (ubuntu-20.04)"
           - "64GB RAM (ubuntu-22.04)"
-      db:
-        description: "[Single Test Only] The DB to use for your tests"
-        type: choice
-        default: "sqlite"
-        options:
-          - cassandra
-          - sqlite
-          - mysql8
-          - postgres12
-          - postgres12_pgx
+#      db:
+#        description: "[Single Test Only] The DB to use for your tests"
+#        type: choice
+#        default: "sqlite"
+#        options:
+#          - cassandra
+#          - sqlite
+#          - mysql8
+#          - postgres12
+#          - postgres12_pgx
 
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
@@ -195,6 +215,7 @@ jobs:
       test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
       runs_on: ${{ steps.generate_output.outputs.runs_on }}
+      dbs: ${{ steps.generate_output.outputs.dbs }}
     steps:
       - id: generate_output
         run: |
@@ -207,6 +228,17 @@ jobs:
             single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
             if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
               runs_on='["ubuntu-latest-16-cores"]'
+            fi 
+            dbs_arr=()
+            if [[ "${{ inputs.cassandra }}" != "true" ]]; then dbs_arr+="cassandra"; fi
+            if [[ "${{ inputs.sqlite }}" != "true" ]]; then dbs_arr+="sqlite"; fi
+            if [[ "${{ inputs.mysql8 }}" != "true" ]]; then dbs_arr+="mysql8"; fi
+            if [[ "${{ inputs.postgres12 }}" != "true" ]]; then dbs_arr+="postgres12"; fi
+            if [[ "${{ inputs.postgres12_pgx }}" != "true" ]]; then dbs_arr+="postgres12_pgx"; fi
+            if [[ ${#dbs_arr[@]} == 0 ]]; then
+              dbs='["sqlite"]'
+            else
+              dbs=$(printf '%s\n' "${dbs_arr[@]}" | jq -R . | jq -s .)
             fi
           fi
           {
@@ -215,7 +247,8 @@ jobs:
             echo "github_timeout=$((timeout+5))"
             echo "test_timeout=${timeout}m"
             echo "single_test_args=$single_test_args"
-            echo "runs_on=$runs_on" 
+            echo "runs_on=$runs_on"
+            echo "dbs=$dbs" 
           } >> "$GITHUB_OUTPUT"
 
   functional-test:
@@ -269,38 +302,38 @@ jobs:
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - uses: ScribeMD/docker-cache@0.3.7
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
 
       - uses: actions/checkout@v4
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ env.COMMIT }}
 
       - uses: actions/setup-go@v5
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           go-version-file: 'go.mod'
           check-latest: true
 
       - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER)) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} up -d ${{ join(matrix.containers, ' ') }}
 
       - name: Run functional test
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results
-        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER)) }}
+        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         run: make upload-test-results
 
       - name: Tear down docker compose
-        if: ${{ (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER)) && always() }}
+        if: ${{ (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER))) && always() }}
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
diff --git a/.github/workflows/run-tests2.yml b/.github/workflows/run-tests2.yml
new file mode 100644
index 00000000000..30ea32071f6
--- /dev/null
+++ b/.github/workflows/run-tests2.yml
@@ -0,0 +1,153 @@
+name: All Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - release/**
+      - cloud/**
+
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: "Commit SHA"
+        required: true
+      run_single_functional_test:
+        description: "Whether to run a single test. If so, the rest of the input fields are required."
+        type: boolean
+        default: false
+      n_runs:
+        description: "[Single Test Only] Number of times to repeat the single test per database type"
+        type: number
+        default: 1
+      test_name:
+        description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
+        type: string
+      timeout_minutes:
+        description: "[Single Test Only] Test timeout in minutes"
+        type: number
+        default: 120
+      test_runner:
+        description: "[Single Test Only] Which runner to use. Choose higher RAM if your n_runs is high."
+        type: choice
+        default: "16GB RAM (ubuntu-20.04)"
+        options:
+          - "16GB RAM (ubuntu-20.04)"
+          - "64GB RAM (ubuntu-22.04)"
+      db:
+        description: "[Single Test Only] The DB to use for your tests"
+        type: choice
+        default: "sqlite"
+        options:
+          - cass_es
+          # - cass_es8
+          - sqlite
+          - mysql8
+          - postgres12
+          - postgres12_pgx
+
+concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
+  group: run-tests-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  # For workflow_dispatch: use the given commit.
+  # For pull_request: use the head of the PR branch (not the merge branch which is the default!)
+  # For push: use the pushed commit.
+  COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }}
+  PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
+  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
+  TEMPORAL_VERSION_CHECK_DISABLED: 1
+  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
+
+jobs:
+  set-up-functional-test:
+    name: Set up functional test
+    runs-on: ubuntu-20.04
+    outputs:
+      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      total_shards: ${{ steps.generate_output.outputs.shards }}
+      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
+      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      runs_on: ${{ steps.generate_output.outputs.runs_on }}
+    steps:
+      - id: generate_output
+        run: |
+          shards=3
+          timeout=30
+          runs_on='["ubuntu-20.04"]'
+          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
+            shards=1
+            timeout=${{ inputs.timeout_minutes }}
+            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
+              runs_on='["ubuntu-latest-16-cores"]'
+            fi
+          fi
+          {
+            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
+            echo "shards=$shards"
+            echo "github_timeout=$((timeout+5))"
+            echo "test_timeout=${timeout}m"
+            echo "single_test_args=$single_test_args"
+            echo "runs_on=$runs_on" 
+          } >> "$GITHUB_OUTPUT"
+
+  functional-test:
+    name: Functional test
+    needs: [set-up-functional-test]
+    strategy:
+      fail-fast: false
+      matrix:
+        runs-on: ${{ fromJson(needs.set-up-functional-test.outputs.runs_on) }}
+        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
+        name:
+          - cass_es
+          # - cass_es8
+          - sqlite
+          - mysql8
+          - postgres12
+          - postgres12_pgx
+        include:
+          - name: cass_es
+            persistence_type: nosql
+            persistence_driver: cassandra
+            containers: [cassandra, elasticsearch]
+          # - name: cass_es8
+          #   persistence_type: nosql
+          #   persistence_driver: cassandra
+          #   containers: [cassandra, elasticsearch8]
+          - name: sqlite
+            persistence_type: sql
+            persistence_driver: sqlite
+            containers: []
+          - name: mysql8
+            persistence_type: sql
+            persistence_driver: mysql8
+            containers: [mysql]
+          - name: postgres12
+            persistence_type: sql
+            persistence_driver: postgres12
+            containers: [postgresql]
+          - name: postgres12_pgx
+            persistence_type: sql
+            persistence_driver: postgres12_pgx
+            containers: [postgresql]
+    runs-on: ${{ matrix.runs-on }}
+    env:
+      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
+      TEST_SHARD_INDEX: ${{ matrix.shard_index }}
+      PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
+      PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
+      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
+      BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
+    steps:
+
+      - name: Run functional test
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
+        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
+        run: echo "make functional-test-coverage ${PERSISTENCE_DRIVER}"
+

From e2e0d152a15b4edfa1dbad7058789b34fdc3a27b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 19:51:47 -0700
Subject: [PATCH 60/76] remove postgres12_pgx test db

---
 .github/workflows/run-tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index ec4d274c7f3..78defffb53f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -33,10 +33,10 @@ on:
         description: "postgres12"
         type: boolean
         default: false
-      postgres12_pgx:
-        description: "postgres12_pgx"
-        type: boolean
-        default: false
+#      postgres12_pgx:
+#        description: "postgres12_pgx"
+#        type: boolean
+#        default: false
       n_runs:
         description: "[Single Test Only] Number of times to repeat the single test per database type"
         type: number

From 2540f4609e1f267c7e6a60d882843fff4483103b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 19:57:15 -0700
Subject: [PATCH 61/76] fix bash syntax error

---
 .github/workflows/run-tests.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 78defffb53f..cf7baa89753 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -230,11 +230,10 @@ jobs:
               runs_on='["ubuntu-latest-16-cores"]'
             fi 
             dbs_arr=()
-            if [[ "${{ inputs.cassandra }}" != "true" ]]; then dbs_arr+="cassandra"; fi
-            if [[ "${{ inputs.sqlite }}" != "true" ]]; then dbs_arr+="sqlite"; fi
-            if [[ "${{ inputs.mysql8 }}" != "true" ]]; then dbs_arr+="mysql8"; fi
-            if [[ "${{ inputs.postgres12 }}" != "true" ]]; then dbs_arr+="postgres12"; fi
-            if [[ "${{ inputs.postgres12_pgx }}" != "true" ]]; then dbs_arr+="postgres12_pgx"; fi
+            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+="cassandra"; fi
+            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+="sqlite"; fi
+            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+="mysql8"; fi
+            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+="postgres12"; fi
             if [[ ${#dbs_arr[@]} == 0 ]]; then
               dbs='["sqlite"]'
             else

From e9ce2a78bb541ab45451976005803d1d3e7077d9 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 20:03:21 -0700
Subject: [PATCH 62/76] try a different jq syntax

---
 .github/workflows/run-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index cf7baa89753..749d1812782 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -237,7 +237,7 @@ jobs:
             if [[ ${#dbs_arr[@]} == 0 ]]; then
               dbs='["sqlite"]'
             else
-              dbs=$(printf '%s\n' "${dbs_arr[@]}" | jq -R . | jq -s .)
+              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
             fi
           fi
           {

From fdcaa7f7da0f7f3dbd04893e7528ab096d97eea2 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 20:26:29 -0700
Subject: [PATCH 63/76] delete bad mktemp --tmpdir flag

---
 develop/buf-breaking.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/develop/buf-breaking.sh b/develop/buf-breaking.sh
index 2299693c0b4..8400d80682a 100755
--- a/develop/buf-breaking.sh
+++ b/develop/buf-breaking.sh
@@ -43,7 +43,7 @@ fi
 # case this is being run manually.
 $MAKE "$INTERNAL_BINPB"
 
-tmp=$(mktemp --tmpdir -d temporal-buf-breaking.XXXXXXXXX)
+tmp=$(mktemp -d temporal-buf-breaking.XXXXXXXXX)
 trap 'rm -rf $tmp' EXIT
 
 check_against_commit() {

From b7d9cccf5a80fa87272f112996999696005ce9eb Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 20:28:23 -0700
Subject: [PATCH 64/76] revert --tempdir flag change

---
 develop/buf-breaking.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/develop/buf-breaking.sh b/develop/buf-breaking.sh
index 8400d80682a..c7ce408d947 100755
--- a/develop/buf-breaking.sh
+++ b/develop/buf-breaking.sh
@@ -43,7 +43,7 @@ fi
 # case this is being run manually.
 $MAKE "$INTERNAL_BINPB"
 
-tmp=$(mktemp -d temporal-buf-breaking.XXXXXXXXX)
+tmp=$(mktemp --tmpdir -d temporal-buf-breaking.XXXXXXXXX)
 trap 'rm -rf $tmp' EXIT
 
 check_against_commit() {
@@ -83,4 +83,4 @@ fi
 
 # Next check against main:
 git -C "$tmp" fetch --depth=1 https://github.com/temporalio/temporal.git "$MAIN_BRANCH"
-check_against_commit FETCH_HEAD "main"
+check_against_commit FETCH_HEAD "main"
\ No newline at end of file

From fc558d90ae73dd9da705ee821c970ebfb0e2f314 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 20:38:14 -0700
Subject: [PATCH 65/76] put back newlines

---
 develop/buf-breaking.sh        | 2 +-
 develop/upload-test-results.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/develop/buf-breaking.sh b/develop/buf-breaking.sh
index c7ce408d947..2299693c0b4 100755
--- a/develop/buf-breaking.sh
+++ b/develop/buf-breaking.sh
@@ -83,4 +83,4 @@ fi
 
 # Next check against main:
 git -C "$tmp" fetch --depth=1 https://github.com/temporalio/temporal.git "$MAIN_BRANCH"
-check_against_commit FETCH_HEAD "main"
\ No newline at end of file
+check_against_commit FETCH_HEAD "main"
diff --git a/develop/upload-test-results.sh b/develop/upload-test-results.sh
index 6204952f2cf..aab7dc7c528 100644
--- a/develop/upload-test-results.sh
+++ b/develop/upload-test-results.sh
@@ -23,4 +23,4 @@ for file in *.junit.xml; do
     -F "run_env[url]=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
     -F "run_env[message]=${BUILDKITE_MESSAGE}" \
     https://analytics-api.buildkite.com/v1/uploads
-done
\ No newline at end of file
+done

From 2db8d8716ca497493c76efd4eedc8a4b8cdf71b7 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 20:39:41 -0700
Subject: [PATCH 66/76] temporarily remove make ci-build-misc

---
 .github/workflows/run-tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index c19ef2522be..4c4a72e50a1 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -108,9 +108,9 @@ jobs:
 
       - run: GOOS=darwin GOARCH=arm64 make clean-bins bins
 
-      - run: make clean-bins ci-build-misc
-
-      - run: make build-tests
+#      - run: make clean-bins ci-build-misc (removed temporarily because ci-build-misc is broken)
+#
+#      - run: make build-tests
 
   unit-test:
     if: ${{ inputs.run_single_functional_test != true }}

From 5951bb34ea045da12a4272f56c91360b01756216 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Tue, 23 Jul 2024 20:41:13 -0700
Subject: [PATCH 67/76] remove cache-docker-images dependency

---
 .github/workflows/run-tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 4c4a72e50a1..c1165e1b638 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -145,7 +145,7 @@ jobs:
   integration-test:
     if: ${{ inputs.run_single_functional_test != true }}
     name: Integration test
-    needs: [misc-checks, cache-docker-images]
+    needs: misc-checks
     strategy:
       fail-fast: false
       matrix:
@@ -235,7 +235,7 @@ jobs:
 
   functional-test:
     name: Functional test
-    needs: [misc-checks, cache-docker-images, set-up-functional-test]
+    needs: [misc-checks, set-up-functional-test]
     strategy:
       fail-fast: false
       matrix:

From 7417f3d0feba41d1ebc73c80de74ad7790fde80c Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Wed, 24 Jul 2024 15:03:07 -0700
Subject: [PATCH 68/76] add prints

---
 .github/workflows/run-tests.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index c1165e1b638..a9a1b41c439 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -232,6 +232,9 @@ jobs:
             echo "runs_on=$runs_on"
             echo "dbs=$dbs" 
           } >> "$GITHUB_OUTPUT"
+      - id: cat_output
+        run: |
+          cat $GITHUB_OUTPUT
 
   functional-test:
     name: Functional test
@@ -285,6 +288,8 @@ jobs:
       TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
+      - name: Print functional test
+        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}"
       - uses: ScribeMD/docker-cache@0.3.7
         if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:

From 16ee81e90cf65c2387502c785dbc82d70356e67f Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Thu, 25 Jul 2024 11:21:59 -0700
Subject: [PATCH 69/76] save changes

---
 .github/workflows/run-tests2.yml | 81 +++++++++++++++++++++++---------
 1 file changed, 60 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/run-tests2.yml b/.github/workflows/run-tests2.yml
index 30ea32071f6..a3da7ba9119 100644
--- a/.github/workflows/run-tests2.yml
+++ b/.github/workflows/run-tests2.yml
@@ -17,6 +17,26 @@ on:
         description: "Whether to run a single test. If so, the rest of the input fields are required."
         type: boolean
         default: false
+      cassandra:
+        description: "cassandra"
+        type: boolean
+        default: false
+      sqlite:
+        description: "sqlite"
+        type: boolean
+        default: false
+      mysql8:
+        description: "mysql8"
+        type: boolean
+        default: false
+      postgres12:
+        description: "postgres12"
+        type: boolean
+        default: false
+      #      postgres12_pgx:
+      #        description: "postgres12_pgx"
+      #        type: boolean
+      #        default: false
       n_runs:
         description: "[Single Test Only] Number of times to repeat the single test per database type"
         type: number
@@ -35,17 +55,16 @@ on:
         options:
           - "16GB RAM (ubuntu-20.04)"
           - "64GB RAM (ubuntu-22.04)"
-      db:
-        description: "[Single Test Only] The DB to use for your tests"
-        type: choice
-        default: "sqlite"
-        options:
-          - cass_es
-          # - cass_es8
-          - sqlite
-          - mysql8
-          - postgres12
-          - postgres12_pgx
+#      db:
+#        description: "[Single Test Only] The DB to use for your tests"
+#        type: choice
+#        default: "sqlite"
+#        options:
+#          - cassandra
+#          - sqlite
+#          - mysql8
+#          - postgres12
+#          - postgres12_pgx
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -72,6 +91,7 @@ jobs:
       test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
       runs_on: ${{ steps.generate_output.outputs.runs_on }}
+      dbs: ${{ steps.generate_output.outputs.dbs }}
     steps:
       - id: generate_output
         run: |
@@ -84,6 +104,16 @@ jobs:
             single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
             if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
               runs_on='["ubuntu-latest-16-cores"]'
+            fi 
+            dbs_arr=()
+            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+="cassandra"; fi
+            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+="sqlite"; fi
+            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+="mysql8"; fi
+            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+="postgres12"; fi
+            if [[ ${#dbs_arr[@]} == 0 ]]; then
+              dbs='["sqlite"]'
+            else
+              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
             fi
           fi
           {
@@ -92,8 +122,12 @@ jobs:
             echo "github_timeout=$((timeout+5))"
             echo "test_timeout=${timeout}m"
             echo "single_test_args=$single_test_args"
-            echo "runs_on=$runs_on" 
+            echo "runs_on=$runs_on"
+            echo "dbs=$dbs" 
           } >> "$GITHUB_OUTPUT"
+      - id: cat_output
+        run: |
+          cat $GITHUB_OUTPUT
 
   functional-test:
     name: Functional test
@@ -105,7 +139,7 @@ jobs:
         shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
         name:
           - cass_es
-          # - cass_es8
+          - cass_es8
           - sqlite
           - mysql8
           - postgres12
@@ -115,10 +149,12 @@ jobs:
             persistence_type: nosql
             persistence_driver: cassandra
             containers: [cassandra, elasticsearch]
-          # - name: cass_es8
-          #   persistence_type: nosql
-          #   persistence_driver: cassandra
-          #   containers: [cassandra, elasticsearch8]
+            es_version: v7
+          - name: cass_es8
+            persistence_type: nosql
+            persistence_driver: cassandra
+            containers: [cassandra, elasticsearch8]
+            es_version: v8
           - name: sqlite
             persistence_type: sql
             persistence_driver: sqlite
@@ -145,9 +181,12 @@ jobs:
       TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
-
-      - name: Run functional test
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && inputs.db == env.PERSISTENCE_DRIVER) }}
+      - name: Print functional test
         timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
-        run: echo "make functional-test-coverage ${PERSISTENCE_DRIVER}"
+        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}"
+
+#      - name: Run functional test
+#        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+#        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
+#        run: echo "make functional-test-coverage ${PERSISTENCE_DRIVER}"
 

From edb53433cb851998d4053c15ad81a332246df3b7 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 9 Sep 2024 09:15:52 -0700
Subject: [PATCH 70/76] echo vars then run functional test n times

---
 .github/workflows/run-tests.yml | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index a9a1b41c439..77055dbf2ee 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -33,10 +33,6 @@ on:
         description: "postgres12"
         type: boolean
         default: false
-#      postgres12_pgx:
-#        description: "postgres12_pgx"
-#        type: boolean
-#        default: false
       n_runs:
         description: "[Single Test Only] Number of times to repeat the single test per database type"
         type: number
@@ -55,16 +51,6 @@ on:
         options:
           - "16GB RAM (ubuntu-20.04)"
           - "64GB RAM (ubuntu-22.04)"
-#      db:
-#        description: "[Single Test Only] The DB to use for your tests"
-#        type: choice
-#        default: "sqlite"
-#        options:
-#          - cassandra
-#          - sqlite
-#          - mysql8
-#          - postgres12
-#          - postgres12_pgx
 
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
@@ -108,9 +94,9 @@ jobs:
 
       - run: GOOS=darwin GOARCH=arm64 make clean-bins bins
 
-#      - run: make clean-bins ci-build-misc (removed temporarily because ci-build-misc is broken)
-#
-#      - run: make build-tests
+      - run: make clean-bins ci-build-misc (removed temporarily because ci-build-misc is broken)
+
+      - run: make build-tests
 
   unit-test:
     if: ${{ inputs.run_single_functional_test != true }}
@@ -289,7 +275,7 @@ jobs:
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - name: Print functional test
-        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}"
+        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
       - uses: ScribeMD/docker-cache@0.3.7
         if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:

From 90c49621fdbbaeefb680e8f59ac5ac6c7b84fa2a Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Mon, 9 Sep 2024 09:30:18 -0700
Subject: [PATCH 71/76] fix syntax

---
 .github/workflows/run-tests.yml  |  12 +-
 .github/workflows/run-tests2.yml | 192 -------------------------------
 2 files changed, 6 insertions(+), 198 deletions(-)
 delete mode 100644 .github/workflows/run-tests2.yml

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 77055dbf2ee..9a8ade3b4ff 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -94,7 +94,7 @@ jobs:
 
       - run: GOOS=darwin GOARCH=arm64 make clean-bins bins
 
-      - run: make clean-bins ci-build-misc (removed temporarily because ci-build-misc is broken)
+      - run: make clean-bins ci-build-misc
 
       - run: make build-tests
 
@@ -199,10 +199,10 @@ jobs:
               runs_on='["ubuntu-latest-16-cores"]'
             fi 
             dbs_arr=()
-            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+="cassandra"; fi
-            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+="sqlite"; fi
-            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+="mysql8"; fi
-            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+="postgres12"; fi
+            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi
+            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi
+            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi
+            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi
             if [[ ${#dbs_arr[@]} == 0 ]]; then
               dbs='["sqlite"]'
             else
@@ -220,7 +220,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
       - id: cat_output
         run: |
-          cat $GITHUB_OUTPUT
+          cat "$GITHUB_OUTPUT"
 
   functional-test:
     name: Functional test
diff --git a/.github/workflows/run-tests2.yml b/.github/workflows/run-tests2.yml
deleted file mode 100644
index a3da7ba9119..00000000000
--- a/.github/workflows/run-tests2.yml
+++ /dev/null
@@ -1,192 +0,0 @@
-name: All Tests
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-      - release/**
-      - cloud/**
-
-  workflow_dispatch:
-    inputs:
-      commit:
-        description: "Commit SHA"
-        required: true
-      run_single_functional_test:
-        description: "Whether to run a single test. If so, the rest of the input fields are required."
-        type: boolean
-        default: false
-      cassandra:
-        description: "cassandra"
-        type: boolean
-        default: false
-      sqlite:
-        description: "sqlite"
-        type: boolean
-        default: false
-      mysql8:
-        description: "mysql8"
-        type: boolean
-        default: false
-      postgres12:
-        description: "postgres12"
-        type: boolean
-        default: false
-      #      postgres12_pgx:
-      #        description: "postgres12_pgx"
-      #        type: boolean
-      #        default: false
-      n_runs:
-        description: "[Single Test Only] Number of times to repeat the single test per database type"
-        type: number
-        default: 1
-      test_name:
-        description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
-        type: string
-      timeout_minutes:
-        description: "[Single Test Only] Test timeout in minutes"
-        type: number
-        default: 120
-      test_runner:
-        description: "[Single Test Only] Which runner to use. Choose higher RAM if your n_runs is high."
-        type: choice
-        default: "16GB RAM (ubuntu-20.04)"
-        options:
-          - "16GB RAM (ubuntu-20.04)"
-          - "64GB RAM (ubuntu-22.04)"
-#      db:
-#        description: "[Single Test Only] The DB to use for your tests"
-#        type: choice
-#        default: "sqlite"
-#        options:
-#          - cassandra
-#          - sqlite
-#          - mysql8
-#          - postgres12
-#          - postgres12_pgx
-
-concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
-  group: run-tests-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-env:
-  # For workflow_dispatch: use the given commit.
-  # For pull_request: use the head of the PR branch (not the merge branch which is the default!)
-  # For push: use the pushed commit.
-  COMMIT: ${{ github.event.inputs.commit || github.event.pull_request.head.sha || github.sha }}
-  PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
-  DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
-  TEMPORAL_VERSION_CHECK_DISABLED: 1
-  BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
-
-jobs:
-  set-up-functional-test:
-    name: Set up functional test
-    runs-on: ubuntu-20.04
-    outputs:
-      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
-      total_shards: ${{ steps.generate_output.outputs.shards }}
-      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
-      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
-      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
-      runs_on: ${{ steps.generate_output.outputs.runs_on }}
-      dbs: ${{ steps.generate_output.outputs.dbs }}
-    steps:
-      - id: generate_output
-        run: |
-          shards=3
-          timeout=30
-          runs_on='["ubuntu-20.04"]'
-          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
-            shards=1
-            timeout=${{ inputs.timeout_minutes }}
-            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
-            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
-              runs_on='["ubuntu-latest-16-cores"]'
-            fi 
-            dbs_arr=()
-            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+="cassandra"; fi
-            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+="sqlite"; fi
-            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+="mysql8"; fi
-            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+="postgres12"; fi
-            if [[ ${#dbs_arr[@]} == 0 ]]; then
-              dbs='["sqlite"]'
-            else
-              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
-            fi
-          fi
-          {
-            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
-            echo "shards=$shards"
-            echo "github_timeout=$((timeout+5))"
-            echo "test_timeout=${timeout}m"
-            echo "single_test_args=$single_test_args"
-            echo "runs_on=$runs_on"
-            echo "dbs=$dbs" 
-          } >> "$GITHUB_OUTPUT"
-      - id: cat_output
-        run: |
-          cat $GITHUB_OUTPUT
-
-  functional-test:
-    name: Functional test
-    needs: [set-up-functional-test]
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on: ${{ fromJson(needs.set-up-functional-test.outputs.runs_on) }}
-        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
-        name:
-          - cass_es
-          - cass_es8
-          - sqlite
-          - mysql8
-          - postgres12
-          - postgres12_pgx
-        include:
-          - name: cass_es
-            persistence_type: nosql
-            persistence_driver: cassandra
-            containers: [cassandra, elasticsearch]
-            es_version: v7
-          - name: cass_es8
-            persistence_type: nosql
-            persistence_driver: cassandra
-            containers: [cassandra, elasticsearch8]
-            es_version: v8
-          - name: sqlite
-            persistence_type: sql
-            persistence_driver: sqlite
-            containers: []
-          - name: mysql8
-            persistence_type: sql
-            persistence_driver: mysql8
-            containers: [mysql]
-          - name: postgres12
-            persistence_type: sql
-            persistence_driver: postgres12
-            containers: [postgresql]
-          - name: postgres12_pgx
-            persistence_type: sql
-            persistence_driver: postgres12_pgx
-            containers: [postgresql]
-    runs-on: ${{ matrix.runs-on }}
-    env:
-      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
-      TEST_SHARD_INDEX: ${{ matrix.shard_index }}
-      PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
-      PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
-      TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
-      BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
-    steps:
-      - name: Print functional test
-        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
-        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}"
-
-#      - name: Run functional test
-#        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
-#        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
-#        run: echo "make functional-test-coverage ${PERSISTENCE_DRIVER}"
-

From 89b47d1748459a643517fca29d5898b37f85b06d Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 4 Oct 2024 12:58:36 -0700
Subject: [PATCH 72/76] offer custom test directory

---
 .github/workflows/run-tests.yml | 13 +++++++++----
 Makefile                        |  2 +-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 9a8ade3b4ff..131cdb5a5a5 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -25,10 +25,10 @@ on:
         description: "sqlite"
         type: boolean
         default: false
-      mysql8:
-        description: "mysql8"
-        type: boolean
-        default: false
+#      mysql8:
+#        description: "mysql8"
+#        type: boolean
+#        default: false
       postgres12:
         description: "postgres12"
         type: boolean
@@ -40,6 +40,10 @@ on:
       test_name:
         description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
         type: string
+      test_directory:
+        description: "[Single Test Only] Directory to run 'go test' in"
+        type: string
+        default: "./tests"
       timeout_minutes:
         description: "[Single Test Only] Test timeout in minutes"
         type: number
@@ -271,6 +275,7 @@ jobs:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
+      FUNCTIONAL_TEST_ROOT: ${{ inputs.test_directory }}
       TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
diff --git a/Makefile b/Makefile
index db0eb0f794f..968d9850cc7 100644
--- a/Makefile
+++ b/Makefile
@@ -82,7 +82,7 @@ ALL_SCRIPTS     := $(shell find . -name "*.sh")
 MAIN_BRANCH    := main
 
 TEST_DIRS       := $(sort $(dir $(filter %_test.go,$(ALL_SRC))))
-FUNCTIONAL_TEST_ROOT          := ./tests
+FUNCTIONAL_TEST_ROOT          ?= ./tests
 FUNCTIONAL_TEST_XDC_ROOT      := ./tests/xdc
 FUNCTIONAL_TEST_NDC_ROOT      := ./tests/ndc
 DB_INTEGRATION_TEST_ROOT      := ./common/persistence/tests

From 20e1f1e0ba3b71b4994a0b270d20a0175ef8911b Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 4 Oct 2024 14:08:25 -0700
Subject: [PATCH 73/76] offer n-times run for unit test also

---
 .github/workflows/run-tests.yml | 200 +++++++++++++++++---------------
 Makefile                        |   7 +-
 2 files changed, 110 insertions(+), 97 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 131cdb5a5a5..b4f4b060677 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -14,48 +14,53 @@ on:
         description: "Commit SHA"
         required: true
       run_single_functional_test:
-        description: "Whether to run a single test. If so, the rest of the input fields are required."
+        description: "Run a single functional test"
         type: boolean
         default: false
-      cassandra:
-        description: "cassandra"
-        type: boolean
-        default: false
-      sqlite:
-        description: "sqlite"
-        type: boolean
-        default: false
-#      mysql8:
-#        description: "mysql8"
-#        type: boolean
-#        default: false
-      postgres12:
-        description: "postgres12"
+      run_single_unit_test:
+        description: "Run a single unit test (INSTEAD of functional test)"
         type: boolean
         default: false
+      unit_test_directory:
+        description: "[Unit Test Only] Directory to run unit tests in"
+        type: string
+        default: "./temporal"
       n_runs:
-        description: "[Single Test Only] Number of times to repeat the single test per database type"
+        description: "Number of times to repeat the single test per database type"
         type: number
         default: 1
       test_name:
-        description: "[Single Test Only] Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
-        type: string
-      test_directory:
-        description: "[Single Test Only] Directory to run 'go test' in"
+        description: "Name of the test to run (i.e. 'TestAcquireShard_DeadlineExceededErrorSuite' or 'TestFunctionalSuite/TestUpdateWorkflow')"
         type: string
-        default: "./tests"
       timeout_minutes:
-        description: "[Single Test Only] Test timeout in minutes"
+        description: "Test timeout in minutes"
         type: number
         default: 120
       test_runner:
-        description: "[Single Test Only] Which runner to use. Choose higher RAM if your n_runs is high."
+        description: "Which runner to use. Choose higher RAM if your n_runs is high."
         type: choice
         default: "16GB RAM (ubuntu-20.04)"
         options:
           - "16GB RAM (ubuntu-20.04)"
           - "64GB RAM (ubuntu-22.04)"
-
+      cassandra:
+        description: "cassandra"
+        type: boolean
+        default: false
+      sqlite:
+        description: "sqlite"
+        type: boolean
+        default: false
+      #      mysql8:
+      #        description: "mysql8"
+      #        type: boolean
+      #        default: false
+#      postgres12:
+#        description: "postgres12"
+#        type: boolean
+#        default: false
+          # if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi
+        # if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -72,6 +77,52 @@ env:
   BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
 
 jobs:
+  set-up-single-test:
+    name: Set up single test
+    runs-on: ubuntu-20.04
+    outputs:
+      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
+      total_shards: ${{ steps.generate_output.outputs.shards }}
+      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
+      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
+      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
+      runs_on: ${{ steps.generate_output.outputs.runs_on }}
+      dbs: ${{ steps.generate_output.outputs.dbs }}
+    steps:
+      - id: generate_output
+        run: |
+          shards=3
+          timeout=30
+          runs_on='["ubuntu-20.04"]'
+          if [[ "${{ inputs.run_single_functional_test }}" == "true" || "${{ inputs.run_single_unit_test }}" == "true" ]]; then
+            shards=1
+            timeout=${{ inputs.timeout_minutes }}
+            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
+            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
+              runs_on='[ "ubuntu-latest-16-cores" ]'
+            fi 
+            dbs_arr=()
+            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi
+            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi
+            if [[ ${#dbs_arr[@]} == 0 ]]; then
+              dbs='["sqlite"]'
+            else
+              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
+            fi
+          fi
+          {
+            echo "shard_indices=[ $(seq -s, 0 $((shards-1))) ]"
+            echo "shards=$shards"
+            echo "github_timeout=$((timeout+5))"
+            echo "test_timeout=${timeout}m"
+            echo "single_test_args=$single_test_args"
+            echo "runs_on=$runs_on"
+            echo "dbs=$dbs" 
+          } >> "$GITHUB_OUTPUT"
+      - id: cat_output
+        run: |
+          cat "$GITHUB_OUTPUT"
+
   misc-checks:
     name: Misc checks
     strategy:
@@ -105,7 +156,7 @@ jobs:
   unit-test:
     if: ${{ inputs.run_single_functional_test != true }}
     name: Unit test
-    needs: misc-checks
+    needs: [misc-checks, set-up-single-test]
     strategy:
       fail-fast: false
       matrix:
@@ -113,6 +164,10 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     env:
       BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}"
+      SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
+      UNIT_TEST_DIR: ${{ inputs.unit_test_directory }}
+      TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
+      RUN_SINGLE_UNIT_TEST: ${{ inputs.run_single_unit_test }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -124,16 +179,22 @@ jobs:
           go-version-file: 'go.mod'
           check-latest: true
 
-      - name: Run unit test
+      - name: Run unit tests
+        if: ${{ !cancelled() && !inputs.run_single_unit_test }}
         timeout-minutes: 15
         run: make unit-test-coverage
 
+      - name: Run single unit test
+        if: ${{ !cancelled() && inputs.run_single_unit_test }}
+        timeout-minutes: 15
+        run: UNIT_TEST_DIRS=$UNIT_TEST_DIR make unit-test-coverage
+
       - name: Upload test results
         if: ${{ !cancelled() }}
         run: make upload-test-results
 
   integration-test:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Integration test
     needs: misc-checks
     strategy:
@@ -177,63 +238,15 @@ jobs:
         run: |
           docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v
 
-
-  set-up-functional-test:
-    name: Set up functional test
-    runs-on: ubuntu-20.04
-    outputs:
-      shard_indices: ${{ steps.generate_output.outputs.shard_indices }}
-      total_shards: ${{ steps.generate_output.outputs.shards }}
-      github_timeout: ${{ steps.generate_output.outputs.github_timeout }}
-      test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
-      single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
-      runs_on: ${{ steps.generate_output.outputs.runs_on }}
-      dbs: ${{ steps.generate_output.outputs.dbs }}
-    steps:
-      - id: generate_output
-        run: |
-          shards=3
-          timeout=30
-          runs_on='["ubuntu-20.04"]'
-          if [[ "${{ inputs.run_single_functional_test }}" == "true" ]]; then
-            shards=1
-            timeout=${{ inputs.timeout_minutes }}
-            single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
-            if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
-              runs_on='["ubuntu-latest-16-cores"]'
-            fi 
-            dbs_arr=()
-            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi
-            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi
-            if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi
-            if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi
-            if [[ ${#dbs_arr[@]} == 0 ]]; then
-              dbs='["sqlite"]'
-            else
-              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
-            fi
-          fi
-          {
-            echo "shard_indices=[$(seq -s, 0 $((shards-1)))]"
-            echo "shards=$shards"
-            echo "github_timeout=$((timeout+5))"
-            echo "test_timeout=${timeout}m"
-            echo "single_test_args=$single_test_args"
-            echo "runs_on=$runs_on"
-            echo "dbs=$dbs" 
-          } >> "$GITHUB_OUTPUT"
-      - id: cat_output
-        run: |
-          cat "$GITHUB_OUTPUT"
-
   functional-test:
+    if: ${{ inputs.run_single_unit_test != true }}
     name: Functional test
-    needs: [misc-checks, set-up-functional-test]
+    needs: [misc-checks, set-up-single-test]
     strategy:
       fail-fast: false
       matrix:
-        runs-on: ${{ fromJson(needs.set-up-functional-test.outputs.runs_on) }}
-        shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }}
+        runs-on: ${{ fromJson(needs.set-up-single-test.outputs.runs_on) }}
+        shard_index: ${{ fromJson(needs.set-up-single-test.outputs.shard_indices) }}
         name:
           - cass_es
           - cass_es8
@@ -270,36 +283,35 @@ jobs:
             containers: [postgresql]
     runs-on: ${{ matrix.runs-on }}
     env:
-      TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }}
+      TEST_TOTAL_SHARDS: ${{ needs.set-up-single-test.outputs.total_shards }}
       TEST_SHARD_INDEX: ${{ matrix.shard_index }}
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      SINGLE_TEST_ARGS: ${{ needs.set-up-functional-test.outputs.single_test_args }}
-      FUNCTIONAL_TEST_ROOT: ${{ inputs.test_directory }}
-      TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }}
+      SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
+      TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
       BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
     steps:
       - name: Print functional test
-        run: echo "${{ needs.set-up-functional-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
+        run: echo "${{ needs.set-up-single-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
       - uses: ScribeMD/docker-cache@0.3.7
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           key: docker-${{ runner.os }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}
 
       - uses: actions/checkout@v4
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ env.COMMIT }}
 
       - uses: actions/setup-go@v5
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
           go-version-file: 'go.mod'
           check-latest: true
 
       - name: Start containerized dependencies
-        if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
+        if: ${{ toJson(matrix.containers) != '[]' && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         uses: hoverkraft-tech/compose-action@v2.0.1
         with:
           compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
@@ -307,16 +319,16 @@ jobs:
           down-flags: -v
 
       - name: Run functional test
-        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
-        timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
+        if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
+        timeout-minutes: ${{ fromJSON(needs.set-up-single-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile
         run: make functional-test-coverage
 
       - name: Upload test results
-        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-functional-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
+        if: ${{ !cancelled() && (inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER))) }}
         run: make upload-test-results
 
   functional-test-xdc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Functional test xdc
     needs: misc-checks
     strategy:
@@ -379,7 +391,7 @@ jobs:
         run: make upload-test-results
 
   functional-test-ndc:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Functional test ndc
     needs: misc-checks
     strategy:
@@ -444,7 +456,7 @@ jobs:
         run: make functional-test-ndc-coverage
 
   test-status:
-    if: ${{ inputs.run_single_functional_test != true }}
+    if: ${{ inputs.run_single_functional_test != true && inputs.run_single_unit_test != true }}
     name: Test Status
     needs:
       - unit-test
diff --git a/Makefile b/Makefile
index 968d9850cc7..237e05c6aeb 100644
--- a/Makefile
+++ b/Makefile
@@ -82,13 +82,13 @@ ALL_SCRIPTS     := $(shell find . -name "*.sh")
 MAIN_BRANCH    := main
 
 TEST_DIRS       := $(sort $(dir $(filter %_test.go,$(ALL_SRC))))
-FUNCTIONAL_TEST_ROOT          ?= ./tests
+FUNCTIONAL_TEST_ROOT          := ./tests
 FUNCTIONAL_TEST_XDC_ROOT      := ./tests/xdc
 FUNCTIONAL_TEST_NDC_ROOT      := ./tests/ndc
 DB_INTEGRATION_TEST_ROOT      := ./common/persistence/tests
 DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests
 INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest ./internal/temporalite
-UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% ./internal/temporalite%,$(TEST_DIRS))
+UNIT_TEST_DIRS ?= $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% ./internal/temporalite%,$(TEST_DIRS))
 
 # github.com/urfave/cli/v2@v2.4.0             - needs to accept comma in values before unlocking https://github.com/urfave/cli/pull/1241.
 PINNED_DEPENDENCIES := \
@@ -376,9 +376,10 @@ $(TEST_OUTPUT_ROOT):
 prepare-coverage-test: $(GOTESTSUM) $(TEST_OUTPUT_ROOT)
 
 unit-test-coverage: prepare-coverage-test
+	@printf $(COLOR) $(UNIT_TEST_DIRS)
 	@printf $(COLOR) "Run unit tests with coverage..."
 	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
-		$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) -coverprofile=$(NEW_COVER_PROFILE)
+		$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) $(SINGLE_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE)
 
 integration-test-coverage: prepare-coverage-test
 	@printf $(COLOR) "Run integration tests with coverage..."

From b70af29dabff075e42d203e08786a0457df230d6 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Fri, 4 Oct 2024 14:16:25 -0700
Subject: [PATCH 74/76] print test command in run

---
 Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 237e05c6aeb..31a7475ca5f 100644
--- a/Makefile
+++ b/Makefile
@@ -376,9 +376,8 @@ $(TEST_OUTPUT_ROOT):
 prepare-coverage-test: $(GOTESTSUM) $(TEST_OUTPUT_ROOT)
 
 unit-test-coverage: prepare-coverage-test
-	@printf $(COLOR) $(UNIT_TEST_DIRS)
 	@printf $(COLOR) "Run unit tests with coverage..."
-	@$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
+	$(GOTESTSUM) --junitfile $(NEW_REPORT) -- \
 		$(UNIT_TEST_DIRS) -shuffle on -timeout=$(TEST_TIMEOUT) -race $(TEST_TAG_FLAG) $(SINGLE_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE)
 
 integration-test-coverage: prepare-coverage-test

From cb7d6c1e6e145ec18cb6867ec249f656472b2cc5 Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Wed, 9 Oct 2024 11:32:44 -0700
Subject: [PATCH 75/76] take dbs as list

---
 .github/workflows/run-tests.yml | 35 ++++++---------------------------
 1 file changed, 6 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b4f4b060677..f13dea671fb 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -43,24 +43,10 @@ on:
         options:
           - "16GB RAM (ubuntu-20.04)"
           - "64GB RAM (ubuntu-22.04)"
-      cassandra:
-        description: "cassandra"
-        type: boolean
-        default: false
-      sqlite:
-        description: "sqlite"
-        type: boolean
-        default: false
-      #      mysql8:
-      #        description: "mysql8"
-      #        type: boolean
-      #        default: false
-#      postgres12:
-#        description: "postgres12"
-#        type: boolean
-#        default: false
-          # if [[ "${{ inputs.mysql8 }}" == "true" ]]; then dbs_arr+=("mysql8"); fi
-        # if [[ "${{ inputs.postgres12 }}" == "true" ]]; then dbs_arr+=("postgres12"); fi
+      test_dbs:
+        description: 'List of DBs to test on (i.e. ["sqlite", "cassandra", "mysql8", "postgres12"])'
+        type: string
+        default: '["sqlite"]'
 
 concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
   group: run-tests-${{ github.head_ref || github.run_id }}
@@ -87,7 +73,7 @@ jobs:
       test_timeout: ${{ steps.generate_output.outputs.test_timeout }}
       single_test_args: ${{ steps.generate_output.outputs.single_test_args }}
       runs_on: ${{ steps.generate_output.outputs.runs_on }}
-      dbs: ${{ steps.generate_output.outputs.dbs }}
+      dbs: ${{ inputs.test_dbs }}
     steps:
       - id: generate_output
         run: |
@@ -100,14 +86,6 @@ jobs:
             single_test_args="-run ${{ inputs.test_name }} -count ${{ inputs.n_runs }}"
             if [[ "${{ inputs.test_runner }}" == "64GB RAM (ubuntu-22.04)" ]]; then
               runs_on='[ "ubuntu-latest-16-cores" ]'
-            fi 
-            dbs_arr=()
-            if [[ "${{ inputs.cassandra }}" == "true" ]]; then dbs_arr+=("cassandra"); fi
-            if [[ "${{ inputs.sqlite }}" == "true" ]]; then dbs_arr+=("sqlite"); fi
-            if [[ ${#dbs_arr[@]} == 0 ]]; then
-              dbs='["sqlite"]'
-            else
-              dbs=$(jq -c -n '$ARGS.positional' --args "${dbs_arr[@]}")
             fi
           fi
           {
@@ -117,7 +95,6 @@ jobs:
             echo "test_timeout=${timeout}m"
             echo "single_test_args=$single_test_args"
             echo "runs_on=$runs_on"
-            echo "dbs=$dbs" 
           } >> "$GITHUB_OUTPUT"
       - id: cat_output
         run: |
@@ -398,7 +375,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [ubuntu-20.04]
-        name: 
+        name:
           - cass_es
           - cass_es8
           - mysql8

From c857565fccca77080110b8bd6f35030585a756df Mon Sep 17 00:00:00 2001
From: Carly de Frondeville <carly.defrondeville@temporal.io>
Date: Wed, 9 Oct 2024 11:43:52 -0700
Subject: [PATCH 76/76] make misc-checks a no-op in single test mode

---
 .github/workflows/run-tests.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index f13dea671fb..9a43a94325d 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -109,6 +109,7 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     steps:
       - uses: actions/checkout@v4
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ env.COMMIT }}
@@ -116,19 +117,25 @@ jobs:
           fetch-depth: 100
 
       - uses: actions/setup-go@v5
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
         with:
           go-version-file: 'go.mod'
           check-latest: true
 
       - uses: arduino/setup-protoc@v3
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
 
       - run: GOOS=windows GOARCH=amd64 make clean-bins bins
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
 
       - run: GOOS=darwin GOARCH=arm64 make clean-bins bins
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
 
       - run: make clean-bins ci-build-misc
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
 
       - run: make build-tests
+        if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
 
   unit-test:
     if: ${{ inputs.run_single_functional_test != true }}