Disable more flaky tests

Follow up to #6634, the following tests are still flaky:

```
TestTaskWriterShutdown
TestVersioningFunctionalSuite/TestDispatchRetry
TestUpdateWorkflowSdkSuite/TestUpdateWorkflow_TimeoutWorkflowAfterUpdateAccepted
```

Also increased XDC timeout to 40 minutes since those tests occasionally
time out.

.github/workflows/run-tests.yml

@@ -78,7 +78,7 @@ jobs:
       - id: generate_output
         run: |
           shards=3
-          timeout=30
+          timeout=40
           runs_on='["ubuntu-20.04"]'
           if [[ "${{ inputs.run_single_functional_test }}" == "true" || "${{ inputs.run_single_unit_test }}" == "true" ]]; then
             shards=1
@@ -119,7 +119,7 @@ jobs:
       - uses: actions/setup-go@v5
         if: ${{ !inputs.run_single_functional_test && !inputs.run_single_unit_test }}
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
           check-latest: true
 
       - uses: arduino/setup-protoc@v3
@@ -147,7 +147,7 @@ jobs:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
     env:
-      BUILDKITE_MESSAGE: "{\"job\": \"unit-test\"}"
+      BUILDKITE_MESSAGE: '{"job": "unit-test"}'
       SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
       UNIT_TEST_DIR: ${{ inputs.unit_test_directory }}
       TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
@@ -160,7 +160,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
           check-latest: true
 
       - name: Run unit tests
@@ -187,7 +187,7 @@ jobs:
         runs-on: [ubuntu-20.04]
     runs-on: ${{ matrix.runs-on }}
     env:
-      BUILDKITE_MESSAGE: "{\"job\": \"integration-test\"}"
+      BUILDKITE_MESSAGE: '{"job": "integration-test"}'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -196,7 +196,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
           check-latest: true
 
       - name: Start containerized dependencies
@@ -273,7 +273,7 @@ jobs:
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
       SINGLE_TEST_ARGS: ${{ needs.set-up-single-test.outputs.single_test_args }}
       TEST_TIMEOUT: ${{ needs.set-up-single-test.outputs.test_timeout }}
-      BUILDKITE_MESSAGE: "{\"job\": \"functional-test\", \"db\": \"${{ matrix.persistence_driver }}\"}"
+      BUILDKITE_MESSAGE: '{"job": "functional-test", "db": "${{ matrix.persistence_driver }}"}'
     steps:
       - name: Print functional test
         run: echo "${{ needs.set-up-single-test.outputs.dbs }}" && echo "$SINGLE_TEST_ARGS"
@@ -291,7 +291,7 @@ jobs:
       - uses: actions/setup-go@v5
         if: ${{ inputs.run_single_functional_test != true || (inputs.run_single_functional_test == true && contains(fromJSON(needs.set-up-single-test.outputs.dbs), env.PERSISTENCE_DRIVER)) }}
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
           check-latest: true
 
       - name: Start containerized dependencies
@@ -345,7 +345,7 @@ jobs:
     env:
       PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
       PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
-      BUILDKITE_MESSAGE: "{\"job\": \"functional-test-xdc\", \"db\": \"${{ matrix.persistence_driver }}\"}"
+      BUILDKITE_MESSAGE: '{"job": "functional-test-xdc", "db": "${{ matrix.persistence_driver }}"}'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -354,7 +354,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
           check-latest: true
 
       - name: Start containerized dependencies
@@ -366,7 +366,7 @@ jobs:
           down-flags: -v
 
       - name: Run functional test xdc
-        timeout-minutes: 30
+        timeout-minutes: 40
         run: make functional-test-xdc-coverage
 
       - name: Upload test results
@@ -423,7 +423,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version-file: "go.mod"
           check-latest: true
 
       - name: Start containerized dependencies

Makefile

@@ -64,9 +64,10 @@ define NEWLINE
 
 endef
 
-# 30 minutes is the upper bound defined for all tests, the longer running ones at the time of writing are XDC tests. The
-# 30 minute timeout is also defined in run-tests.yml
-TEST_TIMEOUT ?= 30m
+# 35 minutes is the upper bound defined for all tests, the longer running ones at the time of writing are XDC tests.
+# This takes into account the 40 minute timeout defined in run-tests.yml minus 5 minutes to avoid timing out the GH
+# action, which causes the a job run to not produce any logs and hurts the debugging experience.
+TEST_TIMEOUT ?= 35m
 
 PROTO_ROOT := proto
 PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto")

service/matching/backlog_manager_test.go

@@ -139,6 +139,8 @@ func TestReadLevelForAllExpiredTasksInBatch(t *testing.T) {
 }
 
 func TestTaskWriterShutdown(t *testing.T) {
+	t.Skip("flaky test")
+
 	controller := gomock.NewController(t)
 
 	tlm := mustCreateTestPhysicalTaskQueueManager(t, controller)

tests/update_workflow_sdk_test.go

@@ -94,6 +94,8 @@ func (s *UpdateWorkflowSdkSuite) TestUpdateWorkflow_TerminateWorkflowAfterUpdate
 // server times out the WF after the update has been accepted but before it has been completed. It checks
 // that the client gets a NotFound error when attempting to fetch the update result (rather than a timeout).
 func (s *UpdateWorkflowSdkSuite) TestUpdateWorkflow_TimeoutWorkflowAfterUpdateAccepted() {
+	s.T().Skip("flaky test")
+
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 	tv := testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(namespace.Name(s.Namespace()))

tests/versioning_test.go

@@ -3509,6 +3509,8 @@ func (s *VersioningIntegSuite) dispatchRetryOld() {
 }
 
 func (s *VersioningIntegSuite) TestDispatchRetry() {
+	s.T().Skip("flaky test")
+
 	s.RunTestWithMatchingBehavior(s.dispatchRetry)
 }