diff --git a/Makefile b/Makefile index b94de374a81..42c7c7660c9 100644 --- a/Makefile +++ b/Makefile @@ -249,6 +249,12 @@ ci-test-job: install-tools dashboard-ui TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso +test-tso: install-tools + # testing TSO function & consistency... + @$(FAILPOINT_ENABLE) + CGO_ENABLED=1 go test -race -tags without_dashboard,tso_full_test,deadlock $(TSO_INTEGRATION_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; } + @$(FAILPOINT_DISABLE) + test-tso-function: install-tools # testing TSO function... @$(FAILPOINT_ENABLE) @@ -267,7 +273,7 @@ test-real-cluster: # testing with the real cluster... cd $(REAL_CLUSTER_TEST_PATH) && $(MAKE) check -.PHONY: test basic-test test-with-cover test-tso-function test-tso-consistency test-real-cluster +.PHONY: test basic-test test-with-cover test-tso test-tso-function test-tso-consistency test-real-cluster #### Daily CI coverage analyze #### diff --git a/tests/server/tso/consistency_test.go b/tests/server/tso/consistency_test.go index 977b18beca9..d1c45df7f17 100644 --- a/tests/server/tso/consistency_test.go +++ b/tests/server/tso/consistency_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/tso" "github.com/tikv/pd/pkg/utils/grpcutil" @@ -63,6 +64,7 @@ func (suite *tsoConsistencyTestSuite) TearDownSuite() { // TestSynchronizedGlobalTSO is used to test the synchronized way of global TSO generation. func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -77,7 +79,6 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() @@ -95,12 +96,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { // Get some local TSOs first oldLocalTSOs := make([]*pdpb.Timestamp, 0, dcLocationNum) for _, dcLocation := range dcLocationConfig { - localTSO := suite.getTimestampByDC(ctx, cluster, dcLocation) + localTSO := suite.getTimestampByDC(ctx, re, cluster, dcLocation) oldLocalTSOs = append(oldLocalTSOs, localTSO) re.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, localTSO)) } // Get a global TSO then - globalTSO := suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) + globalTSO := suite.getTimestampByDC(ctx, re, cluster, tso.GlobalDCLocation) for _, oldLocalTSO := range oldLocalTSOs { re.Equal(1, tsoutil.CompareTimestamp(globalTSO, oldLocalTSO)) } @@ -110,7 +111,7 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { // Get some local TSOs again newLocalTSOs := make([]*pdpb.Timestamp, 0, dcLocationNum) for _, dcLocation := range dcLocationConfig { - newLocalTSOs = append(newLocalTSOs, suite.getTimestampByDC(ctx, cluster, dcLocation)) + newLocalTSOs = append(newLocalTSOs, suite.getTimestampByDC(ctx, re, cluster, dcLocation)) } for _, newLocalTSO := range newLocalTSOs { re.Equal(-1, tsoutil.CompareTimestamp(maxGlobalTSO, newLocalTSO)) @@ -118,7 +119,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSO() { } } -func (suite *tsoConsistencyTestSuite) getTimestampByDC(ctx context.Context, cluster *tests.TestCluster, dcLocation string) *pdpb.Timestamp { +func (suite *tsoConsistencyTestSuite) getTimestampByDC( + ctx context.Context, + re *require.Assertions, + cluster *tests.TestCluster, + dcLocation string, +) *pdpb.Timestamp { req := &pdpb.TsoRequest{ Header: testutil.NewRequestHeader(suite.leaderServer.GetClusterID()), Count: tsoCount, @@ -138,6 +144,7 @@ func (suite *tsoConsistencyTestSuite) getTimestampByDC(ctx context.Context, clus } func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -152,7 +159,6 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() @@ -166,11 +172,12 @@ func (suite *tsoConsistencyTestSuite) TestSynchronizedGlobalTSOOverflow() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/globalTSOOverflow", `return(true)`)) - suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) + suite.getTimestampByDC(ctx, re, cluster, tso.GlobalDCLocation) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/tso/globalTSOOverflow")) } func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { + re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange", `return(true)`)) dcLocationConfig := map[string]string{ "pd1": "dc-1", @@ -184,7 +191,6 @@ func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) suite.leaderServer = cluster.GetLeaderServer() @@ -197,11 +203,12 @@ func (suite *tsoConsistencyTestSuite) TestLocalAllocatorLeaderChange() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - suite.getTimestampByDC(ctx, cluster, tso.GlobalDCLocation) + suite.getTimestampByDC(ctx, re, cluster, tso.GlobalDCLocation) re.NoError(failpoint.Disable("github.com/tikv/pd/server/mockLocalAllocatorLeaderChange")) } func (suite *tsoConsistencyTestSuite) TestLocalTSO() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -232,6 +239,7 @@ func (suite *tsoConsistencyTestSuite) checkTSOUnique(tso *pdpb.Timestamp) bool { } func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { + re := suite.Require() dcLocationConfig := map[string]string{ "pd1": "dc-1", "pd2": "dc-2", @@ -246,7 +254,6 @@ func (suite *tsoConsistencyTestSuite) TestLocalTSOAfterMemberChanged() { re.NoError(err) re.NoError(cluster.RunInitialServers()) - re := suite.Require() cluster.WaitAllLeaders(re, dcLocationConfig) leaderServer := cluster.GetLeaderServer()