From cefd0ea70ad4b43c1c08f7d5343631dbebe547d6 Mon Sep 17 00:00:00 2001 From: Rustam Zagirov Date: Sun, 9 Jul 2023 23:04:54 +0300 Subject: [PATCH 1/2] add flag testify.shuffle --- suite/suite.go | 19 +++++++++++++++++++ suite/suite_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/suite/suite.go b/suite/suite.go index 8b4202d89..7b3290113 100644 --- a/suite/suite.go +++ b/suite/suite.go @@ -3,10 +3,12 @@ package suite import ( "flag" "fmt" + "math/rand" "os" "reflect" "regexp" "runtime/debug" + "strconv" "sync" "testing" "time" @@ -17,6 +19,7 @@ import ( var allTestsFilter = func(_, _ string) (bool, error) { return true, nil } var matchMethod = flag.String("testify.m", "", "regular expression to select tests of the testify suite to run") +var shuffle = flag.String("testify.shuffle", "off", "randomize the execution order of tests") // Suite is a basic testing suite with methods for storing and // retrieving the current *testing.T context. @@ -199,6 +202,22 @@ func Run(t *testing.T, suite TestingSuite) { } tests = append(tests, test) } + if *shuffle != "off" { + var n int64 + var err error + if *shuffle == "on" { + n = time.Now().UnixNano() + } else { + n, err = strconv.ParseInt(*shuffle, 10, 64) + if err != nil { + fmt.Fprintln(os.Stderr, `testing: -shuffle should be "off", "on", or a valid integer:`, err) + return + } + } + fmt.Println("-testify.shuffle", n) + rng := rand.New(rand.NewSource(n)) + rng.Shuffle(len(tests), func(i, j int) { tests[i], tests[j] = tests[j], tests[i] }) + } if suiteSetupDone { defer func() { if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { diff --git a/suite/suite_test.go b/suite/suite_test.go index d684f52b9..10d66831a 100644 --- a/suite/suite_test.go +++ b/suite/suite_test.go @@ -617,3 +617,47 @@ func (s *FailfastSuite) Test_B_Passes() { s.call("Test B Passes") s.Require().True(true) } + +type ShuffleOrderSuite struct { + Suite + callOrder []string +} + +func (s *ShuffleOrderSuite) call(method string) { + s.callOrder = append(s.callOrder, method) +} + +func TestSuiteShuffleOrder(t *testing.T) { + Run(t, new(ShuffleOrderSuite)) +} + +func TestShuffleOrderSuiteShuffleOn(t *testing.T) { + // To test this with testify.shuffle on we launch it in its own process + cmd := exec.Command("go", "test", "-v", "-race", "-run", "TestSuiteShuffleOrder", "-testify.shuffle", "2") + var out bytes.Buffer + cmd.Stdout = &out + t.Logf("Running %s", strings.Join(cmd.Args, " ")) + err := cmd.Run() + t.Log(out.String()) + if err != nil { + t.Log(err) + t.Fail() + } +} + +func (s *ShuffleOrderSuite) TearDownSuite() { + shuffle := flag.Lookup("testify.shuffle").Value.(flag.Getter).Get().(string) + if shuffle == "off" { + assert.Equal(s.T(), "Test A;Test B", strings.Join(s.callOrder, ";")) + } else { + assert.Equal(s.T(), "Test B;Test A", strings.Join(s.callOrder, ";")) + } +} + +func (s *ShuffleOrderSuite) Test_A() { + s.call("Test A") +} + +func (s *ShuffleOrderSuite) Test_B() { + s.call("Test B") +} From 356b4888626421b8ead17f1efbf5b46161d1f7cc Mon Sep 17 00:00:00 2001 From: Rustam Zagirov Date: Tue, 11 Jul 2023 12:03:15 +0300 Subject: [PATCH 2/2] fixed comments from dolmen --- suite/suite.go | 10 +++++++--- suite/suite_test.go | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/suite/suite.go b/suite/suite.go index 7b3290113..0d44a6b8a 100644 --- a/suite/suite.go +++ b/suite/suite.go @@ -19,7 +19,7 @@ import ( var allTestsFilter = func(_, _ string) (bool, error) { return true, nil } var matchMethod = flag.String("testify.m", "", "regular expression to select tests of the testify suite to run") -var shuffle = flag.String("testify.shuffle", "off", "randomize the execution order of tests") +var shuffle = flag.String("testify.shuffle", "off", "randomize the execution order of tests in testify suites, like -test.shuffle") // Suite is a basic testing suite with methods for storing and // retrieving the current *testing.T context. @@ -202,6 +202,9 @@ func Run(t *testing.T, suite TestingSuite) { } tests = append(tests, test) } + + // implementation is similar to the implementation of -test.shuffle in package testing + // https://cs.opensource.google/go/go/+/refs/tags/go1.20.5:src/testing/testing.go;l=1876-1893 if *shuffle != "off" { var n int64 var err error @@ -210,14 +213,15 @@ func Run(t *testing.T, suite TestingSuite) { } else { n, err = strconv.ParseInt(*shuffle, 10, 64) if err != nil { - fmt.Fprintln(os.Stderr, `testing: -shuffle should be "off", "on", or a valid integer:`, err) + t.Fatal(`testing: -testify.shuffle should be "off", "on", or a valid integer:`, err) return } } - fmt.Println("-testify.shuffle", n) + t.Log("-testify.shuffle", n) rng := rand.New(rand.NewSource(n)) rng.Shuffle(len(tests), func(i, j int) { tests[i], tests[j] = tests[j], tests[i] }) } + if suiteSetupDone { defer func() { if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { diff --git a/suite/suite_test.go b/suite/suite_test.go index 10d66831a..5e6fb65ce 100644 --- a/suite/suite_test.go +++ b/suite/suite_test.go @@ -631,9 +631,11 @@ func TestSuiteShuffleOrder(t *testing.T) { Run(t, new(ShuffleOrderSuite)) } +// TestShuffleOrderSuiteShuffleOn relies on the algorithm used by package math/rand. +// If that algorithm changes the seed value may have to be adapted. func TestShuffleOrderSuiteShuffleOn(t *testing.T) { // To test this with testify.shuffle on we launch it in its own process - cmd := exec.Command("go", "test", "-v", "-race", "-run", "TestSuiteShuffleOrder", "-testify.shuffle", "2") + cmd := exec.Command("go", "test", "-v", "-run", "TestSuiteShuffleOrder", "-testify.shuffle", "2") var out bytes.Buffer cmd.Stdout = &out t.Logf("Running %s", strings.Join(cmd.Args, " "))