Run timeout tests via Gradle test task so they can be retried

Since some of these tests occasionally fail on CI, in particular on Windows, they are now run via Gradle so that they are retried on CI automatically and reported as flaky rather than failing the build.
yhkuo41 · Feb 21, 2023 · 4e64cf2 · 4e64cf2
1 parent e0e8778
commit 4e64cf2
Show file tree

Hide file tree

Showing 9 changed files with 35 additions and 6 deletions.
diff --git a/documentation/documentation.gradle.kts b/documentation/documentation.gradle.kts
@@ -13,6 +13,7 @@ plugins {
 	id("org.asciidoctor.jvm.pdf")
 	id("org.ajoberstar.git-publish")
 	id("junitbuild.kotlin-library-conventions")
+	id("junitbuild.testing-conventions")
 }
 
 val modularProjects: List<Project> by rootProject
@@ -37,7 +38,6 @@ dependencies {
 	// in reports generated by the ApiReportGenerator.
 	modularProjects.forEach { apiReport(it) }
 
-	testImplementation(projects.junitJupiter)
 	testImplementation(projects.junitJupiterMigrationsupport)
 	testImplementation(projects.junitPlatformConsole)
 	testImplementation(projects.junitPlatformRunner)
@@ -46,7 +46,6 @@ dependencies {
 	testImplementation(kotlin("stdlib"))
 
 	testImplementation(projects.junitVintageEngine)
-	testRuntimeOnly(libs.bundles.log4j)
 	testRuntimeOnly(libs.apiguardian) {
 		because("it's required to generate API tables")
 	}
@@ -121,6 +120,7 @@ tasks {
 		args.addAll("--include-classname", ".*Tests")
 		args.addAll("--include-classname", ".*Demo")
 		args.addAll("--exclude-tag", "exclude")
+		args.addAll("--exclude-tag", "timeout")
 	}
 
 	register<RunConsoleLauncher>("consoleLauncher") {
@@ -131,11 +131,19 @@ tasks {
 		args.addAll("--include-classname", ".*Tests")
 		args.addAll("--include-classname", ".*Demo")
 		args.addAll("--exclude-tag", "exclude")
+		args.addAll("--exclude-tag", "timeout")
 	}
 
 	test {
+		include("**/*Demo.class")
+		(options as JUnitPlatformOptions).apply {
+			includeEngines("junit-vintage")
+			includeTags("timeout")
+		}
+	}
+
+	check {
 		dependsOn(consoleLauncherTest)
-		exclude("**/*")
 	}
 
 	val generateConsoleLauncherOptions by registering(JavaExec::class) {

diff --git a/documentation/src/test/java/example/AssertionsDemo.java b/documentation/src/test/java/example/AssertionsDemo.java
@@ -27,6 +27,7 @@
 import example.domain.Person;
 import example.util.Calculator;
 
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 
 class AssertionsDemo {
@@ -91,6 +92,9 @@ void exceptionTesting() {
 		assertEquals("/ by zero", exception.getMessage());
 	}
 
+	// end::user_guide[]
+	@Tag("timeout")
+	// tag::user_guide[]
 	@Test
 	void timeoutNotExceeded() {
 		// The following assertion succeeds.
@@ -99,6 +103,9 @@ void timeoutNotExceeded() {
 		});
 	}
 
+	// end::user_guide[]
+	@Tag("timeout")
+	// tag::user_guide[]
 	@Test
 	void timeoutNotExceededWithResult() {
 		// The following assertion succeeds, and returns the supplied object.
@@ -108,6 +115,9 @@ void timeoutNotExceededWithResult() {
 		assertEquals("a result", actualResult);
 	}
 
+	// end::user_guide[]
+	@Tag("timeout")
+	// tag::user_guide[]
 	@Test
 	void timeoutNotExceededWithMethod() {
 		// The following assertion invokes a method reference and returns an object.
@@ -116,6 +126,7 @@ void timeoutNotExceededWithMethod() {
 	}
 
 	// end::user_guide[]
+	@Tag("timeout")
 	@extensions.ExpectToFail
 	// tag::user_guide[]
 	@Test
@@ -129,6 +140,7 @@ void timeoutExceeded() {
 	}
 
 	// end::user_guide[]
+	@Tag("timeout")
 	@extensions.ExpectToFail
 	// tag::user_guide[]
 	@Test

diff --git a/documentation/src/test/java/example/SlowTests.java b/documentation/src/test/java/example/SlowTests.java
@@ -10,15 +10,16 @@
 
 package example;
 
-// tag::user_guide[]
 import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD;
 
 import java.util.stream.IntStream;
 
 import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.parallel.Execution;
 
+@Tag("exclude")
 @Disabled
 class SlowTests {
 
@@ -123,4 +124,3 @@ private void foo() {
 		IntStream.range(1, 100_000_000).mapToDouble(i -> Math.pow(i, i)).map(Math::sqrt).max();
 	}
 }
-// end::user_guide[]
diff --git a/documentation/src/test/java/example/TimeoutDemo.java b/documentation/src/test/java/example/TimeoutDemo.java
@@ -13,10 +13,12 @@
 import java.util.concurrent.TimeUnit;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.Timeout;
 import org.junit.jupiter.api.Timeout.ThreadMode;
 
+@Tag("timeout")
 // tag::user_guide[]
 class TimeoutDemo {
 

diff --git a/documentation/src/test/kotlin/example/KotlinAssertionsDemo.kt b/documentation/src/test/kotlin/example/KotlinAssertionsDemo.kt
@@ -15,6 +15,7 @@ import example.domain.Person
 import example.util.Calculator
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Tag
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertAll
 import org.junit.jupiter.api.assertDoesNotThrow
@@ -76,6 +77,9 @@ class KotlinAssertionsDemo {
         )
     }
 
+    // end::user_guide[]
+    @Tag("timeout")
+    // tag::user_guide[]
     @Test
     fun `timeout not exceeded testing`() {
         val fibonacciCalculator = FibonacciCalculator()
@@ -86,6 +90,7 @@ class KotlinAssertionsDemo {
     }
 
     // end::user_guide[]
+    @Tag("timeout")
     @extensions.ExpectToFail
     // tag::user_guide[]
     @Test

diff --git a/gradle/plugins/src/main/kotlin/junitbuild.testing-conventions.gradle.kts b/gradle/plugins/src/main/kotlin/junitbuild.testing-conventions.gradle.kts
@@ -79,7 +79,6 @@ dependencies {
 	if (!project.name.startsWith("junit-jupiter")) {
 		testImplementation(project(":junit-jupiter"))
 	}
-	testImplementation(testFixtures(project(":junit-jupiter-api")))
 
 	testRuntimeOnly(project(":junit-platform-engine"))
 	testRuntimeOnly(project(":junit-platform-jfr"))

diff --git a/junit-jupiter-engine/junit-jupiter-engine.gradle.kts b/junit-jupiter-engine/junit-jupiter-engine.gradle.kts
@@ -24,6 +24,7 @@ dependencies {
 	testImplementation(libs.junit4)
 	testImplementation(libs.kotlinx.coroutines)
 	testImplementation(libs.groovy4)
+	testImplementation(testFixtures(projects.junitJupiterApi))
 
 	osgiVerification(projects.junitPlatformLauncher)
 }

diff --git a/junit-vintage-engine/junit-vintage-engine.gradle.kts b/junit-vintage-engine/junit-vintage-engine.gradle.kts
@@ -22,6 +22,7 @@ dependencies {
 	testImplementation(projects.junitPlatformLauncher)
 	testImplementation(projects.junitPlatformSuiteEngine)
 	testImplementation(projects.junitPlatformTestkit)
+	testImplementation(testFixtures(projects.junitJupiterApi))
 
 	osgiVerification(projects.junitPlatformLauncher)
 }

diff --git a/platform-tests/platform-tests.gradle.kts b/platform-tests/platform-tests.gradle.kts
@@ -33,6 +33,7 @@ dependencies {
 	testImplementation(libs.joox)
 	testImplementation(libs.openTestReporting.tooling)
 	testImplementation(libs.bundles.xmlunit)
+	testImplementation(testFixtures(projects.junitJupiterApi))
 
 	// --- Test run-time dependencies ---------------------------------------------
 	testRuntimeOnly(projects.junitVintageEngine)