From 0f06f66a270fac0e44c8a1cc6aef6133649eb69d Mon Sep 17 00:00:00 2001
From: Nikita Dubrovskii <nikita@linux.ibm.com>
Date: Mon, 17 Jul 2023 08:14:07 +0200
Subject: [PATCH] kola: support denylist Warn feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

https://github.com/coreos/coreos-assembler/issues/3344

We bubble denylisted tests with 'Warn: true' option as warnings rather than hard failures:
```
kola -p qemu run --parallel 8 ext.config.ntp.* --output-dir tmp/kola

⚠️  Warning kola test pattern "ext.config.ntp.chrony.dhcp-propagation", snoozing expired on Jul 07 2023:
  👉 https://github.com/coreos/fedora-coreos-tracker/issues/1508
⚠️  Warning kola test pattern "ext.config.ntp.timesyncd.dhcp-propagation":
  👉 https://github.com/coreos/fedora-coreos-tracker/issues/1508
=== RUN   ext.config.ntp.timesyncd.dhcp-propagation
=== RUN   ext.config.ntp.chrony.coreos-platform-chrony-config
=== RUN   ext.config.ntp.chrony.dhcp-propagation
--- PASS: ext.config.ntp.chrony.coreos-platform-chrony-config (27.02s)
--- FAIL: ext.config.ntp.chrony.dhcp-propagation (107.62s)
--- FAIL: ext.config.ntp.timesyncd.dhcp-propagation (118.02s)
FAIL, output in tmp/kola
--- ⚠️ IGNORE: "ext.config.ntp.chrony.dhcp-propagation"
--- ⚠️ IGNORE: "ext.config.ntp.timesyncd.dhcp-propagation"
+ rc=0
+ set +x
```
---
 mantle/kola/harness.go | 56 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 5 deletions(-)

diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go
index f4283e72cf..0dae414564 100644
--- a/mantle/kola/harness.go
+++ b/mantle/kola/harness.go
@@ -128,6 +128,7 @@ var (
 	// SkipConsoleWarnings is set via SkipConsoleWarningsTag in kola-denylist.yaml
 	SkipConsoleWarnings bool
 	DenylistedTests     []string // tests which are on the denylist
+	WarnOnErrorTests    []string // denylisted tests we are going to run and warn in case of error
 	Tags                []string // tags to be ran
 
 	// Sharding is a string of the form: hash:m/n where m and n are integers to run only tests which hash to m.
@@ -356,6 +357,7 @@ type DenyListObj struct {
 	Platforms  []string `yaml:"platforms"`
 	SnoozeDate string   `yaml:"snooze"`
 	OsVersion  []string `yaml:"osversion"`
+	Warn       bool     `yaml:"warn"`
 }
 
 type ManifestData struct {
@@ -451,11 +453,19 @@ func ParseDenyListYaml(pltfrm string) error {
 			snoozeDate, err := time.Parse(snoozeFormat, obj.SnoozeDate)
 			if err != nil {
 				return err
-			} else if today.After(snoozeDate) {
-				continue
 			}
-
-			fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+			if today.After(snoozeDate) {
+				if !obj.Warn {
+					continue
+				}
+				fmt.Printf("⚠️  Warning kola test pattern \"%s\", snoozing expired on %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+				WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
+			} else {
+				fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+			}
+		} else if obj.Warn {
+			fmt.Printf("⚠️  Warning kola test pattern \"%s\":\n", obj.Pattern)
+			WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
 		} else {
 			fmt.Printf("⚠️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
 		}
@@ -467,7 +477,7 @@ func ParseDenyListYaml(pltfrm string) error {
 		/// Process "special" patterns which aren't test names, but influence overall behavior
 		if obj.Pattern == SkipConsoleWarningsTag {
 			SkipConsoleWarnings = true
-		} else {
+		} else if !obj.Warn {
 			DenylistedTests = append(DenylistedTests, obj.Pattern)
 		}
 	}
@@ -854,10 +864,29 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 			return reRunErr
 		}
 	}
+
+	// Ignore the error when only denied tests with Warn:true feaute failed
+	if runErr != nil && allTestsWarnOnError(testResults.getResults()) {
+		return nil
+	}
+
 	// If the intial run failed and the rerun passed, we still return an error
 	return runErr
 }
 
+func allTestsWarnOnError(tests []*harness.H) bool {
+	for _, test := range tests {
+		if !test.Failed() {
+			continue
+		}
+		if !IsWarningOnFailure(test.Name()) {
+			return false
+		}
+		fmt.Printf("--- ⚠️ IGNORE: \"%s\"\n", test.Name())
+	}
+	return true
+}
+
 func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool {
 	// Always consider the special AllowRerunSuccessTag that is added
 	// by the test harness in some failure scenarios.
@@ -875,6 +904,9 @@ func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun
 	// Iterate over the tests that were re-ran. If any of them don't
 	// allow rerun success then just exit early.
 	for _, test := range testsToRerun {
+		if IsWarningOnFailure(test) {
+			continue
+		}
 		testAllowsRerunSuccess := false
 		for _, tag := range testsBank[test].Tags {
 			if rerunSuccessTagMap[tag] {
@@ -913,6 +945,20 @@ func GetRerunnableTestName(testName string) (string, bool) {
 	}
 }
 
+func IsWarningOnFailure(testName string) bool {
+	for _, pattern := range WarnOnErrorTests {
+		found, err := filepath.Match(pattern, testName)
+		if err != nil {
+			plog.Fatal(err)
+			return false
+		}
+		if found {
+			return true
+		}
+	}
+	return false
+}
+
 func getRerunnable(tests []*harness.H) []string {
 	var testsToRerun []string
 	for _, h := range tests {