kola: support denylist Warn feature

#3344

We bubble denylisted tests with 'Warn: true' option as warnings rather than hard failures:
```
kola -p qemu run --parallel 8 ext.config.ntp.* --output-dir tmp/kola

⏭️  Skipping kola test pattern "ext.config.ntp.chrony.dhcp-propagation":
  👉 coreos/fedora-coreos-tracker#1508
⚠️  Warning kola test pattern "ext.config.ntp.timesyncd.dhcp-propagation", snoozing expired on Jul 20 2023:
  👉 coreos/fedora-coreos-tracker#1508
=== RUN   ext.config.ntp.chrony.coreos-platform-chrony-config
=== RUN   ext.config.ntp.timesyncd.dhcp-propagation
--- PASS: ext.config.ntp.chrony.coreos-platform-chrony-config (27.56s)
--- WARN: ext.config.ntp.timesyncd.dhcp-propagation (90.72s)
        cluster.go:162: Error: Unit kola-runext.service exited with code 1
        cluster.go:162: 2023-07-27T06:24:18Z cli: Unit kola-runext.service exited with code 1
        harness.go:1236: kolet failed: : kolet run-test-unit failed: Process exited with status 1
FAIL, output in tmp/kola
+ rc=0
+ set +x
```

Co-authored-by: Dusty Mabe <[email protected]>

mantle/harness/harness.go

@@ -76,6 +76,7 @@ type H struct {
 
 	isParallel               bool
 	nonExclusiveTestsStarted bool
+	warningOnFailure         bool
 
 	timeout   time.Duration // Duration for which the test will be allowed to run
 	timedout  bool          // A timeout was reached
@@ -167,6 +168,9 @@ func (h *H) Verbose() bool {
 
 func (c *H) status() testresult.TestResult {
 	if c.Failed() {
+		if c.warningOnFailure {
+			return testresult.Warn
+		}
 		return testresult.Fail
 	} else if c.Skipped() {
 		return testresult.Skip
@@ -276,6 +280,10 @@ func (c *H) NonExclusiveTestStarted() {
 	c.nonExclusiveTestsStarted = true
 }
 
+func (c *H) WarningOnFailure() {
+	c.warningOnFailure = true
+}
+
 // Fail marks the function as having failed but continues execution.
 func (c *H) Fail() {
 	if c.parent != nil {

mantle/harness/testresult/status.go

@@ -16,6 +16,7 @@ package testresult
 
 const (
 	Fail TestResult = "FAIL"
+	Warn TestResult = "WARN"
 	Skip TestResult = "SKIP"
 	Pass TestResult = "PASS"
 )

mantle/kola/harness.go

@@ -128,6 +128,7 @@ var (
 	// SkipConsoleWarnings is set via SkipConsoleWarningsTag in kola-denylist.yaml
 	SkipConsoleWarnings bool
 	DenylistedTests     []string // tests which are on the denylist
+	WarnOnErrorTests    []string // denylisted tests we are going to run and warn in case of error
 	Tags                []string // tags to be ran
 
 	// Sharding is a string of the form: hash:m/n where m and n are integers to run only tests which hash to m.
@@ -356,6 +357,7 @@ type DenyListObj struct {
 	Platforms  []string `yaml:"platforms"`
 	SnoozeDate string   `yaml:"snooze"`
 	OsVersion  []string `yaml:"osversion"`
+	Warn       bool     `yaml:"warn"`
 }
 
 type ManifestData struct {
@@ -451,13 +453,21 @@ func ParseDenyListYaml(pltfrm string) error {
 			snoozeDate, err := time.Parse(snoozeFormat, obj.SnoozeDate)
 			if err != nil {
 				return err
-			} else if today.After(snoozeDate) {
-				continue
 			}
-
-			fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+			if today.After(snoozeDate) {
+				if !obj.Warn {
+					continue
+				}
+				fmt.Printf("⚠️  Warning kola test pattern \"%s\", snoozing expired on %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+				WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
+			} else {
+				fmt.Printf("🕒 Snoozing kola test pattern \"%s\" until %s:\n", obj.Pattern, snoozeDate.Format("Jan 02 2006"))
+			}
+		} else if obj.Warn {
+			fmt.Printf("⚠️  Will warn on failure for kola test pattern \"%s\":\n", obj.Pattern)
+			WarnOnErrorTests = append(WarnOnErrorTests, obj.Pattern)
 		} else {
-			fmt.Printf("⚠️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
+			fmt.Printf("⏭️  Skipping kola test pattern \"%s\":\n", obj.Pattern)
 		}
 
 		if obj.Tracker != "" {
@@ -467,7 +477,7 @@ func ParseDenyListYaml(pltfrm string) error {
 		/// Process "special" patterns which aren't test names, but influence overall behavior
 		if obj.Pattern == SkipConsoleWarningsTag {
 			SkipConsoleWarnings = true
-		} else {
+		} else if !obj.Warn {
 			DenylistedTests = append(DenylistedTests, obj.Pattern)
 		}
 	}
@@ -854,10 +864,28 @@ func runProvidedTests(testsBank map[string]*register.Test, patterns []string, mu
 			return reRunErr
 		}
 	}
+
+	// Ignore the error when only denied tests with Warn:true feature failed
+	if runErr != nil && allFailedTestsAreWarnOnError(testResults.getResults()) {
+		return nil
+	}
+
 	// If the intial run failed and the rerun passed, we still return an error
 	return runErr
 }
 
+func allFailedTestsAreWarnOnError(tests []*harness.H) bool {
+	for _, test := range tests {
+		if !test.Failed() {
+			continue
+		}
+		if !IsWarningOnFailure(test.Name()) {
+			return false
+		}
+	}
+	return true
+}
+
 func allTestsAllowRerunSuccess(testsBank map[string]*register.Test, testsToRerun, rerunSuccessTags []string) bool {
 	// Always consider the special AllowRerunSuccessTag that is added
 	// by the test harness in some failure scenarios.
@@ -913,6 +941,20 @@ func GetRerunnableTestName(testName string) (string, bool) {
 	}
 }
 
+func IsWarningOnFailure(testName string) bool {
+	for _, pattern := range WarnOnErrorTests {
+		found, err := filepath.Match(pattern, testName)
+		if err != nil {
+			plog.Fatal(err)
+			return false
+		}
+		if found {
+			return true
+		}
+	}
+	return false
+}
+
 func getRerunnable(tests []*harness.H) []string {
 	var testsToRerun []string
 	for _, h := range tests {
@@ -1719,6 +1761,10 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig
 		FailFast:    t.FailFast,
 	}
 
+	if IsWarningOnFailure(t.Name) {
+		tcluster.H.WarningOnFailure()
+	}
+
 	// Note that we passed in SkipStartMachine=true in our machine
 	// options. This means NewMachines() didn't block on the machines
 	// being up with SSH access before returning; i.e. it skipped running