qemu.go, harness.go: Add go routine to handle badness

After the tests are complete this go routine verifies contents
of console.txt and journal.txt through the use of CheckConsole.

Ref: #3788

mantle/cmd/kola/testiso.go

@@ -675,59 +675,85 @@ func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir st
 					err = platform.ErrInitramfsEmergency
 				}
 			}
+		}()
+		go func() {
+			err := inst.Wait()
+			// only one Wait() gets process data, so also manually check for signal
+			plog.Debugf("qemu exited err=%v", err)
+			if err == nil && inst.Signaled() {
+				err = errors.New("process killed")
+			}
 			if err != nil {
-				errchan <- err
+				errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion")
 			}
+			time.Sleep(1 * time.Minute)
+			errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion")
 		}()
-	}
-	go func() {
-		err := inst.Wait()
-		// only one Wait() gets process data, so also manually check for signal
-		plog.Debugf("qemu exited err=%v", err)
-		if err == nil && inst.Signaled() {
-			err = errors.New("process killed")
-		}
-		if err != nil {
-			errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion")
-		}
-		time.Sleep(1 * time.Minute)
-		errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion")
-	}()
-	go func() {
-		r := bufio.NewReader(qchan)
-		for _, exp := range expected {
-			l, err := r.ReadString('\n')
-			if err != nil {
-				if err == io.EOF {
-					// this may be from QEMU getting killed or exiting; wait a bit
-					// to give a chance for .Wait() above to feed the channel with a
-					// better error
-					time.Sleep(1 * time.Second)
-					errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp)
-				} else {
-					errchan <- errors.Wrapf(err, "reading from completion channel")
+		go func() {
+			r := bufio.NewReader(qchan)
+			for _, exp := range expected {
+				l, err := r.ReadString('\n')
+				if err != nil {
+					if err == io.EOF {
+						// this may be from QEMU getting killed or exiting; wait a bit
+						// to give a chance for .Wait() above to feed the channel with a
+						// better error
+						time.Sleep(1 * time.Second)
+						errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp)
+					} else {
+						errchan <- errors.Wrapf(err, "reading from completion channel")
+					}
+					return
+				}
+				line := strings.TrimSpace(l)
+				if line != exp {
+					errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp)
+					return
 				}
-				return
+				plog.Debugf("Matched expected message %s", exp)
 			}
-			line := strings.TrimSpace(l)
-			if line != exp {
-				errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp)
-				return
+			plog.Debugf("Matched all expected messages")
+			// OK!
+			errchan <- nil
+		}()
+		go func() {
+			//check for error when switching boot order
+			if booterrchan != nil {
+				if err := <-booterrchan; err != nil {
+					errchan <- err
+				}
 			}
-			plog.Debugf("Matched expected message %s", exp)
-		}
-		plog.Debugf("Matched all expected messages")
-		// OK!
-		errchan <- nil
-	}()
-	go func() {
-		//check for error when switching boot order
-		if booterrchan != nil {
-			if err := <-booterrchan; err != nil {
-				errchan <- err
+		}()
+		go func() {
+			err := <-errchan
+			if err == nil {
+				// No error so far, check the console and journal files
+				files := []string{filepath.Join(outdir, "console.txt"), filepath.Join(outdir, "journal.txt")}
+				for _, file := range files {
+					// Read the contents of the file
+					fileContent, err := inst.ReadFile(file)
+					if err != nil {
+						fmt.Printf("error reading file %s: %v\n", file, err)
+						return
+					}
+					// Check for badness with CheckConsole
+					warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil)
+					if len(badlines) > 0 {
+						err = fmt.Errorf("errors found in console file: %v", badlines)
+						for _, badline := range badlines {
+							if err != nil {
+								fmt.Printf("badness detected in console file: %v\n", badline)
+							}
+						}
+						errchan <- err
+					} else if warnOnly {
+						fmt.Println("warnings found in console file")
+						continue
+					}
+				}
 			}
-		}
-	}()
+		}()
+	}
 	err := <-errchan
 	return time.Since(start), err
 }

mantle/kola/harness.go

@@ -1917,6 +1917,27 @@ func ScpKolet(machines []platform.Machine) error {
 	return fmt.Errorf("Unable to locate kolet binary for %s", mArch)
 }
 
+// CheckConsoleText checks console output for badness
+func CheckConsoleText(consoleOutput []byte) (bool, []string) {
+	var badlines []string
+	warnOnly := true
+	for _, check := range consoleChecks {
+		match := check.match.FindSubmatch(consoleOutput)
+		if match != nil {
+			badline := check.desc
+			if len(match) > 1 {
+				// include first subexpression
+				badline += fmt.Sprintf(" (%s)", match[1])
+			}
+			badlines = append(badlines, badline)
+			if !check.warnOnly {
+				warnOnly = false
+			}
+		}
+	}
+	return warnOnly, badlines
+}
+
 // CheckConsole checks some console output for badness and returns short
 // descriptions of any bad lines it finds along with a boolean
 // indicating if the configuration has the bad lines marked as
@@ -1926,6 +1947,12 @@ func ScpKolet(machines []platform.Machine) error {
 func CheckConsole(output []byte, t *register.Test) (bool, []string) {
 	var badlines []string
 	warnOnly, allowRerunSuccess := true, true
+	// Check for badness using CheckConsoleText
+	addWarnOnly, addBadlines := CheckConsoleText(output)
+	if !addWarnOnly {
+		warnOnly = false
+	}
+	badlines = append(badlines, addBadlines...)
 	for _, check := range consoleChecks {
 		if check.skipFlag != nil && t != nil && t.HasFlag(*check.skipFlag) {
 			continue

mantle/platform/qemu.go

@@ -255,6 +255,17 @@ func (inst *QemuInstance) WaitIgnitionError(ctx context.Context) (string, error)
 	return r.String(), nil
 }
 
+// Read file, output contents and return error if reading fails
+func (inst *QemuInstance) ReadFile(filename string) (string, error) {
+    data, err := os.ReadFile(filename)
+	errchan := make(chan error)
+    if err != nil {
+		errchan <- err
+        return "", err
+    }
+    return string(data), nil
+}
+
 // WaitAll wraps the process exit as well as WaitIgnitionError,
 // returning an error if either fail.
 func (inst *QemuInstance) WaitAll(ctx context.Context) error {