From fcb5eb7ea5431b486e1513b38d170d0c2f760c1e Mon Sep 17 00:00:00 2001 From: Adam Piasecki Date: Mon, 2 Sep 2024 17:46:56 +0100 Subject: [PATCH] testiso.go: Add go routine to handle badness After the tests are complete this go routine verifies contents of console.txt and journal.txt through the use of CheckConsole. Ref: https://github.com/coreos/coreos-assembler/issues/3788 --- mantle/cmd/kola/testiso.go | 118 ++++++++++++++++++++++--------------- mantle/kola/harness.go | 27 +++++++++ mantle/platform/qemu.go | 11 ++++ 3 files changed, 110 insertions(+), 46 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index e5c991fca8..ff57d85b90 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -675,59 +675,85 @@ func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir st err = platform.ErrInitramfsEmergency } } + }() + go func() { + err := inst.Wait() + // only one Wait() gets process data, so also manually check for signal + plog.Debugf("qemu exited err=%v", err) + if err == nil && inst.Signaled() { + err = errors.New("process killed") + } if err != nil { - errchan <- err + errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") } + time.Sleep(1 * time.Minute) + errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") }() - } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") - } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") + go func() { + r := bufio.NewReader(qchan) + for _, exp := range expected { + l, err := r.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line := strings.TrimSpace(l) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) + return } - return + plog.Debugf("Matched expected message %s", exp) } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return + plog.Debugf("Matched all expected messages") + // OK! + errchan <- nil + }() + go func() { + //check for error when switching boot order + if booterrchan != nil { + if err := <-booterrchan; err != nil { + errchan <- err + } } - plog.Debugf("Matched expected message %s", exp) - } - plog.Debugf("Matched all expected messages") - // OK! - errchan <- nil - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err + }() + go func() { + err := <-errchan + if err == nil { + // No error so far, check the console and journal files + files := []string{filepath.Join(outdir, "console.txt"), filepath.Join(outdir, "journal.txt")} + for _, file := range files { + // Read the contents of the file + fileContent, err := inst.ReadFile(file) + if err != nil { + fmt.Printf("error reading file %s: %v\n", file, err) + return + } + // Check for badness with CheckConsole + warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) + if len(badlines) > 0 { + err = fmt.Errorf("errors found in console file: %v", badlines) + for _, badline := range badlines { + if err != nil { + fmt.Printf("badness detected in console file: %v\n", badline) + } + } + errchan <- err + } else if warnOnly { + fmt.Println("warnings found in console file") + continue + } + } } - } - }() + }() + } err := <-errchan return time.Since(start), err } diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 6ecd84b81d..9111722a3b 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -1917,6 +1917,27 @@ func ScpKolet(machines []platform.Machine) error { return fmt.Errorf("Unable to locate kolet binary for %s", mArch) } +// CheckConsoleText checks console output for badness +func CheckConsoleText(consoleOutput []byte) (bool, []string) { + var badlines []string + warnOnly := true + for _, check := range consoleChecks { + match := check.match.FindSubmatch(consoleOutput) + if match != nil { + badline := check.desc + if len(match) > 1 { + // include first subexpression + badline += fmt.Sprintf(" (%s)", match[1]) + } + badlines = append(badlines, badline) + if !check.warnOnly { + warnOnly = false + } + } + } + return warnOnly, badlines +} + // CheckConsole checks some console output for badness and returns short // descriptions of any bad lines it finds along with a boolean // indicating if the configuration has the bad lines marked as @@ -1926,6 +1947,12 @@ func ScpKolet(machines []platform.Machine) error { func CheckConsole(output []byte, t *register.Test) (bool, []string) { var badlines []string warnOnly, allowRerunSuccess := true, true + // Check for badness using CheckConsoleText + addWarnOnly, addBadlines := CheckConsoleText(output) + if !addWarnOnly { + warnOnly = false + } + badlines = append(badlines, addBadlines...) for _, check := range consoleChecks { if check.skipFlag != nil && t != nil && t.HasFlag(*check.skipFlag) { continue diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 6fe76da9fc..b3d109840f 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -255,6 +255,17 @@ func (inst *QemuInstance) WaitIgnitionError(ctx context.Context) (string, error) return r.String(), nil } +// Read file, output contents and return error if reading fails +func (inst *QemuInstance) ReadFile(filename string) (string, error) { + data, err := os.ReadFile(filename) + errchan := make(chan error) + if err != nil { + errchan <- err + return "", err + } + return string(data), nil +} + // WaitAll wraps the process exit as well as WaitIgnitionError, // returning an error if either fail. func (inst *QemuInstance) WaitAll(ctx context.Context) error {