Merge branch 'install-rpu'

conftest.py

@@ -169,6 +169,7 @@ def setup_host(hostname_or_ip, *, config=None):
             vif = host_vm.vifs()[0]
             mac_address = vif.param_get('MAC')
             logging.info("Nested host has MAC %s", mac_address)
+            pxe.arp_clear_for(mac_address)
 
             host_vm.start()
             wait_for(host_vm.is_running, "Wait for nested host VM running")

data.py-dist

@@ -43,6 +43,7 @@ HOSTS_IP_CONFIG = {
     'HOSTS': {
 #        'DEFAULT': '192.16.0.1',
 #        'host2': '192.16.0.2',
+#        'host3': '192.16.0.3',
     },
 #    'NETMASK': '255.255.0.0',
 #    'GATEWAY': '192.16.0.254',

jobs.py

@@ -2,6 +2,7 @@
 
 import argparse
 import json
+import logging
 import subprocess
 import sys
 from lib.commands import ssh
@@ -487,6 +488,8 @@
     # running quicktest on zfsvol generates dangling TAP devices that are hard to
     # cleanup. Bug needs to be fixed before enabling quicktest on zfsvol.
     "tests/storage/zfsvol/test_zfsvol_sr.py::TestZfsvolVm::test_quicktest",
+    # not meant to be run from jobs.py (yet)
+    "tests/install/test_pool.py",
 ]
 
 # Returns the vm filename or None if a host_version is passed and matches the one specified
@@ -710,6 +713,7 @@ def action_run(args):
         sys.exit(1)
 
 def main():
+    logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.DEBUG)
     parser = argparse.ArgumentParser(description="Manage test jobs")
     parser.add_argument("-v", "--host-version", help="host version to match VM filters.")
     subparsers = parser.add_subparsers(dest="action", metavar="action")

lib/commands.py

@@ -62,14 +62,14 @@ def _ellide_log_lines(log):
 def _ssh(hostname_or_ip, cmd, check, simple_output, suppress_fingerprint_warnings,
          background, target_os, decode, options):
     opts = list(options)
-    opts.append('-o "BatchMode yes"')
+    opts.append('-o BatchMode=yes')
     if suppress_fingerprint_warnings:
         # Suppress warnings and questions related to host key fingerprints
         # because on a test network IPs get reused, VMs are reinstalled, etc.
         # Based on https://unix.stackexchange.com/a/365976/257493
-        opts.append('-o "StrictHostKeyChecking no"')
-        opts.append('-o "LogLevel ERROR"')
-        opts.append('-o "UserKnownHostsFile /dev/null"')
+        opts.extend(['-o StrictHostKeyChecking=no',
+                     '-o LogLevel=ERROR',
+                     '-o UserKnownHostsFile=/dev/null'])
 
     command = " ".join(cmd)
     if background and target_os != "windows":
@@ -216,7 +216,7 @@ def local_cmd(cmd, check=True, decode=True):
 
     errorcode_msg = "" if res.returncode == 0 else " - Got error code: %s" % res.returncode
     command = " ".join(cmd)
-    logging.debug(f"[local] {command}{errorcode_msg}{_ellide_log_lines(output_for_logs)}")
+    logging.debug(f"[local] {command}{errorcode_msg}, output: {_ellide_log_lines(output_for_logs)}")
 
     if res.returncode and check:
         raise LocalCommandFailed(res.returncode, output_for_logs, command)

lib/host.py

@@ -434,6 +434,17 @@ def yum_restore_saved_state(self):
         self.saved_packages_list = None
         self.saved_rollback_id = None
 
+    def shutdown(self, verify=False):
+        logging.info("Shutdown host %s" % self)
+        try:
+            self.ssh(['shutdown'])
+        except commands.SSHCommandFailed as e:
+            # ssh connection may get killed by the shutdown and terminate with an error code
+            if "closed by remote host" not in e.stdout:
+                raise
+        if verify:
+            wait_for_not(self.is_enabled, "Wait for host down")
+
     def reboot(self, verify=False):
         logging.info("Reboot host %s" % self)
         try:

lib/installer.py

@@ -1,8 +1,10 @@
 import logging
+import os
 import time
 import xml.etree.ElementTree as ET
 
-from lib.commands import ssh, SSHCommandFailed
+from lib import pxe
+from lib.commands import local_cmd, scp, ssh, SSHCommandFailed
 from lib.common import wait_for
 
 # FIXME should only be used for <7.0
@@ -143,6 +145,56 @@ def monitor_upgrade(*, ip):
                          ).returncode == 1,
              "Wait for installer to terminate")
 
+# FIXME essentially duplicates vm_booted_with_installer fixture
+def perform_upgrade(*, iso, host_vm, host):
+    vif = host_vm.vifs()[0]
+    mac_address = vif.param_get('MAC')
+    logging.info("Host VM has MAC %s", mac_address)
+
+    try:
+        remote_iso = host.pool.push_iso(iso)
+        host_vm.insert_cd(os.path.basename(remote_iso))
+
+        try:
+            pxe.arp_clear_for(mac_address)
+
+            host_vm.start()
+            wait_for(host_vm.is_running, "Wait for host VM running")
+
+            # catch host-vm IP address
+            wait_for(lambda: pxe.arp_addresses_for(mac_address),
+                     "Wait for DHCP server to see Host VM in ARP tables",
+                     timeout_secs=10 * 60)
+            ips = pxe.arp_addresses_for(mac_address)
+            logging.info("Host VM has IPs %s", ips)
+            assert len(ips) == 1
+            host_vm.ip = ips[0]
+
+            # host may not be up if ARP cache was filled
+            wait_for(lambda: local_cmd(["ping", "-c1", host_vm.ip], check=False),
+                     "Wait for host up", timeout_secs=10 * 60, retry_delay_secs=10)
+            wait_for(lambda: local_cmd(["nc", "-zw5", host_vm.ip, "22"], check=False),
+                     "Wait for ssh up on host", timeout_secs=10 * 60, retry_delay_secs=5)
+
+            yield host_vm
+
+            logging.info("Shutting down Host VM")
+            poweroff(host_vm.ip)
+            wait_for(host_vm.is_halted, "Wait for host VM halted")
+
+        except Exception as e:
+            logging.critical("caught exception %s", e)
+            host_vm.shutdown(force=True)
+            raise
+        except KeyboardInterrupt:
+            logging.warning("keyboard interrupt")
+            host_vm.shutdown(force=True)
+            raise
+
+        host_vm.eject_cd()
+    finally:
+        host.pool.remove_iso(remote_iso)
+
 def monitor_restore(*, ip):
     # wait for "yum install" phase to start
     wait_for(lambda: ssh(ip, ["grep",

lib/pxe.py

@@ -45,3 +45,10 @@ def arp_addresses_for(mac_address):
     )
     candidate_ips = output.splitlines()
     return candidate_ips
+
+def arp_clear_for(mac_address):
+    for stray_ip in arp_addresses_for(mac_address):
+        output = ssh(
+            PXE_CONFIG_SERVER,
+            ['arp', '-d', stray_ip]
+        )

scripts/gen-indirect-installs.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+# creation of reference installations to be used for basis of upgrade tests
+
+# * needs at least --hosts=$NEST
+# * ch821 and xs8 needs ISO in the cache
+
+. $(dirname $0)/lib.bash
+
+TESTCLASS="tests/install/test.py::TestNested"
+for conf in uefi-75+821.1-iso-ext; do
+    IFS=- read fw versions media sr < <(echo "$conf")
+    IFS=+ read origversion stepversion < <(echo "$versions")
+    TESTS=(
+        $TESTCLASS::test_install[$fw-$origversion-$media-$sr]
+        $TESTCLASS::test_tune_firstboot[None-$fw-$origversion-host1-$media-$sr]
+        $TESTCLASS::test_boot_inst[$fw-$origversion-host1-$media-$sr]
+
+        $TESTCLASS::test_upgrade[$fw-$origversion-$stepversion-host1-$media-$sr]
+        $TESTCLASS::test_boot_upg[$fw-$origversion-$stepversion-host1-$media-$sr]
+
+        #$TESTCLASS::test_upgrade[$fw-$origversion-$stepversion-83nightly-host1-$media-$sr]
+    )
+    run_pytest "$conf" \
+        --log-file=test-genref-$conf.log \
+        --reruns=5 --only-rerun=TimeoutError \
+        "$@" \
+        "${TESTS[@]}"
+done
+
+report_failures

scripts/lib.bash

@@ -46,6 +46,8 @@ run_pytest() {
 
 # reference configurations, to be upgraded to nightly
 
+# FIXME this is also where we take the list to create all ref
+# installs, 75/76 should be separated
 REFVERSIONS=(
     83rc1 83b2 83b1
     821.1 81 80

tests/install/conftest.py

@@ -116,7 +116,22 @@ def installer_iso(request):
 @pytest.fixture(scope='function')
 def install_disk(request):
     firmware = request.getfixturevalue("firmware")
-    yield {"uefi": "nvme0n1", "bios": "sda"}[firmware]
+    if firmware.startswith("uefi"):
+        yield "nvme0n1"
+    elif firmware.startswith("bios"):
+        yield "sda"
+    else:
+        assert False, f"unknown firmware {firmware!r}"
+
+@pytest.fixture(scope='function')
+def answerfile_maybe_tweak_parttable(request, answerfile):
+    firmware = request.getfixturevalue("firmware")
+    if firmware.endswith("+dell"):
+        answerfile.top_append(dict(TAG="script", stage="installation-start",
+                                   type="url", CONTENTS="file:///root/preinstall-utilitypart.sh"))
+    if firmware.endswith("+mbr"):
+        answerfile.top_append(dict(TAG="script", stage="installation-start",
+                                   type="url", CONTENTS="file:///root/preinstall-mbrparttable.sh"))
 
 # Remasters the ISO sepecified by `installer_iso` mark, with:
 # - network and ssh support activated, and .ssh/authorized_key so tests can
@@ -131,7 +146,7 @@ def install_disk(request):
 #     in contexts where the same IP is reused by successively different MACs
 #     (when cloning VMs from cache)
 @pytest.fixture(scope='function')
-def remastered_iso(installer_iso, answerfile):
+def remastered_iso(installer_iso, answerfile, install_disk):
     iso_file = installer_iso['iso']
     unsigned = installer_iso['unsigned']
 
@@ -230,6 +245,30 @@ def remastered_iso(installer_iso, answerfile):
     chmod +x "$INSTALLIMG/etc/init.d/S12test-pingpxe"
 fi
 
+cat > "$INSTALLIMG/root/preinstall-utilitypart.sh" <<'EOF'
+#!/bin/sh
+set -ex
+
+# Dell utility partition
+sgdisk --zap-all /dev/{install_disk}
+sfdisk /dev/{install_disk} << 'EOP'
+unit: sectors
+p1 : start=     2048, size=    32768, Id=de
+EOP
+EOF
+
+cat > "$INSTALLIMG/root/preinstall-mbrparttable.sh" <<'EOF'
+#!/bin/sh
+set -ex
+
+# Dell utility partition
+sgdisk --zap-all /dev/{install_disk}
+sfdisk /dev/{install_disk} << 'EOP'
+unit: sectors
+p1 : start=     2048, size=    32768, Id=83
+EOP
+EOF
+
 cat > "$INSTALLIMG/root/postinstall.sh" <<'EOF'
 #!/bin/sh
 set -ex
@@ -299,6 +338,8 @@ def vm_booted_with_installer(host, remastered_iso, create_vms):
         host_vm.insert_cd(os.path.basename(remote_iso))
 
         try:
+            pxe.arp_clear_for(mac_address)
+
             host_vm.start()
             wait_for(host_vm.is_running, "Wait for host VM running")
 

tests/install/test-sequences/uefi-82-nosr+pool+rpu83nightly.lst

@@ -0,0 +1 @@
+tests/install/test_pool.py::test_pool_rpu[uefi-821.1-83nightly]

tests/install/test-sequences/uefi-82-nosr+pool.lst

@@ -0,0 +1,2 @@
+tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host2-iso-nosr]
+tests/install/test.py::TestNested::test_boot_inst[uefi-821.1-host2-iso-nosr]

tests/install/test-sequences/uefi-82-nosr.lst

@@ -0,0 +1,3 @@
+tests/install/test.py::TestNested::test_install[uefi-821.1-iso-nosr]
+tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host1-iso-nosr]
+tests/install/test.py::TestNested::test_boot_inst[uefi-821.1-host1-iso-nosr]