Merge pull request #3860 from nickzhq/misc_refactor_I

utils_misc module refactor

virttest/vt_utils/block.py

@@ -24,14 +24,14 @@
 PARTITION_TYPE_PRIMARY = "primary"
 
 
-def get_disks(partition=False):
+def get_disks_info(has_partition=False):
     """
     List all disks or disks with no partition.
 
-    :param partition: if true, list all disks; otherwise,
-                      list only disks with no partition
-    :type partition: boolean
-    :return: the disks info.( e.g. {kname: [kname, size, type, serial, wwn]} )
+    :param has_partition: If true, list disks info;
+                          otherwise, list disks info which do NOT have partition.
+    :type has_partition: Boolean
+    :return: The disks info.( e.g. {kname: [kname, size, type, serial, wwn]} )
     :rtype: dict
     """
     disks_dict = {}
@@ -40,24 +40,30 @@ def get_disks(partition=False):
     if platform.machine() == "s390x":
         driver = "css0"
     block_info = process.run(
-        'ls /sys/dev/block -l | grep "/%s"' % driver, verbose=False
+        'ls /sys/dev/block -l | grep "/%s"' % driver,
+        verbose=False,
+        shell=True,
     ).stdout_text
     for matched in re.finditer(r"/block/(\S+)\s^", block_info, re.M):
         knames = matched.group(1).split("/")
         if len(knames) == 2:
             parent_disks.add(knames[0])
-        if partition is False and knames[0] in parent_disks:
+        if has_partition is False and knames[0] in parent_disks:
             if knames[0] in disks_dict:
                 del disks_dict[knames[0]]
             continue
 
         disks_dict[knames[-1]] = [knames[-1]]
         o = process.run(
-            'lsblk -o KNAME,SIZE | grep "%s "' % knames[-1], verbose=False
+            'lsblk -o KNAME,SIZE | grep "%s "' % knames[-1],
+            verbose=False,
+            shell=True,
         ).stdout_text
         disks_dict[knames[-1]].append(o.split()[-1])
         o = process.run(
-            "udevadm info -q all -n %s" % knames[-1], verbose=False
+            "udevadm info -q all -n %s" % knames[-1],
+            verbose=False,
+            shell=True,
         ).stdout_text
         for parttern in (
             r"DEVTYPE=(\w+)\s^",
@@ -69,6 +75,26 @@ def get_disks(partition=False):
     return disks_dict
 
 
+def get_disks_path(partition_path=False):
+    """
+    List disk path in Linux host.
+
+    :param partition_path: If true, list disk name including partition; otherwise,
+                           list disk name excluding partition.
+    :type partition_path: Boolean
+
+    :return: The disks path in set(). ( e.g. {'/dev/sda2', '/dev/sda1', '/dev/sda'} or {'/dev/sda'} )
+    :rtype: Set
+    """
+    cmd = "ls /dev/[vhs]d*"
+    if not partition_path:
+        cmd = "%s | grep -v [0-9]$" % cmd
+    status, output = process.getstatusoutput(cmd, shell=True)
+    if status != 0:
+        raise RuntimeError("Get disks failed with output %s" % output)
+    return set(output.split())
+
+
 def create_partition(did, size, start, part_type=PARTITION_TYPE_PRIMARY, timeout=360):
     """
     Create single partition on disk.
@@ -175,14 +201,18 @@ def get_disk_size(did):
     :param did: disk kname. e.g. 'sdb', 'sdc'
     :return: disk size.
     """
-    disks_info = get_disks(partition=True)
+    disks_info = get_disks_info(True)
     disk_size = disks_info["%s" % did][1]
     return int(utils_numeric.normalize_data_size(disk_size, "B").split(".")[0])
 
 
-def get_partions_list():
+def get_partitions_list():
     """
-    Get all partition lists.
+    Get all partition list.
+
+    :return: All partition list.
+             e.g. ['sda', 'sda1', 'sda2', 'dm-0', 'dm-1', 'dm-2']
+    :rtype: List
     """
     parts_cmd = "cat /proc/partitions"
     parts_out = process.run(parts_cmd, verbose=False).stdout_text
@@ -201,10 +231,12 @@ def get_disk_by_serial(serial_str):
     """
     Get disk by serial in host.
 
-    :param serial_str: ID_SERIAL of disk, string value
-    :return: Disk name if find one with serial_str, else None
+    :param serial_str: ID_SERIAL of disk, string value.
+    :type serial_str: String
+    :return: Disk name if find one with serial_str, else None.
+    :rtype: String
     """
-    parts_list = get_partions_list()
+    parts_list = get_partitions_list()
     for disk in parts_list:
         cmd = "udevadm info --query=all --name=/dev/{} | grep ID_SERIAL={}".format(
             disk, serial_str
@@ -215,3 +247,30 @@ def get_disk_by_serial(serial_str):
         if not status:
             return disk
     return None
+
+
+def get_drive_path(did, timeout=120):
+    """
+    Get drive path( devname ) on host by drive serial or wwn
+
+    :param did: A drive serial( ID_SERIAL or ID_SERIAL_SHORT )
+                or a wwn( ID_WWN ).
+    :type did: String
+    :param timeout: Time out.
+    :type timeout: Integer
+
+    :return: A drive path( devname )
+    :rtype: String
+
+    :raises: An RuntimeError will be raised when cmd exit code is NOT 0.
+    """
+    cmd = "for dev_path in `ls -d /sys/block/*`; do "
+    cmd += "echo `udevadm info -q property -p $dev_path`; done"
+    status, output = process.getstatusoutput(cmd, timeout=timeout)
+    if status != 0:
+        raise RuntimeError("Command running was failed. Output: %s" % output)
+    p = r"DEVNAME=([^\s]+)\s.*(?:ID_SERIAL|ID_SERIAL_SHORT|ID_WWN)=%s" % did
+    dev = re.search(p, output, re.M)
+    if dev:
+        return dev.groups()[0]
+    return ""

virttest/vt_utils/filesystem.py

@@ -13,6 +13,7 @@
 #
 # Copyright: Red Hat (c) 2023 and Avocado contributors
 # Author: Houqi Zuo <[email protected]>
+import os
 import re
 
 from avocado.utils import process
@@ -121,20 +122,20 @@ def umount(src, dst, fstype=None):
     return True
 
 
-def create_filesyetem(partition_name, fstype, timeout=360):
+def create_filesyetem(partition, fstype, timeout=360):
     """
     create file system.
 
-    :param partition_name: partition name that to be formatted. e.g. sdb1
+    :param partition: partition name that to be formatted. e.g. /dev/sdb1
     :param fstype: filesystem type for the disk.
     :param timeout: Timeout for cmd execution in seconds.
     """
     if fstype == "xfs":
         mkfs_cmd = "mkfs.%s -f" % fstype
     else:
         mkfs_cmd = "mkfs.%s -F" % fstype
-    format_cmd = "yes|%s '/dev/%s'" % (mkfs_cmd, partition_name)
-    process.system(format_cmd, timeout=timeout, verbose=False)
+    format_cmd = "yes|%s '%s'" % (mkfs_cmd, partition)
+    process.run(format_cmd, timeout=timeout, verbose=False)
 
 
 def resize_filesystem(partition, size):
@@ -147,7 +148,7 @@ def resize_filesystem(partition, size):
     :param size: resize file system to size.
                  size unit can be 'B', 'K', 'M', 'G'.
                  support transfer size with SIZE_AVAILABLE,
-                 enlarge to maximun available size.
+                 enlarge to maximum available size.
     """
 
     def get_start_size():
@@ -193,7 +194,7 @@ def resize_ext_fs(size):
             ) * bsize
             size = utils_numeric.normalize_data_size(str(size).split(".")[0], "K")
             resize_fs_cmd = "resize2fs %s %sK" % (partition, int(size.split(".")[0]))
-        process.system(resize_fs_cmd, verbose=False)
+        process.run(resize_fs_cmd, verbose=False)
         if flag:
             mount(partition, mountpoint, fstype=fstype)
 
@@ -216,3 +217,65 @@ def get_mpoint_fstype(partition):
     mount_list = process.run("cat /proc/mounts", verbose=False).stdout_text
     mount_info = re.search(r"%s\s(.+?)\s(.+?)\s" % partition, mount_list)
     return mount_info.groups()
+
+
+def format_disk(
+    did,
+    all_disks_did,
+    partition=False,
+    mountpoint=None,
+    size=None,
+    fstype="ext3",
+):
+    """
+    Create a partition on disk in Linux host and format and mount it.
+
+    :param did: Disk kname, serial or wwn.
+    :type did: String
+    :param all_disks_did: All disks did lists each include disk kname,
+                          serial and wwn.
+    :type all_disks_did: List
+    :param partition: If true, can format all disks; otherwise,
+                      only format the ones with no partition originally.
+    :type partition: Boolean
+    :param mountpoint: Mount point for the disk.
+    :type mountpoint: String
+    :param size: Partition size( such as 6G, 500M ).
+    :type size: String
+    :param fstype: Filesystem type for the disk.
+    :type fstype: String
+
+    :return: If disk is usable, return True. Otherwise, return False.
+    :rtype: Boolean
+    """
+    disks = block.get_disks_path(partition)
+    for line in disks:
+        kname = line.split("/")[-1]
+        did_list = all_disks_did[kname]
+        if did not in did_list:
+            # Continue to search target disk
+            continue
+        if not size:
+            size_output = process.run(
+                "lsblk -o KNAME,SIZE|grep %s" % kname,
+                verbose=False,
+                shell=True,
+            ).stdout_text
+            size = size_output.splitlines()[0].split()[1]
+        all_disks_before = block.get_disks_path(True)
+        devname = line
+        block.create_partition(
+            devname.split("/")[-1],
+            size,
+            "0M",
+        )
+        all_disks_after = block.get_disks_path(True)
+        partname = (all_disks_after - all_disks_before).pop()
+        create_filesyetem(partname, fstype)
+        if not mountpoint:
+            process.run("mkdir /mnt/%s" % kname)
+            mountpoint = os.path.join("/mnt", kname)
+        mount(src=partname, dst=mountpoint, fstype=fstype)
+        if is_mounted(src=partname, dst=mountpoint, fstype=fstype):
+            return True
+        return False

virttest/vt_utils/interrupted_thread.py

@@ -0,0 +1,90 @@
+#
+# Library for interrupted thread related helper functions
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; specifically version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat (c) 2024 and Avocado contributors
+# Author: Houqi Zuo <[email protected]>
+import sys
+import threading
+
+from virttest import error_context
+
+
+class InterruptedThread(threading.Thread):
+    """
+    Run a function in a background thread.
+    """
+
+    def __init__(self, target, args=(), kwargs={}):
+        """
+        Initialize the instance.
+
+        :param target: Function to run in the thread.
+        :type target: Function object
+        :param args: Arguments to pass to target.
+        :type args: Tuple
+        :param kwargs: Keyword arguments to pass to target.
+        :type kwargs: Dictionary
+        """
+        threading.Thread.__init__(self)
+        self._target = target
+        self._args = args
+        self._kwargs = kwargs
+
+    def run(self):
+        """
+        Run target (passed to the constructor).  No point in calling this
+        function directly.  Call start() to make this function run in a new
+        thread.
+
+        :raises: An Exception from run() of threading.Thread.
+        """
+        self._e = None
+        self._retval = None
+        try:
+            try:
+                self._retval = self._target(*self._args, **self._kwargs)
+            except Exception:
+                self._e = sys.exc_info()
+                raise
+        finally:
+            # Avoid circular references (start() may be called only once so
+            # it's OK to delete these)
+            del self._target, self._args, self._kwargs
+
+    def join(self, timeout=None, suppress_exception=False):
+        """
+        Join the thread.  If target raised an exception, re-raise it.
+        Otherwise, return the value returned by target.
+
+        :param timeout: Timeout value to pass to threading.Thread.join().
+        :type timeout: Integer
+        :param suppress_exception: If True, don't re-raise the exception.
+        :type suppress_exception: Boolean
+        """
+        threading.Thread.join(self, timeout)
+        try:
+            if self._e:
+                if not suppress_exception:
+                    # Because the exception was raised in another thread, we
+                    # need to explicitly insert the current context into it
+                    s = error_context.exception_context(self._e[1])
+                    s = error_context.join_contexts(error_context.get_context(), s)
+                    error_context.set_exception_context(self._e[1], s)
+                    raise self._e.with_traceback(*self._e)
+            else:
+                return self._retval
+        finally:
+            # Avoid circular references (join() may be called multiple times
+            # so we can't delete these)
+            self._e = None
+            self._retval = None