Merge branch '10.10.11-8.3' into driver-disks

Signed-off-by: Yann Dirson <[email protected]>

answerfile.py

@@ -92,6 +92,8 @@ def processAnswerfile(self):
             else:
                 raise AnswerfileException("Unknown mode, %s" % install_type)
 
+            results['repo-gpgcheck'] = getBoolAttribute(self.top_node, ['repo-gpgcheck'], default=True)
+            results['gpgcheck'] = getBoolAttribute(self.top_node, ['gpgcheck'], default=True)
             results.update(self.parseCommon())
         elif self.operation == 'restore':
             results = self.parseRestore()
@@ -133,6 +135,7 @@ def parseFreshInstall(self):
         results['preserve-settings'] = False
         results['backup-existing-installation'] = False
 
+        results.update(self.parseRaid())
         results.update(self.parseDisks())
         results.update(self.parseInterface())
         results.update(self.parseRootPassword())
@@ -266,7 +269,21 @@ def parseSource(self):
             if rtype == 'url':
                 address = util.URL(address)
 
-            results['sources'].append({'media': rtype, 'address': address})
+            # workaround getBoolAttribute() not allowing "None" as
+            # default, by using a getStrAttribute() call first to
+            # handle the default situation where the attribute is not
+            # specified
+            repo_gpgcheck = (None if getStrAttribute(i, ['repo-gpgcheck'], default=None) is None
+                             else getBoolAttribute(i, ['repo-gpgcheck']))
+            gpgcheck = (None if getStrAttribute(i, ['gpgcheck'], default=None) is None
+                        else getBoolAttribute(i, ['gpgcheck']))
+
+            results['sources'].append({
+                'media': rtype, 'address': address,
+                'repo_gpgcheck': repo_gpgcheck,
+                'gpgcheck': gpgcheck,
+            })
+            logger.log("parsed source %s" % results['sources'][-1])
 
         return results
 
@@ -293,6 +310,16 @@ def parseDriverSource(self):
             results['extra-repos'].append((rtype, address))
         return results
 
+    def parseRaid(self):
+        results = {}
+        for raid_node in getElementsByTagName(self.top_node, ['raid']):
+            disk_device = normalize_disk(getStrAttribute(raid_node, ['device'], mandatory=True))
+            disks = [normalize_disk(getText(node)) for node in getElementsByTagName(raid_node, ['disk'])]
+            if 'raid' not in results:
+                results['raid'] = {}
+            results['raid'][disk_device] = disks
+        return results
+
     def parseDisks(self):
         results = {}
 
@@ -324,7 +351,7 @@ def parseDisks(self):
 
         results['sr-type'] = getMapAttribute(self.top_node, ['sr-type', 'srtype'],
                                              [('lvm', SR_TYPE_LVM),
-                                              ('ext', SR_TYPE_EXT)], default='lvm')
+                                              ('ext', SR_TYPE_EXT)], default='ext')
         return results
 
     def parseFCoEInterface(self):

backend.py

@@ -33,6 +33,7 @@
 import version
 from version import *
 from constants import *
+from diskutil import getRemovableDeviceList
 
 MY_PRODUCT_BRAND = PRODUCT_BRAND or PLATFORM_NAME
 
@@ -104,6 +105,7 @@ def getPrepSequence(ans, interactive):
         Task(util.getUUID, As(ans), ['installation-uuid']),
         Task(util.getUUID, As(ans), ['control-domain-uuid']),
         Task(util.randomLabelStr, As(ans), ['disk-label-suffix']),
+        Task(diskutil.create_raid, A(ans, 'raid'), []),
         Task(inspectTargetDisk, A(ans, 'primary-disk', 'installation-to-overwrite', 'preserve-first-partition','sr-on-primary'), ['target-boot-mode', 'boot-partnum', 'primary-partnum', 'backup-partnum', 'logs-partnum', 'swap-partnum', 'storage-partnum']),
         ]
 
@@ -161,7 +163,7 @@ def getMainRepoSequence(ans, repos):
 def getRepoSequence(ans, repos):
     seq = []
     for repo in repos:
-        seq.append(Task(repo.installPackages, A(ans, 'mounts'), [],
+        seq.append(Task(repo.installPackages, A(ans, 'mounts', 'kernel-alt'), [],
                      progress_scale=100,
                      pass_progress_callback=True,
                      progress_text="Installing %s..." % repo.name()))
@@ -171,6 +173,7 @@ def getRepoSequence(ans, repos):
 
 def getFinalisationSequence(ans):
     seq = [
+        Task(importYumAndRpmGpgKeys, A(ans, 'mounts'), []),
         Task(writeResolvConf, A(ans, 'mounts', 'manual-hostname', 'manual-nameservers'), []),
         Task(writeMachineID, A(ans, 'mounts'), []),
         Task(writeKeyboardConfiguration, A(ans, 'mounts', 'keymap'), []),
@@ -192,6 +195,7 @@ def getFinalisationSequence(ans):
                                   'boot-partnum', 'primary-partnum', 'target-boot-mode', 'branding',
                                   'disk-label-suffix', 'bootloader-location', 'write-boot-entry', 'install-type',
                                   'serial-console', 'boot-serial', 'host-config', 'fcoe-interfaces'), []),
+        Task(postInstallAltKernel, A(ans, 'mounts', 'kernel-alt'), []),
         Task(touchSshAuthorizedKeys, A(ans, 'mounts'), []),
         Task(setRootPassword, A(ans, 'mounts', 'root-password'), [], args_sensitive=True),
         Task(setTimeZone, A(ans, 'mounts', 'timezone'), []),
@@ -376,7 +380,7 @@ def handleRepos(repos, ans):
     main_repositories = []
     update_repositories = []
 
-    def add_repos(main_repositories, update_repositories, repos):
+    def add_repos(main_repositories, update_repositories, repos, repo_gpgcheck, gpgcheck):
         """Add repositories to the appropriate list, ensuring no duplicates,
         that the main repository is at the beginning, and that the order of the
         rest is maintained."""
@@ -393,20 +397,28 @@ def add_repos(main_repositories, update_repositories, repos):
                 else:
                     repo_list.append(repo)
 
+                if repo_list is main_repositories: # i.e., if repo is a "main repository"
+                    repo.setRepoGpgCheck(repo_gpgcheck)
+                    repo.setGpgCheck(gpgcheck)
+
+    default_repo_gpgcheck = answers.get('repo-gpgcheck', True)
+    default_gpgcheck = answers.get('gpgcheck', True)
     # A list of sources coming from the answerfile
     if 'sources' in answers_pristine:
         for i in answers_pristine['sources']:
             repos = repository.repositoriesFromDefinition(i['media'], i['address'])
-            add_repos(main_repositories, update_repositories, repos)
+            repo_gpgcheck = default_repo_gpgcheck if i['repo_gpgcheck'] is None else i['repo_gpgcheck']
+            gpgcheck = default_gpgcheck if i['gpgcheck'] is None else i['gpgcheck']
+            add_repos(main_repositories, update_repositories, repos, repo_gpgcheck, gpgcheck)
 
     # A single source coming from an interactive install
     if 'source-media' in answers_pristine and 'source-address' in answers_pristine:
         repos = repository.repositoriesFromDefinition(answers_pristine['source-media'], answers_pristine['source-address'])
-        add_repos(main_repositories, update_repositories, repos)
+        add_repos(main_repositories, update_repositories, repos, default_repo_gpgcheck, default_gpgcheck)
 
     for media, address in answers_pristine['extra-repos']:
         repos = repository.repositoriesFromDefinition(media, address)
-        add_repos(main_repositories, update_repositories, repos)
+        add_repos(main_repositories, update_repositories, repos, default_repo_gpgcheck, default_gpgcheck)
 
     if not main_repositories or main_repositories[0].identifier() != MAIN_REPOSITORY_NAME:
         raise RuntimeError("No main repository found")
@@ -421,8 +433,19 @@ def add_repos(main_repositories, update_repositories, repos):
         if r.accessor().canEject():
             r.accessor().eject()
 
+    # XCP-ng: so, very unfortunately we don't remember with precision why this was added and
+    # no commit message or comment can help us here.
+    # It may be related to the fact that the "all_repositories" above doesn't contain
+    # the installation CD-ROM or USB stick in the case of a netinstall.
+    # Question: why it is needed at all since there's no repository on the netinstall
+    # installation media?
+    if answers.get('netinstall'):
+        for device in getRemovableDeviceList():
+            util.runCmd2(['eject', device])
+
     if interactive and (constants.HAS_SUPPLEMENTAL_PACKS or
                         "driver-repos" in answers):
+
         # Add supp packs in a loop
         while True:
             media_ans = dict(answers_pristine)
@@ -1139,7 +1162,11 @@ def installBootLoader(mounts, disk, boot_partnum, primary_partnum, target_boot_m
                 setEfiBootEntry(mounts, disk, boot_partnum, install_type, branding)
         else:
             if location == constants.BOOT_LOCATION_MBR:
-                installGrub2(mounts, disk, False)
+                if diskutil.is_raid(disk):
+                    for member in diskutil.getDeviceSlaves(disk):
+                        installGrub2(mounts, member, False)
+                else:
+                    installGrub2(mounts, disk, False)
             else:
                 installGrub2(mounts, root_partition, True)
 
@@ -1523,15 +1550,15 @@ def configureNetworking(mounts, admin_iface, admin_bridge, admin_config, hn_conf
             print >>mc, "NETMASK='%s'" % admin_config.netmask
             if admin_config.gateway:
                 print >>mc, "GATEWAY='%s'" % admin_config.gateway
-            if manual_nameservers:
-                print >>mc, "DNS='%s'" % (','.join(nameservers),)
-            if domain:
-                print >>mc, "DOMAIN='%s'" % domain
         print >>mc, "MODEV6='%s'" % netinterface.NetInterface.getModeStr(admin_config.modev6)
         if admin_config.modev6 == netinterface.NetInterface.Static:
             print >>mc, "IPv6='%s'" % admin_config.ipv6addr
             if admin_config.ipv6_gateway:
                 print >>mc, "IPv6_GATEWAY='%s'" % admin_config.ipv6_gateway
+        if manual_nameservers:
+            print >>mc, "DNS='%s'" % (','.join(nameservers),)
+        if domain:
+            print >>mc, "DOMAIN='%s'" % domain
         if admin_config.vlan:
             print >>mc, "VLAN='%d'" % admin_config.vlan
         mc.close()
@@ -1573,12 +1600,18 @@ def configureNetworking(mounts, admin_iface, admin_bridge, admin_config, hn_conf
     # now we need to write /etc/sysconfig/network
     nfd = open("%s/etc/sysconfig/network" % mounts["root"], "w")
     nfd.write("NETWORKING=yes\n")
-    if admin_config.modev6:
+    ipv6 = admin_config.modev6 is not None
+    if ipv6:
         nfd.write("NETWORKING_IPV6=yes\n")
         util.runCmd2(['chroot', mounts['root'], 'systemctl', 'enable', 'ip6tables'])
     else:
         nfd.write("NETWORKING_IPV6=no\n")
         netutil.disable_ipv6_module(mounts["root"])
+
+    with open("%s/etc/sysctl.d/91-net-ipv6.conf" % mounts["root"], "w") as ipv6_conf:
+        for i in ['all', 'default']:
+            ipv6_conf.write('net.ipv6.conf.%s.disable_ipv6=%d\n' % (i, int(not ipv6)))
+
     nfd.write("IPV6_AUTOCONF=no\n")
     nfd.write('NTPSERVERARGS="iburst prefer"\n')
     nfd.close()
@@ -1662,6 +1695,57 @@ def touchSshAuthorizedKeys(mounts):
     fh = open("%s/root/.ssh/authorized_keys" % mounts['root'], 'a')
     fh.close()
 
+def importYumAndRpmGpgKeys(mounts):
+    # Python script that uses yum functions to import the GPG key for our repositories
+    import_yum_keys = """#!/bin/env python
+from __future__ import print_function
+from yum import YumBase
+
+def retTrue(*args, **kwargs):
+    return True
+
+base = YumBase()
+for repo in base.repos.repos.itervalues():
+    if repo.id.startswith('xcp-ng'):
+        print("*** Importing GPG key for repository %s - %s" % (repo.id, repo.name))
+        base.getKeyForRepo(repo, callback=retTrue)
+"""
+    internal_tmp_filepath = '/tmp/import_yum_keys.py'
+    external_tmp_filepath = mounts['root'] + internal_tmp_filepath
+    with open(external_tmp_filepath, 'w') as f:
+        f.write(import_yum_keys)
+    # bind mount /dev, necessary for NSS initialization without which RPM won't work
+    util.bindMount('/dev', "%s/dev" % mounts['root'])
+    try:
+        util.runCmd2(['chroot', mounts['root'], 'python', internal_tmp_filepath])
+        util.runCmd2(['chroot', mounts['root'], 'rpm', '--import', '/etc/pki/rpm-gpg/RPM-GPG-KEY-xcpng'])
+    finally:
+        util.umount("%s/dev" % mounts['root'])
+        os.unlink(external_tmp_filepath)
+
+def postInstallAltKernel(mounts, kernel_alt):
+    """ Install our alternate kernel. Must be called after the bootloader installation. """
+    if not kernel_alt:
+        logger.log('kernel-alt not installed')
+        return
+
+    util.bindMount("/proc", "%s/proc" % mounts['root'])
+    util.bindMount("/sys", "%s/sys" % mounts['root'])
+    util.bindMount("/dev", "%s/dev" % mounts['root'])
+
+    try:
+        rc, out = util.runCmd2(['chroot', mounts['root'], 'rpm', '-q', 'kernel-alt', '--qf', '%{version}'],
+                               with_stdout=True)
+        version = out
+        # Generate the initrd as it was disabled during initial installation
+        util.runCmd2(['chroot', mounts['root'], 'dracut', '-f', '/boot/initrd-%s.img' % version, version])
+
+        # Update grub
+        util.runCmd2(['chroot', mounts['root'], 'python', '/usr/lib/python2.7/site-packages/xcp/updategrub.py', 'add', 'kernel-alt', version])
+    finally:
+        util.umount("%s/dev" % mounts['root'])
+        util.umount("%s/sys" % mounts['root'])
+        util.umount("%s/proc" % mounts['root'])
 
 ################################################################################
 # OTHER HELPERS

constants.py

@@ -58,9 +58,9 @@ def error_string(error, logname, with_hd):
     ) = range(3)
 
     ERROR_STRINGS = {
-        ERROR_STRING_UNKNOWN_ERROR_WITH_HD: "An unrecoverable error has occurred.  The details of the error can be found in the log file, which has been written to /tmp/%s (and /root/%s on your hard disk if possible).\n\nPlease refer to your user guide or contact a Technical Support Representative for more details.",
-        ERROR_STRING_UNKNOWN_ERROR_WITHOUT_HD: "An unrecoverable error has occurred.  The details of the error can be found in the log file, which has been written to /tmp/%s.\n\nPlease refer to your user guide or contact a Technical Support Representative for more details.",
-        ERROR_STRING_KNOWN_ERROR: "An unrecoverable error has occurred.  The error was:\n\n%s\n\nPlease refer to your user guide, or contact a Technical Support Representative, for further details."
+        ERROR_STRING_UNKNOWN_ERROR_WITH_HD: "An unrecoverable error has occurred.  The details of the error can be found in the log file, which has been written to /tmp/%s (and /root/%s on your hard disk if possible).",
+        ERROR_STRING_UNKNOWN_ERROR_WITHOUT_HD: "An unrecoverable error has occurred.  The details of the error can be found in the log file, which has been written to /tmp/%s.",
+        ERROR_STRING_KNOWN_ERROR: "An unrecoverable error has occurred.  The error was:\n\n%s"
         }
 
     if error == "":
@@ -154,6 +154,9 @@ def error_string(error, logname, with_hd):
 HYPERVISOR_CAPS_FILE = "/sys/hypervisor/properties/capabilities"
 SAFE_2_UPGRADE = "var/preserve/safe2upgrade"
 
+# NTP server domains to treat as 'default' servers
+DEFAULT_NTP_DOMAINS = [".centos.pool.ntp.org", ".xenserver.pool.ntp.org"]
+
 # timer to exit installer after fatal error
 AUTO_EXIT_TIMER = 10 * 1000
 

disktools.py

@@ -490,7 +490,7 @@ def diskDevice(partitionDevice):
 
 def determineMidfix(device):
     DISK_PREFIX = '/dev/'
-    P_STYLE_DISKS = [ 'cciss', 'ida', 'rd', 'sg', 'i2o', 'amiraid', 'iseries', 'emd', 'carmel', 'mapper/', 'nvme', 'md' ]
+    P_STYLE_DISKS = [ 'cciss', 'ida', 'rd', 'sg', 'i2o', 'amiraid', 'iseries', 'emd', 'carmel', 'mapper/', 'nvme', 'md', 'mmcblk' ]
     PART_STYLE_DISKS = [ 'disk/by-id' ]
 
     for key in P_STYLE_DISKS:

diskutil.py

@@ -113,6 +113,9 @@ def mpath_disable():
 for major in range(48, 56):
     disk_nodes += [ (major, x * 8) for x in range(32) ]
 
+# /dev/mmcblk: mmcblk has major 179, each device usually (per kernel) has 7 minors
+disk_nodes += [ (179, x * 8) for x in range(32) ]
+
 def getDiskList():
     # read the partition tables:
     parts = open("/proc/partitions")
@@ -153,6 +156,21 @@ def getDiskList():
 
     return disks
 
+def create_raid(configuration):
+    if configuration:
+        for raid_device, members in configuration.viewitems():
+            # allows for idempotence
+            if not os.path.exists(raid_device):
+                for dev in members:
+                    util.runCmd2(['sgdisk', '--zap-all', dev])
+                    util.runCmd2(['mdadm', '--zero-superblock', '--force', dev])
+                    # let it fail without catching
+                cmd = ['mdadm', '--create', raid_device, '--run', '--metadata=1.0', '--level=mirror',
+                       '--raid-devices=%s' % (len(members))] + members
+                rc, out, err = util.runCmd2(cmd, with_stdout=True, with_stderr=True)
+                if rc != 0:
+                    raise Exception('Error running: %s\n%s\n\n%s' % (' '.join(cmd), out, err))
+
 def getPartitionList():
     disks = getDiskList()
     rv  = []
@@ -280,6 +298,9 @@ def getDiskDeviceSize(dev):
         return int(__readOneLineFile__("/sys/block/%s/device/block/size" % dev))
     elif os.path.exists("/sys/block/%s/size" % dev):
         return int(__readOneLineFile__("/sys/block/%s/size" % dev))
+    else:
+        raise Exception("%s not found as %s or %s" % (dev, "/sys/block/%s/device/block/size",
+                                                      "/sys/block/%s/size"))
 
 def getDiskSerialNumber(dev):
     # For Multipath nodes return info about 1st slave