Fixes #35269 - Support system image download for installation media

* Include proxy.fetch_system_image
* Add system_image_path variable for template reference
* Adapt PXELinux template
* Add tftp_http_port setting
* Add tftp.setTFTPBootFiles test scenarios

app/models/concerns/orchestration/tftp.rb

@@ -111,10 +111,25 @@ def setTFTPBootFiles
     logger.info "Fetching required TFTP boot files for #{host.name}"
     valid = []
 
-    host.operatingsystem.pxe_files(host.medium_provider).each do |bootfile_info|
-      bootfile_info.each do |prefix, path|
-        valid << each_unique_feasible_tftp_proxy do |proxy|
-          proxy.fetch_boot_file(:prefix => prefix.to_s, :path => path)
+    # Check host.medium_provider path for iso image
+    is_image_path = File.extname(host.medium_uri.to_s).downcase.end_with?(".iso")
+
+    valid << each_unique_feasible_tftp_proxy do |proxy|
+      bootfiles = host.operatingsystem.pxe_files(host.medium_provider)
+      if is_image_path
+        max_request_timeout = Setting[:proxy_request_timeout]
+        image_url = host.medium_uri.to_s
+        tftp_files = pxe_files_from_bootfiles(bootfiles, image_url)
+        tftp_dst = pxe_dst_from_bootfiles(bootfiles)
+        image_dst = host.operatingsystem.system_image_path(host.medium_provider, true, false)
+        image_status = poll_fetch_system_image(proxy, image_url, image_dst, tftp_files, tftp_dst, max_request_timeout)
+        logger.error "Timeout fetching system image #{image_url}. See smart proxy log for details." unless image_status
+        image_status
+      else
+        bootfiles.each do |bootfile_info|
+          bootfile_info.each do |prefix, path|
+            proxy.fetch_boot_file(:prefix => prefix.to_s, :path => path)
+          end
         end
       end
     end
@@ -197,4 +212,32 @@ def each_unique_feasible_tftp_proxy
     end
     results.all?
   end
+
+  # Extract pxe files from bootfiles
+  def pxe_files_from_bootfiles(bootfiles, image_url)
+    pxe_files = []
+    bootfiles.each { |pxe_url| pxe_files.append(pxe_url.values.first.delete_prefix(image_url)) }
+    pxe_files
+  end
+
+  # Extract pxe destination file name from bootfiles
+  def pxe_dst_from_bootfiles(bootfiles)
+    bootfiles.first.keys.first
+  end
+
+  def poll_fetch_system_image(proxy, image_url, image_dst, tftp_files, tftp_dst, max_request_time)
+    retries = poll_system_image_retries
+    pause_time = max_request_time / poll_system_image_retries
+    request_status = proxy.fetch_system_image(:url => image_url, :path => image_dst, :files => tftp_files, :tftp_path => tftp_dst)
+    until retries <= 0 || request_status == 200
+      sleep(pause_time)
+      request_status = proxy.fetch_system_image(:url => image_url, :path => image_dst, :files => tftp_files, :tftp_path => tftp_dst)
+      retries -= 1
+    end
+    request_status == 200
+  end
+
+  def poll_system_image_retries
+    10
+  end
 end

app/models/operatingsystem.rb

@@ -92,7 +92,7 @@ class Operatingsystem < ApplicationRecord
     property :password_hash, String, desc: 'Encrypted hash of the operating system password'
   end
   class Jail < Safemode::Jail
-    allow :id, :name, :major, :minor, :family, :to_s, :==, :release, :release_name, :kernel, :initrd, :pxe_type, :boot_files_uri, :password_hash, :mediumpath, :bootfile
+    allow :id, :name, :major, :minor, :family, :to_s, :==, :release, :release_name, :kernel, :initrd, :pxe_type, :boot_files_uri, :password_hash, :mediumpath, :bootfile, :system_image_path
   end
 
   def self.title_name
@@ -236,6 +236,27 @@ def bootfile(medium_provider, type)
     pxe_prefix(medium_provider) + "-" + pxe_file_names(medium_provider)[type.to_sym]
   end
 
+  apipie :method, 'Returns path to system image based on given medium provider' do
+    required :medium_provider, 'MediumProviders::Provider', 'Medium provider responsible to provide location of installation medium for a given entity (host or host group)'
+    # optional :include_suffix, Boolean, 'Include the .iso file suffix if true (defaults true)'
+    # optional :include_base_path, Boolean, 'Include the Smart Proxy base address if true (defaults true)'
+    returns String, 'Path to the system image file'
+  end
+  def system_image_path(medium_provider, include_suffix = true, include_base_path = true)
+    unless medium_provider.is_a? MediumProviders::Provider
+      raise Foreman::Exception.new(N_('Please provide a medium provider. It can be found as @medium_provider in templates, or Foreman::Plugin.medium_providers_registry.find_provider(host)'))
+    end
+    include_base_path ? base_path = system_image_base_path : base_path = ""
+    include_suffix ? suffix = ".iso" : suffix = ""
+
+    "#{base_path}#{name.downcase}/#{medium_provider.unique_id}#{suffix}"
+  end
+
+  # Base path for system_image url
+  def system_image_base_path
+    "/tftp/system_image/"
+  end
+
   # Does this OS family support a build variant that is constructed from a prebuilt archive
   def supports_image
     false

app/models/smart_proxy.rb

@@ -109,6 +109,14 @@ def setting(feature, setting)
     smart_proxy_feature_by_name(feature).try(:settings).try(:[], setting)
   end
 
+  def tftp_http_port
+    setting(:TFTP, 'http_port')
+  end
+
+  def tftp_http_port!
+    tftp_http_port || raise(::Foreman::Exception.new(N_("HTTP boot requires proxy with httpboot feature and http_port exposed setting")))
+  end
+
   def httpboot_http_port
     setting(:HTTPBoot, 'http_port')
   end
@@ -202,12 +210,14 @@ def get_features
     sections only: %w[all additional]
     prop_group :basic_model_props, ApplicationRecord, meta: { friendly_name: 'Smart Proxy' }
     property :hostname, String, desc: 'Returns name of the host with proxy'
+    property :tftp_http_port, Integer, desc: 'Returns proxy port for TFTP boot images'
+    property :tftp_http_port!, Integer, desc: 'Same as tftp_http_port, but raises Foreman::Exception if no port is set'
     property :httpboot_http_port, Integer, desc: 'Returns proxy port for HTTP boot'
     property :httpboot_http_port!, Integer, desc: 'Same as httpboot_http_port, but raises Foreman::Exception if no port is set'
     property :httpboot_https_port, Integer, desc: 'Returns proxy port for HTTPS boot'
     property :httpboot_https_port!, Integer, desc: 'Same as httpboot_https_port, but raises Foreman::Exception if no port is set'
   end
   class Jail < ::Safemode::Jail
-    allow :id, :name, :hostname, :httpboot_http_port, :httpboot_https_port, :httpboot_http_port!, :httpboot_https_port!, :url
+    allow :id, :name, :hostname, :tftp_http_port, :httpboot_http_port, :httpboot_https_port, :tftp_http_port!, :httpboot_http_port!, :httpboot_https_port!, :url
   end
 end

app/services/foreman/renderer/configuration.rb

@@ -100,6 +100,7 @@ class Configuration
         :static,
         :template_name,
         :xen,
+        :system_image_path,
       ]
 
       DEFAULT_ALLOWED_GLOBAL_SETTINGS = [

app/services/foreman/renderer/scope/variables/base.rb

@@ -12,7 +12,7 @@ def self.included(base)
           delegate :diskLayout, :disk_layout_source, :medium, :architecture, :ptable, :use_image, :arch,
             :image_file, :default_image_file, to: :host, allow_nil: true
           delegate :mediumpath, :additional_media, :supports_image, :major, :preseed_path, :preseed_server,
-            :xen, :kernel, :initrd, to: :operatingsystem, allow_nil: true
+            :xen, :kernel, :initrd, :system_image_path, to: :operatingsystem, allow_nil: true
           delegate :name, to: :architecture, allow_nil: true, prefix: true
           delegate :content, to: :disk_layout_source, allow_nil: true, prefix: true
 
@@ -97,6 +97,7 @@ def xenserver_attributes
 
           def pxe_config
             return unless @medium_provider
+            @system_image_path = system_image_path(@medium_provider)
             @kernel = kernel(@medium_provider)
             @initrd = initrd(@medium_provider)
             @kernel_uri, @initrd_uri = operatingsystem.boot_files_uri(@medium_provider)

app/services/proxy_api/tftp.rb

@@ -38,6 +38,22 @@ def fetch_boot_file(args)
       raise ProxyException.new(url, e, N_("Unable to fetch TFTP boot file"))
     end
 
+    # Requests that the proxy downloads and extracts an image from the media's source
+    # [+args+] : Hash containing
+    #   :url       => String containing the URL of the image to download
+    #   :path      => String containing the location on the smart proxy to store the image
+    #   :files     => Array of Strings containing boot file paths to extract from the image
+    #   :tftp_path => String containing the location within the TFTP tree to store the files
+    # Returns       : Integer response status
+    def fetch_system_image(args)
+      response = post(args, "fetch_system_image")
+      response.code
+    rescue RestClient::Locked
+      423
+    rescue => e
+      raise ProxyException.new(url, e, N_("Unable to fetch and extract TFTP system image"))
+    end
+
     # returns the TFTP boot server for this proxy
     def bootServer
       if (response = parse(get("serverName"))) && response["serverName"].present?

app/views/unattended/provisioning_templates/PXELinux/preseed_default_pxelinux_autoinstall.erb

@@ -19,15 +19,22 @@ test_on:
 #   System locale
 #
 <%
-  image_path = @preseed_path.sub(/\/?$/, '.iso')
+  if @preseed_path.downcase.end_with?('.iso')
+    image_path = @system_image_path
+    tftp = @host.subnet.tftp
+    image_host = "#{tftp}:#{tftp.tftp_http_port}"
+  else
+    image_path = @preseed_path.sub(/\/?$/, '.iso')
+    image_host = foreman_request_addr.split(':').first
+  end
   options = []
   if host_param('blacklist')
     options << host_param('blacklist').split(',').collect{|x| "#{x.strip}.blacklist=yes"}.join(' ')
   end
   options << 'root=/dev/ram0'
   options << 'ramdisk_size=1500000'
   options << 'fsck.mode=skip'
-  options << "url=http://#{@preseed_server}#{image_path}"
+  options << "url=http://#{image_host}#{image_path}"
   options << 'cloud-config-url=/dev/null'
   options << "ds=nocloud-net;s=http://#{foreman_request_addr}/userdata/"
   options << 'autoinstall'

test/models/orchestration/tftp_test.rb

@@ -360,4 +360,43 @@ class TFTPOrchestrationTest < ActiveSupport::TestCase
     assert_equal expected.strip, template
     assert h.build
   end
+
+  describe "test #setTFTPBootFiles" do
+    setup do
+      @host = FactoryBot.build_stubbed(:host, :managed, :with_tftp_dual_stack_orchestration, :build => true)
+      arch = FactoryBot.build_stubbed(:architecture)
+      @medium = FactoryBot.build_stubbed(:medium, :name => 'Great OS Local', :path => 'http://my-example.com/official/great/os.iso')
+      @host.architecture = arch
+      @host.operatingsystem = FactoryBot.build_stubbed(:debian7_0, :media => [@medium], :architectures => [arch])
+      @host.medium = @medium
+    end
+
+    test "should download system image" do
+      ProxyAPI::TFTP.any_instance.stubs(:fetch_system_image).returns(200)
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).once
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).returns(true)
+      @host.provision_interface.send(:setTFTPBootFiles)
+    end
+
+    test "should fail system image download due to timeout" do
+      ProxyAPI::TFTP.any_instance.stubs(:fetch_system_image).returns(202)
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).once
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).returns(true)
+      @host.provision_interface.send(:setTFTPBootFiles)
+    end
+
+    test "should fail system image download due to timeout locked server" do
+      ProxyAPI::TFTP.any_instance.stubs(:fetch_system_image).returns(402)
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).once
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).returns(false)
+      @host.provision_interface.send(:setTFTPBootFiles)
+    end
+
+    test "should fail system image download due to proxy error" do
+      ProxyAPI::TFTP.any_instance.stubs(:fetch_system_image).returns(501)
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).once
+      Nic::Managed.any_instance.expects(:poll_fetch_system_image).returns(false)
+      @host.provision_interface.send(:setTFTPBootFiles)
+    end
+  end
 end

test/unit/foreman/renderer/snapshots/ProvisioningTemplate/PXELinux/Preseed_default_PXELinux_Autoinstall.ubuntu_autoinst4dhcp.snap.txt

@@ -20,6 +20,6 @@ DEFAULT linux cloud-init autoinstall
 LABEL linux cloud-init autoinstall
     KERNEL boot/ubuntu-mirror-rf32u3HGTMZf-vmlinuz
     INITRD boot/ubuntu-mirror-rf32u3HGTMZf-initrd
-    APPEND root=/dev/ram0 ramdisk_size=1500000 fsck.mode=skip url=http://archive.ubuntu.com:80/ubuntu.iso cloud-config-url=/dev/null ds=nocloud-net;s=http://foreman.example.com/userdata/ autoinstall ip=dhcp console-setup/ask_detect=false localechooser/translation/warn-light=true localechooser/translation/warn-severe=true hostname=snapshot-ipv4-dhcp-ubuntu20 domain=snap.example.com
+    APPEND root=/dev/ram0 ramdisk_size=1500000 fsck.mode=skip url=http://foreman.example.com/ubuntu.iso cloud-config-url=/dev/null ds=nocloud-net;s=http://foreman.example.com/userdata/ autoinstall ip=dhcp console-setup/ask_detect=false localechooser/translation/warn-light=true localechooser/translation/warn-severe=true hostname=snapshot-ipv4-dhcp-ubuntu20 domain=snap.example.com