Fixes #35959 - Add structured APT content mode

Note that this feature must be enabled/disabled using:

foreman-rake katello:enable_structured_content_for_deb

app/controllers/katello/api/v2/activation_keys_controller.rb

@@ -238,10 +238,10 @@ def content_override
           validate_content_overrides_enabled(override)
         end
         specified_labels.uniq!
-        existing_labels = organization.contents.where(label: specified_labels).uniq
+        existing_labels = organization.contents.where(label: specified_labels).pluck(:label).uniq
 
         unless specified_labels.size == existing_labels.size
-          missing_labels = specified_labels - existing_labels.pluck(:label)
+          missing_labels = specified_labels - existing_labels
           msg = "Content label(s) \"#{missing_labels.join(", ")}\" were not found in the Organization \"#{organization}\""
           fail HttpErrors::BadRequest, _(msg)
         end

app/controllers/katello/api/v2/repositories_controller.rb

@@ -650,6 +650,7 @@ def construct_repo_from_params(repo_params) # rubocop:disable Metrics/AbcSize
         root.deb_releases = repo_params[:deb_releases] if repo_params[:deb_releases]
         root.deb_components = repo_params[:deb_components] if repo_params[:deb_components]
         root.deb_architectures = repo_params[:deb_architectures] if repo_params[:deb_architectures]
+        root.content_id = 'INITIAL_DUMMY_VALUE' unless Setting['deb_enable_structured_apt']
       end
 
       if root.ansible_collection?

app/lib/actions/candlepin/product/content_create.rb

@@ -10,13 +10,24 @@ class ContentCreate < Candlepin::Abstract
           param :content_url
           param :owner
           param :os_versions
+          param :repository_id
         end
 
         def run
+          content_url = input[:content_url]
+          if input[:type] == ::Katello::Repository::DEB_TYPE
+            # We must retrieve the repository in the run phase, so the latest Pulp version_href is
+            # already set. This is needed to retrieve the latest repository.deb_content_url_options!
+            repository = ::Katello::Repository.find(input[:repository_id])
+            if repository.deb_using_structured_apt?
+              content_url += repository.deb_content_url_options
+            end
+          end
+
           output[:response] = ::Katello::Resources::Candlepin::Content.
               create(input[:owner],
                      name: input[:name],
-                     contentUrl: input[:content_url],
+                     contentUrl: content_url,
                      type: input[:type],
                      arches: input[:arches],
                      label: input[:label],

app/lib/actions/candlepin/product/content_update.rb

@@ -3,7 +3,7 @@ module Candlepin
     module Product
       class ContentUpdate < Candlepin::Abstract
         input_format do
-          param :content_id
+          param :repository_id
           param :name
           param :type
           param :arches
@@ -15,19 +15,36 @@ class ContentUpdate < Candlepin::Abstract
           param :metadata_expire
         end
 
-        def run
+        def finalize
+          content_url = input[:content_url]
+          # We must retrieve the repository in the finalize phase, because Katello::Product::ContentCreate
+          # only updates the repository.content_id in the finalize phase!
+          repository = ::Katello::Repository.find(input[:repository_id])
+          if repository.deb_using_structured_apt?
+            content_url += repository.deb_content_url_options
+          end
+
           output[:response] = ::Katello::Resources::Candlepin::Content.
               update(input[:owner],
-                     id: input[:content_id],
+                     id: repository.content_id,
                      name: input[:name],
-                     contentUrl: input[:content_url],
+                     contentUrl: content_url,
                      gpgUrl: input[:gpg_key_url] || '', #candlepin ignores nil
                      type: input[:type],
                      arches: input[:arches] || '',
                      requiredTags: input[:os_versions],
                      label: input[:label],
                      metadataExpire: input[:metadata_expire] || 1,
                      vendor: ::Katello::Provider::CUSTOM)
+
+          repository.content.update!(
+            name: input[:name],
+            content_url: content_url,
+            content_type: input[:type],
+            label: input[:label],
+            gpg_url: input[:gpg_key_url],
+            vendor: ::Katello::Provider::CUSTOM
+          )
         end
       end
     end

app/lib/actions/katello/cdn_configuration/update.rb

@@ -21,7 +21,7 @@ def plan(cdn_configuration, options)
             full_path = if cdn_configuration.redhat_cdn? || cdn_configuration.custom_cdn?
                           root.product.repo_url(root.library_instance.generate_content_path)
                         elsif cdn_configuration.network_sync?
-                          resource.repository_url(content_label: root.content.label, arch: root.arch, major: root.major, minor: root.minor)
+                          resource.repository_url(content_label: root.library_instance.content.label, arch: root.arch, major: root.major, minor: root.minor)
                         end
             plan_action(::Actions::Katello::Repository::Update, root, url: full_path)
           end

app/lib/actions/katello/product/content_create.rb

@@ -4,51 +4,60 @@ module Product
       class ContentCreate < Actions::Base
         middleware.use Actions::Middleware::RemoteAction
 
-        def plan(root)
+        def plan(repository)
+          root = repository.root
           sequence do
-            if root.content_id.nil?
+            if repository.content_id.nil? || repository.content_id == 'INITIAL_DUMMY_VALUE'
               content_create = plan_action(Candlepin::Product::ContentCreate,
-                                           owner:         root.product.organization.label,
+                                           repository_id: repository.id,
+                                           owner:         root.organization.label,
                                            name:          root.name,
                                            type:          root.content_type,
                                            arches:        root.format_arches,
                                            label:         root.custom_content_label,
-                                           os_versions:   root.os_versions&.join(','),
-                                           content_url:   root.custom_content_path)
+                                           content_url:   root.custom_content_path,
+                                           os_versions:   root.os_versions&.join(','))
               content_id = content_create.output[:response][:id]
               plan_action(Candlepin::Product::ContentAdd, owner: root.product.organization.label,
                                     product_id: root.product.cp_id,
                                     content_id: content_id)
 
             else
-              content_id = root.content_id
+              content_id = repository.content_id
             end
 
+            plan_self(root_repository_id: root.id, repository_id: repository.id, content_id: content_id)
+
             if root.gpg_key
               plan_action(Candlepin::Product::ContentUpdate,
-                          owner:       root.organization.label,
-                          content_id:  content_id,
-                          name:        root.name,
-                          type:        root.content_type,
-                          arches:      root.format_arches,
-                          label:       root.custom_content_label,
-                          content_url: root.custom_content_path,
-                          gpg_key_url: root.library_instance.yum_gpg_key_url)
+                          repository_id: repository.id,
+                          owner:         root.organization.label,
+                          name:          root.name,
+                          type:          root.content_type,
+                          arches:        root.format_arches,
+                          label:         root.custom_content_label,
+                          content_url:   root.custom_content_path,
+                          gpg_key_url:   root.library_instance.yum_gpg_key_url)
             end
-
-            plan_self(root_repository_id: root.id, content_id: content_id)
           end
         end
 
         def finalize
           root = ::Katello::RootRepository.find(input[:root_repository_id])
-          root.update(:content_id => input[:content_id])
+          content_url = root.custom_content_path
+          if root.deb_using_structured_apt?
+            repository = ::Katello::Repository.find(input[:repository_id])
+            content_url += repository.deb_content_url_options
+            repository.update(:content_id => input[:content_id])
+          else
+            root.update(:content_id => input[:content_id])
+          end
 
-          content = ::Katello::Content.where(organization_id: root.product.organization_id, cp_content_id: root.content_id).first_or_create do |new_content|
+          content = ::Katello::Content.where(organization_id: root.product.organization_id, cp_content_id: input[:content_id]).first_or_create do |new_content|
             new_content.name = root.name
             new_content.content_type = root.content_type
             new_content.label = root.custom_content_label
-            new_content.content_url = root.custom_content_path
+            new_content.content_url = content_url
             new_content.vendor = ::Katello::Provider::CUSTOM
           end
 

app/lib/actions/katello/product/content_destroy.rb

@@ -2,22 +2,23 @@ module Actions
   module Katello
     module Product
       class ContentDestroy < Actions::Base
-        def plan(root_repository)
+        def plan(repository)
+          root_repository = repository.root
           fail _("Cannot delete redhat product content") if root_repository.product.redhat?
           sequence do
             plan_action(Candlepin::Product::ContentRemove,
                         owner: root_repository.product.organization.label,
                         product_id: root_repository.product.cp_id,
-                        content_id: root_repository.content_id)
+                        content_id: repository.content_id)
 
-            katello_content_id = root_repository.content&.id
+            katello_content_id = repository.content&.id
             ::Katello::ProductContent.where(product_id: root_repository.product_id,
                                             content_id: katello_content_id).destroy_all
 
-            if root_repository.repositories.count <= 1
+            if root_repository.repositories.count <= 1 || repository.deb_using_structured_apt?
               plan_action(Candlepin::Product::ContentDestroy,
                           owner: root_repository.product.organization.label,
-                          content_id: root_repository.content_id)
+                          content_id: repository.content_id)
 
               ::Katello::Content.find_by_id(katello_content_id)&.destroy!
             end

app/lib/actions/katello/product/destroy.rb

@@ -65,7 +65,7 @@ def plan_content_destruction(product, skip_environment_update)
             if repo.root.repositories.where.not(id: repo.id).empty? &&
                 !repo.redhat? &&
                 !skip_environment_update
-              plan_action(::Actions::Katello::Product::ContentDestroy, repo.root)
+              plan_action(::Actions::Katello::Product::ContentDestroy, repo)
             end
           end
         end

app/lib/actions/katello/repository/clone_contents.rb

@@ -25,6 +25,20 @@ def plan(source_repositories, new_repository, options)
 
             index_options = {id: new_repository.id, force_index: true}
             index_options[:source_repository_id] = source_repositories.first.id if source_repositories.count == 1 && filters.empty? && rpm_filenames.nil?
+
+            if new_repository.deb_using_structured_apt? && generate_metadata
+              plan_action(Candlepin::Product::ContentUpdate,
+                          owner:           new_repository.organization.label,
+                          repository_id:   new_repository.id,
+                          name:            new_repository.root.name,
+                          type:            new_repository.root.content_type,
+                          arches:          new_repository.root.format_arches,
+                          label:           new_repository.root.custom_content_label,
+                          content_url:     new_repository.root.custom_content_path,
+                          gpg_key_url:     new_repository.yum_gpg_key_url,
+                          metadata_expire: new_repository.root.metadata_expire)
+            end
+
             plan_action(Katello::Repository::IndexContent, index_options)
           end
         end

app/lib/actions/katello/repository/create.rb

@@ -2,41 +2,38 @@ module Actions
   module Katello
     module Repository
       class Create < Actions::EntryAction
-        # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         def plan(repository, args = {})
           clone = args[:clone] || false
           force_repo_create = args[:force_repo_create] || false
           repository.save!
-          root = repository.root
 
           action_subject(repository)
 
-          org = repository.organization
           sequence do
             # Container push repositories will already be in pulp. The version_href is
             # directly updated after a push.
-            unless root.is_container_push && repository.in_default_view?
+            unless repository.root.is_container_push && repository.in_default_view?
               create_action = plan_action(Pulp3::Orchestration::Repository::Create,
                                           repository, SmartProxy.pulp_primary, force_repo_create)
               return if create_action.error
             end
 
-            # when creating a clone, the following actions are handled by the
-            # publish/promote process
-            unless clone
-              view_env = org.default_content_view.content_view_environment(org.library)
-              if repository.product.redhat?
-                plan_action(Actions::Candlepin::Environment::AddContentToEnvironment, :view_env_cp_id => view_env.cp_id, :content_id => repository.content_id)
-              else
-                unless root.content
-                  content_create = plan_action(Katello::Product::ContentCreate, root)
-                  plan_action(Actions::Candlepin::Environment::AddContentToEnvironment, :view_env_cp_id => view_env.cp_id, :content_id => content_create.input[:content_id])
-                end
-              end
+            content_id = false
+            if !clone && repository.product.redhat?
+              content_id = repository.content_id
+            elsif create_content?(repository, clone)
+              content_create = plan_action(Katello::Product::ContentCreate, repository)
+              content_id = content_create.input[:content_id]
+            end
+
+            if !clone && content_id
+              view_env = repository.organization.default_content_view.content_view_environment(repository.organization.library)
+              plan_action(Actions::Candlepin::Environment::AddContentToEnvironment, :view_env_cp_id => view_env.cp_id, :content_id => content_id)
             end
 
             # Container push repos do not need metadata generation or ACS (they do not sync)
-            unless root.is_container_push && repository.in_default_view?
+            unless repository.root.is_container_push && repository.in_default_view?
               concurrence do
                 plan_self(:repository_id => repository.id, :clone => clone)
                 if !clone && repository.url.present? && !repository.url.start_with?('uln')
@@ -65,6 +62,16 @@ def run
         def humanized_name
           _("Create")
         end
+
+        def create_content?(repository, clone)
+          if repository.deb_using_structured_apt?
+            return true
+          elsif clone || repository.content
+            return false
+          else
+            return true
+          end
+        end
       end
     end
   end

app/lib/actions/katello/repository/destroy.rb

@@ -90,9 +90,9 @@ def delete_empty_repo_filters(repository)
         end
 
         def handle_custom_content(repository, remove_from_content_view_versions)
-          #if this is the last instance of a custom repo, destroy the content
-          if remove_from_content_view_versions || repository.root.repositories.where.not(id: repository.id).empty?
-            plan_action(::Actions::Katello::Product::ContentDestroy, repository.root)
+          #if this is the last instance of a custom repo or a deb repo using structured APT, destroy the content
+          if remove_from_content_view_versions || repository.root.repositories.where.not(id: repository.id).empty? || repository.deb_using_structured_apt?
+            plan_action(::Actions::Katello::Product::ContentDestroy, repository)
           end
         end
 

app/lib/actions/katello/repository/finish_upload.rb

@@ -15,6 +15,19 @@ def plan(repository, options = {})
           generate_metadata = options.fetch(:generate_metadata, true)
           plan_action(Katello::Repository::MetadataGenerate, repository, :dependency => import_upload_task, :force_publication => true) if generate_metadata
 
+          if repository.deb_using_structured_apt? && generate_metadata
+            plan_action(::Actions::Candlepin::Product::ContentUpdate,
+                        owner:           repository.organization.label,
+                        repository_id:   repository.id,
+                        name:            repository.root.name,
+                        type:            repository.root.content_type,
+                        arches:          repository.root.format_arches,
+                        label:           repository.root.custom_content_label,
+                        content_url:     repository.root.custom_content_path,
+                        gpg_key_url:     repository.yum_gpg_key_url,
+                        metadata_expire: repository.root.metadata_expire)
+          end
+
           recent_range = 5.minutes.ago.utc.iso8601
           plan_action(Katello::Repository::FilteredIndexContent,
                       id: repository.id,

app/lib/actions/katello/repository/sync.rb

@@ -46,6 +46,7 @@ def plan(repo, options = {})
                                         pulp_sync_options)
               output = sync_action.output
 
+              update_deb_content(repo)
               plan_action(Katello::Repository::IndexContent, :id => repo.id, :force_index => skip_metadata_check)
               plan_action(Katello::Foreman::ContentUpdate, repo.environment, repo.content_view, repo)
               plan_action(Katello::Repository::FetchPxeFiles, :id => repo.id)
@@ -94,6 +95,21 @@ def presenter
           Helpers::Presenter::Delegated.new(self, found)
         end
 
+        def update_deb_content(repo)
+          return unless repo.deb_using_structured_apt?
+
+          plan_action(::Actions::Candlepin::Product::ContentUpdate,
+                      owner:           repo.organization.label,
+                      repository_id:   repo.id,
+                      name:            repo.root.name,
+                      type:            repo.root.content_type,
+                      arches:          repo.root.format_arches,
+                      label:           repo.root.custom_content_label,
+                      content_url:     repo.root.custom_content_path,
+                      gpg_key_url:     repo.yum_gpg_key_url,
+                      metadata_expire: repo.root.metadata_expire)
+        end
+
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end