Fixes #37871 - Granular capsule update content counts

app/controllers/katello/api/v2/capsule_content_controller.rb

@@ -26,13 +26,6 @@ def counts
       render json: @capsule.content_counts.to_json
     end
 
-    api :POST, '/capsules/:id/content/update_counts', N_('Update content counts for the smart proxy')
-    param :id, Integer, :desc => N_('Id of the smart proxy'), :required => true
-    def update_counts
-      task = async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, @capsule)
-      respond_for_async :resource => task
-    end
-
     api :GET, '/capsules/:id/content/lifecycle_environments', N_('List the lifecycle environments attached to the smart proxy')
     param_group :lifecycle_environments
     def lifecycle_environments
@@ -86,6 +79,24 @@ def sync
       respond_for_async :resource => task
     end
 
+    api :POST, '/capsules/:id/content/update_counts', N_('Update content counts for the smart proxy')
+    param :id, Integer, :desc => N_('Id of the smart proxy'), :required => true
+    param :environment_id, Integer, :desc => N_('Id of the environment to limit the content counting on')
+    param :content_view_id, Integer, :desc => N_('Id of the content view to limit the content counting on')
+    param :repository_id, Integer, :desc => N_('Id of the repository to limit the content counting on')
+    def update_counts
+      find_environment if params[:environment_id]
+      find_content_view if params[:content_view_id]
+      find_repository if params[:repository_id]
+      count_options = {
+        :environment_id => @environment.try(:id),
+        :content_view_id => @content_view.try(:id),
+        :repository_id => @repository.try(:id)
+      }
+      task = async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, @capsule, count_options)
+      respond_for_async :resource => task
+    end
+
     api :GET, '/capsules/:id/content/sync', N_('Get current smart proxy synchronization status')
     param :id, Integer, :desc => N_('Id of the smart proxy'), :required => true
     param :organization_id, Integer, :desc => N_('Id of the organization to get the status for'), :required => false

app/lib/actions/katello/capsule_content/sync_capsule.rb

@@ -5,7 +5,11 @@ class SyncCapsule < ::Actions::EntryAction
         # rubocop:disable Metrics/MethodLength
         execution_plan_hooks.use :update_content_counts, :on => :success
         def plan(smart_proxy, options = {})
-          plan_self(:smart_proxy_id => smart_proxy.id)
+          plan_self(:smart_proxy_id => smart_proxy.id,
+                    :environment_id => options[:environment_id],
+                    :content_view_id => options[:content_view_id],
+                    :repository_id => options[:repository_id],
+                    :skip_content_counts_update => options[:skip_content_counts_update])
           action_subject(smart_proxy)
           environment = options[:environment]
           content_view = options[:content_view]
@@ -69,9 +73,10 @@ def repos_to_sync(smart_proxy, environment, content_view, repository, skip_metat
         end
 
         def update_content_counts(_execution_plan)
-          if Setting[:automatic_content_count_updates]
+          if Setting[:automatic_content_count_updates] && !input[:skip_content_counts_update]
             smart_proxy = ::SmartProxy.unscoped.find(input[:smart_proxy_id])
-            ::ForemanTasks.async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, smart_proxy)
+            options = {environment_id: input[:environment_id], content_view_id: input[:content_view_id], repository_id: input[:repository_id]}
+            ::ForemanTasks.async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, smart_proxy, options)
           else
             Rails.logger.info "Skipping content counts update as automatic content count updates are disabled. To enable automatic content count updates, set the 'automatic_content_count_updates' setting to true.
 To update content counts manually, run the 'Update Content Counts' action."

app/lib/actions/katello/capsule_content/update_content_counts.rb

@@ -2,8 +2,9 @@ module Actions
   module Katello
     module CapsuleContent
       class UpdateContentCounts < Actions::EntryAction
-        def plan(smart_proxy)
-          plan_self(:smart_proxy_id => smart_proxy.id)
+        def plan(smart_proxy, options = {})
+          input[:options] = options
+          plan_self(:smart_proxy_id => smart_proxy.id, environment_id: options[:environment_id], content_view_id: options[:content_view_id], repository_id: options[:repository_id])
         end
 
         def humanized_name
@@ -12,7 +13,22 @@ def humanized_name
 
         def run
           smart_proxy = ::SmartProxy.unscoped.find(input[:smart_proxy_id])
-          smart_proxy.update_content_counts!
+          env = find_env(input[:environment_id])
+          content_view = find_content_view(input[:content_view_id])
+          repository = find_repository(input[:repository_id])
+          smart_proxy.update_content_counts! env, content_view, repository
+        end
+
+        def find_env(environment_id)
+          ::Katello::KTEnvironment.find(environment_id) if environment_id
+        end
+
+        def find_content_view(content_view_id)
+          ::Katello::ContentView.find(content_view_id) if content_view_id
+        end
+
+        def find_repository(repository_id)
+          ::Katello::Repository.find(repository_id) if repository_id
         end
 
         def rescue_strategy

app/lib/actions/katello/content_view/capsule_sync.rb

@@ -12,9 +12,25 @@ def plan(content_view, environment)
               smart_proxies = SmartProxy.unscoped.with_environment(environment).select { |sp| sp.authorized?(:manage_capsule_content) && sp.authorized?(:view_capsule_content) }
               unless smart_proxies.blank?
                 plan_action(::Actions::BulkAction, ::Actions::Katello::CapsuleContent::Sync, smart_proxies.sort,
-                            :content_view_id => content_view.id, :environment_id => environment.id)
+                            :content_view_id => content_view.id, :environment_id => environment.id, :skip_content_counts_update => true)
               end
             end
+            #For Content view triggered capsule sync, we need to update content counts in one action in finalize, instead of one action per CV, per env, per smart proxy
+            plan_self(:content_view_id => content_view.id, :environment_id => environment.id)
+          end
+        end
+
+        def finalize
+          if Setting[:automatic_content_count_updates]
+            environment = ::Katello::KTEnvironment.find(input[:environment_id])
+            smart_proxies = SmartProxy.unscoped.with_environment(environment).select { |sp| sp.authorized?(:manage_capsule_content) && sp.authorized?(:view_capsule_content) }
+            options = {environment_id: input[:environment_id], content_view_id: input[:content_view_id]}
+            smart_proxies.each do |smart_proxy|
+              ::ForemanTasks.async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, smart_proxy, options)
+            end
+          else
+            Rails.logger.info "Skipping content counts update as automatic content count updates are disabled. To enable automatic content count updates, set the 'automatic_content_count_updates' setting to true.
+To update content counts manually, run the 'Update Content Counts' action."
           end
         end
       end

app/models/katello/concerns/smart_proxy_extensions.rb

@@ -140,8 +140,80 @@ def load_balanced?
         URI.parse(self.url).host != self.registration_host
       end
 
-      def update_content_counts!
+      def update_content_counts!(environment = nil, content_view = nil, repository = nil)
         # {:content_view_versions=>{87=>{:repositories=>{1=>{:metadata=>{},:counts=>{:rpms=>98, :module_streams=>9898}}}}}
+        if environment.nil? && content_view.nil? && repository.nil?
+          global_content_counts
+        else
+          smart_proxy_helper = ::Katello::SmartProxyHelper.new(self)
+          repos = repository ? [repository] : smart_proxy_helper.repositories_available_to_capsule(environment, content_view)
+          self.with_lock do
+            repos_content_count(repos)
+          end
+        end
+      end
+
+      #{"content_view_versions"=>
+      #   {"5"=>
+      #     {"repositories"=>
+      #       {"20"=>{"counts"=>{"rpm"=>32, "erratum"=>4}, "metadata"=>{"env_id"=>2, "product_id"=>1, "content_type"=>"yum", "library_instance_id"=>14}},
+      #        "21"=>{"counts"=>{"file"=>3}, "metadata"=>{"env_id"=>2, "product_id"=>1, "content_type"=>"file", "library_instance_id"=>15}},
+      #        "22"=>{"counts"=>{"file"=>3}, "metadata"=>{"env_id"=>3, "product_id"=>1, "content_type"=>"file", "library_instance_id"=>15}},
+      #        "23"=>{"counts"=>{"rpm"=>32, "erratum"=>4}, "metadata"=>{"env_id"=>3, "product_id"=>1, "content_type"=>"yum", "library_instance_id"=>14}}}}}}
+      #
+
+      def repos_content_count(repos)
+        new_content_counts = initialize_content_counts
+        repos.each do |repo|
+          process_repository(repo, new_content_counts)
+        end
+        remove_unavailable_versions(new_content_counts)
+        update(content_counts: new_content_counts)
+      end
+
+      def initialize_content_counts
+        (content_counts.deep_dup || { content_view_versions: {} }).with_indifferent_access
+      end
+
+      def process_repository(repo, content_counts)
+        repo_mirror_service = repo.backend_service(self).with_mirror_adapter
+        repo_content_counts = repo_mirror_service.latest_content_counts
+        translated_counts = translate_counts(repo, repo_mirror_service, repo_content_counts)
+        content_counts[:content_view_versions][repo.content_view_version_id.to_s] ||= { repositories: {}}
+        content_counts[:content_view_versions][repo.content_view_version_id.to_s][:repositories][repo.id.to_s] = translated_counts
+      end
+
+      def translate_counts(repo, repo_mirror_service, repo_content_counts)
+        translated_counts = {metadata: {}, counts: {}}
+        translated_counts[:metadata] = {
+          env_id: repo.environment_id,
+          library_instance_id: repo.library_instance_or_self.id,
+          product_id: repo.product_id,
+          content_type: repo.content_type
+        }
+        repo_content_counts&.each do |name, count|
+          count = count[:count]
+          if name == 'rpm.package' && repo.content_counts['srpm'] > 0
+            translated_counts[:counts]['srpm'] = repo_mirror_service.count_by_pulpcore_type(::Katello::Pulp3::Srpm)
+            translated_counts[:counts]['rpm'] = count - translated_counts[:counts]['srpm']
+          elsif name == 'container.manifest' && repo.content_counts['docker_manifest_list'] > 0
+            translated_counts[:counts]['docker_manifest_list'] = repo_mirror_service.count_by_pulpcore_type(::Katello::Pulp3::DockerManifestList)
+            translated_counts[:counts]['docker_manifest'] = count - translated_counts[:counts]['docker_manifest_list']
+          else
+            translated_counts[:counts][::Katello::Pulp3::PulpContentUnit.katello_name_from_pulpcore_name(name, repo)] = count
+          end
+        end
+        translated_counts
+      end
+
+      def remove_unavailable_versions(content_counts)
+        version_ids_available_to_proxy = Katello::ContentViewVersion.in_environment(lifecycle_environments)&.pluck(:id)&.uniq
+        version_ids_in_count_map = content_counts[:content_view_versions].keys&.map(&:to_i)
+        version_ids_to_remove = version_ids_in_count_map - version_ids_available_to_proxy
+        version_ids_to_remove.each { |id| content_counts[:content_view_versions].delete(id.to_s) }
+      end
+
+      def global_content_counts
         new_content_counts = { content_view_versions: {} }
         smart_proxy_helper = ::Katello::SmartProxyHelper.new(self)
         repos = smart_proxy_helper.repositories_available_to_capsule

app/services/katello/smart_proxy_helper.rb

@@ -47,9 +47,9 @@ def combined_repos_available_to_capsule(environment = nil, content_view = nil, r
 
     def repositories_available_to_capsule(environments = nil, content_view = nil)
       environments = @smart_proxy.lifecycle_environments if environments.nil?
-      yum_repos = Katello::Repository.in_environment(environments)
-      yum_repos = yum_repos.in_content_views([content_view]) if content_view
-      yum_repos.smart_proxy_syncable
+      repos = Katello::Repository.in_environment(environments)
+      repos = repos.in_content_views([content_view]) if content_view
+      repos.smart_proxy_syncable
     end
 
     def unsyncable_content_types

test/actions/katello/content_view_test.rb

@@ -442,7 +442,9 @@ class CapsuleSyncTest < TestBase
       plan_action(action, content_view, library)
       assert_action_planned_with(action, ::Actions::BulkAction, ::Actions::Katello::CapsuleContent::Sync,
                                  [smart_proxy_service_1.smart_proxy, smart_proxy_service_2.smart_proxy].sort,
-                                 :content_view_id => content_view.id, :environment_id => library.id)
+                                 :content_view_id => content_view.id,
+                                 :environment_id => library.id,
+                                 :skip_content_counts_update => true)
     end
   end
 

webpack/scenes/SmartProxy/ExpandedSmartProxyRepositories.js

@@ -19,7 +19,6 @@ const ExpandedSmartProxyRepositories = ({
         filteredData[key] = entry;
       }
     });
-
     return filteredData;
   };
   const envContentCounts = filterDataByEnvId();
@@ -48,6 +47,43 @@ const ExpandedSmartProxyRepositories = ({
     /* eslint-enable max-len */
     return cellList;
   };
+
+  const getDataListItems = () => {
+    if (Object.keys(envContentCounts).length) {
+      return Object.keys(envContentCounts).map((repo, index) => (
+        <DataListItem key={`${repo.id}-${index}`}>
+          <DataListItemRow>
+            <DataListItemCells
+              dataListCells={dataListCellLists(envContentCounts[repo], repo)}
+            />
+          </DataListItemRow>
+        </DataListItem>
+      ));
+    }
+
+    if (repositories?.length) {
+      return repositories.map((repo, index) => (
+        <DataListItem key={`${repo.id}-${index}`}>
+          <DataListItemRow>
+            <DataListItemCells
+              dataListCells={dataListCellListsNotSynced(repo)}
+            />
+          </DataListItemRow>
+        </DataListItem>
+      ));
+    }
+
+    return (
+      <DataListItem key="empty">
+        <DataListItemRow>
+          <DataListItemCells
+            dataListCells={[<DataListCell key="cv-empty"><InactiveText text={__('Content view version is empty')} /></DataListCell>]}
+          />
+        </DataListItemRow>
+      </DataListItem>
+    );
+  };
+
   if (syncedToCapsule) {
     return (
       <DataList aria-label="Expanded repository details" isCompact>
@@ -64,24 +100,7 @@ const ExpandedSmartProxyRepositories = ({
             />
           </DataListItemRow>
         </DataListItem>
-        {Object.keys(envContentCounts).length ?
-          Object.keys(envContentCounts).map((repo, index) => (
-            <DataListItem key={`${repo.id}-${index}`}>
-              <DataListItemRow>
-                <DataListItemCells
-                  dataListCells={dataListCellLists(envContentCounts[repo], repo)}
-                />
-              </DataListItemRow>
-            </DataListItem>
-          )) :
-          <DataListItem key="empty">
-            <DataListItemRow>
-              <DataListItemCells
-                dataListCells={[<DataListCell key="cv-empty"><InactiveText text={__('Content view version is empty')} /></DataListCell>]}
-              />
-            </DataListItemRow>
-          </DataListItem>
-          }
+        {getDataListItems()}
       </DataList>
     );
   }
@@ -112,7 +131,7 @@ const ExpandedSmartProxyRepositories = ({
         <DataListItem key="empty">
           <DataListItemRow>
             <DataListItemCells
-              dataListCells={[<DataListCell key="cv-empty"><InactiveText text={__('Content view version is empty')} /></DataListCell>]}
+              dataListCells={[<DataListCell key="cv-empty"><InactiveText text={__('Content view version is empty or content counts are not up to date')} /></DataListCell>]}
             />
           </DataListItemRow>
         </DataListItem>