Use GitLab for syncing upstream->downstream.

This pull request allows using gitlab for
syncing upstream->downstream.

The rest of function remains. Like
git clonedownstream uses dist-git

Signed-off-by: Petr "Stone" Hracek <[email protected]>

container_workflow_tool/cli_common.py

@@ -59,6 +59,10 @@ def get_parser(self):
         parsers['git'].add_argument('--rebuild-reason', help='Use a custom reason for rebuilding')
         parsers['git'].add_argument('--commit-msg', help='Use a custom message instead of the default one')
         parsers['git'].add_argument('--check-script', help='Script/command to be run when checking repositories')
+        parsers['git'].add_argument(
+            '--gitlab', action='store_true', default=False,
+            help='File a merge request to corresponding repository instead of directly to dist-git'
+        )
         parsers['build'].add_argument(
             '--repo-url', help='Set the url of a .repo file to be used when building the image'
         )
@@ -106,6 +110,7 @@ def git_usage(self):
         --commit-msg     - Use a custom message instead of the default one
         --rebuild-reason - Use a custom reason for rebuilding
         --check-script   - Script/command to be run when checking repositories
+        --gitlab         - Use GitLab for filling merge requests instead of direct pushing to dist-git
     """
         return action_help
 

container_workflow_tool/distgit.py

@@ -5,7 +5,7 @@
 from git import Repo
 from git.exc import GitCommandError
 
-import container_workflow_tool.utility as u
+from container_workflow_tool import utility
 from container_workflow_tool.utility import RebuilderError
 from container_workflow_tool.dockerfile import DockerfileHandler
 from container_workflow_tool.sync import SyncHandler
@@ -49,7 +49,7 @@ def check_script(self, component, script_path, component_path):
             self.logger.info(template.format(name=component, status="Affected"))
             err = ret.stderr.decode('utf-8').strip()
             if err:
-                self.logger.error(u._2sp(err))
+                self.logger.error(utility._2sp(err))
         else:
             self.logger.info(template.format(name=component, status="OK"))
 
@@ -63,7 +63,7 @@ def dist_git_changes(self, images, rebase=False):
             rebase (bool, optional): Specify if a rebase should be done instead
         """
         try:
-            for image in (images):
+            for image in images:
                 name = image["name"]
                 component = image["component"]
                 branch = image["git_branch"]
@@ -92,18 +92,17 @@ def dist_git_changes(self, images, rebase=False):
                     ups_name = name.split('-')[0]
                     # Clone upstream repository
                     ups_path = os.path.join('upstreams/', ups_name)
-                    self._clone_upstream(url, ups_path, commands=commands)
+                    self.clone_upstream(url, ups_path, commands=commands)
                     # Save the upstream commit hash
                     ups_hash = Repo(ups_path).commit().hexsha
-                    self._pull_upstream(component, path, url, repo, ups_name, commands)
+                    self.pull_upstream(component, path, url, repo, ups_name, commands)
                     self.df_handler.update_dockerfile(
                         df_path, from_tag, downstream_from=downstream_from
                     )
                     repo.git.add("Dockerfile")
                     # It is possible for the git repository to have no changes
                     if repo.is_dirty():
-                        commit = self.get_commit_msg(rebase, image, ups_hash
-                        )
+                        commit = self.get_commit_msg(rebase, image, ups_hash)
                         if commit:
                             repo.git.commit("-m", commit)
                         else:
@@ -122,8 +121,8 @@ def _clone_downstream(self, component, branch):
             self.logger.info("Using existing downstream repo: " + component)
             repo = Repo(component)
         else:
-            hostname_url = u._get_hostname_url(self.conf)
-            packager = u._get_packager(self.conf)
+            hostname_url = utility._get_hostname_url(self.conf)
+            packager = utility._get_packager(self.conf)
             # if packager is fedpkg then namespace is `container` else `containers`
             namespace = "container" if packager == "fedpkg" else "containers"
             component_path = f"{namespace}/{component}"
@@ -162,7 +161,7 @@ def push_changes(self, tmp, images):
                     # commit_msg is set so it is always returned
                     commit = self.get_commit_msg(None, image)
                     repo.git.commit("-am", commit)
-                if self._get_unpushed_commits(repo):
+                if self.are_unpushed_commits_available(repo):
                     self.logger.info("Pushing: " + component)
 
                     repo.git.push()
@@ -176,7 +175,7 @@ def push_changes(self, tmp, images):
         if failed:
             self.logger.error("Failed pushing images:")
             for image in failed:
-                self.logger.error(u._2sp(image["component"]))
+                self.logger.error(utility._2sp(image["component"]))
             self.logger.error("Please check the failures and push the changes manually.")
 
     # TODO: Multiple future branches?
@@ -204,5 +203,5 @@ def merge_future_branches(self, images):
         if failed:
             self.logger.error("Failed merging images:")
             for image in failed:
-                self.logger.error(u._2sp(image["component"]))
+                self.logger.error(utility._2sp(image["component"]))
             self.logger.error("Please check the failures and push the changes manually.")

container_workflow_tool/git_operations.py

@@ -1,6 +1,6 @@
 # MIT License
 #
-# Copyright (c) 2020 SCL team at Red Hat
+# Copyright (c) 2023 SCL team at Red Hat
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -55,7 +55,7 @@ def set_commit_msg(self, msg):
         """
         self.commit_msg = msg
 
-    def _do_git_reset(self, repo):
+    def do_git_reset(self, repo):
         file_list = ['--', '.gitignore'] + self.conf.ignore_files
         repo.git.reset(file_list)
         # One file at a time to make sure all files get reset even on error
@@ -68,7 +68,7 @@ def _do_git_reset(self, repo):
                 repo.git.clean('-xfd', f)
                 self.logger.debug("Removing untracked ignored file: " + f)
 
-    def _clone_upstream(self, url, ups_path, commands=None):
+    def clone_upstream(self, url, ups_path, commands=None):
         """
         :params: url is URL to repofile from upstream. https://github.com/sclorg
         :param: ups_path is path where URL is cloned locally
@@ -113,20 +113,24 @@ def _clone_upstream(self, url, ups_path, commands=None):
         os.chdir(oldcwd)
         return repo
 
-    def are_unpushed_commits_available(self, repo) -> bool:
+    def are_unpushed_commits_available(self, repo, branch_name="") -> bool:
         """
         Get unpushed commits
         :param repo: repo object to check for unpushed commits
+        :param branch_name: In case of gitlab, forked_branch and branch_name has to be defined.
+                            branch_name is e.g. rhel-8.7.0 and 'repo.active_branch.name' is 'rhel-8.7.0-<ubi_name>'
         :return: List of commits or empty array
         """
         branch = repo.active_branch.name
         # Get a list of commits that have not been pushed to remote
         select = "origin/" + branch + ".." + branch
+        if branch_name != "":
+            select = "origin/" + branch_name + ".." + branch
         if len(list(repo.iter_commits(select))) == 0:
             return False
         return True
 
-    def show_git_changes(self, tmp, components=None, diff=False):
+    def show_git_changes(self, tmp, components=None, diff=False, branch_name=""):
         """Shows changes made to tracked files in local downstream repositories
 
         Walks through all repositories and calls 'git-show' or 'git-diff' on each of them.
@@ -135,6 +139,8 @@ def show_git_changes(self, tmp, components=None, diff=False):
             tmp (str): Path to the directory that is used to store git repositories
             components (list of str, optional): List of components to show changes for
             diff (boolean, optional): Controls whether the method calls git-show or git-diff
+            branch_name (str, optional): In case of gitlab, forked_branch and branch_name has to be defined.
+                            branch_name is e.g. rhel-8.7.0 and 'repo.active_branch.name' is 'rhel-8.7.0-<ubi_name>'
         """
         # Function to check if a path contains a git repository
         def is_git(x): return os.path.isdir(os.path.join(x, '.git'))
@@ -155,7 +161,7 @@ def is_git(x): return os.path.isdir(os.path.join(x, '.git'))
             repo = Repo(path)
             # Only show changes if there are unpushed commits to show
             # or we only want the diff of unstaged changes
-            if self.are_unpushed_commits_available(repo) or diff:
+            if self.are_unpushed_commits_available(repo, branch_name=branch_name) or diff:
                 # Clears the screen
                 print(chr(27) + "[2J")
                 # Force pager for short git diffs
@@ -229,7 +235,7 @@ def get_commit_msg(self, rebase, image=None, ups_hash=None):
             commit += "\n created from upstream commit: " + ups_hash
         return commit
 
-    def _pull_upstream(self, component, path, url, repo, ups_name, commands):
+    def pull_upstream(self, component, path, url, repo, ups_name, commands):
         """Pulls an upstream repo and copies it into downstream"""
         ups_path = os.path.join('upstreams/', ups_name)
         cp_path = os.path.join(ups_path, path)
@@ -268,7 +274,7 @@ def _pull_upstream(self, component, path, url, repo, ups_name, commands):
             self.update_test_openshift_yaml(test_openshift_yaml_file, path, short_name=ups_name)
 
         repo.git.add("*")
-        self._do_git_reset(repo)
+        self.do_git_reset(repo)
         # TODO: Configurable?
         df_ext = self.df_ext
         df_path = os.path.join(component, "Dockerfile")

container_workflow_tool/main.py

@@ -9,7 +9,6 @@
 import re
 import tempfile
 import pprint
-import getpass
 import logging
 
 from git import Repo, GitError
@@ -53,6 +52,7 @@ def __init__(self,
 
         self.conf_name = config
         self.rebuild_reason = rebuild_reason
+        self.gitlab_usage = None
         self.do_image = None
         self.exclude_image = None
         self.do_set = None
@@ -117,6 +117,8 @@ def _setup_args(self, args):
             self.set_commit_msg(args.commit_msg)
         if getattr(args, 'rebuild_reason', None) is not None and args.rebuild_reason:
             self.rebuild_reason = args.rebuild_reason
+        if getattr(args, 'gitlab', None) is not None and args.gitlab:
+            self.gitlab_usage = args.gitlab
         if getattr(args, 'check_script', None) is not None and args.check_script:
             self.check_script = args.check_script
         if getattr(args, 'disable_klist', None) is not None and args.disable_klist:
@@ -125,6 +127,8 @@ def _setup_args(self, args):
             self.latest_release = args.latest_release
         if getattr(args, 'output_file', None) is not None and args.output_file:
             self.output_file = args.output_file
+        if getattr(args, 'gitlab', None) is not None and args.gitlab:
+            self.gitlab_usage = args.gitlab
 
         # Image set to build
         if getattr(args, 'image_set', None) is not None and args.image_set:
@@ -150,7 +154,7 @@ def git_ops(self):
         if not self._git_ops:
             self._git_ops = GitOperations(self.base_image, self.conf,
                                           self.rebuild_reason,
-                                          self.logger.getChild("-git-ops"))
+                                          self.logger.getChild("git-ops"))
         return self._git_ops
 
     @property
@@ -502,10 +506,7 @@ def pull_downstream(self):
         Additionally runs a script against each repository if check_script is set,
         checking its exit value.
         """
-        self._check_kerb_ticket()
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
+        tmp, images = self.preparation()
         for i in images:
             self.distgit._clone_downstream(i["component"], i["git_branch"])
         # If check script is set, run the script provided for each config entry
@@ -521,24 +522,19 @@ def pull_upstream(self):
         Additionally runs a script against each repository if check_script is set,
         checking its exit value.
         """
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
+        tmp, images = self.preparation()
         for i in images:
             # Use unversioned name as a path for the repository
             ups_name = i["name"].split('-')[0]
-            self.distgit._clone_upstream(i["git_url"],
-                                         ups_name,
-                                         commands=i["commands"])
+            self.git_ops.clone_upstream(i["git_url"], ups_name, commands=i["commands"])
         # If check script is set, run the script provided for each config entry
         if self.check_script:
             for i in images:
                 ups_name = i["name"].split('-')[0]
                 self.distgit.check_script(i["component"], self.check_script,
                                           os.path.join(ups_name, i["git_path"]))
 
-    def push_changes(self):
-        """Pushes changes for all components into downstream dist-git repository"""
+    def preparation(self):
         # Check for kerberos ticket
         self._check_kerb_ticket()
         tmp = self._get_tmp_workdir(setup_dir=False)
@@ -547,7 +543,12 @@ def push_changes(self):
             raise RebuilderError(msg)
         self._change_workdir(tmp)
         images = self._get_images()
+        return tmp, images
+
+    def push_changes(self):
+        """Pushes changes for all components into downstream dist-git repository"""
 
+        tmp, images = self.preparation()
         self.distgit.push_changes(tmp, images)
 
     def dist_git_rebase(self):
@@ -557,21 +558,7 @@ def dist_git_rebase(self):
         """
         self.dist_git_changes(rebase=True)
 
-    def dist_git_changes(self, rebase: bool = False):
-        """Method to merge changes from upstream into downstream
-
-        Pulls both downstream and upstream repositories into a temporary directory.
-        Merge is done by copying tracked files from upstream into downstream.
-
-        Args:
-            rebase (bool, optional): Specifies whether a rebase should be done instead.
-        """
-        # Check for kerberos ticket
-        self._check_kerb_ticket()
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
-        self.distgit.dist_git_changes(images, rebase)
+    def git_changes_report(self, tmp):
         self.logger.info("\nGit location: " + tmp)
         if self.args:
             tmp_str = ' --tmp ' + self.tmp_workdir if self.tmp_workdir else '"'
@@ -583,13 +570,23 @@ def dist_git_changes(self, rebase: bool = False):
                 "cwt git push && cwt build"
                 "[base/core/s2i] --repo-url link-to-repo-file")
 
+    def dist_git_changes(self, rebase: bool = False):
+        """Method to merge changes from upstream into downstream
+
+        Pulls both downstream and upstream repositories into a temporary directory.
+        Merge is done by copying tracked files from upstream into downstream.
+
+        Args:
+            rebase (bool, optional): Specifies whether a rebase should be done instead.
+        """
+        tmp, images = self.preparation()
+        self.distgit.dist_git_changes(images, rebase)
+        self.git_changes_report(tmp=tmp)
+
     def merge_future_branches(self):
         """Merges current branch with future branches"""
         # Check for kerberos ticket
-        self._check_kerb_ticket()
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
-        images = self._get_images()
+        tmp, images = self.preparation()
         self.distgit.merge_future_branches(images)
 
     def show_git_changes(self, components: List = None):
@@ -599,9 +596,8 @@ def show_git_changes(self, components: List = None):
             components (list of str, optional): List of components to show changes for
         Walks through all downstream repositories and calls 'git-show' on each of them.
         """
+        tmp, _ = self.preparation()
         if not components:
             images = self._get_images()
             components = [i["component"] for i in images]
-        tmp = self._get_tmp_workdir()
-        self._change_workdir(tmp)
         self.distgit.show_git_changes(tmp, components)

container_workflow_tool/utility.py

@@ -2,8 +2,9 @@
 import argparse
 import os
 import logging
-
+from pathlib import Path
 import textwrap
+import contextlib
 
 
 class RebuilderError(Exception):
@@ -73,6 +74,22 @@ def _split_config_path(config: str):
     return config_path, image_set
 
 
+@contextlib.contextmanager
+def cwd(path):
+    """
+    Checks out a git repository into a temporary directory.
+    Changes CWD to the temporary directory.
+    Yields the temporary directory.
+    On exit, the temporary directory is removed and CWD is restored.
+    """
+    prev_cwd = Path.cwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(prev_cwd)
+
+
 def setup_logger(logger_id, level=logging.INFO):
     logger = logging.getLogger(logger_id)
     logger.setLevel(level)

image-requirements/install-requirements.yaml

@@ -13,4 +13,5 @@
           - krb5-devel
           - krb5-workstation
           - golang-github-cpuguy83-md2man
+          - python3-gitlab
       become: true