From da4e7179f0332249525daf3163ff0afc6552855f Mon Sep 17 00:00:00 2001 From: Gianluca Brindisi Date: Mon, 26 Feb 2024 14:51:10 +0100 Subject: [PATCH 1/4] Add option to filter target repositories by their visibility --- env.py | 22 +++++++ evergreen.py | 3 + test_env.py | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+) diff --git a/env.py b/env.py index b5826ef..7705ebf 100644 --- a/env.py +++ b/env.py @@ -23,6 +23,7 @@ def get_env_vars() -> ( str, str | None, bool | None, + list[str] | None, ] ): """ @@ -44,6 +45,7 @@ def get_env_vars() -> ( dry_run (bool): Whether or not to actually open issues/pull requests commit_message (str): The commit message of the follow up group_dependencies (bool): Whether to group dependencies in the dependabot.yml file + filter_visibility (list[str]): Run the action only on repositories with the specified listed visibility """ # Load from .env file if it exists dotenv_path = join(dirname(__file__), ".env") @@ -150,6 +152,25 @@ def get_env_vars() -> ( else: dry_run_bool = False + filter_visibility = os.getenv("FILTER_VISIBILITY") + filter_visibility_list = [] + if filter_visibility: + filter_visibility_list = list( + set( + [ + visibility.strip().lower() + for visibility in filter_visibility.split(",") + ] + ) + ) + for visibility in filter_visibility_list: + if visibility not in ["public", "private", "internal"]: + raise ValueError( + "FILTER_VISIBILITY environment variable not 'public', 'private', or 'internal'" + ) + else: + filter_visibility_list = ["public", "private", "internal"] # all + project_id = os.getenv("PROJECT_ID") if project_id and not project_id.isnumeric(): raise ValueError("PROJECT_ID environment variable is not numeric") @@ -167,4 +188,5 @@ def get_env_vars() -> ( commit_message, project_id, group_dependencies_bool, + filter_visibility_list, ) diff --git a/evergreen.py b/evergreen.py index 59e617a..9341629 100644 --- a/evergreen.py +++ b/evergreen.py @@ -28,6 +28,7 @@ def main(): # pragma: no cover commit_message, project_id, group_dependencies, + filter_visibility, ) = env.get_env_vars() # Auth to GitHub.com or GHE @@ -53,6 +54,8 @@ def main(): # pragma: no cover continue if repo.archived: continue + if repo.visibility.lower() not in filter_visibility: + continue try: if repo.file_contents(".github/dependabot.yml").size > 0: continue diff --git a/test_env.py b/test_env.py index 3029b2d..dec62c4 100644 --- a/test_env.py +++ b/test_env.py @@ -40,6 +40,7 @@ def test_get_env_vars_with_org(self): "Create dependabot configuration", "123", False, + ["public", "private", "internal"], ) result = get_env_vars() self.assertEqual(result, expected_result) @@ -77,6 +78,7 @@ def test_get_env_vars_with_repos(self): "Create dependabot configuration", "123", False, + ["public", "private", "internal"], ) result = get_env_vars() self.assertEqual(result, expected_result) @@ -106,6 +108,7 @@ def test_get_env_vars_optional_values(self): "Create dependabot.yaml", None, False, + ["public", "private", "internal"], ) result = get_env_vars() self.assertEqual(result, expected_result) @@ -155,6 +158,168 @@ def test_get_env_vars_with_repos_no_dry_run(self): "Create dependabot.yaml", None, False, + ["public", "private", "internal"], + ) + result = get_env_vars() + self.assertEqual(result, expected_result) + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "ENABLE_SECURITY_UPDATES": "false", + }, + clear=True, + ) + def test_get_env_vars_with_repos_disabled_security_updates(self): + """Test that all environment variables are set correctly when DRY_RUN is false""" + expected_result = ( + "my_organization", + [], + "my_token", + "", + [], + "pull", + "Enable Dependabot", + "Dependabot could be enabled for this repository. \ +Please enable it by merging this pull request so that \ +we can keep our dependencies up to date and secure.", + None, + False, + "Create dependabot.yaml", + None, + False, + ["public", "private", "internal"], + ) + result = get_env_vars() + self.assertEqual(result, expected_result) + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "ENABLE_SECURITY_UPDATES": "false", + "FILTER_VISIBILITY": "private,internal", + }, + clear=True, + ) + def test_get_env_vars_with_repos_filter_visibility_multiple_values(self): + """Test that filter_visibility is set correctly when multiple values are provided""" + expected_result = ( + "my_organization", + [], + "my_token", + "", + [], + "pull", + "Enable Dependabot", + "Dependabot could be enabled for this repository. \ +Please enable it by merging this pull request so that \ +we can keep our dependencies up to date and secure.", + None, + False, + "Create dependabot.yaml", + None, + False, + ["private", "internal"], + ) + result = get_env_vars() + self.assertEqual(result, expected_result) + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "ENABLE_SECURITY_UPDATES": "false", + "FILTER_VISIBILITY": "public", + }, + clear=True, + ) + def test_get_env_vars_with_repos_filter_visibility_single_value(self): + """Test that filter_visibility is set correctly when a single value is provided""" + expected_result = ( + "my_organization", + [], + "my_token", + "", + [], + "pull", + "Enable Dependabot", + "Dependabot could be enabled for this repository. \ +Please enable it by merging this pull request so that \ +we can keep our dependencies up to date and secure.", + None, + False, + "Create dependabot.yaml", + None, + False, + ["public"], + ) + result = get_env_vars() + self.assertEqual(result, expected_result) + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "ENABLE_SECURITY_UPDATES": "false", + "FILTER_VISIBILITY": "foobar", + }, + clear=True, + ) + def test_get_env_vars_with_repos_filter_visibility_invalid_single_value(self): + """Test that filter_visibility throws an error when an invalid value is provided""" + with self.assertRaises(ValueError): + get_env_vars() + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "ENABLE_SECURITY_UPDATES": "false", + "FILTER_VISIBILITY": "public, foobar, private", + }, + clear=True, + ) + def test_get_env_vars_with_repos_filter_visibility_invalid_multiple_value(self): + """Test that filter_visibility throws an error when an invalid value is provided""" + with self.assertRaises(ValueError): + get_env_vars() + + @patch.dict( + os.environ, + { + "ORGANIZATION": "my_organization", + "GH_TOKEN": "my_token", + "ENABLE_SECURITY_UPDATES": "false", + "FILTER_VISIBILITY": "private,private,public", + }, + clear=True, + ) + def test_get_env_vars_with_repos_filter_visibility_no_duplicates(self): + """Test that filter_visibility is set correctly when there are duplicate values""" + expected_result = ( + "my_organization", + [], + "my_token", + "", + [], + "pull", + "Enable Dependabot", + "Dependabot could be enabled for this repository. \ +Please enable it by merging this pull request so that \ +we can keep our dependencies up to date and secure.", + None, + False, + "Create dependabot.yaml", + None, + False, + ["private", "public"], ) result = get_env_vars() self.assertEqual(result, expected_result) From 716e81d4850361e622f9333b237e54ba2ac9f99e Mon Sep 17 00:00:00 2001 From: Gianluca Brindisi Date: Mon, 26 Feb 2024 14:56:54 +0100 Subject: [PATCH 2/4] Add documentation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ea30a9..df6444b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Below are the allowed configuration options: | `GH_ENTERPRISE_URL` | False | "" | The `GH_ENTERPRISE_URL` is used to connect to an enterprise server instance of GitHub. github.com users should not enter anything here. | | `ORGANIZATION` | Required to have `ORGANIZATION` or `REPOSITORY` | | The name of the GitHub organization which you want this action to work from. ie. github.com/github would be `github` | | `REPOSITORY` | Required to have `ORGANIZATION` or `REPOSITORY` | | The name of the repository and organization which you want this action to work from. ie. `github/evergreen` or a comma separated list of multiple repositories `github/evergreen,super-linter/super-linter` | -| `EXEMPT_REPOS` | False | "" | These repositories will be exempt from this action considering them for dependabot enablement. ex: If my org is set to `github` then I might want to exempt a few of the repos but get the rest by setting `EXEMPT_REPOS` to ``github/evergreen,github/contributors` | +| `EXEMPT_REPOS` | False | "" | These repositories will be exempt from this action considering them for dependabot enablement. ex: If my org is set to `github` then I might want to exempt a few of the repos but get the rest by setting `EXEMPT_REPOS` to `github/evergreen,github/contributors` | | `TYPE` | False | pull | Type refers to the type of action you want taken if this workflow determines that dependabot could be enabled. Valid values are `pull` or `issue`.| | `TITLE` | False | "Enable Dependabot" | The title of the issue or pull request that will be created if dependabot could be enabled. | | `BODY` | False | **Pull Request:** "Dependabot could be enabled for this repository. Please enable it by merging this pull request so that we can keep our dependencies up to date and secure." **Issue:** "Please update the repository to include a Dependabot configuration file. This will ensure our dependencies remain updated and secure.Follow the guidelines in [creating Dependabot configuration files](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file) to set it up properly.Here's an example of the code:" | The body of the issue or pull request that will be created if dependabot could be enabled. | @@ -47,6 +47,7 @@ Below are the allowed configuration options: | `PROJECT_ID` | False | "" | If set, this will assign the issue or pull request to the project with the given ID. ( The project ID on GitHub can be located by navigating to the respective project and observing the URL's end.) **The `ORGANIZATION` variable is required** | | `DRY_RUN` | False | false | If set to true, this action will not create any issues or pull requests. It will only log the repositories that could have dependabot enabled. This is useful for testing. | | `GROUP_DEPENDENCIES` | False | false | If set to true, dependabot configuration will group dependencies updates based on [dependency type](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) (production or development, where supported) | +| `FILTER_VISIBILITY` | False | "public,private,internal" | Use this flag to filter repositories in scope by their visibility (`public`, `private`, `internal`). By default all repository are targeted. ex: to ignore public repositories set this value to `private,internal`. | ### Example workflows From 57cab27dc271e5ca5d1e6b89a959a00dceb1a3e8 Mon Sep 17 00:00:00 2001 From: Gianluca Brindisi Date: Wed, 28 Feb 2024 10:56:44 +0100 Subject: [PATCH 3/4] Sorted visibility list to give tests consistency --- env.py | 16 +++++++++------- test_env.py | 12 ++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/env.py b/env.py index e983a30..07e6bec 100644 --- a/env.py +++ b/env.py @@ -171,12 +171,14 @@ def get_env_vars() -> ( filter_visibility = os.getenv("FILTER_VISIBILITY") filter_visibility_list = [] if filter_visibility: - filter_visibility_list = list( - set( - [ - visibility.strip().lower() - for visibility in filter_visibility.split(",") - ] + filter_visibility_list = sorted( + list( + set( + [ + visibility.strip().lower() + for visibility in filter_visibility.split(",") + ] + ) ) ) for visibility in filter_visibility_list: @@ -185,7 +187,7 @@ def get_env_vars() -> ( "FILTER_VISIBILITY environment variable not 'public', 'private', or 'internal'" ) else: - filter_visibility_list = ["public", "private", "internal"] # all + filter_visibility_list = sorted(["public", "private", "internal"]) # all project_id = os.getenv("PROJECT_ID") if project_id and not project_id.isnumeric(): diff --git a/test_env.py b/test_env.py index f983c04..9726ce3 100644 --- a/test_env.py +++ b/test_env.py @@ -40,7 +40,7 @@ def test_get_env_vars_with_org(self): "Create dependabot configuration", "123", False, - ["public", "private", "internal"], + ["internal", "private", "public"], True, # enable_security_updates ) result = get_env_vars() @@ -79,7 +79,7 @@ def test_get_env_vars_with_repos(self): "Create dependabot configuration", "123", False, - ["public", "private", "internal"], + ["internal", "private", "public"], True, # enable_security_updates ) result = get_env_vars() @@ -110,7 +110,7 @@ def test_get_env_vars_optional_values(self): "Create dependabot.yaml", None, False, - ["public", "private", "internal"], + ["internal", "private", "public"], True, # enable_security_updates ) result = get_env_vars() @@ -161,7 +161,7 @@ def test_get_env_vars_with_repos_no_dry_run(self): "Create dependabot.yaml", None, False, - ["public", "private", "internal"], + ["internal", "private", "public"], True, # enable_security_updates ) result = get_env_vars() @@ -194,7 +194,7 @@ def test_get_env_vars_with_repos_disabled_security_updates(self): "Create dependabot.yaml", None, False, - ["public", "private", "internal"], + ["internal", "private", "public"], False, # enable_security_updates ) result = get_env_vars() @@ -228,7 +228,7 @@ def test_get_env_vars_with_repos_filter_visibility_multiple_values(self): "Create dependabot.yaml", None, False, - ["private", "internal"], + ["internal", "private"], False, # enable_security_updates ) result = get_env_vars() From dc50af9526f0504121813cddba0ebd98c5c24c65 Mon Sep 17 00:00:00 2001 From: Gianluca Brindisi Date: Wed, 28 Feb 2024 11:04:02 +0100 Subject: [PATCH 4/4] Reworked the visibility parsing function to make it easier to test --- env.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/env.py b/env.py index 07e6bec..3596f4c 100644 --- a/env.py +++ b/env.py @@ -171,21 +171,14 @@ def get_env_vars() -> ( filter_visibility = os.getenv("FILTER_VISIBILITY") filter_visibility_list = [] if filter_visibility: - filter_visibility_list = sorted( - list( - set( - [ - visibility.strip().lower() - for visibility in filter_visibility.split(",") - ] - ) - ) - ) - for visibility in filter_visibility_list: - if visibility not in ["public", "private", "internal"]: + filter_visibility_set = set() + for visibility in filter_visibility.split(","): + if visibility.strip().lower() not in ["public", "private", "internal"]: raise ValueError( "FILTER_VISIBILITY environment variable not 'public', 'private', or 'internal'" ) + filter_visibility_set.add(visibility.strip().lower()) + filter_visibility_list = sorted(list(filter_visibility_set)) else: filter_visibility_list = sorted(["public", "private", "internal"]) # all