From 240975dca3b7312101fccdd5bc0acc02babc45c3 Mon Sep 17 00:00:00 2001 From: Sean Shookman Date: Thu, 26 Mar 2020 13:29:33 -0500 Subject: [PATCH] Adds ALL keyword to Repository Component for pushing and pulling ALL artifacts (#153) --- CHANGELOG.md | 7 +++ VERSION | 2 +- docs/artifacts.md | 15 ++++++- skelebot/components/repository/repository.py | 23 +++++----- skelebot/components/repository/s3Repo.py | 1 - test/test_components_repository_repository.py | 43 ++++++++++++++++++- 6 files changed, 75 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d55dae2..e782ec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,14 @@ Documenting All Changes to the Skelebot Project --- +## v1.18.2 +#### Changed +- **Repository ALL Keyword** | Adds simple keyword in repository for pushing or pulling ALL artifacts at once + +--- + ## v1.18.1 +#### Merged: 2020-03-12 #### Changed - **Artifactory BugFix** | issue with ArtifactoryPath.glob function fixed by avoiding using the function altogether diff --git a/VERSION b/VERSION index 5ce8b39..753029c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.18.1 \ No newline at end of file +1.18.2 \ No newline at end of file diff --git a/docs/artifacts.md b/docs/artifacts.md index 4e3e9ec..7d23308 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -85,12 +85,25 @@ Pulling the latest compatible version can be accomplished by specifying "LATEST" > skelebot pull artifact-name LATEST ``` +#### ALL Artifacts + +For the puposes of automation, or general laziness, Skelebot offers a way to push and pull all of your artifacts in a single command. The keyword `ALL` can be used in place of the artifact name to iterate over each artifact during the push or pull process. + +``` +> skelebot push ALL +``` + +When used in conjunction with the `LATEST` keyword this can create a powerful pull command that obtains all of the compatible artifacts for the current version of your project. + +``` +> skelebot pull ALL LATEST +``` + #### Override Artifact By default the pull command will place the artifact (with the version number) in the root directory of the project. However, you can tell Skelebot to place the artifact in the location that is provided in the config. This is done with the override parameter (`-o --override`) and would replace the existing artifact in that location automatically, so caution is advised when using this parameter. - ``` > skelebot pull artifact-name 1.0.0 --override ``` diff --git a/skelebot/components/repository/repository.py b/skelebot/components/repository/repository.py index 3118bfe..86d6a89 100644 --- a/skelebot/components/repository/repository.py +++ b/skelebot/components/repository/repository.py @@ -99,7 +99,7 @@ def addParsers(self, subparsers): with the project version and pull artifacts with the provided version number """ - artifactNames = [] + artifactNames = ["ALL"] for artifact in self.artifacts: artifactNames.append(artifact.name) @@ -124,12 +124,6 @@ def addParsers(self, subparsers): def execute(self, config, args): - # Obtain the artifact that matches the provided name - selectedArtifact = None - for artifact in self.artifacts: - if (artifact.name == args.artifact): - selectedArtifact = artifact - # Obtain the user and token if required user = None token = None @@ -140,7 +134,14 @@ def execute(self, config, args): # Obtain the configured artifact repository artifactRepo = self.s3 if self.s3 is not None else self.artifactory - if (args.job == "push"): # Push from Disk to Repo - artifactRepo.push(selectedArtifact, config.version, args.force, user, token) - elif (args.job == "pull"): # Pull from Repo to Disk - artifactRepo.pull(selectedArtifact, args.version, config.version, args.override, user, token) + # Obtain the artifact(s) that matches the provided name + for artifact in self.artifacts: + if ((args.artifact == artifact.name) | (args.artifact == "ALL")): + + # Push or Pull the Artifact + if (args.job == "push"): # Push from Disk to Repo + print("Pushing version {version} of {artifact}".format(version=config.version, artifact=artifact.name)) + artifactRepo.push(artifact, config.version, args.force, user, token) + elif (args.job == "pull"): # Pull from Repo to Disk + print("Pulling version {version} of {artifact}".format(version=args.version, artifact=artifact.name)) + artifactRepo.pull(artifact, args.version, config.version, args.override, user, token) diff --git a/skelebot/components/repository/s3Repo.py b/skelebot/components/repository/s3Repo.py index 51a9f74..19e6fcc 100644 --- a/skelebot/components/repository/s3Repo.py +++ b/skelebot/components/repository/s3Repo.py @@ -35,7 +35,6 @@ def connect(self): def push(self, artifact, version, force=False, user=None, password=None): """ Push an artifact to S3 with the given version number """ - bucket_parts = self.bucket.split("/") bucket_parts.append(artifact.getVersionedName(version)) bucket = bucket_parts[0] diff --git a/test/test_components_repository_repository.py b/test/test_components_repository_repository.py index dbbf924..6d97188 100644 --- a/test/test_components_repository_repository.py +++ b/test/test_components_repository_repository.py @@ -29,12 +29,13 @@ class TestRepository(TestCase): def setUp(self): artifact = sb.components.repository.repository.Artifact("test", "test.pkl") + artifact2 = sb.components.repository.repository.Artifact("test2", "test2.pkl") artifactoryRepo = sb.components.repository.artifactoryRepo.ArtifactoryRepo("artifactory.test.com", "ml", "test") s3Repo = sb.components.repository.s3Repo.S3Repo("my-bucket", "us-east-1", "test") s3Repo_path = sb.components.repository.s3Repo.S3Repo("my-bucket/sub/folder", "us-east-1", "test") - self.artifactory = sb.components.repository.repository.Repository([artifact], s3=None, artifactory=artifactoryRepo) - self.s3 = sb.components.repository.repository.Repository([artifact], s3=s3Repo, artifactory=None) + self.artifactory = sb.components.repository.repository.Repository([artifact, artifact2], s3=None, artifactory=artifactoryRepo) + self.s3 = sb.components.repository.repository.Repository([artifact, artifact2], s3=s3Repo, artifactory=None) self.s3_subfolder = sb.components.repository.repository.Repository([artifact], s3=s3Repo_path, artifactory=None) def test_repository_load(self): @@ -144,6 +145,44 @@ def test_execute_push_s3(self, mock_boto3_session): self.s3.execute(config, args) mock_client.upload_file.assert_called_with("test.pkl", "my-bucket", "test_v1.0.0.pkl") + @mock.patch('shutil.copyfile') + @mock.patch('os.remove') + @mock.patch('artifactory.ArtifactoryPath') + def test_execute_push_artifactory_all(self, mock_artifactory, mock_remove, mock_copy): + config = sb.objects.config.Config(version="1.0.0") + args = argparse.Namespace(job="push", force=True, artifact='ALL', user='sean', token='abc123') + + self.artifactory.execute(config, args) + + mock_artifactory.assert_has_calls([ + mock.call("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')), + mock.call("artifactory.test.com/ml/test/test2_v1.0.0.pkl", auth=('sean', 'abc123')) + ], any_order=True) + mock_copy.assert_has_calls([ + mock.call("test.pkl", "test_v1.0.0.pkl"), + mock.call("test2.pkl", "test2_v1.0.0.pkl") + ], any_order=True) + mock_remove.assert_has_calls([ + mock.call("test_v1.0.0.pkl"), + mock.call("test2_v1.0.0.pkl") + ], any_order=True) + + @mock.patch('boto3.Session') + def test_execute_push_s3_all(self, mock_boto3_session): + mock_client = mock.Mock() + mock_session = mock.Mock() + mock_session.client.return_value = mock_client + mock_boto3_session.return_value = mock_session + + config = sb.objects.config.Config(version="1.0.0") + args = argparse.Namespace(job="push", force=True, artifact='ALL', user='sean', token='abc124') + + self.s3.execute(config, args) + mock_client.upload_file.assert_has_calls([ + mock.call("test.pkl", "my-bucket", "test_v1.0.0.pkl"), + mock.call("test2.pkl", "my-bucket", "test2_v1.0.0.pkl") + ], any_order=True) + @mock.patch('boto3.Session') def test_execute_push_s3_subfolder(self, mock_boto3_session): mock_client = mock.Mock()