Adds ALL keyword to Repository Component for pushing and pulling ALL …

…artifacts (#153)

CHANGELOG.md

@@ -3,7 +3,14 @@ Documenting All Changes to the Skelebot Project
 
 ---
 
+## v1.18.2
+#### Changed
+- **Repository ALL Keyword** | Adds simple keyword in repository for pushing or pulling ALL artifacts at once
+
+---
+
 ## v1.18.1
+#### Merged: 2020-03-12
 #### Changed
 - **Artifactory BugFix** | issue with ArtifactoryPath.glob function fixed by avoiding using the function altogether
 

VERSION

@@ -1 +1 @@
-1.18.1
+1.18.2

docs/artifacts.md

@@ -85,12 +85,25 @@ Pulling the latest compatible version can be accomplished by specifying "LATEST"
 > skelebot pull artifact-name LATEST
 ```
 
+#### ALL Artifacts
+
+For the puposes of automation, or general laziness, Skelebot offers a way to push and pull all of your artifacts in a single command. The keyword `ALL` can be used in place of the artifact name to iterate over each artifact during the push or pull process.
+
+```
+> skelebot push ALL
+```
+
+When used in conjunction with the `LATEST` keyword this can create a powerful pull command that obtains all of the compatible artifacts for the current version of your project.
+
+```
+> skelebot pull ALL LATEST
+```
+
 #### Override Artifact
 
 By default the pull command will place the artifact (with the version number) in the root directory of the project. However, you can tell Skelebot to place the artifact in the location that is provided in the config.
 This is done with the override parameter (`-o --override`) and would replace the existing artifact in that location automatically, so caution is advised when using this parameter.
 
-
 ```
 > skelebot pull artifact-name 1.0.0 --override
 ```

skelebot/components/repository/repository.py

@@ -99,7 +99,7 @@ def addParsers(self, subparsers):
         with the project version and pull artifacts with the provided version number
         """
 
-        artifactNames = []
+        artifactNames = ["ALL"]
         for artifact in self.artifacts:
             artifactNames.append(artifact.name)
 
@@ -124,12 +124,6 @@ def addParsers(self, subparsers):
 
     def execute(self, config, args):
 
-        # Obtain the artifact that matches the provided name
-        selectedArtifact = None
-        for artifact in self.artifacts:
-            if (artifact.name == args.artifact):
-                selectedArtifact = artifact
-
         # Obtain the user and token if required
         user = None
         token = None
@@ -140,7 +134,14 @@ def execute(self, config, args):
         # Obtain the configured artifact repository
         artifactRepo = self.s3 if self.s3 is not None else self.artifactory
 
-        if (args.job == "push"): # Push from Disk to Repo
-            artifactRepo.push(selectedArtifact, config.version, args.force, user, token)
-        elif (args.job == "pull"): # Pull from Repo to Disk
-            artifactRepo.pull(selectedArtifact, args.version, config.version, args.override, user, token)
+        # Obtain the artifact(s) that matches the provided name
+        for artifact in self.artifacts:
+            if ((args.artifact == artifact.name) | (args.artifact == "ALL")):
+
+                # Push or Pull the Artifact
+                if (args.job == "push"): # Push from Disk to Repo
+                    print("Pushing version {version} of {artifact}".format(version=config.version, artifact=artifact.name))
+                    artifactRepo.push(artifact, config.version, args.force, user, token)
+                elif (args.job == "pull"): # Pull from Repo to Disk
+                    print("Pulling version {version} of {artifact}".format(version=args.version, artifact=artifact.name))
+                    artifactRepo.pull(artifact, args.version, config.version, args.override, user, token)

skelebot/components/repository/s3Repo.py

@@ -35,7 +35,6 @@ def connect(self):
     def push(self, artifact, version, force=False, user=None, password=None):
         """ Push an artifact to S3 with the given version number """
 
-
         bucket_parts = self.bucket.split("/")
         bucket_parts.append(artifact.getVersionedName(version))
         bucket = bucket_parts[0]

test/test_components_repository_repository.py

@@ -29,12 +29,13 @@ class TestRepository(TestCase):
 
     def setUp(self):
         artifact = sb.components.repository.repository.Artifact("test", "test.pkl")
+        artifact2 = sb.components.repository.repository.Artifact("test2", "test2.pkl")
         artifactoryRepo = sb.components.repository.artifactoryRepo.ArtifactoryRepo("artifactory.test.com", "ml", "test")
         s3Repo = sb.components.repository.s3Repo.S3Repo("my-bucket", "us-east-1", "test")
         s3Repo_path = sb.components.repository.s3Repo.S3Repo("my-bucket/sub/folder", "us-east-1", "test")
 
-        self.artifactory = sb.components.repository.repository.Repository([artifact], s3=None, artifactory=artifactoryRepo)
-        self.s3 = sb.components.repository.repository.Repository([artifact], s3=s3Repo, artifactory=None)
+        self.artifactory = sb.components.repository.repository.Repository([artifact, artifact2], s3=None, artifactory=artifactoryRepo)
+        self.s3 = sb.components.repository.repository.Repository([artifact, artifact2], s3=s3Repo, artifactory=None)
         self.s3_subfolder = sb.components.repository.repository.Repository([artifact], s3=s3Repo_path, artifactory=None)
 
     def test_repository_load(self):
@@ -144,6 +145,44 @@ def test_execute_push_s3(self, mock_boto3_session):
         self.s3.execute(config, args)
         mock_client.upload_file.assert_called_with("test.pkl", "my-bucket", "test_v1.0.0.pkl")
 
+    @mock.patch('shutil.copyfile')
+    @mock.patch('os.remove')
+    @mock.patch('artifactory.ArtifactoryPath')
+    def test_execute_push_artifactory_all(self, mock_artifactory, mock_remove, mock_copy):
+        config = sb.objects.config.Config(version="1.0.0")
+        args = argparse.Namespace(job="push", force=True, artifact='ALL', user='sean', token='abc123')
+
+        self.artifactory.execute(config, args)
+
+        mock_artifactory.assert_has_calls([
+            mock.call("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=('sean', 'abc123')),
+            mock.call("artifactory.test.com/ml/test/test2_v1.0.0.pkl", auth=('sean', 'abc123'))
+        ], any_order=True)
+        mock_copy.assert_has_calls([
+            mock.call("test.pkl", "test_v1.0.0.pkl"),
+            mock.call("test2.pkl", "test2_v1.0.0.pkl")
+        ], any_order=True)
+        mock_remove.assert_has_calls([
+            mock.call("test_v1.0.0.pkl"),
+            mock.call("test2_v1.0.0.pkl")
+        ], any_order=True)
+
+    @mock.patch('boto3.Session')
+    def test_execute_push_s3_all(self, mock_boto3_session):
+        mock_client = mock.Mock()
+        mock_session = mock.Mock()
+        mock_session.client.return_value = mock_client
+        mock_boto3_session.return_value = mock_session
+
+        config = sb.objects.config.Config(version="1.0.0")
+        args = argparse.Namespace(job="push", force=True, artifact='ALL', user='sean', token='abc124')
+
+        self.s3.execute(config, args)
+        mock_client.upload_file.assert_has_calls([
+            mock.call("test.pkl", "my-bucket", "test_v1.0.0.pkl"),
+            mock.call("test2.pkl", "my-bucket", "test2_v1.0.0.pkl")
+        ], any_order=True)
+
     @mock.patch('boto3.Session')
     def test_execute_push_s3_subfolder(self, mock_boto3_session):
         mock_client = mock.Mock()