Artifactory Pull Latest Compatible Version (#110)

* Working on the basics of latest compatible version pulling

* making progress, still needs LCV logic

* Latest Compatible Version Logic

* Adds override parameter

* Unit tests

* Updates docs

CHANGELOG.md

@@ -2,16 +2,25 @@
 Documenting All Changes to the Skelebot Project
 
 
+---
+
+## v1.9.0
+#### Added
+- **Latest Compatible Version** | Added logic to artifactory pull commands to allow for dynamic look-up of the latest compatible version of an artifact
+- **Override Artifact** | Added an override parameter to the artifact component to allow for pull commands to write to the existing artifact file location
+
 ---
 
 ## v1.8.5
+#### Merged: 2019-11-14
 #### Added
 - **Contact Param** | Added a global contact param for displaying the project's contact email
 
 ---
 
 ## v1.8.4
 #### Merged: 2019-11-11
+#### Released: 2019-11-12
 #### Added
 - **Plugin Quarantine** | Exception handling for plugins will quarantine any plugins that fail to load properly
 

VERSION

@@ -1 +1 @@
-1.8.5
+1.9.0

docs/artifacts.md

@@ -43,6 +43,26 @@ display when executing the commands, or they can be passed as parameters in the
  - `--user (-u)` - The Artifactory username
  - `--token (-t)` - The token associated to the Artifactory username
 
+#### Latest Compatible Version
+
+Skelebot allows for the pulling of artifacts based on the project's current version in order to obtain the "latest compatible version". The latest compatible version is the highest version number of the same major version that is not above the project's current version.
+
+This can be accomplished by specifying "LATEST" as the version number when executing the pull command.
+
+```
+> skelebot pull artifact-name LATEST
+```
+
+#### Override Artifact
+
+By default the pull command will place the artifact (with the version number) in the root directory of the project. However, you can tell Skelebot to place the artifact in the location that is provided in the config.
+This is done with the override parameter (`-o --override`) and would replace the existing artifact in that location automatically, so caution is advised when using this parameter.
+
+
+```
+> skelebot pull artifact-name 1.0.0 --override
+```
+
 ---
 
 <center><< <a href="publishing.html">Publishing</a>  |  <a href="hdfs-kerberos.html">HDFS Kerberos</a> >></center>

skelebot/components/artifactory.py

@@ -7,17 +7,18 @@
 from schema import Schema, And, Optional
 from ..objects.component import Activation, Component
 from ..objects.skeleYaml import SkeleYaml
+from ..objects.semver import Semver
 
-ERROR_ALREADY_PUSHED = """This artifact version has already been pushed.
-Please bump the version before pushing (skelebot bump) or force push (-f)."""
+ERROR_NOT_COMPATIBLE = "No Compatible Version Found"
+ERROR_ALREADY_PUSHED = "This artifact version already exists. Please bump the version or use the force parameter (-f) to overwrite the artifact."
 
 def pushArtifact(artifactFile, user, token, file, url, force):
     """Pushes the given file to the url with the provided user/token auth"""
 
     # Error and exit if artifact already exists and we are not forcing an override
     try:
         if (not force) and (artifactory.ArtifactoryPath(url, auth=(user, token)).exists()):
-            raise Exception(ERROR_ALREADY_PUSHED)
+            raise RuntimeError(ERROR_ALREADY_PUSHED)
     except MissingSchema:
         pass
 
@@ -32,18 +33,43 @@ def pushArtifact(artifactFile, user, token, file, url, force):
         os.remove(file)
         raise
 
-def pullArtifact(user, token, file, url):
+def pullArtifact(user, token, file, url, override, original):
     """Pulls the given file from the url with the provided user/token auth"""
 
     if (artifactory.ArtifactoryPath(url, auth=(user, token)).exists()):
         print("Pulling {file} from {url}".format(file=file, url=url))
         path = artifactory.ArtifactoryPath(url, auth=(user, token))
         with path.open() as fd:
-            with open(file, "wb") as out:
+            dest = original if (override) else file
+            with open(dest, "wb") as out:
                 out.write(fd.read())
     else:
         print("Artifact Not Found: {url}".format(url=url))
 
+def findCompatibleArtifact(user, token, listUrl, currentVersion, filename, ext):
+    """Searches the artifact folder to find the latest compatible artifact version"""
+
+    print("Searching for Latest Compatible Artifact")
+    compatibleSemver = None
+    currentSemver = Semver(currentVersion)
+
+    # Find the artifacts in the folder with the same name and major version
+    path = artifactory.ArtifactoryPath(listUrl, auth=(user, token))
+    if (path.exists()):
+        pathGlob = "{filename}_v{major}.*.{ext}".format(filename=filename, ext=ext, major=currentSemver.major)
+        for artifact in path.glob(pathGlob):
+            artifactSemver = Semver(str(artifact).split("_v")[1].split(ext)[0])
+
+            # Identify the latest compatible version
+            if (currentSemver.isBackwardCompatible(artifactSemver)) and ((compatibleSemver is None) or (compatibleSemver < artifactSemver)):
+                compatibleSemver = artifactSemver
+
+    # Raise an error if no compatible version is found
+    if (compatibleSemver is None):
+        raise RuntimeError(ERROR_NOT_COMPATIBLE)
+
+    return "{filename}_v{version}.{ext}".format(filename=filename, version=compatibleSemver, ext=ext)
+
 class Artifact(SkeleYaml):
     """
     Artifact Class
@@ -132,6 +158,7 @@ def addParsers(self, subparsers):
         parser.add_argument("version", help="The version of the artifact to pull")
         parser.add_argument("-u", "--user", help="Auth user for Artifactory")
         parser.add_argument("-t", "--token", help="Auth token for Artifactory")
+        parser.add_argument("-o", "--override", action='store_true',  help="Override the model in the existing directory")
 
         return subparsers
 
@@ -171,13 +198,21 @@ def execute(self, config, args):
         # Generate the local artifact file and the final Artifactory url path
         ext = selectedArtifact.file.split(".")[1]
         version = config.version if (args.job == "push") else args.version
-        file = "{filename}_v{version}.{ext}"
-        file = file.format(filename=selectedArtifact.name, version=version, ext=ext)
+
+        file = None
+        if (version == "LATEST"):
+            listUrl = "{url}/{repo}/{path}/"
+            listUrl = listUrl.format(url=self.url, repo=self.repo, path=self.path)
+            file = findCompatibleArtifact(user, token, listUrl, config.version, selectedArtifact.name, ext)
+        else:
+            file = "{filename}_v{version}.{ext}"
+            file = file.format(filename=selectedArtifact.name, version=version, ext=ext)
+
         url = "{url}/{repo}/{path}/{file}"
         url = url.format(url=self.url, repo=self.repo, path=self.path, file=file)
 
         # Push the artifact with the config version, or pull with the arg version
         if (args.job == "push"):
             pushArtifact(selectedArtifact.file, user, token, file, url, args.force)
         elif (args.job == "pull"):
-            pullArtifact(user, token, file, url)
+            pullArtifact(user, token, file, url, args.override, selectedArtifact.file)

skelebot/objects/semver.py

@@ -0,0 +1,51 @@
+class Semver():
+    """
+    Semver Class
+
+    Object to translate a semantic version string into an object with separate values for major, minor, and patch that allows for operations to
+    be performed such as comparing two Semvers.
+    """
+
+    major = None
+    minor = None
+    patch = None
+
+    def __init__(self, version):
+        parts = version.split(".")
+        self.major = parts[0]
+        self.minor = parts[1]
+        self.patch = parts[2]
+
+    def __str__(self):
+        return "{maj}.{min}.{pat}".format(maj=self.major, min=self.minor, pat=self.patch)
+
+    def __lt__(self, other):
+        lt = False
+        if (self.major < other.major) or ((self.major == other.major) and ((self.minor < other.minor) or ((self.minor == other.minor) and (self.patch < other.patch)))):
+            lt = True
+
+        return lt
+
+    def __eq__(self, other):
+        eq = True
+        if (self.patch != other.patch) or (self.minor != other.minor) or (self.major != other.major):
+            eq = False
+
+        return eq
+
+    def __le__(self, other):
+        le = False
+        if (self < other) or (self == other):
+            le = True
+
+        return le
+
+    def isBackwardCompatible(self, semver):
+        """Determine whether this Semver backword compatible with the Semver provided"""
+
+        compat = False
+        if (self.major == semver.major) and (self >= semver):
+            compat = True
+
+        return compat
+

test/test_components_artifactory.py

@@ -40,8 +40,7 @@ def test_execute_push_conflict(self, mock_artifactory, mock_rename):
         config = sb.objects.config.Config(version="1.0.0")
         args = argparse.Namespace(job="push", force=False, artifact='test', user='sean', token='abc123')
 
-        expectedException = """This artifact version has already been pushed.
-Please bump the version before pushing (skelebot bump) or force push (-f)."""
+        expectedException = "This artifact version already exists. Please bump the version or use the force parameter (-f) to overwrite the artifact."
 
         try:
             self.artifactory.execute(config, args)
@@ -88,13 +87,62 @@ def test_execute_pull(self, mock_artifactory, mock_open, mock_input):
         mock_input.return_value = "abc"
 
         config = sb.objects.config.Config(version="1.0.0")
-        args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None)
+        args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None, override=False)
 
         self.artifactory.execute(config, args)
 
         mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v0.1.0.pkl", auth=("abc", "abc"))
         mock_open.assert_called_with("test_v0.1.0.pkl", "wb")
 
+    @mock.patch('skelebot.components.artifactory.input')
+    @mock.patch('builtins.open')
+    @mock.patch('artifactory.ArtifactoryPath')
+    def test_execute_pull_lcv(self, mock_artifactory, mock_open, mock_input):
+        mock_apath = mock_artifactory.return_value
+        mock_input.return_value = "abc"
+        mock_apath.glob.return_value = ["artifact_v1.1.0", "artifact_v0.2.4", "artifact_v1.0.0", "artifact_v2.0.1"]
+
+        config = sb.objects.config.Config(version="1.0.9")
+        args = argparse.Namespace(job="pull", version='LATEST', artifact='test', user=None, token=None, override=False)
+
+        self.artifactory.execute(config, args)
+
+        mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v1.0.0.pkl", auth=("abc", "abc"))
+        mock_open.assert_called_with("test_v1.0.0.pkl", "wb")
+
+    @mock.patch('skelebot.components.artifactory.input')
+    @mock.patch('builtins.open')
+    @mock.patch('artifactory.ArtifactoryPath')
+    def test_execute_pull_lcv_not_found(self, mock_artifactory, mock_open, mock_input):
+        mock_apath = mock_artifactory.return_value
+        mock_input.return_value = "abc"
+        mock_apath.glob.return_value = ["artifact_v1.1.0", "artifact_v0.2.4", "artifact_v1.0.0", "artifact_v2.0.1"]
+
+        config = sb.objects.config.Config(version="3.0.9")
+        args = argparse.Namespace(job="pull", version='LATEST', artifact='test', user=None, token=None, override=False)
+
+        try:
+            self.artifactory.execute(config, args)
+            self.fail("Exception Not Thrown")
+        except RuntimeError as err:
+            self.assertEqual(str(err), "No Compatible Version Found")
+
+    @mock.patch('skelebot.components.artifactory.input')
+    @mock.patch('builtins.open')
+    @mock.patch('artifactory.ArtifactoryPath')
+    def test_execute_pull_override_and_lcv(self, mock_artifactory, mock_open, mock_input):
+        mock_apath = mock_artifactory.return_value
+        mock_input.return_value = "abc"
+        mock_apath.glob.return_value = ["artifact_v1.1.0", "artifact_v0.2.4", "artifact_v1.0.0", "artifact_v2.0.1"]
+
+        config = sb.objects.config.Config(version="0.6.9")
+        args = argparse.Namespace(job="pull", version='LATEST', artifact='test', user=None, token=None, override=True)
+
+        self.artifactory.execute(config, args)
+
+        mock_artifactory.assert_called_with("artifactory.test.com/ml/test/test_v0.2.4.pkl", auth=("abc", "abc"))
+        mock_open.assert_called_with("test.pkl", "wb")
+
     @mock.patch('skelebot.components.artifactory.input')
     @mock.patch('artifactory.ArtifactoryPath')
     def test_execute_pull_not_found(self, mock_artifactory, mock_input):
@@ -103,7 +151,7 @@ def test_execute_pull_not_found(self, mock_artifactory, mock_input):
         path.exists.return_value = False
 
         config = sb.objects.config.Config(version="1.0.0")
-        args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None)
+        args = argparse.Namespace(job="pull", version='0.1.0', artifact='test', user=None, token=None, override=False)
 
         self.artifactory.execute(config, args)