Code artifact deps (#219)

* pull wheel are install as file

* unit tests

* update docs

* something is wrong with pipenv and CircleCI...

* simplify the dep string

* woops

.circleci/config.yml

@@ -9,7 +9,7 @@ jobs:
       - run:
           name: pip env
           command: |
-            sudo pip install pipenv
+            sudo pip install pipenv==2022.4.8
             pipenv install
       - run:
           name: install dependencies

CHANGELOG.md

@@ -3,7 +3,15 @@ Documenting All Changes to the Skelebot Project
 
 ---
 
+## v1.29.0
+#### Changed
+- **CodeArtifact Dependencies** | Adds an option to pull CodeArtifact Python packages into a libs folder for install during docker build
+
+---
+
 ## v1.28.0
+#### Merged: 2022-04-15
+#### Released: 2022-04-15
 #### Changed
 - **In Memory Pull** | Allows S3Repo class to pull artifacts and return them directly in python
 

VERSION

@@ -1 +1 @@
-1.28.0
+1.29.0

docs/dependencies.md

@@ -42,6 +42,23 @@ dependencies:
 - github:myGitHub/fakeRepo
 ```
 
+### CodeArtifact Python Packages
+
+Skelebot also supports pulling Python packages that are stored in AWS CodeArtifact. This requires a good deal of information in order to authenticate and pull the correct asset for the package.
+
+```
+language: Python
+dependencies:
+- ca_file:{domain}:{owner}:{repo}:{pkg}:{version}:{profile}
+```
+These values are needed in order to preperly obtain the package and include it in the Docker image.
+- domain - The domain name of the AWS CodeArtifact repository (ex: my_domain)
+- owner - The owner of the AWS CodeArtifact repository (ex: 111122223333)
+- repo - The repository in CodeArtifact where the package is located (ex: my_repo)
+- pkg - The name of the Python package to be installed (ex: my_package)
+- version - The version of the package to be pulled (ex: 1.0.0)
+- profile - [OPTIONAL] The AWS profile on your machine that has access to this CodeArtifact repository (ex: dev)
+
 NOTES:
 
 - When installing via `file:` or `github:` the ability to specify a version is not available.

skelebot/systems/generators/dockerfile.py

@@ -2,10 +2,12 @@
 
 import os
 import re
+from subprocess import call
 from ..execution import commandBuilder
 
 FILE_PATH = "{path}/Dockerfile"
 
+PY_DOWNLOAD_CA = "aws codeartifact get-package-version-asset --domain {domain} --domain-owner {owner} --repository {repo} --package {pkg} --package-version {version}{profile} --format pypi --asset {asset} libs/{asset}"
 PY_INSTALL = "RUN [\"pip\", \"install\", \"{dep}\"]\n"
 PY_INSTALL_VERSION = "RUN [\"pip\", \"install\", \"{depName}=={version}\"]\n"
 PY_INSTALL_GITHUB = "RUN [\"pip\", \"install\", \"git+{depPath}\"]\n"
@@ -46,13 +48,27 @@ def buildDockerfile(config):
     # Add language dependencies
     if (config.language == "Python"):
         for dep in config.dependencies:
-            depSplit = dep.split(":", maxsplit=1)
-            if ("github:" in dep):
-                docker += PY_INSTALL_GITHUB.format(depPath=depSplit[1])
-            elif ("file:" in dep):
+            depSplit = dep.split(":")
+            if (dep.startswith("github:")):
+                docker += PY_INSTALL_GITHUB.format(depPath=dep.split(":", maxsplit=1)[1])
+            elif (dep.startswith("file:")):
                 docker += PY_INSTALL_FILE.format(depPath=depSplit[1])
-            elif ("req:" in dep):
+            elif (dep.startswith("req:")):
                 docker += PY_INSTALL_REQ.format(depPath=depSplit[1])
+            elif (dep.startswith("ca_file:")):
+                domain = depSplit[1]
+                owner = depSplit[2]
+                repo = depSplit[3]
+                pkg = depSplit[4]
+                version = depSplit[5]
+                asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl"
+                profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else ""
+                cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile)
+                status = call(cmd, shell=True)
+                if (status != 0):
+                    raise Exception("Failed to Obtain CodeArtifact Package")
+
+                docker += PY_INSTALL_FILE.format(depPath=f"libs/{asset}")
             # if using PIP version specifiers, will be handled as a standard case
             elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep):
                 verSplit = dep.split("=")
@@ -62,9 +78,9 @@ def buildDockerfile(config):
     if (config.language == "R"):
         for dep in config.dependencies:
             depSplit = dep.split(":")
-            if ("github:" in dep):
+            if (dep.startswith("github:")):
                 docker += R_INSTALL_GITHUB.format(depPath=depSplit[1], depName=depSplit[2])
-            elif ("file:" in dep):
+            elif (dep.startswith("file:")):
                 docker += R_INSTALL_FILE.format(depPath=depSplit[1], depName=depSplit[2])
             elif ("=" in dep):
                 verSplit = dep.split("=")
@@ -74,11 +90,25 @@ def buildDockerfile(config):
 
     if (config.language == "R+Python"):
         for dep in config.dependencies["Python"]:
-            depSplit = dep.split(":", maxsplit=1)
-            if ("github:" in dep):
-                docker += PY_R_INSTALL_GITHUB.format(depPath=depSplit[1])
-            elif ("file:" in dep):
+            depSplit = dep.split(":")
+            if (dep.startswith("github:")):
+                docker += PY_R_INSTALL_GITHUB.format(depPath=dep.split(":", maxsplit=1)[1])
+            elif (dep.startswith("file:")):
                 docker += PY_R_INSTALL_FILE.format(depPath=depSplit[1])
+            elif (dep.startswith("ca_file:")):
+                domain = depSplit[1]
+                owner = depSplit[2]
+                repo = depSplit[3]
+                pkg = depSplit[4]
+                version = depSplit[5]
+                asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl"
+                profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else ""
+                cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile)
+                status = call(cmd, shell=True)
+                if (status != 0):
+                    raise Exception("Failed to Obtain CodeArtifact Package")
+
+                docker += PY_R_INSTALL_FILE.format(depPath=f"libs/{asset}")
             # if using PIP version specifiers, will be handled as a standard case
             elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep):
                 verSplit = dep.split("=")
@@ -87,9 +117,9 @@ def buildDockerfile(config):
                 docker += PY_R_INSTALL.format(dep=dep)
         for dep in config.dependencies["R"]:
             depSplit = dep.split(":")
-            if ("github:" in dep):
+            if (dep.startswith("github:")):
                 docker += R_INSTALL_GITHUB.format(depPath=depSplit[1], depName=depSplit[2])
-            elif ("file:" in dep):
+            elif (dep.startswith("file:")):
                 docker += R_INSTALL_FILE.format(depPath=depSplit[1], depName=depSplit[2])
             elif ("=" in dep):
                 verSplit = dep.split("=")

test/test_systems_generators_dockerfile.py

@@ -1,3 +1,9 @@
+#try:
+    #self.artifactory.execute(config, args)
+    #self.fail("Exception Not Thrown")
+#except RuntimeError as err:
+    #self.assertEqual(str(err), "No Compatible Version Found")
+
 import os
 import unittest
 from unittest import mock
@@ -208,19 +214,42 @@ def test_buildDockerfile_cmd_path(self, mock_getcwd, mock_expanduser):
         self.assertTrue(data is not None)
         self.assertEqual(data, expectedDockerfile)
 
+    @mock.patch('skelebot.systems.generators.dockerfile.call')
+    @mock.patch('os.path.expanduser')
+    @mock.patch('os.getcwd')
+    def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_expanduser, mock_call):
+        folderPath = "{path}/test/files".format(path=self.path)
+        filePath = "{folder}/Dockerfile".format(folder=folderPath)
+
+        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
+        mock_getcwd.return_value = folderPath
+        mock_call.return_value = 1
+        config = sb.systems.generators.yaml.loadConfig()
+        config.language = "Python"
+        config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod")
+
+        try:
+            sb.systems.generators.dockerfile.buildDockerfile(config)
+            self.fail("Exception Not Thrown")
+        except Exception as exc:
+            self.assertEqual(str(exc), "Failed to Obtain CodeArtifact Package")
+
+    @mock.patch('skelebot.systems.generators.dockerfile.call')
     @mock.patch('os.path.expanduser')
     @mock.patch('os.getcwd')
-    def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser):
+    def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser, mock_call):
         folderPath = "{path}/test/files".format(path=self.path)
         filePath = "{folder}/Dockerfile".format(folder=folderPath)
 
         mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
         mock_getcwd.return_value = folderPath
+        mock_call.return_value = 0
         config = sb.systems.generators.yaml.loadConfig()
         config.language = "Python"
         config.dependencies.append("github:github.com/repo")
         config.dependencies.append("github:https://github.com/securerepo")
         config.dependencies.append("file:libs/proj")
+        config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod")
         config.dependencies.append("req:requirements.txt")
         config.dependencies.append("dtable=9.0")
 
@@ -240,6 +269,8 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser):
 RUN ["pip", "install", "git+https://github.com/securerepo"]
 COPY libs/proj libs/proj
 RUN ["pip", "install", "/app/libs/proj"]
+COPY libs/ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl
+RUN ["pip", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"]
 COPY requirements.txt requirements.txt
 RUN ["pip", "install", "-r", "/app/requirements.txt"]
 RUN ["pip", "install", "dtable==9.0"]
@@ -248,11 +279,14 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser):
 RUN rm -rf dist/
 CMD /bin/bash -c \"bash build.sh --env local --log info\"\n"""
 
+        expectedCMD = "aws codeartifact get-package-version-asset --domain cars --domain-owner 12345 --repository python-pkg --package ml-lib --package-version 0.1.0 --profile prod --format pypi --asset ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl"
+
         sb.systems.generators.dockerfile.buildDockerfile(config)
 
         data = None
         with open(filePath, "r") as file:
             data = file.read()
+        mock_call.assert_called_with(expectedCMD, shell=True)
         self.assertTrue(data is not None)
         self.assertEqual(data, expectedDockerfile)
 
@@ -447,21 +481,58 @@ def test_buildDockerfile_custom(self, mock_getcwd, mock_expanduser):
         self.assertTrue(data is not None)
         self.assertEqual(data, expectedDockerfile)
 
+    @mock.patch('skelebot.systems.generators.dockerfile.call')
     @mock.patch('os.path.expanduser')
     @mock.patch('os.getcwd')
-    def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser):
+    def test_buildDockerfile_R_py_ca_file_error(self, mock_getcwd, mock_expanduser, mock_call):
         folderPath = "{path}/test/files".format(path=self.path)
         filePath = "{folder}/Dockerfile".format(folder=folderPath)
 
         mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
         mock_getcwd.return_value = folderPath
+        mock_call.return_value = -1
         config = sb.systems.generators.yaml.loadConfig()
         config.language = "R+Python"
         config.dependencies = {
             "Python":[
                 "numpy", "pandas",
                 "github:github.com/repo", "github:https://github.com/securerepo",
                 "file:libs/proj",
+                "ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod",
+                "dtable>=9.0", "dtable=9.0"
+            ],
+            "R":[
+                "data.table", "here",
+                "github:github.com/repo:cool-lib",
+                "file:libs/proj:cool-proj",
+                "dtable=9.0"
+            ]
+        }
+
+        try:
+            sb.systems.generators.dockerfile.buildDockerfile(config)
+            self.fail("Exception Not Thrown")
+        except Exception as exc:
+            self.assertEqual(str(exc), "Failed to Obtain CodeArtifact Package")
+
+    @mock.patch('skelebot.systems.generators.dockerfile.call')
+    @mock.patch('os.path.expanduser')
+    @mock.patch('os.getcwd')
+    def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser, mock_call):
+        folderPath = "{path}/test/files".format(path=self.path)
+        filePath = "{folder}/Dockerfile".format(folder=folderPath)
+
+        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
+        mock_getcwd.return_value = folderPath
+        mock_call.return_value = 0
+        config = sb.systems.generators.yaml.loadConfig()
+        config.language = "R+Python"
+        config.dependencies = {
+            "Python":[
+                "numpy", "pandas",
+                "github:github.com/repo", "github:https://github.com/securerepo",
+                "file:libs/proj",
+                "ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod",
                 "dtable>=9.0", "dtable=9.0"
             ],
             "R":[
@@ -486,6 +557,8 @@ def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser):
 RUN ["pip3", "install", "git+https://github.com/securerepo"]
 COPY libs/proj libs/proj
 RUN ["pip3", "install", "/app/libs/proj"]
+COPY libs/ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl
+RUN ["pip3", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"]
 RUN ["pip3", "install", "dtable>=9.0"]
 RUN ["pip3", "install", "dtable==9.0"]
 RUN ["Rscript", "-e", "install.packages('data.table', repo='https://cloud.r-project.org'); library(data.table)"]
@@ -502,11 +575,14 @@ def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser):
 CMD /bin/bash -c "/./krb/init.sh user && bash build.sh --env local --log info\"\n"""
         sb.systems.generators.dockerfile.buildDockerfile(config)
 
+        expectedCMD = "aws codeartifact get-package-version-asset --domain cars --domain-owner 12345 --repository python-pkg --package ml-lib --package-version 0.1.0 --profile prod --format pypi --asset ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl"
+
         data = None
         with open(filePath, "r") as file:
             data = file.read()
         self.assertTrue(data is not None)
         self.assertEqual(data, expectedDockerfile)
+        mock_call.assert_called_with(expectedCMD, shell=True)
 
     @mock.patch('os.path.expanduser')
     @mock.patch('os.getcwd')