test for install python dependencies from git and files (#100)

* test for install python dependencies from git and files * bump version * change log * update doc * Update docs/dependencies.md Co-Authored-By: Sean Shookman <[email protected]> * Update docs/dependencies.md Co-Authored-By: Sean Shookman <[email protected]>
carsdotcom · Oct 17, 2019 · 7e1bdee · 7e1bdee
1 parent e4fcb79
commit 7e1bdee
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,7 +3,14 @@ Documenting All Changes to the Skelebot Project
 
 ---
 
+## v1.7.4
+#### Changed
+- **Install Python Dependencies** | Install python dependencies from git and files
+
+---
+
 ## v1.7.3
+#### Merged: 2019-10-17
 #### Changed
 - **CI/CD** | Changed PyPI GitHub Action to use token instead of password
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.7.3
+1.7.4
diff --git a/docs/dependencies.md b/docs/dependencies.md
@@ -20,17 +20,26 @@ By default Python dependencies are installed using pip install.
 
 Versions for packages in R can be specified by appending `={version}` to the end of the dependency name.
 
-Versions for packages in Python can be specified by appending `=={version}` to the end of the dependency name.
+Versions for packages in Python can be specified by appending `={version}` or `=={version}` to the end of the dependency name.
 
-R also supports dependencies to be installed from the local file system as well as from GitHub using the following structure.
+R and Python also both support dependencies to be installed from the local file system as well as from GitHub using the following structure.
 
 ```
+language: R 
 dependencies:
 - {type}:{source}:{name}
 - file:libs/myPackage.tgz:mypack
 - github:myGitHub/fakeRepo:fakeRepo
 ```
 
+```
+language: Python
+dependencies:
+- {type}:{source}
+- file:libs/myPackage.tgz
+- github:myGitHub/fakeRepo
+```
+
 NOTE: When installing via `file:` or `github:` the ability to specify a version is not available.
 
 ---

diff --git a/skelebot/systems/generators/dockerfile.py b/skelebot/systems/generators/dockerfile.py
@@ -6,6 +6,10 @@
 FILE_PATH = "{path}/Dockerfile"
 
 PY_INSTALL = "RUN [\"pip\", \"install\", \"{dep}\"]\n"
+PY_INSTALL_VERSION = "RUN [\"pip\", \"install\", \"{depName}=={version}\"]\n"
+PY_INSTALL_GITHUB = "RUN [\"pip\", \"install\", \"git+{depPath}\"]\n"
+PY_INSTALL_FILE = "COPY {depPath} {depPath}\n"
+PY_INSTALL_FILE += "RUN [\"pip\", \"install\", \"/app/{depPath}\"]\n"
 R_INSTALL = "RUN [\"Rscript\", \"-e\", \"install.packages('{dep}', repo='https://cloud.r-project.org'); library({dep})\"]\n"
 R_INSTALL_VERSION = "RUN [\"Rscript\", \"-e\", \"library(devtools); install_version('{depName}', version='{version}', repos='http://cran.us.r-project.org'); library({depName})\"]\n"
 R_INSTALL_GITHUB = "RUN [\"Rscript\", \"-e\", \"library(devtools); install_github('{depPath}'); library({depName})\"]\n"
@@ -30,7 +34,16 @@ def buildDockerfile(config):
     # Add language dependencies
     if (config.language == "Python"):
         for dep in config.dependencies:
-            docker += PY_INSTALL.format(dep=dep)
+            depSplit = dep.split(":")
+            if ("github:" in dep):
+                docker += PY_INSTALL_GITHUB.format(depPath=depSplit[1])
+            elif ("file:" in dep):
+                docker += PY_INSTALL_FILE.format(depPath=depSplit[1])
+            elif ("=" in dep) & ("==" not in dep): # if specify version with '==', will be handled as a standard case
+                verSplit = dep.split("=")
+                docker += PY_INSTALL_VERSION.format(depName=verSplit[0], version=verSplit[1])
+            else:
+                docker += PY_INSTALL.format(dep=dep)
     if (config.language == "R"):
         for dep in config.dependencies:
             depSplit = dep.split(":")

diff --git a/test/test_systems_generators_dockerfile.py b/test/test_systems_generators_dockerfile.py
@@ -95,6 +95,49 @@ def test_buildDockerfile_base(self, mock_getcwd, mock_expanduser):
 
     @mock.patch('os.path.expanduser')
     @mock.patch('os.getcwd')
+    def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser):
+        folderPath = "{path}/test/files".format(path=self.path)
+        filePath = "{folder}/Dockerfile".format(folder=folderPath)
+
+        mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path)
+        mock_getcwd.return_value = folderPath
+        config = sb.systems.generators.yaml.loadConfig()
+        config.language = "Python"
+        config.dependencies.append("github:github.com/repo")
+        config.dependencies.append("file:libs/proj")
+        config.dependencies.append("dtable=9.0")
+
+        expectedDockerfile = """
+# This Dockerfile was generated by Skelebot
+# Editing this file manually is not advised as all changes will be overwritten by Skelebot
+
+FROM skelebot/python-base
+MAINTAINER Mega Man <[email protected]>
+WORKDIR /app
+RUN ["pip", "install", "pyyaml"]
+RUN ["pip", "install", "artifactory"]
+RUN ["pip", "install", "argparse"]
+RUN ["pip", "install", "coverage"]
+RUN ["pip", "install", "pytest"]
+RUN ["pip", "install", "git+github.com/repo"]
+COPY libs/proj libs/proj
+RUN ["pip", "install", "/app/libs/proj"]
+RUN ["pip", "install", "dtable==9.0"]
+RUN rm -rf build/
+RUN rm -rf dist/
+COPY . /app
+CMD /bin/bash -c \"bash build.sh --env local --log info\"\n"""
+
+        sb.systems.generators.dockerfile.buildDockerfile(config)
+
+        data = None
+        with open(filePath, "r") as file:
+            data = file.read()
+        self.assertTrue(data is not None)
+        print(data)
+        self.assertEqual(data, expectedDockerfile)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('os.getcwd')
     def test_buildDockerfile_krb(self, mock_getcwd, mock_expanduser):
         folderPath = "{path}/test/files".format(path=self.path)
         filePath = "{folder}/Dockerfile".format(folder=folderPath)

diff --git a/test/test_systems_parsing_skeleParser.py b/test/test_systems_parsing_skeleParser.py
@@ -49,7 +49,7 @@ def test_description(self):
 -----------------------------------
 Version: 0.1.0
 Environment: test
-Skelebot Version: 1.7.3
+Skelebot Version: 1.7.4
 -----------------------------------"""
 
         self.assertEqual(description, expectedDescription)