Pyproject deps (#226)

* Add support for parsing pyproject.toml

* Update requirements.txt

* Update docs

* Update changelog and bump version.

* Create libs folder for codeartifact downloads if it doesn't yet exist. (#224)

* Simplify and generate pyproject parsing.

CHANGELOG.md

@@ -3,7 +3,14 @@ Documenting All Changes to the Skelebot Project
 
 ---
 
+## v1.31.0
+#### Changed
+- **Dependencies** | Added the ability for python dependencies to be specified and installed via a `pyproject.toml` file
+
+---
+
 ## v1.30.1
+#### Merged: 2022-05-24
 #### Changed
 - **CodeArtifact Dependencies** | Create libs folder if it doesn't already exist
 

VERSION

@@ -1 +1 @@
-1.30.1
+1.31.0

docs/dependencies.md

@@ -23,7 +23,8 @@ Versions for packages in R can be specified by appending `={version}` to the end
 Versions for packages in Python can be specified by appending `={version}` or `=={version}` to the end of the dependency name.
 
 R and Python also both support dependencies to be installed from the local file system as well as from GitHub using the following structure.
-Python also allows for installs using a text file via `req:requirements.txt` syntax.
+
+Python also allows for installs using a text file via `req:requirements.txt` syntax or using a [`pyproject.toml`](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) via `proj:pyproject.toml` syntax. When using the later, all specified required and optional set(s) of dependencies will be installed.
 
 ```
 language: R
@@ -40,6 +41,7 @@ dependencies:
 - file:libs/myPackage.tgz
 - req:requirements.txt
 - github:myGitHub/fakeRepo
+- proj:pyproject.toml
 ```
 
 ### CodeArtifact Python Packages

requirements.txt

@@ -6,3 +6,4 @@ colorama~=0.4.1
 coverage~=4.5.4
 pytest~=5.1
 boto3~=1.10
+tomli >= 1.1.0 ; python_version < "3.11"

skelebot-exp.yaml

@@ -14,6 +14,7 @@ dependencies:
 - coverage~=4.5.4
 - pytest~=5.1
 - boto3~=1.10
+- tomli>=1.1.0
 ignores:
 - '**/*.zip'
 - '**/*.RData'

skelebot/systems/generators/dockerfile.py

@@ -5,6 +5,12 @@
 from subprocess import call
 from ..execution import commandBuilder
 
+try:
+    import tomllib
+except ModuleNotFoundError:
+    # python <= 3.11
+    import tomli as tomllib
+
 FILE_PATH = "{path}/Dockerfile"
 
 PY_DOWNLOAD_CA = "aws codeartifact get-package-version-asset --domain {domain} --domain-owner {owner} --repository {repo} --package {pkg} --package-version {version}{profile} --format pypi --asset {asset} libs/{asset}"
@@ -33,6 +39,22 @@
 
 """
 
+def parse_pyproj(pyproject_file):
+    """Parse all required and optional dependencies from pyproject file."""
+    with open(os.path.join(os.getcwd(), pyproject_file), "rb") as f:
+        pyproj = tomllib.load(f).get("project", {})
+
+    deps = pyproj.get("dependencies", []).copy()
+    for opt_deps in pyproj.get('optional-dependencies', {}).values():
+        deps += opt_deps
+
+    # Replace any double quotes in dependencies with single quotes so we don't
+    # break the Dockerfile
+    deps = [d.replace('"', "'") for d in deps]
+
+    return '", "'.join(deps)
+
+
 def buildDockerfile(config):
     """Generates the Dockerfile based on values from the Config object"""
 
@@ -70,6 +92,9 @@ def buildDockerfile(config):
                     raise Exception("Failed to Obtain CodeArtifact Package")
 
                 docker += PY_INSTALL_FILE.format(depPath=f"libs/{asset}")
+            elif (dep.startswith("proj:")):
+                deps = parse_pyproj(depSplit[1])
+                docker += PY_INSTALL.format(dep=deps)
             # if using PIP version specifiers, will be handled as a standard case
             elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep):
                 verSplit = dep.split("=")

test/files/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "test"
+description = "Just a test"
+dependencies = [
+    "requests",
+    "numpy==1.15.4",
+    "pandas~=1.1",
+    'scikit-learn<=2.0.0 ; python_version<="3.6"',
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest ~= 6.2",
+    "pytest-cov ~= 3.0"
+]
+custom = [
+    "fake-package == 1.2.3",
+    "not-real"
+]

test/test_systems_generators_dockerfile.py

@@ -254,6 +254,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser,
         config.dependencies.append("file:libs/proj")
         config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod")
         config.dependencies.append("req:requirements.txt")
+        config.dependencies.append("proj:pyproject.toml")
         config.dependencies.append("dtable=9.0")
 
         expectedDockerfile = """
@@ -276,6 +277,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser,
 RUN ["pip", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"]
 COPY requirements.txt requirements.txt
 RUN ["pip", "install", "-r", "/app/requirements.txt"]
+RUN ["pip", "install", "requests", "numpy==1.15.4", "pandas~=1.1", "scikit-learn<=2.0.0 ; python_version<=\'3.6\'", "pytest ~= 6.2", "pytest-cov ~= 3.0", "fake-package == 1.2.3", "not-real"]
 RUN ["pip", "install", "dtable==9.0"]
 COPY . /app
 RUN rm -rf build/