From 1387b842ca9e0270ac534ab70249b39c06c2edf1 Mon Sep 17 00:00:00 2001 From: Joao Moreira Date: Tue, 24 May 2022 15:01:24 -0400 Subject: [PATCH] Pyproject deps (#226) * Add support for parsing pyproject.toml * Update requirements.txt * Update docs * Update changelog and bump version. * Create libs folder for codeartifact downloads if it doesn't yet exist. (#224) * Simplify and generate pyproject parsing. --- CHANGELOG.md | 7 ++++++ VERSION | 2 +- docs/dependencies.md | 4 +++- requirements.txt | 1 + skelebot-exp.yaml | 1 + skelebot/systems/generators/dockerfile.py | 25 ++++++++++++++++++++++ test/files/pyproject.toml | 19 ++++++++++++++++ test/test_systems_generators_dockerfile.py | 2 ++ 8 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 test/files/pyproject.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f22ff5..fb981a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,14 @@ Documenting All Changes to the Skelebot Project --- +## v1.31.0 +#### Changed +- **Dependencies** | Added the ability for python dependencies to be specified and installed via a `pyproject.toml` file + +--- + ## v1.30.1 +#### Merged: 2022-05-24 #### Changed - **CodeArtifact Dependencies** | Create libs folder if it doesn't already exist diff --git a/VERSION b/VERSION index 329bff2..8b96872 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.30.1 \ No newline at end of file +1.31.0 \ No newline at end of file diff --git a/docs/dependencies.md b/docs/dependencies.md index 47f8436..aebec60 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -23,7 +23,8 @@ Versions for packages in R can be specified by appending `={version}` to the end Versions for packages in Python can be specified by appending `={version}` or `=={version}` to the end of the dependency name. R and Python also both support dependencies to be installed from the local file system as well as from GitHub using the following structure. -Python also allows for installs using a text file via `req:requirements.txt` syntax. + +Python also allows for installs using a text file via `req:requirements.txt` syntax or using a [`pyproject.toml`](https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html) via `proj:pyproject.toml` syntax. When using the later, all specified required and optional set(s) of dependencies will be installed. ``` language: R @@ -40,6 +41,7 @@ dependencies: - file:libs/myPackage.tgz - req:requirements.txt - github:myGitHub/fakeRepo +- proj:pyproject.toml ``` ### CodeArtifact Python Packages diff --git a/requirements.txt b/requirements.txt index 4f05c21..fda7039 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ colorama~=0.4.1 coverage~=4.5.4 pytest~=5.1 boto3~=1.10 +tomli >= 1.1.0 ; python_version < "3.11" diff --git a/skelebot-exp.yaml b/skelebot-exp.yaml index aa4ddee..87c64e4 100644 --- a/skelebot-exp.yaml +++ b/skelebot-exp.yaml @@ -14,6 +14,7 @@ dependencies: - coverage~=4.5.4 - pytest~=5.1 - boto3~=1.10 +- tomli>=1.1.0 ignores: - '**/*.zip' - '**/*.RData' diff --git a/skelebot/systems/generators/dockerfile.py b/skelebot/systems/generators/dockerfile.py index eba7885..75018c1 100644 --- a/skelebot/systems/generators/dockerfile.py +++ b/skelebot/systems/generators/dockerfile.py @@ -5,6 +5,12 @@ from subprocess import call from ..execution import commandBuilder +try: + import tomllib +except ModuleNotFoundError: + # python <= 3.11 + import tomli as tomllib + FILE_PATH = "{path}/Dockerfile" PY_DOWNLOAD_CA = "aws codeartifact get-package-version-asset --domain {domain} --domain-owner {owner} --repository {repo} --package {pkg} --package-version {version}{profile} --format pypi --asset {asset} libs/{asset}" @@ -33,6 +39,22 @@ """ +def parse_pyproj(pyproject_file): + """Parse all required and optional dependencies from pyproject file.""" + with open(os.path.join(os.getcwd(), pyproject_file), "rb") as f: + pyproj = tomllib.load(f).get("project", {}) + + deps = pyproj.get("dependencies", []).copy() + for opt_deps in pyproj.get('optional-dependencies', {}).values(): + deps += opt_deps + + # Replace any double quotes in dependencies with single quotes so we don't + # break the Dockerfile + deps = [d.replace('"', "'") for d in deps] + + return '", "'.join(deps) + + def buildDockerfile(config): """Generates the Dockerfile based on values from the Config object""" @@ -70,6 +92,9 @@ def buildDockerfile(config): raise Exception("Failed to Obtain CodeArtifact Package") docker += PY_INSTALL_FILE.format(depPath=f"libs/{asset}") + elif (dep.startswith("proj:")): + deps = parse_pyproj(depSplit[1]) + docker += PY_INSTALL.format(dep=deps) # if using PIP version specifiers, will be handled as a standard case elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep): verSplit = dep.split("=") diff --git a/test/files/pyproject.toml b/test/files/pyproject.toml new file mode 100644 index 0000000..9d8f6c4 --- /dev/null +++ b/test/files/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "test" +description = "Just a test" +dependencies = [ + "requests", + "numpy==1.15.4", + "pandas~=1.1", + 'scikit-learn<=2.0.0 ; python_version<="3.6"', +] + +[project.optional-dependencies] +test = [ + "pytest ~= 6.2", + "pytest-cov ~= 3.0" +] +custom = [ + "fake-package == 1.2.3", + "not-real" +] diff --git a/test/test_systems_generators_dockerfile.py b/test/test_systems_generators_dockerfile.py index fa8e232..1e93f3e 100644 --- a/test/test_systems_generators_dockerfile.py +++ b/test/test_systems_generators_dockerfile.py @@ -254,6 +254,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, config.dependencies.append("file:libs/proj") config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod") config.dependencies.append("req:requirements.txt") + config.dependencies.append("proj:pyproject.toml") config.dependencies.append("dtable=9.0") expectedDockerfile = """ @@ -276,6 +277,7 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_mkdir, mock_expanduser, RUN ["pip", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"] COPY requirements.txt requirements.txt RUN ["pip", "install", "-r", "/app/requirements.txt"] +RUN ["pip", "install", "requests", "numpy==1.15.4", "pandas~=1.1", "scikit-learn<=2.0.0 ; python_version<=\'3.6\'", "pytest ~= 6.2", "pytest-cov ~= 3.0", "fake-package == 1.2.3", "not-real"] RUN ["pip", "install", "dtable==9.0"] COPY . /app RUN rm -rf build/