diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 44826e3..ab4da4e 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -1,43 +1,31 @@ -FROM continuumio/miniconda3:23.10.0-1 +FROM mambaorg/micromamba:bookworm-slim -# install OS dependencies USER root -ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8 LC_ALL=C.UTF-8 -RUN apt update --fix-missing && \ - apt install -y --no-install-recommends \ - software-properties-common pkg-config build-essential unzip git gcc g++ && \ - apt autoremove -y && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /usr/local/src/* - -# create a limited user -ARG NEW_USER=appuser -RUN groupadd -g 999 $NEW_USER && \ - useradd -ms /bin/bash -r -u 1000 -g $NEW_USER $NEW_USER - -# create the working directory with the entrypoint -RUN mkdir /submission && \ - chown -R $NEW_USER /submission -RUN mkdir /data && \ - chown -R $NEW_USER /data -USER $NEW_USER - -# create conda environment -COPY environment.yml /tmp/ -RUN conda env create -n condaenv -f /tmp/environment.yml && \ - conda clean --all --yes - -# copy the tests into the container -COPY ./tests /tests - -# install the entrypoint as root -USER root -COPY ./entrypoint.sh /entrypoint.sh -COPY ./validate.py /validate.py -RUN chmod +x /entrypoint.sh -USER $NEW_USER - -# Execute the entrypoint.sh script inside the container when we do docker run -WORKDIR /submission -CMD ["/bin/bash", "/entrypoint.sh"] + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + PYTHONUNBUFFERED=1 \ + SHELL=/bin/bash + +COPY apt.txt apt.txt +RUN apt-get update --fix-missing \ + && apt-get install -y apt-utils 2> /dev/null \ + && xargs -a apt.txt apt-get install -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /apt.txt + +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/env.yml +RUN micromamba install --name base --yes --file /tmp/env.yml && \ + micromamba clean --all --force-pkgs-dirs --yes + +RUN mkdir -p /code_execution/submission +RUN chown -R ${MAMBA_USER}:${MAMBA_USER} /code_execution + +COPY entrypoint.sh /entrypoint.sh + +WORKDIR /code_execution +USER ${MAMBA_USER} +COPY runtime/tests /code_execution/tests + +CMD ["bash", "/entrypoint.sh"] \ No newline at end of file diff --git a/runtime/apt.txt b/runtime/apt.txt new file mode 100644 index 0000000..d7c64fb --- /dev/null +++ b/runtime/apt.txt @@ -0,0 +1,5 @@ +curl +libxml2 +tzdata +wget +zip \ No newline at end of file diff --git a/runtime/entrypoint.sh b/runtime/entrypoint.sh index 389265f..a071d09 100644 --- a/runtime/entrypoint.sh +++ b/runtime/entrypoint.sh @@ -1,52 +1,30 @@ #!/bin/bash -set -e - -exit_code=0 - -run_user_code() { - if [ -f "main.sh" ] - then - echo "Running main.sh ..." - sh main.sh - else - echo "ERROR: Could not find main.sh in submission.zip" - exit_code=1 - fi -} -split () { - echo "********************************************************************************" -} +set -euxo pipefail +main () { + expected_filename=main.sh -{ - sleep 10 - source activate condaenv - cd /submission + cd /code_execution - echo "Unpacking submission..." - unzip -o /submission/submission.zip -d ./ - python /validate.py pre_create . + submission_files=$(zip -sf ./submission/submission.zip) + if ! grep -q ${expected_filename}<<<$submission_files; then + echo "Submission zip archive must include $expected_filename" + return 1 + fi - run_user_code - python /validate.py post_query . + echo Unpacking submission + unzip ./submission/submission.zip -d ./src - echo "Exporting submission.csv result..." + echo Printing submission contents + find src - # Valid scripts must create a "submission.csv" file within the same directory as main - if [ -f "submission.csv" ] - then - echo "Script completed its run." - else - echo "ERROR: Script did not produce a submission.csv file in the main directory." - exit_code=1 - fi + LOGURU_LEVEL=INFO sh main.sh +} + +main |& tee "/code_execution/submission/log.txt" +exit_code=${PIPESTATUS[0]} - echo "Running acceptance tests..." - # conda run -n condaenv --no-capture-output pytest --verbose --rootdir=. /tests/test_submission.py - echo "================ END ================" -} |& tee "/submission/log.txt" +cp /code_execution/submission/log.txt /tmp/log -# copy for additional log uses -cp /submission/log.txt /tmp/log -exit $exit_code +exit $exit_code \ No newline at end of file diff --git a/runtime/environment.yml b/runtime/environment.yml index 744c223..439d5ed 100644 --- a/runtime/environment.yml +++ b/runtime/environment.yml @@ -1,11 +1,23 @@ -name: condaenv +name: spacecraftpose channels: - conda-forge dependencies: - - python=3.11.6 - - numpy - - pandas - - pip - - pip: - - loguru - - tqdm + - keras=2.13 + - lightgbm=4.1 + - loguru=0.7 + - geopandas=0.14 + - numba=0.58 + - numpy=1.26 + - pandas=2.1 + - pip=23.3 + - pydantic=1.10 + - pytest=7.4 + - python=3.10 + - pytorch=1.*=cpu* + - pytorch-lightning + - scikit-learn=1.3 + - scipy=1.11 + - statsmodels=0.14 + - tensorflow=2.13.*=cpu* + - tqdm=4.66 + - xgboost=1.7.*=cpu* \ No newline at end of file diff --git a/runtime/requirements.txt b/runtime/requirements.txt deleted file mode 100644 index 55b033e..0000000 --- a/runtime/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pytest \ No newline at end of file diff --git a/runtime/run-tests.sh b/runtime/run-tests.sh deleted file mode 100644 index 58750bf..0000000 --- a/runtime/run-tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -echo "Running Python tests" -pytest tests/test_installs.py diff --git a/runtime/tests/test_installs.py b/runtime/tests/test_installs.py deleted file mode 100644 index 3adfadf..0000000 --- a/runtime/tests/test_installs.py +++ /dev/null @@ -1,12 +0,0 @@ -import importlib -import pytest - -packages = [ - "pandas", - "numpy", -] - - -@pytest.mark.parametrize("package", packages) -def test_package_can_be_imported(package): - importlib.import_module(package) diff --git a/runtime/tests/test_packages.py b/runtime/tests/test_packages.py new file mode 100644 index 0000000..82b906f --- /dev/null +++ b/runtime/tests/test_packages.py @@ -0,0 +1,19 @@ +import importlib + +import pytest + +packages = [ + "keras", + "numpy", + "pandas", + "scipy", + "sklearn", + "tensorflow", + "torch", +] + + +@pytest.mark.parametrize("package_name", packages, ids=packages) +def test_import(package_name): + """Test that certain dependencies are importable.""" + importlib.import_module(package_name) diff --git a/runtime/tests/test_submission.py b/runtime/tests/test_submission.py index 5640b19..162dfc0 100644 --- a/runtime/tests/test_submission.py +++ b/runtime/tests/test_submission.py @@ -1,10 +1,13 @@ +import os + from pathlib import Path -from loguru import logger import pytest -SUBMISSION_PATH = Path("/submission/submission.csv") +SUBMISSION_PATH = Path("/code_execution/submission/submission.csv") +CHECK_SUBMISSION = os.environ.get("CHECK_SUBMISSION", "false") == "true" +@pytest.mark.skipif(not CHECK_SUBMISSION, reason="Not checking submission yet") def test_submission_exists(): assert SUBMISSION_PATH.exists() diff --git a/runtime/validate.py b/runtime/validate.py deleted file mode 100644 index 5350d0e..0000000 --- a/runtime/validate.py +++ /dev/null @@ -1,8 +0,0 @@ -from pathlib import Path -import sys - - -if __name__ == "__main__": - phase = sys.argv[1] - working_dir = Path(sys.argv[2]).resolve() - sys.exit(0) \ No newline at end of file