-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit c068c6d
Showing
18 changed files
with
2,114 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
name: build | ||
|
||
on: | ||
push: | ||
branches: [main] | ||
paths: ["runtime/**", ".github/workflows/build-containers.yml"] | ||
pull_request: | ||
paths: ["runtime/**", ".github/workflows/build-containers.yml"] | ||
workflow_dispatch: | ||
|
||
permissions: | ||
id-token: write | ||
contents: read | ||
|
||
jobs: | ||
build: | ||
name: Build, Test, and Publish Image | ||
runs-on: ubuntu-latest | ||
env: | ||
LOGIN_SERVER: ${{ secrets.LOGIN_SERVER }} | ||
IMAGE: spacecraft-pose-object-detection | ||
SHA_TAG: ${{ github.sha }} | ||
LATEST_TAG: latest | ||
PUBLISH_IMAGE: ${{ secrets.PUBLISH_IMAGE }} | ||
|
||
steps: | ||
- uses: actions/checkout@v2 | ||
|
||
- name: Build Image | ||
run: | | ||
docker build runtime \ | ||
--tag $LOGIN_SERVER/$IMAGE:$SHA_TAG \ | ||
--tag $LOGIN_SERVER/$IMAGE:$LATEST_TAG | ||
- name: Check image size | ||
run: | | ||
docker image list $LOGIN_SERVER/$IMAGE | ||
- name: Tests packages in container | ||
run: | | ||
docker run $LOGIN_SERVER/$IMAGE:$SHA_TAG pytest tests/test_packages.py | ||
- name: Log into Azure | ||
if: ${{ github.ref == 'refs/heads/main' && env.PUBLISH_IMAGE == '1' }} | ||
uses: azure/login@v1 | ||
with: | ||
client-id: ${{secrets.AZURE_CLIENT_ID}} | ||
tenant-id: ${{secrets.AZURE_TENANT_ID}} | ||
subscription-id: ${{secrets.AZURE_SUBSCRIPTION_ID}} | ||
|
||
- name: Log into ACR with Docker | ||
if: ${{ github.ref == 'refs/heads/main' && env.PUBLISH_IMAGE == '1' }} | ||
uses: azure/docker-login@v1 | ||
with: | ||
login-server: ${{ env.LOGIN_SERVER }} | ||
username: ${{ secrets.REGISTRY_USERNAME }} | ||
password: ${{ secrets.REGISTRY_PASSWORD }} | ||
|
||
- name: Push image to ACR | ||
if: ${{ github.ref == 'refs/heads/main' && env.PUBLISH_IMAGE == '1' }} | ||
run: | | ||
docker push $LOGIN_SERVER/$IMAGE --all-tags |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,168 @@ | ||
data/ | ||
submission/ | ||
submission_src/ | ||
data_dev/ | ||
|
||
|
||
## https://github.com/github/gitignore/blob/main/Python.gitignore | ||
|
||
# Byte-compiled / optimized / DLL files | ||
__pycache__/ | ||
*.py[cod] | ||
*$py.class | ||
|
||
# C extensions | ||
*.so | ||
|
||
# Distribution / packaging | ||
.Python | ||
build/ | ||
develop-eggs/ | ||
dist/ | ||
downloads/ | ||
eggs/ | ||
.eggs/ | ||
lib/ | ||
lib64/ | ||
parts/ | ||
sdist/ | ||
var/ | ||
wheels/ | ||
share/python-wheels/ | ||
*.egg-info/ | ||
.installed.cfg | ||
*.egg | ||
MANIFEST | ||
|
||
# PyInstaller | ||
# Usually these files are written by a python script from a template | ||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | ||
*.manifest | ||
*.spec | ||
|
||
# Installer logs | ||
pip-log.txt | ||
pip-delete-this-directory.txt | ||
|
||
# Unit test / coverage reports | ||
htmlcov/ | ||
.tox/ | ||
.nox/ | ||
.coverage | ||
.coverage.* | ||
.cache | ||
nosetests.xml | ||
coverage.xml | ||
*.cover | ||
*.py,cover | ||
.hypothesis/ | ||
.pytest_cache/ | ||
cover/ | ||
|
||
# Translations | ||
*.mo | ||
*.pot | ||
|
||
# Django stuff: | ||
*.log | ||
local_settings.py | ||
db.sqlite3 | ||
db.sqlite3-journal | ||
|
||
# Flask stuff: | ||
instance/ | ||
.webassets-cache | ||
|
||
# Scrapy stuff: | ||
.scrapy | ||
|
||
# Sphinx documentation | ||
docs/_build/ | ||
|
||
# PyBuilder | ||
.pybuilder/ | ||
target/ | ||
|
||
# Jupyter Notebook | ||
.ipynb_checkpoints | ||
|
||
# IPython | ||
profile_default/ | ||
ipython_config.py | ||
|
||
# pyenv | ||
# For a library or package, you might want to ignore these files since the code is | ||
# intended to run in multiple environments; otherwise, check them in: | ||
# .python-version | ||
|
||
# pipenv | ||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | ||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | ||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | ||
# install all needed dependencies. | ||
#Pipfile.lock | ||
|
||
# poetry | ||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. | ||
# This is especially recommended for binary packages to ensure reproducibility, and is more | ||
# commonly ignored for libraries. | ||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control | ||
#poetry.lock | ||
|
||
# pdm | ||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. | ||
#pdm.lock | ||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it | ||
# in version control. | ||
# https://pdm.fming.dev/#use-with-ide | ||
.pdm.toml | ||
|
||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm | ||
__pypackages__/ | ||
|
||
# Celery stuff | ||
celerybeat-schedule | ||
celerybeat.pid | ||
|
||
# SageMath parsed files | ||
*.sage.py | ||
|
||
# Environments | ||
.env | ||
.venv | ||
env/ | ||
venv/ | ||
ENV/ | ||
env.bak/ | ||
venv.bak/ | ||
|
||
# Spyder project settings | ||
.spyderproject | ||
.spyproject | ||
|
||
# Rope project settings | ||
.ropeproject | ||
|
||
# mkdocs documentation | ||
/site | ||
|
||
# mypy | ||
.mypy_cache/ | ||
.dmypy.json | ||
dmypy.json | ||
|
||
# Pyre type checker | ||
.pyre/ | ||
|
||
# pytype static type analyzer | ||
.pytype/ | ||
|
||
# Cython debug symbols | ||
cython_debug/ | ||
|
||
# PyCharm | ||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can | ||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore | ||
# and can be added to the global gitignore or merged into this file. For a more nuclear | ||
# option (not recommended) you can uncomment the following to ignore the entire idea folder. | ||
.idea/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
.PHONY: build build-full pull pack-benchmark pack-submission repack-submission repack-example test-submission | ||
|
||
# ================================================================================================ | ||
# Settings | ||
# ================================================================================================ | ||
TAG = latest | ||
LOCAL_TAG = local | ||
|
||
TASK = object-detection | ||
REPO = spacecraft-pose-${TASK} | ||
REGISTRY_IMAGE = spacecraftpose.azurecr.io/${REPO}:${TAG} | ||
LOCAL_IMAGE = ${REPO}:${LOCAL_TAG} | ||
CONTAINER_NAME = spacecraft-pose-${TASK} | ||
|
||
# if not TTY (for example GithubActions CI) no interactive tty commands for docker | ||
ifneq (true, ${GITHUB_ACTIONS_NO_TTY}) | ||
TTY_ARGS = -it | ||
endif | ||
|
||
# To run a submission, use local version if that exists; otherwise, use official version | ||
# setting SUBMISSION_IMAGE as an environment variable will override the image | ||
SUBMISSION_IMAGE ?= $(shell docker images -q ${LOCAL_IMAGE}) | ||
ifeq (,${SUBMISSION_IMAGE}) | ||
SUBMISSION_IMAGE := $(shell docker images -q ${REGISTRY_IMAGE}) | ||
endif | ||
|
||
# Give write access to the submission folder to everyone so Docker user can write when mounted | ||
_submission_write_perms: | ||
mkdir -p submission/ | ||
chmod -R 0777 submission/ | ||
|
||
# ================================================================================================ | ||
# Commands for building the container if you are changing the requirements | ||
# ================================================================================================ | ||
|
||
## Builds the container locally | ||
build: | ||
docker build -t ${LOCAL_IMAGE} runtime | ||
|
||
build-full: | ||
docker build --no-cache -t ${LOCAL_IMAGE} runtime | ||
|
||
## Ensures that your locally built container can import all the Python packages successfully when it runs | ||
test-container: build _submission_write_perms | ||
docker run \ | ||
${TTY_ARGS} \ | ||
--mount type=bind,source="$(shell pwd)"/runtime/tests,target=/tests,readonly \ | ||
${LOCAL_IMAGE} \ | ||
pytest tests/test_packages.py | ||
|
||
## Start your locally built container and open a bash shell within the running container; same as submission setup except has network access | ||
interact-container: build _submission_write_perms | ||
docker run \ | ||
--mount type=bind,source="$(shell pwd)"/data,target=/code_execution/data,readonly \ | ||
--mount type=bind,source="$(shell pwd)"/submission,target=/code_execution/submission \ | ||
--shm-size 8g \ | ||
-it \ | ||
--entrypoint /bin/bash \ | ||
${LOCAL_IMAGE} | ||
|
||
## Pulls the official container from Azure Container Registry | ||
pull: | ||
docker pull ${REGISTRY_IMAGE} | ||
|
||
## Creates a submission/submission.zip file from the source code in examples_src | ||
pack-example: | ||
# Don't overwrite so no work is lost accidentally | ||
ifneq (,$(wildcard ./submission/submission.zip)) | ||
$(error You already have a submission/submission.zip file. Rename or remove that file (e.g., rm submission/submission.zip).) | ||
endif | ||
mkdir -p submission/ | ||
cd example_src; zip -r ../submission/submission.zip ./* | ||
|
||
## Creates a submission/submission.zip file from the source code in example_benchmark | ||
pack-benchmark: | ||
# Don't overwrite so no work is lost accidentally | ||
ifneq (,$(wildcard ./submission/submission.zip)) | ||
$(error You already have a submission/submission.zip file. Rename or remove that file (e.g., rm submission/submission.zip).) | ||
endif | ||
mkdir -p submission/ | ||
cd example_benchmark; zip -r ../submission/submission.zip ./* | ||
|
||
## Creates a submission/submission.zip file from the source code in submission_src | ||
pack-submission: | ||
# Don't overwrite so no work is lost accidentally | ||
ifneq (,$(wildcard ./submission/submission.zip)) | ||
$(error You already have a submission/submission.zip file. Rename or remove that file (e.g., rm submission/submission.zip).) | ||
endif | ||
mkdir -p submission/ | ||
cd submission_src; zip -r ../submission/submission.zip ./* | ||
|
||
repack-submission: clean pack-submission | ||
|
||
repack-example: clean pack-example | ||
|
||
## Runs container using code from `submission/submission.zip` and data from `data/` | ||
test-submission: _submission_write_perms | ||
# if submission file does not exist | ||
ifeq (,$(wildcard ./submission/submission.zip)) | ||
$(error To test your submission, you must first put a "submission.zip" file in the "submission" folder. \ | ||
If you want to use the benchmark, you can run `make pack-benchmark` first) | ||
endif | ||
docker run \ | ||
${TTY_ARGS} \ | ||
--network none \ | ||
--mount type=bind,source="$(shell pwd)"/data,target=/code_execution/data,readonly \ | ||
--mount type=bind,source="$(shell pwd)"/submission,target=/code_execution/submission \ | ||
--shm-size 8g \ | ||
--name ${CONTAINER_NAME} \ | ||
--rm \ | ||
${SUBMISSION_IMAGE} | ||
|
||
## Delete temporary Python cache and bytecode files | ||
clean: | ||
find . -type f -name "*.py[co]" -delete | ||
find . -type d -name "__pycache__" -delete | ||
rm -rf ./submission/* |
Oops, something went wrong.