-
Notifications
You must be signed in to change notification settings - Fork 5
/
justfile
203 lines (153 loc) · 6.52 KB
/
justfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
set dotenv-load := true
# just has no idiom for setting a default value for an environment variable
# so we shell out, as we need VIRTUAL_ENV in the justfile environment
export VIRTUAL_ENV := `echo ${VIRTUAL_ENV:-.venv}`
export BIN := VIRTUAL_ENV + if os_family() == "unix" { "/bin" } else { "/Scripts" }
export PIP := BIN + if os_family() == "unix" { "/python -m pip" } else { "/python.exe -m pip" }
# enforce our chosen pip compile flags
export COMPILE := BIN + "/pip-compile --allow-unsafe --generate-hashes"
export DEFAULT_PYTHON := if os_family() == "unix" { "python3.8" } else { "python" }
# list available commands
default:
@"{{ just_executable() }}" --list
# clean up temporary files
clean:
rm -rf .venv
# ensure valid virtualenv
virtualenv:
#!/usr/bin/env bash
set -euo pipefail
# allow users to specify python version in .env
PYTHON_VERSION=${PYTHON_VERSION:-$DEFAULT_PYTHON}
# create venv and upgrade pip
test -d $VIRTUAL_ENV || { $PYTHON_VERSION -m venv $VIRTUAL_ENV && $PIP install --upgrade pip; }
# ensure we have pip-tools so we can run pip-compile
test -e $BIN/pip-compile || $PIP install $(grep pip-tools== requirements.dev.txt)
_compile src dst *args: virtualenv
#!/usr/bin/env bash
set -euo pipefail
# exit if src file is older than dst file (-nt = 'newer than', but we negate with || to avoid error exit code)
test "${FORCE:-}" = "true" -o {{ src }} -nt {{ dst }} || exit 0
$BIN/pip-compile --allow-unsafe --output-file={{ dst }} {{ src }} {{ args }}
# update requirements.prod.txt if setup.py has changed
requirements-prod *args:
"{{ just_executable() }}" _compile pyproject.toml requirements.prod.txt {{ args }}
# update requirements.dev.txt if requirements.dev.in has changed
requirements-dev *args: requirements-prod
"{{ just_executable() }}" _compile requirements.dev.in requirements.dev.txt {{ args }}
# update requirements.tools.txt in requirements.tools.in has changed
requirements-tools *args: requirements-prod
"{{ just_executable() }}" _compile requirements.tools.in requirements.tools.txt {{ args }}
# ensure prod requirements installed and up to date
prodenv: requirements-prod
#!/usr/bin/env bash
set -euo pipefail
# exit if .txt file has not changed since we installed them (-nt == "newer than', but we negate with || to avoid error exit code)
test requirements.prod.txt -nt $VIRTUAL_ENV/.prod || exit 0
$PIP install -r requirements.prod.txt
touch $VIRTUAL_ENV/.prod
# && dependencies are run after the recipe has run. Needs just>=0.9.9. This is
# a killer feature over Makefiles.
#
# ensure dev requirements installed and up to date
devenv: prodenv requirements-dev && install-precommit
#!/usr/bin/env bash
set -euo pipefail
# exit if .txt file has not changed since we installed them (-nt == "newer than', but we negate with || to avoid error exit code)
test requirements.dev.txt -nt $VIRTUAL_ENV/.dev || exit 0
$PIP install -r requirements.dev.txt
touch $VIRTUAL_ENV/.dev
# ensure precommit is installed
install-precommit:
#!/usr/bin/env bash
set -euo pipefail
BASE_DIR=$(git rev-parse --show-toplevel)
test -f $BASE_DIR/.git/hooks/pre-commit || $BIN/pre-commit install
# upgrade dev or prod dependencies (specify package to upgrade single package, all by default)
upgrade env package="": virtualenv
#!/usr/bin/env bash
set -euo pipefail
opts="--upgrade"
test -z "{{ package }}" || opts="--upgrade-package {{ package }}"
FORCE=true "{{ just_executable() }}" requirements-{{ env }} $opts
# *ARGS is variadic, 0 or more. This allows us to do `just test -k match`, for example.
# Run the tests
test *ARGS: devenv
$BIN/coverage run --module pytest {{ ARGS }}
$BIN/coverage report || $BIN/coverage html
test-fast *ARGS: devenv
$BIN/python -m pytest tests -m "not slow_test" {{ ARGS }}
test-verbose *ARGS: devenv
$BIN/python -m pytest tests/test_integration.py -o log_cli=true -o log_cli_level=INFO {{ ARGS }}
test-no-docker *ARGS: devenv
$BIN/python -m pytest -m "not needs_docker" {{ ARGS }}
# run db migrations locally
migrate:
$BIN/python -m jobrunner.cli.migrate
package-build: virtualenv
rm -rf dist
$PIP install build
$BIN/python -m build
package-test type: package-build
#!/usr/bin/env bash
set -euo pipefail
VENV="test-{{ type }}"
distribution_suffix="{{ if type == "wheel" { "whl" } else { "tar.gz" } }}"
# build a fresh venv
python -m venv $VENV
# ensure a modern pip
$VENV/bin/pip install pip --upgrade
# install the wheel distribution
$VENV/bin/pip install dist/*."$distribution_suffix"
# Minimal check that it has actually built correctly
# Note: this uses the command installed into the virtualenv, not python -m,
# to make sure we test the installed package and don't accidentally rely on
# imports picking code on-disk.
$VENV/bin/local_run --help
# check we haven't packaged tests with it
if unzip -Z -1 dist/*.whl | grep -q "^tests"; then
echo "Built wheel contains tests!"
exit 1
fi
# clean up after ourselves
rm -rf $VENV
# test the license shenanigins work when run from a console
test-stata: devenv
rm -f tests/fixtures/stata_project/output/env.txt
$BIN/python -c 'from jobrunner.cli.local_run import main; main("tests/fixtures/stata_project", ["stata"])'
cat tests/fixtures/stata_project/output/env.txt
echo
# runs the format (black), sort (isort) and lint (flake8) check but does not change any files
check: devenv
$BIN/black --check .
$BIN/isort --check-only --diff .
$BIN/flake8
# fix formatting and import sort ordering
fix: devenv
$BIN/black .
$BIN/isort .
just --fmt --unstable --justfile justfile
just --fmt --unstable --justfile docker/justfile
# Run the dev project
run repo action: devenv
$BIN/add_job {{ repo }} {{ action }}
# required by docker-compose.yaml
dotenv:
cp dotenv-sample .env
update-wheels: devenv
#!/usr/bin/env bash
set -euo pipefail
if test -d lib; then
git -C lib pull
else
git clone [email protected]:opensafely/job-runner-dependencies.git lib
fi
# wipe clean to remove old packages and start fresh each time
rm lib/* -rf
# restore README.md, as we don't want to delete that
git -C lib checkout README.md
# Now install the dependencies
$BIN/pip install -U -r requirements.prod.txt -r requirements.tools.txt --target lib
cp requirements.prod.txt requirements.tools.txt lib/
# remove any .so libs
find lib/ -name \*.so -exec rm {} \;