forked from NVIDIA-Merlin/models
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tox.ini
135 lines (125 loc) · 6.53 KB
/
tox.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
; For all tests that run in Github Actions, see
; .github/workflows/cpu-ci.yml for the workflow definition.
[tox]
envlist = py38-gpu,py38-multi-gpu
[testenv]
commands =
pip install --upgrade pip
pip install -e .[all]
[testenv:py38-gpu]
; Runs in: Github Actions
; Runs GPU-based tests.
deps =
--no-deps -rrequirements/test.txt
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
sitepackages=true
commands =
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH:main}
python -m pytest --cov-report term --cov merlin -m "unit {env:EXTRA_PYTEST_MARKERS}"
[testenv:py38-multi-gpu]
; Runs in: Github Actions
; Runs GPU-based tests.
allowlist_externals =
horovodrun
sh
passenv =
OPAL_PREFIX
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
CPATH={env:CPATH}{:}{envdir}/hugectr/include
LD_LIBRARY_PATH=${envdir}/hugectr/include/lib{:}/usr/local/lib/python3.8/dist-packages/tensorflow{:}{env:LD_LIBRARY_PATH}
LIBRARY_PATH=${envdir}/hugectr/lib{:}{env:LIBRARY_PATH}
sitepackages=true
commands =
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{env:MERLIN_BRANCH:main}
sh examples/usecases/multi-gpu/install_sparse_operation_kit.sh {envdir}
horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh python -m pytest -m "horovod {env:EXTRA_PYTEST_MARKERS}" -rxs tests/unit
[testenv:py38-horovod-cpu]
setenv =
HOROVOD_WITH_MPI=1
HOROVOD_WITH_TENSORFLOW=1
PATH={env:PATH}{:}{envdir}/env/bin
LD_LIBRARY_PATH={env:LD_LIBRARY_PATH}{:}{envdir}/env/lib
commands =
conda update --yes --name base --channel defaults conda
conda env create --prefix {envdir}/env --file requirements/horovod-cpu-environment.yml --force
{envdir}/env/bin/python -m pip install 'horovod==0.27.0' --no-cache-dir
{envdir}/env/bin/horovodrun --check-build
{envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
{envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
{envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{env:MERLIN_BRANCH:main}
{envdir}/env/bin/horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh pytest -m "horovod {env:EXTRA_PYTEST_MARKERS}" -rxs tests/unit
[testenv:py38-nvtabular-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
deps =
-rrequirements/base.txt
-rrequirements/dev.txt
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/NVTabular.git nvtabular-{env:GIT_COMMIT}
python -m pip install --upgrade "./nvtabular-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./nvtabular-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest nvtabular-{env:GIT_COMMIT}/tests/unit
[testenv:py38-systems-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
deps =
-rrequirements/base.txt
-rrequirements/dev.txt
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/systems.git systems-{env:GIT_COMMIT}
python -m pip install --upgrade "./systems-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./systems-{env:GIT_COMMIT}/requirements/test-cpu.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest -m "not notebook" systems-{env:GIT_COMMIT}/tests/unit
[testenv:py38-transformers4rec-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/Transformers4Rec.git Transformers4Rec-{env:GIT_COMMIT}
python -m pip install --upgrade -r "./Transformers4Rec-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade "./Transformers4Rec-{env:GIT_COMMIT}"
python -m pip install --no-deps git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --no-deps git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install --no-deps .
python -m pytest Transformers4Rec-{env:GIT_COMMIT}/tests/unit
[testenv:docs]
; Runs in: Github Actions
; Generates documentation with sphinx. There are other steps in the Github Actions workflow
; to publish the documentation on release.
changedir = {toxinidir}
deps =
-rrequirements/docs.txt
-rrequirements/test.txt
commands =
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git
python -m sphinx.cmd.build -E -P -b html docs/source docs/build/html
[testenv:docs-multi]
; Run the multi-version build that is shown on GitHub Pages.
changedir = {toxinidir}
deps =
-rrequirements/docs.txt
-rrequirements/test.txt
commands =
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git
sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
sphinx-multiversion docs/source docs/build/html