-
-
Notifications
You must be signed in to change notification settings - Fork 739
/
Vagrantfile
434 lines (404 loc) · 19.3 KB
/
Vagrantfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Automated creation of testing environments / binaries on misc. platforms
$cpus = Integer(ENV.fetch('VMCPUS', '8')) # create VMs with that many cpus
$xdistn = Integer(ENV.fetch('XDISTN', '8')) # dispatch tests to that many pytest workers
$wmem = $xdistn * 256 # give the VM additional memory for workers [MB]
def packages_debianoid(user)
return <<-EOF
export DEBIAN_FRONTEND=noninteractive
# this is to avoid grub asking about which device it should install to:
echo "set grub-pc/install_devices /dev/sda" | debconf-communicate
apt-get -y -qq update
apt-get -y -qq dist-upgrade
# for building borgbackup and dependencies:
apt install -y pkg-config
apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true
apt install -y libfuse-dev fuse || true
apt install -y libfuse3-dev fuse3 || true
apt install -y locales || true
sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
usermod -a -G fuse #{user}
chgrp fuse /dev/fuse
chmod 666 /dev/fuse
apt install -y fakeroot build-essential git curl
apt install -y python3-dev python3-setuptools virtualenv
# for building python:
apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
EOF
end
def packages_freebsd
return <<-EOF
# in case the VM has no hostname set
hostname freebsd
# install all the (security and other) updates, base system
freebsd-update --not-running-from-cron fetch install
# for building borgbackup and dependencies:
pkg install -y xxhash liblz4 zstd pkgconf
pkg install -y fusefs-libs || true
pkg install -y fusefs-libs3 || true
pkg install -y rust
pkg install -y git bash # fakeroot causes lots of troubles on freebsd
# for building python (for the tests we use pyenv built pythons):
pkg install -y python310 py310-sqlite3
# make sure there is a python3 command
ln -sf /usr/local/bin/python3.10 /usr/local/bin/python3
python3 -m ensurepip
pip3 install virtualenv
# make bash default / work:
chsh -s bash vagrant
mount -t fdescfs fdesc /dev/fd
echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab
# make FUSE work
echo 'fuse_load="YES"' >> /boot/loader.conf
echo 'vfs.usermount=1' >> /etc/sysctl.conf
kldload fusefs
sysctl vfs.usermount=1
pw groupmod operator -M vagrant
# /dev/fuse has group operator
chmod 666 /dev/fuse
# install all the (security and other) updates, packages
pkg update
yes | pkg upgrade
echo 'export BORG_OPENSSL_PREFIX=/usr' >> ~vagrant/.bash_profile
# (re)mount / with acls
mount -o acls /
EOF
end
def packages_openbsd
return <<-EOF
pkg_add bash
chsh -s bash vagrant
pkg_add xxhash
pkg_add lz4
pkg_add zstd
pkg_add git # no fakeroot
pkg_add rust
pkg_add openssl%3.0
pkg_add py3-pip
pkg_add py3-virtualenv
EOF
end
def packages_netbsd
return <<-EOF
echo 'http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf
pkgin update
pkgin -y upgrade
pkg_add zstd lz4 xxhash git
pkg_add rust
pkg_add bash
chsh -s bash vagrant
echo "export PROMPT_COMMAND=" >> ~vagrant/.bash_profile # bug in netbsd 9.3, .bash_profile broken for screen
echo "export PROMPT_COMMAND=" >> ~root/.bash_profile # bug in netbsd 9.3, .bash_profile broken for screen
pkg_add pkg-config
# pkg_add fuse # llfuse supports netbsd, but is still buggy.
# https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space
pkg_add py311-sqlite3 py311-pip py311-virtualenv py311-expat
ln -s /usr/pkg/bin/python3.11 /usr/pkg/bin/python
ln -s /usr/pkg/bin/python3.11 /usr/pkg/bin/python3
ln -s /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip
ln -s /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3
ln -s /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv
ln -s /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3
EOF
end
def packages_macos
return <<-EOF
# install all the (security and other) updates
sudo softwareupdate --ignore iTunesX
sudo softwareupdate --ignore iTunes
sudo softwareupdate --ignore Safari
sudo softwareupdate --ignore "Install macOS High Sierra"
sudo softwareupdate --install --all
which brew || CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
brew update > /dev/null
brew install pkg-config readline xxhash [email protected] zstd lz4 xz
brew install --cask macfuse
# brew upgrade # upgrade everything (takes rather long)
echo 'export LDFLAGS=-L/usr/local/opt/[email protected]/lib' >> ~vagrant/.bash_profile
echo 'export CPPFLAGS=-I/usr/local/opt/[email protected]/include' >> ~vagrant/.bash_profile
echo 'export PKG_CONFIG_PATH=/usr/local/opt/[email protected]/lib/pkgconfig' >> ~vagrant/.bash_profile
echo 'export [email protected]' >> ~vagrant/.bash_profile
EOF
end
def packages_openindiana
return <<-EOF
# needs separate provisioning step + reboot:
#pkg update
pkg install gcc-13 git pkg-config libxxhash
ln -sf /usr/bin/python3.9 /usr/bin/python3
python3 -m ensurepip
ln -sf /usr/bin/pip3.9 /usr/bin/pip3
pip3 install virtualenv
# let borg's pkg-config find openssl:
pfexec pkg set-mediator -V 3.1 openssl
EOF
end
def install_pyenv(boxname)
return <<-EOF
echo 'export PYTHON_CONFIGURE_OPTS="${PYTHON_CONFIGURE_OPTS} --enable-shared"' >> ~/.bash_profile
echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bash_profile
echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bash_profile
. ~/.bash_profile
curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
echo 'eval "$(pyenv init --path)"' >> ~/.bash_profile
echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
echo 'eval "$(pyenv init -)"' >> ~/.bashrc
echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bashrc
EOF
end
def fix_pyenv_macos(boxname)
return <<-EOF
echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile
EOF
end
def install_pythons(boxname)
return <<-EOF
. ~/.bash_profile
echo "PYTHON_CONFIGURE_OPTS: ${PYTHON_CONFIGURE_OPTS}"
pyenv install 3.12.6 # tests, binary build
pyenv install 3.11.3 # tests
pyenv install 3.10.2 # tests
pyenv install 3.9.4 # tests
pyenv rehash
EOF
end
def build_sys_venv(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
virtualenv --python=python3 borg-env
EOF
end
def build_pyenv_venv(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
# use the latest 3.12 release
pyenv global 3.12.6
pyenv virtualenv 3.12.6 borg-env
ln -s ~/.pyenv/versions/borg-env .
EOF
end
def install_borg(fuse)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
pip install -U wheel # upgrade wheel, might be too old
cd borg
pip install -r requirements.d/development.lock.txt
python3 scripts/make.py clean
pip install -e .[#{fuse}]
EOF
end
def install_pyinstaller()
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
pip install 'pyinstaller==6.10.0'
EOF
end
def build_binary_with_pyinstaller(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
cd borg
pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec
echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile
cd .. && tar -czvf borg.tgz borg-dir
EOF
end
def run_tests(boxname, skip_env)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg/borg
. ../borg-env/bin/activate
if which pyenv 2> /dev/null; then
# for testing, use the earliest point releases of the supported python versions:
pyenv global 3.9.4 3.10.2 3.11.3 3.12.6
pyenv local 3.9.4 3.10.2 3.11.3 3.12.6
fi
# otherwise: just use the system python
# some OSes can only run specific test envs, e.g. because they miss FUSE support:
export TOX_SKIP_ENV='#{skip_env}'
if which fakeroot 2> /dev/null; then
echo "Running tox WITH fakeroot -u"
fakeroot -u tox --skip-missing-interpreters
else
echo "Running tox WITHOUT fakeroot -u"
tox --skip-missing-interpreters
fi
EOF
end
def fs_init(user)
return <<-EOF
# clean up (wrong/outdated) stuff we likely got via rsync:
rm -rf /vagrant/borg/borg/.tox 2> /dev/null
rm -rf /vagrant/borg/borg/borgbackup.egg-info 2> /dev/null
rm -rf /vagrant/borg/borg/__pycache__ 2> /dev/null
find /vagrant/borg/borg/src -name '__pycache__' -exec rm -rf {} \\; 2> /dev/null
chown -R #{user} /vagrant/borg
touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile
echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile
echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile
echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile
EOF
end
Vagrant.configure(2) do |config|
# use rsync to copy content to the folder
config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "--exclude", ".python-version"], :rsync__chown => false
# do not let the VM access . on the host machine via the default shared folder!
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.provider :virtualbox do |v|
#v.gui = true
v.cpus = $cpus
end
config.vm.define "noble" do |b|
b.vm.box = "bento/ubuntu-24.04"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("noble")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("noble", ".*none.*")
end
config.vm.define "jammy" do |b|
b.vm.box = "ubuntu/jammy64"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jammy")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jammy", ".*none.*")
end
config.vm.define "bookworm" do |b|
b.vm.box = "debian/bookworm64"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("bookworm")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("bookworm")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("bookworm")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("bookworm")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bookworm", ".*none.*")
end
config.vm.define "bullseye" do |b|
b.vm.box = "debian/bullseye64"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("bullseye")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("bullseye")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("bullseye")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("bullseye")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bullseye", ".*none.*")
end
config.vm.define "freebsd13" do |b|
b.vm.box = "generic/freebsd13"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.ssh.shell = "sh"
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd13")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd13")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd13")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd13")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd13", ".*(fuse3|none).*")
end
config.vm.define "freebsd14" do |b|
b.vm.box = "generic/freebsd14"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.ssh.shell = "sh"
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd14")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd14")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd14")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd14")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd14", ".*(fuse3|none).*")
end
config.vm.define "openbsd7" do |b|
b.vm.box = "generic/openbsd7"
b.vm.provider :virtualbox do |v|
v.memory = 1024 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd7")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd7", ".*fuse.*")
end
config.vm.define "netbsd9" do |b|
b.vm.box = "generic/netbsd9"
b.vm.provider :virtualbox do |v|
v.memory = 4096 + $wmem # need big /tmp tmpfs in RAM!
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd9")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false)
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd9", ".*fuse.*")
end
config.vm.define "macos1012" do |b|
b.vm.box = "macos-sierra"
b.vm.provider :virtualbox do |v|
v.memory = 8192 + $wmem
v.customize ['modifyvm', :id, '--ostype', 'MacOS_64']
v.customize ['modifyvm', :id, '--paravirtprovider', 'default']
v.customize ['modifyvm', :id, '--nested-hw-virt', 'on']
# Adjust CPU settings according to
# https://github.com/geerlingguy/macos-virtualbox-vm
v.customize ['modifyvm', :id, '--cpuidset',
'00000001', '000306a9', '00020800', '80000201', '178bfbff']
# Disable USB variant requiring Virtualbox proprietary extension pack
v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off']
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages macos", :type => :shell, :privileged => false, :inline => packages_macos
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("macos1012")
b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_macos("macos1012")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("macos1012")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("macos1012")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("macos1012")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("macos1012", ".*(fuse3|none).*")
end
# rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of
# permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again.
config.vm.define "openindiana" do |b|
b.vm.box = "openindiana/hipster"
b.vm.provider :virtualbox do |v|
v.memory = 2048 + $wmem
end
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana", ".*fuse.*")
end
end