diff --git a/.circleci/config.yml b/.circleci/config.yml index 56b0b1e308..9922d20588 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,6 +89,25 @@ executors: <<: *default-environment PYTHON_VERSION_SHORT: << parameters.py-version >> + arm: + parameters: + # We don't need both of those here but define them anyway to be compatible with the other + # executors + py-version: + <<: *py-version-template + resource: + description: "resource type for underlying VM" + default: "medium" + type: enum + enum: ["small", "medium", "medium+", "large", "xlarge"] + working_directory: ~/raiden + environment: + <<: *default-environment + PYTHON_VERSION_SHORT: << parameters.py-version >> + machine: + image: ubuntu-2004:202101-01 + resource_class: arm.medium + commands: conditional-skip: @@ -527,25 +546,24 @@ jobs: parameters: py-version: <<: *py-version-template + executor: + description: "executor type to use" + default: "docker" + type: enum + enum: ["docker", "arm"] resource: description: "resource type for underlying VM" default: "medium" type: enum enum: ["small", "medium", "medium+", "large", "xlarge"] - remote-host: - description: "name of the arm server" - default: "" - type: enum - enum: ["", "armv7", "armv8"] architecture: description: "defines the architecture for which the binary is built" default: "x86_64" type: enum - enum: ["x86_64", "armv7l", "aarch64"] - + enum: ["x86_64", "aarch64"] executor: - name: docker + name: << parameters.executor >> py-version: "3.7" resource: << parameters.resource >> @@ -558,17 +576,8 @@ jobs: - conditional-skip - config-path - when: - condition: << parameters.remote-host >> - steps: - - run: - name: "set remote docker host" - command: echo 'export DOCKER_HOST=localhost:23760' >> ${BASH_ENV} - - run: - name: SSH Port Forwarding To ARM Server - command: | - ssh -f -N -L 127.0.0.1:23760:localhost:2376 ci@<< parameters.remote-host >>.ci.raiden.network -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no - - unless: - condition: << parameters.remote-host >> + condition: + equal: [ docker, << parameters.executor >> ] steps: - setup_remote_docker: docker_layer_caching: true @@ -578,6 +587,11 @@ jobs: source .circleci/get_archive_tag.sh make bundle-docker echo ${RELEASE_TYPE}/raiden-${ARCHIVE_TAG}-linux-<>.tar.gz > dist/archive/_LATEST-${RELEASE_TYPE}-linux-<>.txt + - run: + name: Test if Binary can be launched + command: | + tar -C /tmp -xf dist/archive/raiden-${ARCHIVE_TAG}-linux-<>.tar.gz + /tmp/raiden-${ARCHIVE_TAG}-linux-<> --help - persist_to_workspace: root: "~" paths: @@ -610,6 +624,10 @@ jobs: pyinstaller --noconfirm --clean raiden.spec zip -9 --junk-paths dist/archive/raiden-${ARCHIVE_TAG}-macOS-x86_64.zip dist/raiden-${ARCHIVE_TAG}-macOS-x86_64 echo ${RELEASE_TYPE}/raiden-${ARCHIVE_TAG}-macOS-x86_64.zip > dist/archive/_LATEST-${RELEASE_TYPE}-macOS-x86_64.txt + - run: + name: Test if Binary can be launched + command: | + dist/raiden-${ARCHIVE_TAG}-macOS-x86_64 --help - persist_to_workspace: root: "~" paths: @@ -858,24 +876,26 @@ workflows: filters: <<: *only-tagged-version-filter - - prepare-python-linux: + - prepare-python-macos: py-version: '3.7' requires: - prepare-system-linux filters: <<: *only-tagged-version-filter - - prepare-python-macos: - py-version: '3.7' + - build-binary-linux: + name: build-binary-linux-x86_64 requires: - - prepare-system-macos + - prepare-system-linux filters: <<: *only-tagged-version-filter - build-binary-linux: - name: build-binary-linux-x86 + name: build-binary-linux-aarch64 + executor: arm + architecture: aarch64 requires: - - prepare-python-linux + - prepare-system-linux filters: <<: *only-tagged-version-filter @@ -894,7 +914,8 @@ workflows: - deploy-digitalocean: context: "raiden-py" requires: - - build-binary-linux-x86 + - build-binary-linux-x86_64 + - build-binary-linux-aarch64 - build-binary-macos filters: <<: *only-tagged-version-filter @@ -902,7 +923,8 @@ workflows: - deploy-github-release: context: "raiden-general-2021" requires: - - build-binary-linux-x86 + - build-binary-linux-x86_64 + - build-binary-linux-aarch64 - build-binary-macos filters: <<: *only-tagged-version-filter @@ -910,7 +932,8 @@ workflows: - deploy-pypi: context: "raiden-py" requires: - - build-binary-linux-x86 + - build-binary-linux-x86_64 + - build-binary-linux-aarch64 - build-binary-macos filters: <<: *only-tagged-version-filter diff --git a/Makefile b/Makefile index 6bc3846760..2e8a00634f 100644 --- a/Makefile +++ b/Makefile @@ -129,7 +129,7 @@ endif bundle-docker: ARCHITECTURE_TAG = $(shell docker run --rm python:3.7 uname -m) bundle-docker: ARCHIVE_TAG ?= v$(shell python setup.py --version) bundle-docker: - docker build -t pyinstallerbuilder --build-arg GETH_URL_LINUX=$(GETH_URL_LINUX) --build-arg SOLC_URL_LINUX=$(SOLC_URL_LINUX) --build-arg ARCHITECTURE_TAG=$(ARCHITECTURE_TAG) --build-arg ARCHIVE_TAG=$(ARCHIVE_TAG) $(GITHUB_ACCESS_TOKEN_ARG) -f docker/build.Dockerfile . + docker build -t pyinstallerbuilder --build-arg ARCHITECTURE_TAG=$(ARCHITECTURE_TAG) --build-arg ARCHIVE_TAG=$(ARCHIVE_TAG) $(GITHUB_ACCESS_TOKEN_ARG) -f docker/build.Dockerfile . -(docker rm builder) docker create --name builder pyinstallerbuilder mkdir -p dist/archive diff --git a/docker/build.Dockerfile b/docker/build.Dockerfile index 03a6c269b9..685544f829 100644 --- a/docker/build.Dockerfile +++ b/docker/build.Dockerfile @@ -1,9 +1,5 @@ FROM python:3.7-stretch -# these are defined in .circleci/config.yml and passed here in the makefile -ARG SOLC_URL_LINUX -ARG GETH_URL_LINUX - # install dependencies RUN apt-get update RUN apt-get install -y git-core wget xz-utils build-essential \ @@ -11,15 +7,6 @@ RUN apt-get install -y git-core wget xz-utils build-essential \ libavdevice-dev libavfilter-dev libopus-dev libvpx-dev pkg-config \ libsrtp2-dev -RUN wget -nv -O /usr/bin/solc ${SOLC_URL_LINUX} && \ - chmod +x /usr/bin/solc -RUN wget -nv -O /tmp/geth.tar.gz ${GETH_URL_LINUX} && \ - cd /tmp && \ - tar xf geth.tar.gz && \ - mv geth-linux-amd64-*/geth /usr/bin/geth && \ - rm geth.tar.gz - - RUN python3 -m venv /venv ENV PATH="/venv/bin:$PATH" diff --git a/raiden.spec b/raiden.spec index be360081f4..1ec297ebbb 100644 --- a/raiden.spec +++ b/raiden.spec @@ -12,18 +12,40 @@ PyInstaller spec file to build single file or dir distributions """ # Set to false to produce an exploded single-dir -ONEFILE = int(os.environ.get('ONEFILE', True)) - - -def Entrypoint(dist, group, name, scripts=None, pathex=None, hiddenimports=None, hookspath=None, - excludes=None, runtime_hooks=None, datas=None): +ONEFILE = int(os.environ.get("ONEFILE", True)) + +# Hack: This is a list of prefixes to be removed from the `binaries`. +# We do this to prevent including unnecessary libraries (pyav audio / video dependencies) +BINARIES_PREFIX_BLOCKLIST = [ + "libav", + "libaom", + "libswscale", + "libswrescale", + "libvorbis", + "libx264", + "libx265", +] + + +def Entrypoint( + dist, + group, + name, + scripts=None, + pathex=None, + hiddenimports=None, + hookspath=None, + excludes=None, + runtime_hooks=None, + datas=None, +): import pkg_resources # get toplevel packages of distribution from metadata def get_toplevel(dist): distribution = pkg_resources.get_distribution(dist) - if distribution.has_metadata('top_level.txt'): - return list(distribution.get_metadata('top_level.txt').split()) + if distribution.has_metadata("top_level.txt"): + return list(distribution.get_metadata("top_level.txt").split()) else: return [] @@ -42,54 +64,75 @@ def Entrypoint(dist, group, name, scripts=None, pathex=None, hiddenimports=None, # insert path of the egg at the verify front of the search path pathex = [ep.dist.location] + pathex # script name must not be a valid module name to avoid name clashes on import - script_path = os.path.join(workpath, name + '-script.py') + script_path = os.path.join(workpath, name + "-script.py") print("creating script for entry point", dist, group, name) - with open(script_path, 'w') as fh: + with open(script_path, "w") as fh: print("import", ep.module_name, file=fh) - print("%s.%s()" % (ep.module_name, '.'.join(ep.attrs)), file=fh) + print("%s.%s()" % (ep.module_name, ".".join(ep.attrs)), file=fh) for package in packages: print("import", package, file=fh) - return Analysis( + analysis = Analysis( [script_path] + scripts, pathex=pathex, hiddenimports=hiddenimports, hookspath=hookspath, excludes=excludes, runtime_hooks=runtime_hooks, - datas=datas + datas=datas, ) - - -if hasattr(pdb, 'pdb'): + # `Analysis.binaries` behaves set-like and matches on the first tuple item (`name`). + # Since library names include the version we first build a list of the concrete names + # by prefix matching and then subtract that via the set-like behaviour. + binaries_to_remove = [ + (name, None, None) + for name, *_ in analysis.binaries + if any(name.startswith(blocklist_item) for blocklist_item in BINARIES_PREFIX_BLOCKLIST) + ] + analysis.binaries -= binaries_to_remove + return analysis + + +if hasattr(pdb, "pdb"): # pdbpp moves the stdlib pdb to the `pdb` attribute of it's own patched pdb module raise RuntimeError( - 'pdbpp is installed which causes broken PyInstaller builds. Please uninstall it.', + "pdbpp is installed which causes broken PyInstaller builds. Please uninstall it.", ) # We don't need Tk and friends -sys.modules['FixTk'] = None +sys.modules["FixTk"] = None -executable_name = 'raiden-{}-{}-{}'.format( - os.environ.get('ARCHIVE_TAG', 'v' + get_system_spec()['raiden']), - 'macOS' if platform.system() == 'Darwin' else platform.system().lower(), - platform.machine() +executable_name = "raiden-{}-{}-{}".format( + os.environ.get("ARCHIVE_TAG", "v" + get_system_spec()["raiden"]), + "macOS" if platform.system() == "Darwin" else platform.system().lower(), + platform.machine(), ) a = Entrypoint( - 'raiden', - 'console_scripts', - 'raiden', - hookspath=['tools/pyinstaller_hooks'], + "raiden", + "console_scripts", + "raiden", + hookspath=["tools/pyinstaller_hooks"], runtime_hooks=[ - 'tools/pyinstaller_hooks/runtime_gevent_monkey.py', - 'tools/pyinstaller_hooks/runtime_encoding.py', - 'tools/pyinstaller_hooks/runtime_raiden_contracts.py', + "tools/pyinstaller_hooks/runtime_gevent_monkey.py", + "tools/pyinstaller_hooks/runtime_encoding.py", + "tools/pyinstaller_hooks/runtime_raiden_contracts.py", ], - hiddenimports=['scrypt', 'eth_tester'], + hiddenimports=[], datas=[], - excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'], + excludes=[ + "_tkinter", + "FixTk", + "ipython", + "jupyter", + "jupyter_core", + "notebook", + "tcl", + "tk", + "tkinter", + "Tkinter", + ], ) pyz = PYZ(a.pure, a.zipped_data, cipher=None) @@ -103,10 +146,10 @@ if ONEFILE: a.datas, name=executable_name, debug=False, - strip=True, + strip=False, upx=False, runtime_tmpdir=None, - console=True + console=True, ) else: exe = EXE( @@ -117,7 +160,7 @@ else: debug=True, strip=False, upx=False, - console=True + console=True, ) coll = COLLECT( exe, @@ -126,5 +169,5 @@ else: a.datas, strip=False, upx=False, - name='raiden' + name="raiden", ) diff --git a/tools/pyinstaller_hooks/hook-av.py b/tools/pyinstaller_hooks/hook-av.py new file mode 100644 index 0000000000..d2a98cd348 --- /dev/null +++ b/tools/pyinstaller_hooks/hook-av.py @@ -0,0 +1,4 @@ +from PyInstaller.utils.hooks import collect_submodules + + +hiddenimports = collect_submodules("av") diff --git a/tools/pyinstaller_hooks/runtime_gevent_monkey.py b/tools/pyinstaller_hooks/runtime_gevent_monkey.py index 0d4cb6375f..afea202cbd 100644 --- a/tools/pyinstaller_hooks/runtime_gevent_monkey.py +++ b/tools/pyinstaller_hooks/runtime_gevent_monkey.py @@ -1,7 +1,3 @@ -from gevent.monkey import patch_all # isort:skip # noqa +from gevent.monkey import patch_all -patch_all() # isort:skip # noqa - -from raiden.network.transport.matrix.rtc.utils import setup_asyncio_event_loop - -setup_asyncio_event_loop() +patch_all()