[WIP] Fix ARM builds

.circleci/config.yml

@@ -89,6 +89,25 @@ executors:
       <<: *default-environment
       PYTHON_VERSION_SHORT: << parameters.py-version >>
 
+  arm:
+    parameters:
+      # We don't need both of those here but define them anyway to be compatible with the other
+      # executors
+      py-version:
+        <<: *py-version-template
+      resource:
+        description: "resource type for underlying VM"
+        default: "medium"
+        type: enum
+        enum: ["small", "medium", "medium+", "large", "xlarge"]
+    working_directory: ~/raiden
+    environment:
+      <<: *default-environment
+      PYTHON_VERSION_SHORT: << parameters.py-version >>
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.medium
+
 
 commands:
   conditional-skip:
@@ -527,25 +546,24 @@ jobs:
     parameters:
       py-version:
         <<: *py-version-template
+      executor:
+        description: "executor type to use"
+        default: "docker"
+        type: enum
+        enum: ["docker", "arm"]
       resource:
         description: "resource type for underlying VM"
         default: "medium"
         type: enum
         enum: ["small", "medium", "medium+", "large", "xlarge"]
-      remote-host:
-        description: "name of the arm server"
-        default: ""
-        type: enum
-        enum: ["", "armv7", "armv8"]
       architecture:
         description: "defines the architecture for which the binary is built"
         default: "x86_64"
         type: enum
-        enum: ["x86_64", "armv7l", "aarch64"]
-
+        enum: ["x86_64", "aarch64"]
 
     executor:
-      name: docker
+      name: << parameters.executor >>
       py-version: "3.7"
       resource: << parameters.resource >>
 
@@ -558,17 +576,8 @@ jobs:
       - conditional-skip
       - config-path
       - when:
-          condition: << parameters.remote-host >>
-          steps:
-            - run:
-                name: "set remote docker host"
-                command: echo 'export DOCKER_HOST=localhost:23760' >> ${BASH_ENV}
-            - run:
-                name: SSH Port Forwarding To ARM Server
-                command: |
-                  ssh -f -N -L 127.0.0.1:23760:localhost:2376 ci@<< parameters.remote-host >>.ci.raiden.network -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
-      - unless:
-          condition: << parameters.remote-host >>
+          condition:
+            equal: [ docker, << parameters.executor >> ]
           steps:
             - setup_remote_docker:
                 docker_layer_caching: true
@@ -578,6 +587,11 @@ jobs:
             source .circleci/get_archive_tag.sh
             make bundle-docker
             echo ${RELEASE_TYPE}/raiden-${ARCHIVE_TAG}-linux-<<parameters.architecture>>.tar.gz > dist/archive/_LATEST-${RELEASE_TYPE}-linux-<<parameters.architecture>>.txt
+      - run:
+          name: Test if Binary can be launched
+          command: |
+            tar -C /tmp -xf dist/archive/raiden-${ARCHIVE_TAG}-linux-<<parameters.architecture>>.tar.gz
+            /tmp/raiden-${ARCHIVE_TAG}-linux-<<parameters.architecture>> --help
       - persist_to_workspace:
           root: "~"
           paths:
@@ -610,6 +624,10 @@ jobs:
             pyinstaller --noconfirm --clean raiden.spec
             zip -9 --junk-paths dist/archive/raiden-${ARCHIVE_TAG}-macOS-x86_64.zip dist/raiden-${ARCHIVE_TAG}-macOS-x86_64
             echo ${RELEASE_TYPE}/raiden-${ARCHIVE_TAG}-macOS-x86_64.zip > dist/archive/_LATEST-${RELEASE_TYPE}-macOS-x86_64.txt
+      - run:
+          name: Test if Binary can be launched
+          command: |
+            dist/raiden-${ARCHIVE_TAG}-macOS-x86_64 --help
       - persist_to_workspace:
           root: "~"
           paths:
@@ -858,24 +876,26 @@ workflows:
           filters:
             <<: *only-tagged-version-filter
 
-      - prepare-python-linux:
+      - prepare-python-macos:
           py-version: '3.7'
           requires:
             - prepare-system-linux
           filters:
             <<: *only-tagged-version-filter
 
-      - prepare-python-macos:
-          py-version: '3.7'
+      - build-binary-linux:
+          name: build-binary-linux-x86_64
           requires:
-            - prepare-system-macos
+            - prepare-system-linux
           filters:
             <<: *only-tagged-version-filter
 
       - build-binary-linux:
-          name: build-binary-linux-x86
+          name: build-binary-linux-aarch64
+          executor: arm
+          architecture: aarch64
           requires:
-            - prepare-python-linux
+            - prepare-system-linux
           filters:
             <<: *only-tagged-version-filter
 
@@ -894,23 +914,26 @@ workflows:
       - deploy-digitalocean:
           context: "raiden-py"
           requires:
-            - build-binary-linux-x86
+            - build-binary-linux-x86_64
+            - build-binary-linux-aarch64
             - build-binary-macos
           filters:
             <<: *only-tagged-version-filter
 
       - deploy-github-release:
           context: "raiden-general-2021"
           requires:
-            - build-binary-linux-x86
+            - build-binary-linux-x86_64
+            - build-binary-linux-aarch64
             - build-binary-macos
           filters:
             <<: *only-tagged-version-filter
 
       - deploy-pypi:
           context: "raiden-py"
           requires:
-            - build-binary-linux-x86
+            - build-binary-linux-x86_64
+            - build-binary-linux-aarch64
             - build-binary-macos
           filters:
             <<: *only-tagged-version-filter

Makefile

@@ -129,7 +129,7 @@ endif
 bundle-docker: ARCHITECTURE_TAG = $(shell docker run --rm python:3.7 uname -m)
 bundle-docker: ARCHIVE_TAG ?= v$(shell python setup.py --version)
 bundle-docker:
-	docker build -t pyinstallerbuilder --build-arg GETH_URL_LINUX=$(GETH_URL_LINUX) --build-arg SOLC_URL_LINUX=$(SOLC_URL_LINUX) --build-arg ARCHITECTURE_TAG=$(ARCHITECTURE_TAG) --build-arg ARCHIVE_TAG=$(ARCHIVE_TAG) $(GITHUB_ACCESS_TOKEN_ARG) -f docker/build.Dockerfile .
+	docker build -t pyinstallerbuilder --build-arg ARCHITECTURE_TAG=$(ARCHITECTURE_TAG) --build-arg ARCHIVE_TAG=$(ARCHIVE_TAG) $(GITHUB_ACCESS_TOKEN_ARG) -f docker/build.Dockerfile .
 	-(docker rm builder)
 	docker create --name builder pyinstallerbuilder
 	mkdir -p dist/archive

docker/build.Dockerfile

@@ -1,25 +1,12 @@
 FROM python:3.7-stretch
 
-# these are defined in .circleci/config.yml and passed here in the makefile
-ARG SOLC_URL_LINUX
-ARG GETH_URL_LINUX
-
 # install dependencies
 RUN apt-get update
 RUN apt-get install -y git-core wget xz-utils build-essential \
     automake pkg-config libtool libffi-dev python3-dev libgmp-dev \
     libavdevice-dev libavfilter-dev libopus-dev libvpx-dev pkg-config \
     libsrtp2-dev
 
-RUN wget -nv -O /usr/bin/solc ${SOLC_URL_LINUX} && \
-    chmod +x /usr/bin/solc
-RUN wget -nv -O /tmp/geth.tar.gz ${GETH_URL_LINUX} && \
-    cd /tmp && \
-    tar xf geth.tar.gz && \
-    mv geth-linux-amd64-*/geth /usr/bin/geth && \
-    rm geth.tar.gz
-
-
 RUN python3 -m venv /venv
 ENV PATH="/venv/bin:$PATH"
 

raiden.spec

@@ -12,18 +12,40 @@ PyInstaller spec file to build single file or dir distributions
 """
 
 # Set to false to produce an exploded single-dir
-ONEFILE = int(os.environ.get('ONEFILE', True))
-
-
-def Entrypoint(dist, group, name, scripts=None, pathex=None, hiddenimports=None, hookspath=None,
-               excludes=None, runtime_hooks=None, datas=None):
+ONEFILE = int(os.environ.get("ONEFILE", True))
+
+# Hack: This is a list of prefixes to be removed from the `binaries`.
+#       We do this to prevent including unnecessary libraries (pyav audio / video dependencies)
+BINARIES_PREFIX_BLOCKLIST = [
+    "libav",
+    "libaom",
+    "libswscale",
+    "libswrescale",
+    "libvorbis",
+    "libx264",
+    "libx265",
+]
+
+
+def Entrypoint(
+    dist,
+    group,
+    name,
+    scripts=None,
+    pathex=None,
+    hiddenimports=None,
+    hookspath=None,
+    excludes=None,
+    runtime_hooks=None,
+    datas=None,
+):
     import pkg_resources
 
     # get toplevel packages of distribution from metadata
     def get_toplevel(dist):
         distribution = pkg_resources.get_distribution(dist)
-        if distribution.has_metadata('top_level.txt'):
-            return list(distribution.get_metadata('top_level.txt').split())
+        if distribution.has_metadata("top_level.txt"):
+            return list(distribution.get_metadata("top_level.txt").split())
         else:
             return []
 
@@ -42,54 +64,75 @@ def Entrypoint(dist, group, name, scripts=None, pathex=None, hiddenimports=None,
     # insert path of the egg at the verify front of the search path
     pathex = [ep.dist.location] + pathex
     # script name must not be a valid module name to avoid name clashes on import
-    script_path = os.path.join(workpath, name + '-script.py')
+    script_path = os.path.join(workpath, name + "-script.py")
     print("creating script for entry point", dist, group, name)
-    with open(script_path, 'w') as fh:
+    with open(script_path, "w") as fh:
         print("import", ep.module_name, file=fh)
-        print("%s.%s()" % (ep.module_name, '.'.join(ep.attrs)), file=fh)
+        print("%s.%s()" % (ep.module_name, ".".join(ep.attrs)), file=fh)
         for package in packages:
             print("import", package, file=fh)
 
-    return Analysis(
+    analysis = Analysis(
         [script_path] + scripts,
         pathex=pathex,
         hiddenimports=hiddenimports,
         hookspath=hookspath,
         excludes=excludes,
         runtime_hooks=runtime_hooks,
-        datas=datas
+        datas=datas,
     )
-
-
-if hasattr(pdb, 'pdb'):
+    # `Analysis.binaries` behaves set-like and matches on the first tuple item (`name`).
+    # Since library names include the version we first build a list of the concrete names
+    # by prefix matching and then subtract that via the set-like behaviour.
+    binaries_to_remove = [
+        (name, None, None)
+        for name, *_ in analysis.binaries
+        if any(name.startswith(blocklist_item) for blocklist_item in BINARIES_PREFIX_BLOCKLIST)
+    ]
+    analysis.binaries -= binaries_to_remove
+    return analysis
+
+
+if hasattr(pdb, "pdb"):
     # pdbpp moves the stdlib pdb to the `pdb` attribute of it's own patched pdb module
     raise RuntimeError(
-        'pdbpp is installed which causes broken PyInstaller builds. Please uninstall it.',
+        "pdbpp is installed which causes broken PyInstaller builds. Please uninstall it.",
     )
 
 
 # We don't need Tk and friends
-sys.modules['FixTk'] = None
+sys.modules["FixTk"] = None
 
-executable_name = 'raiden-{}-{}-{}'.format(
-    os.environ.get('ARCHIVE_TAG', 'v' + get_system_spec()['raiden']),
-    'macOS' if platform.system() == 'Darwin' else platform.system().lower(),
-    platform.machine()
+executable_name = "raiden-{}-{}-{}".format(
+    os.environ.get("ARCHIVE_TAG", "v" + get_system_spec()["raiden"]),
+    "macOS" if platform.system() == "Darwin" else platform.system().lower(),
+    platform.machine(),
 )
 
 a = Entrypoint(
-    'raiden',
-    'console_scripts',
-    'raiden',
-    hookspath=['tools/pyinstaller_hooks'],
+    "raiden",
+    "console_scripts",
+    "raiden",
+    hookspath=["tools/pyinstaller_hooks"],
     runtime_hooks=[
-        'tools/pyinstaller_hooks/runtime_gevent_monkey.py',
-        'tools/pyinstaller_hooks/runtime_encoding.py',
-        'tools/pyinstaller_hooks/runtime_raiden_contracts.py',
+        "tools/pyinstaller_hooks/runtime_gevent_monkey.py",
+        "tools/pyinstaller_hooks/runtime_encoding.py",
+        "tools/pyinstaller_hooks/runtime_raiden_contracts.py",
     ],
-    hiddenimports=['scrypt', 'eth_tester'],
+    hiddenimports=[],
     datas=[],
-    excludes=['FixTk', 'tcl', 'tk', '_tkinter', 'tkinter', 'Tkinter'],
+    excludes=[
+        "_tkinter",
+        "FixTk",
+        "ipython",
+        "jupyter",
+        "jupyter_core",
+        "notebook",
+        "tcl",
+        "tk",
+        "tkinter",
+        "Tkinter",
+    ],
 )
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=None)
@@ -103,10 +146,10 @@ if ONEFILE:
         a.datas,
         name=executable_name,
         debug=False,
-        strip=True,
+        strip=False,
         upx=False,
         runtime_tmpdir=None,
-        console=True
+        console=True,
     )
 else:
     exe = EXE(
@@ -117,7 +160,7 @@ else:
         debug=True,
         strip=False,
         upx=False,
-        console=True
+        console=True,
     )
     coll = COLLECT(
         exe,
@@ -126,5 +169,5 @@ else:
         a.datas,
         strip=False,
         upx=False,
-        name='raiden'
+        name="raiden",
     )

tools/pyinstaller_hooks/hook-av.py

@@ -0,0 +1,4 @@
+from PyInstaller.utils.hooks import collect_submodules
+
+
+hiddenimports = collect_submodules("av")

tools/pyinstaller_hooks/runtime_gevent_monkey.py

@@ -1,7 +1,3 @@
-from gevent.monkey import patch_all  # isort:skip # noqa
+from gevent.monkey import patch_all
 
-patch_all()  # isort:skip # noqa
-
-from raiden.network.transport.matrix.rtc.utils import setup_asyncio_event_loop
-
-setup_asyncio_event_loop()
+patch_all()