From 383d2e5ca723e34f1d2267fe34559ac3042b031c Mon Sep 17 00:00:00 2001 From: Er2 Date: Sat, 20 May 2023 12:48:15 +0300 Subject: [PATCH 1/3] Improve CI This commit improves GitHub Actions by using matrix and sending artifacts. --- .github/workflows/build.yml | 198 ++++++++++++++++---------------- .github/workflows/tests.yml | 112 +++++++++++------- scripts/build-android-armv7a.sh | 9 -- scripts/build-macos-amd64.sh | 8 -- scripts/build-ubuntu-amd64.sh | 8 -- scripts/build-ubuntu-i386.sh | 10 -- scripts/tests-macos-amd64.sh | 7 -- scripts/tests-ubuntu-amd64.sh | 10 -- scripts/tests-ubuntu-i386.sh | 11 -- 9 files changed, 165 insertions(+), 208 deletions(-) delete mode 100755 scripts/build-android-armv7a.sh delete mode 100755 scripts/build-macos-amd64.sh delete mode 100755 scripts/build-ubuntu-amd64.sh delete mode 100755 scripts/build-ubuntu-i386.sh delete mode 100755 scripts/tests-macos-amd64.sh delete mode 100755 scripts/tests-ubuntu-amd64.sh delete mode 100755 scripts/tests-ubuntu-i386.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39c8aadd9..3a6817dcc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,111 +1,107 @@ name: Build on: [push, pull_request] +env: + WAF_FLAGS: -T release --disable-warns --prefix=build_hl2 jobs: - build-linux-i386: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Build linux-i386 - run: | - scripts/build-ubuntu-i386.sh - - build-linux-amd64: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - name: Build linux-amd64 - run: | - scripts/build-ubuntu-amd64.sh - - build-android-armv7a: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - name: Build android-armv7a - run: | - scripts/build-android-armv7a.sh - - build-windows-i386: - runs-on: windows-2019 - - steps: - - uses: actions/checkout@v2 - - name: Build windows-i386 - run: | - git submodule init && git submodule update - ./waf.bat configure -T debug - ./waf.bat build - - build-windows-amd64: - runs-on: windows-2019 - - steps: - - uses: actions/checkout@v2 - - name: Build windows-amd64 - run: | - git submodule init && git submodule update - ./waf.bat configure -T debug -8 - ./waf.bat build - - build-dedicated-windows-i386: - runs-on: windows-2019 - - steps: - - uses: actions/checkout@v2 - - name: Build dedicated windows-i386 - run: | - git submodule init && git submodule update - ./waf.bat configure -T debug -d - ./waf.bat build - - build-dedicated-windows-amd64: - runs-on: windows-2019 - + build-linux: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + bits: ['', '--64bits'] + android: [''] + flags: ['', '-d'] + include: + - os: ubuntu-20.04 + bits: '' + android: '--android=armeabi-v7a-hard,4.9,21 --togles' + flags: '' + runs-on: ${{ matrix.os }} + env: + DEPS: libopenal-dev libpng-dev libjpeg-dev libfreetype6-dev libfontconfig1-dev libcurl4-gnutls-dev libsdl2-dev zlib1g-dev libbz2-dev libedit-dev steps: - - uses: actions/checkout@v2 - - name: Build dedicated windows-amd64 - run: | - git submodule init && git submodule update - ./waf.bat configure -T debug -d -8 - ./waf.bat build - - build-dedicated-linux-i386: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - name: Build dedicated linux-i386 - run: | - scripts/build-ubuntu-i386.sh -d - - build-dedicated-linux-amd64: - runs-on: ubuntu-20.04 - - steps: - - uses: actions/checkout@v2 - - name: Build dedicated linux-amd64 - run: | - scripts/build-ubuntu-amd64.sh -d - - build-macos-amd64: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install common dependencies + run: sudo apt-get update && sudo apt-get install -y g++-multilib gcc-multilib + - name: Install 32bit dependencies + if: ${{ matrix.bits == '' && !matrix.android }} + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update + sudo apt-get install -y aptitude + sudo aptitude install -y $(echo $DEPS | sed -r 's/[a-z0-9_\-]+/&:i386/g') + echo "PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig" >> $GITHUB_ENV + - name: Install 64bit dependencies + if: ${{ matrix.bits != '' }} + run: | + sudo apt-get install -y $DEPS + - name: Install Android dependencies + if: ${{ matrix.android != '' }} + run: | + wget https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip -o /dev/null + unzip android-ndk-r10e-linux-x86_64.zip + export NDK_HOME=$PWD/android-ndk-r10e/ + echo "NDK_HOME=$NDK_HOME" >> $GITHUB_ENV + echo "ANDROID_NDK_HOME=$NDK_HOME" >> $GITHUB_ENV + + - name: Configure + run: ./waf configure ${{ matrix.bits }} ${{ matrix.android }} ${{ matrix.flags }} $WAF_FLAGS + - name: Build ${{ matrix.os }} + run: ./waf install + - name: Tar binaries + run: tar cvf build.tar build_hl2 + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: Linux${{matrix.bits}}${{matrix.flags}}${{matrix.android}}.tar + path: build.tar + + build-macos: + strategy: + fail-fast: false + matrix: + flags: ['', '-d'] runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - name: Build macos-amd64 - run: | - scripts/build-macos-amd64.sh + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install dependencies + run: brew install sdl2 + + - name: Configure + run: ./waf configure --64bits ${{ matrix.flags }} $WAF_FLAGS + - name: Build macos-amd64 + run: ./waf install + - name: Tar binaries + run: tar cvf build.tar build_hl2 + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: macOS--64bits${{matrix.flags}}.tar + path: build.tar + + build-windows: + strategy: + fail-fast: false + matrix: + os: [windows-2019] + bits: ['', '--64bits'] + flags: ['', '-d'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive - build-dedicated-macos-amd64: - runs-on: macos-latest + - name: Configure + run: ./waf configure ${{ matrix.bits }} ${{ matrix.flags }} ${{ env.WAF_FLAGS }} + - name: Build ${{ matrix.os }} + run: ./waf install - steps: - - uses: actions/checkout@v2 - - name: Build dedicated macos-amd64 - run: | - scripts/build-macos-amd64.sh -d diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8adc5ab0..74d878d51 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,59 +1,83 @@ name: Tests on: [push, pull_request] +env: + WAF_FLAGS: -T release --disable-warns --prefix=build_tests --tests --sanitize=address,undefined jobs: - tests-linux-i386: - runs-on: ubuntu-20.04 - + tests-linux: + strategy: + matrix: + os: [ubuntu-latest, ubuntu-20.04] + bits: ['', '--64bits'] + android: [''] + runs-on: ${{ matrix.os }} + env: + DEPS: libbz2-dev steps: - - uses: actions/checkout@v2 - - name: Run tests linux-i386 - run: | - scripts/tests-ubuntu-i386.sh + - uses: actions/checkout@v3 + with: + submodules: recursive - tests-linux-amd64: - runs-on: ubuntu-20.04 + - name: Install common dependencies + run: sudo apt-get update && sudo apt-get install -y g++-multilib gcc-multilib + - name: Install 32bit dependencies + if: ${{ matrix.bits == '' && !matrix.android }} + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update + sudo apt-get install aptitude + sudo aptitude install -y $(echo $DEPS | sed -r 's/[a-z0-9_\-]+/&:i386/g') + echo "PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig" >> $GITHUB_ENV + - name: Install 64bit dependencies + if: ${{ matrix.bits != '' }} + run: | + sudo apt-get install -y $DEPS - steps: - - uses: actions/checkout@v2 - - name: Run tests linux-amd64 - run: | - scripts/tests-ubuntu-amd64.sh + - name: Configure + run: ./waf configure ${{ matrix.bits }} $WAF_FLAGS + - name: Build ${{ matrix.os }} + run: ./waf install + - name: Run tests + run: | + cd build_tests + LD_LIBRARY_PATH=bin/ ./unittest - tests-macos-amd64: + tests-macos: runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - name: Run tests macos-amd64 - run: | - scripts/tests-macos-amd64.sh + - uses: actions/checkout@v3 + with: + submodules: recursive - tests-windows-i386: - runs-on: windows-2019 + - name: Configure + run: ./waf configure --64bits $WAF_FLAGS + - name: Build macos-amd64 + run: ./waf install + - name: Run tests + run: | + cd build_tests + DYLD_LIBRARY_PATH=bin/ ./unittest + tests-windows: + strategy: + matrix: + os: [windows-2019] + bits: ['', '--64bits'] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Run tests windows-i386 - run: | - git submodule init && git submodule update - ./waf.bat configure -T release --tests --prefix=out/ - ./waf.bat install - cd out - $env:Path = "bin"; - ./unittest.exe - - tests-windows-amd64: - runs-on: windows-2019 + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Configure + run: ./waf configure ${{ matrix.bits }} ${{ env.WAF_FLAGS }} + + - name: Build ${{ matrix.os }} + run: ./waf install + - name: Run tests + run: | + cd build_tests + $env:Path = "bin"; + ./unittest.exe - steps: - - uses: actions/checkout@v2 - - name: Run tests windows-amd64 - run: | - git submodule init && git submodule update - ./waf.bat configure -T release --tests --prefix=out/ -8 - ./waf.bat install - cd out - $env:Path = "bin"; - ./unittest.exe diff --git a/scripts/build-android-armv7a.sh b/scripts/build-android-armv7a.sh deleted file mode 100755 index 4c7201ff9..000000000 --- a/scripts/build-android-armv7a.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -git submodule init && git submodule update -wget https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip -o /dev/null -unzip android-ndk-r10e-linux-x86_64.zip -export ANDROID_NDK_HOME=$PWD/android-ndk-r10e/ -export NDK_HOME=$PWD/android-ndk-r10e/ -./waf configure -T debug --android=armeabi-v7a-hard,4.9,21 --togles --disable-warns && -./waf build diff --git a/scripts/build-macos-amd64.sh b/scripts/build-macos-amd64.sh deleted file mode 100755 index de0e8cfd1..000000000 --- a/scripts/build-macos-amd64.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -git submodule init && git submodule update - -brew install sdl2 - -./waf configure -T debug --64bits --disable-warns $* && -./waf build diff --git a/scripts/build-ubuntu-amd64.sh b/scripts/build-ubuntu-amd64.sh deleted file mode 100755 index e598240dc..000000000 --- a/scripts/build-ubuntu-amd64.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -git submodule init && git submodule update -sudo apt-get update -sudo apt-get install -f -y libopenal-dev g++-multilib gcc-multilib libpng-dev libjpeg-dev libfreetype6-dev libfontconfig1-dev libcurl4-gnutls-dev libsdl2-dev zlib1g-dev libbz2-dev libedit-dev - -./waf configure -T debug --64bits --disable-warns $* && -./waf build diff --git a/scripts/build-ubuntu-i386.sh b/scripts/build-ubuntu-i386.sh deleted file mode 100755 index c9609cae7..000000000 --- a/scripts/build-ubuntu-i386.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -git submodule init && git submodule update -sudo dpkg --add-architecture i386 -sudo apt-get update -sudo apt-get install -y aptitude -sudo aptitude install -y libopenal-dev:i386 g++-multilib gcc-multilib libpng-dev:i386 libjpeg-dev:i386 libfreetype6-dev:i386 libfontconfig1-dev:i386 libcurl4-gnutls-dev:i386 libsdl2-dev:i386 zlib1g-dev:i386 libbz2-dev:i386 libedit-dev:i386 - -PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig ./waf configure -T debug --disable-warns $* && -./waf build diff --git a/scripts/tests-macos-amd64.sh b/scripts/tests-macos-amd64.sh deleted file mode 100755 index ff4e46b27..000000000 --- a/scripts/tests-macos-amd64.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -git submodule init && git submodule update -./waf configure -T release --sanitize=address,undefined --disable-warns --tests -8 --prefix=out/ $* && -./waf install && -cd out && -DYLD_LIBRARY_PATH=bin/ ./unittest || exit 1 diff --git a/scripts/tests-ubuntu-amd64.sh b/scripts/tests-ubuntu-amd64.sh deleted file mode 100755 index 1cef058e9..000000000 --- a/scripts/tests-ubuntu-amd64.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -git submodule init && git submodule update -sudo apt-get update -sudo apt-get install -y libbz2-dev - -./waf configure -T release --sanitize=address,undefined --disable-warns --tests --prefix=out/ --64bits $* && -./waf install && -cd out && -LD_LIBRARY_PATH=bin/ ./unittest diff --git a/scripts/tests-ubuntu-i386.sh b/scripts/tests-ubuntu-i386.sh deleted file mode 100755 index f14fc3616..000000000 --- a/scripts/tests-ubuntu-i386.sh +++ /dev/null @@ -1,11 +0,0 @@ -install#!/bin/sh - -git submodule init && git submodule update -sudo dpkg --add-architecture i386 -sudo apt-get update -sudo apt-get install -y g++-multilib gcc-multilib libbz2-dev:i386 - -PKG_CONFIG_PATH=/usr/lib/i386-linux-gnu/pkgconfig ./waf configure -T release --sanitize=address,undefined --disable-warns --tests --prefix=out/ $* && -./waf install && -cd out && -LD_LIBRARY_PATH=bin/ ./unittest From c7895b28ed86e3f7ebe57a7897446d7489f61fd7 Mon Sep 17 00:00:00 2001 From: Er2 Date: Sat, 20 May 2023 13:26:30 +0300 Subject: [PATCH 2/3] Use strip to reduce artifacts size --- .github/workflows/build.yml | 38 ++++++++++++++++--------- scripts/waifulib/strip_on_install_v2.py | 21 ++++++++++---- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3a6817dcc..c7103d5ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,7 +2,7 @@ name: Build on: [push, pull_request] env: - WAF_FLAGS: -T release --disable-warns --prefix=build_hl2 + WAF_FLAGS: -T debug --disable-warns --prefix=build_hl2 jobs: build-linux: @@ -52,14 +52,20 @@ jobs: - name: Configure run: ./waf configure ${{ matrix.bits }} ${{ matrix.android }} ${{ matrix.flags }} $WAF_FLAGS - name: Build ${{ matrix.os }} - run: ./waf install - - name: Tar binaries - run: tar cvf build.tar build_hl2 - - name: Upload artifact + run: ./waf install --strip-to-file + - name: Upload build + uses: actions/upload-artifact@v3 + with: + name: linux${{matrix.bits}}${{matrix.flags}}${{matrix.android}} + path: | + build_hl2 + !build_hl2/**/*.debug + - name: Upload debug symbols uses: actions/upload-artifact@v3 with: - name: Linux${{matrix.bits}}${{matrix.flags}}${{matrix.android}}.tar - path: build.tar + name: debug-Linux${{matrix.bits}}${{matrix.flags}}${{matrix.android}} + path: | + build_hl2/**/*.debug build-macos: strategy: @@ -78,14 +84,20 @@ jobs: - name: Configure run: ./waf configure --64bits ${{ matrix.flags }} $WAF_FLAGS - name: Build macos-amd64 - run: ./waf install - - name: Tar binaries - run: tar cvf build.tar build_hl2 - - name: Upload artifact + run: ./waf install --strip-to-file + - name: Upload build + uses: actions/upload-artifact@v3 + with: + name: macOS--64bits${{matrix.flags}}${{matrix.android}} + path: | + build_hl2 + !build_hl2/**/*.dSYM + - name: Upload debug symbols uses: actions/upload-artifact@v3 with: - name: macOS--64bits${{matrix.flags}}.tar - path: build.tar + name: debug-macOS--64bits${{matrix.flags}}${{matrix.android}} + path: | + build_hl2/**/*.dSYM build-windows: strategy: diff --git a/scripts/waifulib/strip_on_install_v2.py b/scripts/waifulib/strip_on_install_v2.py index 3258b2eb0..d8b94055d 100644 --- a/scripts/waifulib/strip_on_install_v2.py +++ b/scripts/waifulib/strip_on_install_v2.py @@ -25,6 +25,8 @@ def configure(conf): if conf.env.DEST_BINFMT == 'mac-o': conf.env.STRIPFLAGS += ['-x'] + conf.find_program('dsymutil', var='DSYMUTIL') + conf.env.STRIPFLAGS += ['-S'] # we will use .dSYM instead return # macOS don't have objcopy # a1ba: I am lazy to add `export OBJCOPY=` everywhere in my scripts @@ -56,12 +58,19 @@ def copy_fun(self, src, tgt): c3 = Logs.colors.YELLOW c4 = Logs.colors.BLUE try: - if self.generator.bld.options.strip_to_file and self.env.DEST_BINFMT == 'elf': - ocopy_cmd = self.env.OBJCOPY + ['--only-keep-debug', tgt, tgt_debug] - self.generator.bld.cmd_and_log(ocopy_cmd, output=Context.BOTH, quiet=Context.BOTH) - if not self.generator.bld.progress_bar: - Logs.info('%s+ objcopy --only-keep-debug %s%s%s %s%s%s', c1, c4, tgt, c1, c3, tgt_debug, c1) - + if self.generator.bld.options.strip_to_file: + if self.env.DEST_BINFMT == 'elf': + ocopy_cmd = self.env.OBJCOPY + ['--only-keep-debug', tgt, tgt_debug] + self.generator.bld.cmd_and_log(ocopy_cmd, output=Context.BOTH, quiet=Context.BOTH) + if not self.generator.bld.progress_bar: + Logs.info('%s+ objcopy --only-keep-debug %s%s%s %s%s%s', c1, c4, tgt, c1, c3, tgt_debug, c1) + elif self.env.DEST_BINFMT == 'mac-o': + tgt_debug = os.path.splitext(tgt)[0] + '.dSYM' + dsymutil_cmd = self.env.DSYMUTIL + [tgt, '-o', tgt_debug] + self.generator.bld.cmd_and_log(dsymutil_cmd, output=Context.BOTH, quiet=Context.BOTH) + if not self.generator.bld.progress_bar: + Logs.info('%s+ dsymutil %s%s%s -o %s%s%s', c1, c4, tgt, c1, c3, tgt_debug, c1) + strip_cmd = self.env.STRIP + self.env.STRIPFLAGS + [tgt] self.generator.bld.cmd_and_log(strip_cmd, output=Context.BOTH, quiet=Context.BOTH) if not self.generator.bld.progress_bar: From 6ea7368483b40dbdfec431873edafacdacb9e8e4 Mon Sep 17 00:00:00 2001 From: Er2 Date: Sun, 21 May 2023 14:28:24 +0300 Subject: [PATCH 3/3] add objcopy for android --- scripts/waifulib/xcompile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/waifulib/xcompile.py b/scripts/waifulib/xcompile.py index 2db9b7f9d..f5994cbab 100644 --- a/scripts/waifulib/xcompile.py +++ b/scripts/waifulib/xcompile.py @@ -211,6 +211,11 @@ def strip(self): return 'llvm-strip' return os.path.join(self.gen_binutils_path(), 'strip') + def objcopy(self): + if self.is_host(): + return 'llvm-objcopy' + return os.path.join(self.gen_binutils_path(), 'objcopy') + def system_stl(self): # TODO: proper STL support return [ @@ -341,6 +346,7 @@ def configure(conf): conf.environ['CC'] = android.cc() conf.environ['CXX'] = android.cxx() conf.environ['STRIP'] = android.strip() + conf.environ['OBJCOPY'] = android.objcopy() conf.env.CFLAGS += android.cflags() conf.env.CXXFLAGS += android.cflags(True) conf.env.LINKFLAGS += android.linkflags()