Build windows deps ourselves

.github/workflows/build-windows.yml

@@ -36,8 +36,69 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  deps:
+    name: ${{ matrix.name }} deps
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        # make i686 deps and x86_64 deps
+        include:
+          - {
+            name: "64-bit",
+            winarch: x86_64,
+            msystem: MINGW64
+          }
+          - {
+            name: "32-bit",
+            winarch: i686,
+            msystem: MINGW32
+          }
+
+    steps:
+      - run: git config --global core.autocrlf input  # do not introduce carriage returns
+      - uses: actions/[email protected]
+
+      - name: Test for Win Deps cache hit
+        id: windep-cache
+        uses: actions/[email protected]
+        with:
+          path: ${{ github.workspace }}/pygame_win_deps_${{ matrix.winarch }}
+          # The hash of all files in buildconfig manylinux-build and windependencies is
+          # the key to the cache. If anything changes here, the deps are built again
+          key: windep-${{ hashFiles('buildconfig/manylinux-build/**') }}-${{ hashFiles('buildconfig/windependencies/*.sh') }}-${{ matrix.winarch }}
+          lookup-only: true
+
+      - uses: msys2/setup-msys2@v2
+        with:
+          msystem: ${{ matrix.msystem }}
+          install: >-
+            mingw-w64-${{ matrix.winarch }}-gcc
+            mingw-w64-${{ matrix.winarch }}-cmake
+            mingw-w64-${{ matrix.winarch }}-meson
+            mingw-w64-${{ matrix.winarch }}-autotools
+            mingw-w64-${{ matrix.winarch }}-nasm
+            mingw-w64-${{ matrix.winarch }}-python
+            mingw-w64-${{ matrix.winarch }}-python-setuptools
+
+      # build win deps on cache miss
+      - name: Build Win Deps
+        if: steps.windep-cache.outputs.cache-hit != 'true'
+        shell: msys2 {0}
+        run: |
+          export WIN_ARCH="${{ matrix.winarch }}"
+          cd buildconfig/windependencies
+          bash ./build_win_deps.sh
+
+      # Uncomment when you want to manually verify the deps by downloading them
+      - name: Upload win deps
+        uses: actions/upload-artifact@v3
+        with:
+          name: pygame-win-deps-${{ matrix.winarch }}
+          path: ${{ github.workspace }}/pygame_win_deps_${{ matrix.winarch }}
+
   build:
     name: ${{ matrix.winarch }}
+    needs: deps
     runs-on: windows-latest
     strategy:
       fail-fast: false  # if a particular matrix build fails, don't skip the rest

buildconfig/manylinux-build/docker_base/gettext/build-gettext.sh

@@ -9,12 +9,11 @@ GETTEXT=gettext-0.21
 curl -sL --retry 10 https://ftp.gnu.org/gnu/gettext/${GETTEXT}.tar.gz > ${GETTEXT}.tar.gz
 sha512sum -c gettext.sha512
 
-if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-      # linux
-      export GETTEXT_CONFIGURE=
-elif [[ "$OSTYPE" == "darwin"* ]]; then
+if [[ "$OSTYPE" == "darwin"* ]]; then
       # Mac OSX, ship libintl.h on mac.
       export GETTEXT_CONFIGURE=--with-included-gettext
+else
+      export GETTEXT_CONFIGURE=
 fi
 
 tar xzf ${GETTEXT}.tar.gz

buildconfig/manylinux-build/docker_base/libtiff/build-tiff.sh

@@ -13,7 +13,7 @@ cd $TIFF
 
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then
     ./configure $PG_BASE_CONFIGURE_FLAGS --disable-lzma --disable-webp --disable-zstd
-elif [[ "$OSTYPE" == "darwin"* ]]; then
+else
     # Use CMake on macOS because arm64 builds fail with weird errors in ./configure
     cmake . $PG_BASE_CMAKE_FLAGS -Dlzma=OFF -Dwebp=OFF -Dzstd=OFF
 fi

buildconfig/manylinux-build/docker_base/ogg/build-ogg.sh

@@ -22,6 +22,13 @@ cd ..
 tar xzf ${VORBIS}.tar.gz
 cd $VORBIS
 
+# some hackery needed to make libvorbis build under mingw
+case "$OSTYPE" in
+  msys|mingw32|mingw64)
+    sed -i '/LIBRARY/d' win32/*.def
+    ;;
+esac
+
 cmake . $PG_BASE_CMAKE_FLAGS
 make
 make install

buildconfig/manylinux-build/docker_base/portmidi/build-portmidi.sh

@@ -3,10 +3,11 @@ set -e -x
 
 cd $(dirname `readlink -f "$0"`)
 
-PORTMIDI_VER="2.0.4"
+# 2.0.4 has compilation issues on windows, so pin to latest commit on github
+PORTMIDI_VER="7a5de5b7597c46f963d72a83defe7592f901e5f1"
 PORTMIDI="portmidi-${PORTMIDI_VER}"
 
-curl -sL --retry 10 https://github.com/PortMidi/portmidi/archive/refs/tags/v${PORTMIDI_VER}.tar.gz> ${PORTMIDI}.tar.gz
+curl -sL --retry 10 https://github.com/PortMidi/portmidi/archive/$PORTMIDI_VER.tar.gz > ${PORTMIDI}.tar.gz
 sha512sum -c portmidi.sha512
 
 tar xzf ${PORTMIDI}.tar.gz

buildconfig/manylinux-build/docker_base/portmidi/portmidi.sha512

@@ -1 +1 @@
-d9f22d161e1dd9a4bde1971bb2b6e5352da51545f4fe5ecad11c55e7a535f0d88efce18d1c8fd91e93b70a7926150f86a0f53972ad92370e86556a8dd72dc194  portmidi-2.0.4.tar.gz
+b8ccc72dc7e199266e4bfbaa777b1979c5fc1d97e7dcb10707ec0c2325abe79c036e6939b79f97fea6c9602620d41e6659cd60b0bea4c59d8fb95e478ba0bd75  portmidi-7a5de5b7597c46f963d72a83defe7592f901e5f1.tar.gz

buildconfig/manylinux-build/docker_base/sdl_libs/build-sdl2-libs.sh

@@ -20,10 +20,18 @@ curl -sL --retry 10 https://github.com/libsdl-org/SDL_mixer/releases/download/re
 curl -sL --retry 10 https://github.com/libsdl-org/SDL_ttf/releases/download/release-$TTF2_VER/$TTF2.tar.gz > ${TTF2}.tar.gz
 sha512sum -c sdl2.sha512
 
-# On mac/manylinux we have to make use of standard dynamic linking rather than
-# dlopen-ing the library itself. This is important for when auditwheel/delocate
-# moves libraries into the wheel.
-PG_DEPS_SHARED=0
+case "$OSTYPE" in
+  msys|mingw32|mingw64)
+    # on windows, do SDL-style shared deps (SDLs default strategy)
+    PG_DEPS_SHARED=1
+    ;;
+  *)
+    # On mac/manylinux we have to make use of standard dynamic linking rather than
+    # dlopen-ing the library itself. This is important for when auditwheel/delocate
+    # moves libraries into the wheel.
+    PG_DEPS_SHARED=0
+    ;;
+esac
 
 # Build SDL
 tar xzf ${SDL2}.tar.gz

buildconfig/windependencies/build_win_deps.sh

@@ -0,0 +1,84 @@
+# This uses manylinux build scripts to build dependencies on windows.
+
+set -e -x
+
+# This is needed for tar to work in some places
+export MSYS=winsymlinks:lnk
+
+# The below three lines convert something like D:\path\goes\here to /d/path/goes/here
+export BASE_DIR=$(echo "$GITHUB_WORKSPACE" | tr '[:upper:]' '[:lower:]')
+export BASE_DIR="${BASE_DIR//\\//}"  # //\\// replaces all \ with / in the variable
+export BASE_DIR="/${BASE_DIR//:/}"  # remove colon from drive part, add leading /
+
+export PG_DEP_PREFIX="$BASE_DIR/pygame_win_deps_$WIN_ARCH"
+
+export PKG_CONFIG_PATH="$PG_DEP_PREFIX/lib/pkgconfig:$PKG_CONFIG_PATH"
+
+mkdir $PG_DEP_PREFIX
+
+# for great speed.
+export MAKEFLAGS="-j 4"
+
+# for scripts using ./configure
+export CC="gcc"
+export CXX="g++"
+
+# With this we
+# 1) Force install prefix to $PG_DEP_PREFIX
+# 2) use lib directory within $PG_DEP_PREFIX (and not lib64)
+# 3) make release binaries
+# 4) build shared libraries
+# 5) make cmake use gcc/g++/make
+export PG_BASE_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PG_DEP_PREFIX \
+    -DCMAKE_INSTALL_LIBDIR:PATH=lib \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_SHARED_LIBS=true \
+    -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_MAKE_PROGRAM=make"
+
+export CMAKE_GENERATOR="MSYS Makefiles"
+
+export PG_BASE_CONFIGURE_FLAGS="--prefix=$PG_DEP_PREFIX"
+
+export PG_BASE_MESON_FLAGS="--prefix=$PG_DEP_PREFIX \
+    -Dlibdir=lib \
+    -Dbuildtype=release \
+    -Ddefault_library=shared"
+
+cd ../manylinux-build/docker_base
+
+# Now start installing dependencies
+# ---------------------------------
+
+# install some buildtools
+# bash buildtools/install.sh
+
+# sdl_image deps
+bash zlib-ng/build-zlib-ng.sh
+bash libpng/build-png.sh # depends on zlib
+bash libjpegturbo/build-jpeg-turbo.sh
+bash libtiff/build-tiff.sh
+bash libwebp/build-webp.sh
+
+# freetype (also sdl_ttf dep)
+bash brotli/build-brotli.sh
+bash bzip2/build-bzip2.sh
+bash freetype/build-freetype.sh
+
+# sdl_mixer deps
+bash libxmp/build-libxmp.sh
+bash ogg/build-ogg.sh
+bash flac/build-flac.sh
+bash mpg123/build-mpg123.sh
+bash opus/build-opus.sh # needs libogg (which is a container format)
+bash wavpack/build-wavpack.sh
+
+# fluidsynth (for sdl_mixer)
+# bash gettext/build-gettext.sh
+bash glib/build-glib.sh
+bash sndfile/build-sndfile.sh
+bash fluidsynth/build-fluidsynth.sh
+
+bash sdl_libs/build-sdl2-libs.sh
+
+# for pygame.midi
+bash portmidi/build-portmidi.sh

buildconfig/windependencies/clean_windows_deps.py

@@ -0,0 +1,58 @@
+"""
+Cleans self-built windows deps, to only retain what's needed for the build
+"""
+
+import sys
+import pathlib
+import shutil
+
+base_dir = pathlib.Path(sys.argv[1])
+
+# delete 'share' and 'man'
+shutil.rmtree(base_dir / "share")
+shutil.rmtree(base_dir / "man")
+
+
+# delete all headers and import lib in 'include' and 'lib' except the ones we
+# need
+def clean_dir(dir: pathlib.Path, allowlist: set[str]):
+    for path in dir.iterdir():
+        if path.name in allowlist:
+            continue
+
+        if path.is_dir():
+            shutil.rmtree(path)
+        else:
+            path.unlink()
+
+
+clean_dir(
+    base_dir / "include",
+    {
+        "SDL2",
+        "freetype2",
+        "portmidi.h",
+        "porttime.h",
+        "pmutil.h",
+    },
+)
+
+clean_dir(
+    base_dir / "lib",
+    {
+        "libSDL2.dll.a",
+        "libSDL2_ttf.dll.a",
+        "libSDL2_image.dll.a",
+        "libSDL2_mixer.dll.a",
+        "libportmidi.dll.a",
+        "libfreetype.dll.a",
+    },
+)
+
+
+# copy all dlls from 'bin' into 'lib', and then delete 'bin'
+bindir = base_dir / "bin"
+for dll in bindir.glob("*.dll"):
+    shutil.copy(dll, base_dir / "lib")
+
+shutil.rmtree(bindir)