Merge pull request #618 from KomodoPlatform/patch-macos-build-clang

build: use native clang as darwin compiler
KomodoPlatform · Aug 19, 2024 · 6697cef · 6697cef
2 parents 3b3b2d5 + 9a79ff0
commit 6697cef
Show file tree

Hide file tree

Showing 21 changed files with 138 additions and 99 deletions.
diff --git a/.github/workflows/komodo_linux_ci.yml b/.github/workflows/komodo_linux_ci.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install deps (Linux)
 
@@ -54,7 +54,7 @@ jobs:
     needs: linux-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install deps (Dice, Token, Faucet, Rewards CC)
         run: |
@@ -91,7 +91,7 @@ jobs:
     needs: linux-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install deps (OraclesCC)
         run: |
@@ -128,7 +128,7 @@ jobs:
     needs: linux-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install deps (BasicRPC)
         run: |
@@ -165,7 +165,7 @@ jobs:
     needs: linux-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install deps (ChannelsCC)
         run: |
@@ -202,7 +202,7 @@ jobs:
     needs: linux-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Install deps (HeirCC)
         run: |

diff --git a/.github/workflows/komodo_mac_ci.yml b/.github/workflows/komodo_mac_ci.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: macos-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # Workaround for https://github.com/actions/setup-python/issues/577
       - name: Clean up binaries and links (macOS)
@@ -31,29 +31,30 @@ jobs:
       - name: Install deps (macOS)
         run: |
           brew update
-          brew upgrade || true
           brew tap discoteq/discoteq; brew install flock
           brew install autoconf autogen automake
-          brew install gcc@8
           brew install binutils
           brew install protobuf
           brew install coreutils
           brew install wget
           brew install python3
-          brew install gmp
 
+      # https://www.jessesquires.com/blog/2020/01/06/selecting-an-xcode-version-on-github-ci/
+      # https://github.com/actions/runner-images/issues/2619 (bsdtar issues)
+      # flag for some CC tests transactions - so DO NOT USE THIS CI ARTIFACTS IN PRODUCTION!!!
       - name: Build (macOS)
         run: |
-          # flag for some CC tests transactions - so DO NOT USE THIS CI ARTIFACTS IN PRODUCTION!!!
-          export CPATH=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/
           ./zcutil/build-mac-dtest.sh -j4
-          tar -czvf komodo-macos.tar.gz src/komodod src/komodo-cli
+          gtar -czvf komodo-macos.tar.gz src/komodod src/komodo-cli src/wallet-utility src/komodo-tx
+        # env:
+        #   DEVELOPER_DIR: /Applications/Xcode_13.2.1.app/Contents/Developer
 
       - name: Upload komodo-macos.tar.gz as artifact
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: komodo-macos
-          path: ./komodo-macos.tar.gz
+          path: |
+            ./komodo-macos.tar.gz
 
   macos-test-dice-token-reards-faucet-cc:
     if: ${{ false }}
@@ -63,7 +64,7 @@ jobs:
     needs: macos-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install deps (Dice, Token, Faucet, Rewards CC)
         run: |
@@ -72,7 +73,7 @@ jobs:
           python3 -m pip install slick-bitcoinrpc pytest wget jsonschema
 
       - name: Download komodo-macos.tar.gz
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: komodo-macos
 
@@ -93,15 +94,15 @@ jobs:
     needs: macos-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install deps (OraclesCC)
         run: |
           brew install python3 curl
           python3 -m pip install setuptools wheel
           python3 -m pip install slick-bitcoinrpc pytest wget jsonschema
       - name: Download komodo-macos.tar.gz
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: komodo-macos
 
@@ -122,15 +123,15 @@ jobs:
     needs: macos-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install deps (BasicRPC)
         run: |
           brew install python3 curl
           python3 -m pip install setuptools wheel
           python3 -m pip install slick-bitcoinrpc pytest wget jsonschema
       - name: Download komodo-macos.tar.gz
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: komodo-macos
 
@@ -151,15 +152,15 @@ jobs:
     needs: macos-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install deps (ChannelsCC)
         run: |
           brew install python3 curl
           python3 -m pip install setuptools wheel
           python3 -m pip install slick-bitcoinrpc pytest wget jsonschema
       - name: Download komodo-macos.tar.gz
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: komodo-macos
 
@@ -180,15 +181,15 @@ jobs:
     needs: macos-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install deps (HeirCC)
         run: |
           brew install python3 curl
           python3 -m pip install setuptools wheel
           python3 -m pip install slick-bitcoinrpc pytest wget jsonschema
       - name: Download komodo-macos.tar.gz
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: komodo-macos
 

diff --git a/.github/workflows/komodo_win_ci.yml b/.github/workflows/komodo_win_ci.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Install deps(mingw)
       env:
         DEBIAN_FRONTEND: noninteractive
@@ -80,7 +80,7 @@ jobs:
     runs-on: windows-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Download komodo_win.zip
         uses: actions/download-artifact@v1
@@ -111,7 +111,7 @@ jobs:
     needs: windows-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Download komodo_win.zip
         uses: actions/download-artifact@v1
@@ -142,7 +142,7 @@ jobs:
     needs: windows-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Download komodo_win.zip
         uses: actions/download-artifact@v1
@@ -173,7 +173,7 @@ jobs:
     needs: windows-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Download komodo_win.zip
         uses: actions/download-artifact@v1
@@ -204,7 +204,7 @@ jobs:
     needs: windows-build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Download komodo_win.zip
         uses: actions/download-artifact@v1

diff --git a/.github/workflows/komodod_cd.yml b/.github/workflows/komodod_cd.yml
@@ -30,7 +30,7 @@ jobs:
         id: shortify_commit
 
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Install deps (Linux)
         env:
@@ -79,7 +79,7 @@ jobs:
     if: ${{ github.event_name != 'workflow_dispatch' }}
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install deps (macOS)
         run: |
           rm '/usr/local/bin/2to3'
@@ -111,7 +111,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install deps (Windows)
         env:
           DEBIAN_FRONTEND: noninteractive

diff --git a/README.md b/README.md
@@ -94,7 +94,7 @@ brew update
 brew upgrade
 brew tap discoteq/discoteq; brew install flock
 brew install autoconf autogen automake
-brew update && brew install gcc@8
+brew update
 brew install binutils
 brew install protobuf
 brew install coreutils

diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk
@@ -1,22 +1,23 @@
-build_darwin_CC = gcc-8
-build_darwin_CXX = g++-8
+build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path)
+build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path)
 build_darwin_AR: = $(shell xcrun -f ar)
 build_darwin_RANLIB: = $(shell xcrun -f ranlib)
 build_darwin_STRIP: = $(shell xcrun -f strip)
 build_darwin_OTOOL: = $(shell xcrun -f otool)
 build_darwin_NM: = $(shell xcrun -f nm)
 build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
 build_darwin_SHA256SUM = shasum -a 256
-build_darwin_DOWNLOAD = curl --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -f -o
+build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
-darwin_CC= gcc-8
-darwin_CXX= g++-8
+darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -fvisibility=hidden -fvisibility-inlines-hidden --sysroot $(shell xcrun --show-sdk-path)
+darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -fvisibility=hidden -fvisibility-inlines-hidden -stdlib=libc++ --sysroot $(shell xcrun --show-sdk-path)
 darwin_AR:=$(shell xcrun -f ar)
 darwin_RANLIB:=$(shell xcrun -f ranlib)
 darwin_STRIP:=$(shell xcrun -f strip)
 darwin_LIBTOOL:=$(shell xcrun -f libtool)
 darwin_OTOOL:=$(shell xcrun -f otool)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_native_binutils=
 darwin_native_toolchain=
diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk
@@ -1,17 +1,50 @@
-OSX_MIN_VERSION=10.12
-OSX_SDK_VERSION=10.12
-OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk
-LD64_VERSION=253.9
-darwin_CC=gcc-8 -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION)
-darwin_CXX=g++-8 -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION)
+OSX_MIN_VERSION=10.15
+OSX_SDK_VERSION=10.15.6
+XCODE_VERSION=12.1
+XCODE_BUILD_ID=12A7403
+LD64_VERSION=609
+
+OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
+
+# Flag explanations:
+#
+#     -mlinker-version
+#
+#         Ensures that modern linker features are enabled. See here for more
+#         details: https://github.com/bitcoin/bitcoin/pull/19407.
+#
+#     -B$(build_prefix)/bin
+#
+#         Explicitly point to our binaries (e.g. cctools) so that they are
+#         ensured to be found and preferred over other possibilities.
+#
+#     -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#
+#         Forces clang to use the libc++ headers from our SDK and completely
+#         forget about the libc++ headers from the standard directories
+#
+#         TODO: Once we start requiring a clang version that has the
+#         -stdlib++-isystem<directory> flag first introduced here:
+#         https://reviews.llvm.org/D64089, we should use that instead. Read the
+#         differential summary there for more details.
+#
+darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin
+darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
 
-darwin_release_CFLAGS=-g -O2
+darwin_release_CFLAGS=-O2
 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 
 darwin_debug_CFLAGS=-g -O0
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
 
+darwin_native_binutils=native_cctools
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 darwin_native_toolchain=native_cctools
+else
+darwin_native_toolchain=
+endif
+
+darwin_cmake_system=Darwin
diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk
@@ -25,7 +25,9 @@ endef
 
 ifeq ($(build_os),darwin)
 define $(package)_preprocess_cmds
-   sed -i -e "s/WinIoCtl.h/winioctl.h/g" src/dbinc/win_db.h
+   sed -i -e "s/WinIoCtl.h/winioctl.h/g" src/dbinc/win_db.h && \
+   sed -i.old 's/__atomic_compare_exchange\\(/__atomic_compare_exchange_db(/' src/dbinc/atomic.h && \
+   sed -i.old 's/atomic_init/atomic_init_db/' src/dbinc/atomic.h src/mp/mp_region.c src/mp/mp_mvcc.c src/mp/mp_fget.c src/mutex/mut_method.c src/mutex/mut_tas.c
 endef
 else ifeq ($(host_os),mingw32)
 define $(package)_preprocess_cmds

diff --git a/depends/packages/libcurl.mk b/depends/packages/libcurl.mk
@@ -7,6 +7,7 @@ $(package)_sha256_hash=816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f
 $(package)_config_opts=--with-openssl --disable-shared --enable-static --prefix=$(host_prefix)
 $(package)_config_opts_linux=--host=x86_64-unknown-linux-gnu
 $(package)_config_opts_mingw32=--enable-mingw --host=x86_64-w64-mingw32
+$(package)_config_opts_darwin=--without-libidn2 --without-zstd --without-nghttp2 --without-librtmp --without-brotli
 $(package)_cflags_darwin=-mmacosx-version-min=$(OSX_MIN_VERSION)
 $(package)_conf_tool=./configure
 

diff --git a/depends/packages/libsodium.mk b/depends/packages/libsodium.mk
@@ -9,7 +9,7 @@ $(package)_config_opts=
 define $(package)_set_vars
   $(package)_build_env=DO_NOT_UPDATE_CONFIG_SCRIPTS=1
   ifeq ($(build_os),darwin)
-  $(package)_build_env+=MACOSX_DEPLOYMENT_TARGET="10.11"
+  $(package)_build_env+=MACOSX_DEPLOYMENT_TARGET="$(OSX_MIN_VERSION)"
   $(package)_cc=clang
   $(package)_cxx=clang++
   endif