Skip to content

Main

Main #1122

Workflow file for this run

name: Main
on:
pull_request:
branches: [main]
push:
branches: [main]
release:
types: [published]
schedule:
- cron: '30 20 * * *' # Warning: Timezone dep - 20:00 is 1:00
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
build:
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- name: ubu22-x86-gcc12-clang17
os: ubuntu-22.04
compiler: gcc-12
clang-runtime: '17'
cling: Off
cppyy: On
- name: ubu22-x86-gcc12-clang16
os: ubuntu-22.04
compiler: gcc-12
clang-runtime: '16'
cling: Off
cppyy: On
- name: ubu22-x86-gcc12-cling
os: ubuntu-22.04
compiler: gcc-12
clang-runtime: '13'
cling: On
cling-version: '1.0'
cppyy: On
- name: osx13-x86-clang-clang17
os: macos-13
compiler: clang
clang-runtime: '17'
cling: Off
cppyy: On
- name: osx13-x86-clang-clang16
os: macos-13
compiler: clang
clang-runtime: '16'
cling: Off
cppyy: On
- name: osx13-x86-clang-cling
os: macos-13
compiler: clang
clang-runtime: '13'
cling: On
cling-version: '1.0'
cppyy: On
- name: osx14-arm-clang-clang17
os: macos-14
compiler: clang
clang-runtime: '17'
cling: Off
cppyy: On
- name: osx14-arm-clang-clang16
os: macos-14
compiler: clang
clang-runtime: '16'
cling: Off
cppyy: On
- name: osx14-arm-clang-cling
os: macos-14
compiler: clang
clang-runtime: '13'
cling: On
cling-version: '1.0'
cppyy: On
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Save PR Info on Unix systems
if: ${{ runner.os != 'windows' }}
run: |
mkdir -p ./pr
echo ${{ github.event.number }} > ./pr/NR
echo ${{ github.repository }} > ./pr/REPO
cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]')
if [[ "$cling_on" == "ON" ]]; then
export CLING_HASH=$(git ls-remote https://github.com/root-project/cling.git refs/tags/v${{ matrix.cling-version }} | tr '\t' '-')
export LLVM_HASH=$(git ls-remote https://github.com/root-project/llvm-project.git cling-llvm${{ matrix.clang-runtime}} | tr '\t' '-')
else
export CLING_HASH="Repl"
# May need to revert back to both having same llvm_hash, as below cause llvm to be rebuilt everytime commit is made to llvm/llvm-project for release a.x
# which could be quite often for new releases
export LLVM_HASH=$(git ls-remote https://github.com/llvm/llvm-project.git refs/heads/release/${{ matrix.clang-runtime}}.x | tr '\t' '-')
fi
echo "CLING_HASH=$CLING_HASH" >> $GITHUB_ENV
echo "LLVM_HASH=$LLVM_HASH" >> $GITHUB_ENV
- uses: nelonoel/[email protected]
- name: Setup default Build Type on *nux
if: runner.os != 'windows'
run: |
echo "BUILD_TYPE=Release" >> $GITHUB_ENV
echo "CODE_COVERAGE=0" >> $GITHUB_ENV
os="${{ matrix.os }}"
if [[ "${os}" == "macos"* ]]; then
echo "ncpus=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV
else
echo "ncpus=$(nproc --all)" >> $GITHUB_ENV
fi
- name: Setup compiler on Linux
if: runner.os == 'Linux'
run: |
# https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
vers="${compiler#*-}"
os_codename="`cat /etc/os-release | grep UBUNTU_CODENAME | cut -d = -f 2`"
sudo apt update
if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then
sudo apt install -y gcc-${vers} g++-${vers}
echo "CC=gcc-${vers}" >> $GITHUB_ENV
echo "CXX=g++-${vers}" >> $GITHUB_ENV
else
if ! sudo apt install -y clang-${vers}; then
curl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
echo "deb https://apt.llvm.org/${os_codename}/ llvm-toolchain-${os_codename}-${vers} main" | sudo tee -a /etc/apt/sources.list
sudo apt update
sudo apt install -y clang-${vers}
fi
echo "CC=clang-${vers}" >> $GITHUB_ENV
echo "CXX=clang++-${vers}" >> $GITHUB_ENV
fi
env:
compiler: ${{ matrix.compiler }}
- name: Setup compiler on macOS
if: runner.os == 'macOS'
run: |
vers="${compiler#*-}"
echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV
if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then
brew install "gcc@$vers"
echo "CC=gcc-${vers}" >> $GITHUB_ENV
echo "CXX=g++-${vers}" >> $GITHUB_ENV
else
#Use clang-15 and clang++-15 compiler that is installed on runner, instead of one
#provided by MacOS (could brew install clang-16/17 to use consistent version
#of clang)
echo "CC=$(brew --prefix llvm@15)/bin/clang" >> $GITHUB_ENV
echo "CXX=$(brew --prefix llvm@15)/bin/clang++" >> $GITHUB_ENV
fi
env:
compiler: ${{ matrix.compiler }}
- name: Install deps on Linux
if: runner.os == 'Linux'
run: |
# Install deps
sudo apt-get update --yes
sudo apt-get install --yes --no-install-recommends ncurses-dev libtinfo-dev libtinfo5 lld
sudo apt autoremove
sudo apt clean
- name: Install deps on MacOS
if: runner.os == 'macOS'
run: |
brew update
# workaround for https://github.com/actions/setup-python/issues/577
for pkg in $(brew list | grep '^python@'); do
brew unlink "$pkg"
brew link --overwrite "$pkg"
done
brew upgrade
cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]')
if [[ "${cling_on}" == "ON" ]]; then
brew install libomp
fi
pip install distro pytest
- name: Restore Cache LLVM/Clang runtime build directory
uses: actions/cache/restore@v3
id: cache
with:
path: |
clang-dev
${{ matrix.cling=='On' && 'cling' || '' }}
key: ${{ env.CLING_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x-patch-${{ hashFiles(format('patches/llvm/clang{0}-*.patch', matrix.clang-runtime)) || 'none' }}
- name: Build runtime LLVM/Clang on Linux if the cache is invalid
if: ${{ runner.os != 'windows' && steps.cache.outputs.cache-hit != 'true' }}
run: |
cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]')
if [[ "${cling_on}" == "ON" ]]; then
git clone https://github.com/root-project/cling.git
cd ./cling
git checkout tags/v${{ matrix.cling-version }}
cd ..
git clone --depth=1 -b cling-llvm${{ matrix.clang-runtime }} https://github.com/root-project/llvm-project.git clang-dev
else # repl
git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git clang-dev
fi
cd clang-dev
mkdir build
if [[ "${cling_on}" == "ON" ]]; then
cd build
#FIXME: Cannot build with openmp project on osx Github runners
#despite being able to install locally
cmake -DLLVM_ENABLE_PROJECTS=clang \
-DLLVM_EXTERNAL_PROJECTS=cling \
-DLLVM_EXTERNAL_CLING_SOURCE_DIR=../../cling \
-DLLVM_TARGETS_TO_BUILD="host;NVPTX" \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_ENABLE_LLD=ON \
-DCLANG_ENABLE_STATIC_ANALYZER=OFF \
-DCLANG_ENABLE_ARCMT=OFF \
-DCLANG_ENABLE_FORMAT=OFF \
-DCLANG_ENABLE_BOOTSTRAP=OFF \
../llvm
cmake --build . --target clang --parallel ${{ env.ncpus }}
cmake --build . --target cling --parallel ${{ env.ncpus }}
# Now build gtest.a and gtest_main for CppInterOp to run its tests.
cmake --build . --target gtest_main --parallel ${{ env.ncpus }}
cd ../
else
mkdir inst
# Apply patches
git apply -v ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch
echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:"
cd build
cmake -DLLVM_ENABLE_PROJECTS="clang;openmp" \
-DLLVM_TARGETS_TO_BUILD="host;NVPTX" \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_ENABLE_LLD=ON \
-DCLANG_ENABLE_STATIC_ANALYZER=OFF \
-DCLANG_ENABLE_ARCMT=OFF \
-DCLANG_ENABLE_FORMAT=OFF \
-DCLANG_ENABLE_BOOTSTRAP=OFF \
-DCMAKE_INSTALL_PREFIX=../inst \
../llvm
cmake --build . --target all --parallel ${{ env.ncpus }}
cmake --build . --target install --parallel ${{ env.ncpus }}
cd ..
rm -rf $(find . -maxdepth 1 ! -name "inst" ! -name ".")
fi
cd ..
- name: Save Cache LLVM/Clang runtime build directory
uses: actions/cache/save@v3
if: ${{ steps.cache.outputs.cache-hit != 'true' }}
with:
path: |
clang-dev
${{ matrix.cling=='On' && 'cling' || '' }}
key: ${{ steps.cache.outputs.cache-primary-key }}
- name: Set PATH_TO_LLVM_BUILD etc to Env
run: |
export PATH_TO_LLVM_BUILD="$(realpath clang-dev/inst)"
export PATH=$PATH_TO_LLVM_BUILD/bin:$PATH
export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH
echo "PATH_TO_LLVM_BUILD=$PATH_TO_LLVM_BUILD" >> $GITHUB_ENV
echo "PATH=$PATH" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
- name: Create clang-dev.tar.bz2 for artifact/release asset
if: ${{ runner.os == 'Linux' && matrix.clang-runtime == '17' }}
run: |
#TODO: Clean unneeded folders...
# Tar prebuild clang-dev for dev binder containers
tar -cjf clang-dev.tar.bz2 ./clang-dev
- name: Add assets to Release
id: add_release_assets
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/v')
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
token: ${{ secrets.GITHUB_TOKEN }}
files: |
clang-dev.tar.bz2
- name: Archive clang-dev artifact
if: ${{ runner.os == 'Linux' && matrix.clang-runtime == '17' }}
uses: actions/upload-artifact@v3
with:
name: clang-dev
path: clang-dev.tar.bz2
retention-days: ${{ github.event_name=='schedule' && 2 || 7 }}
- name: Setup and activate mamba virtual environment on Unix Systems
if: runner.os != 'Windows' #TODO: exclude on release
run: |
wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
bash Miniforge3.sh -b -p "${HOME}/conda"
source "${HOME}/conda/etc/profile.d/conda.sh"
source "${HOME}/conda/etc/profile.d/mamba.sh"
mamba create -y -n .venv python=3.10.6
mamba activate .venv
mamba install --quiet --yes -c conda-forge \
cmake \
'xeus>=2.0' \
xeus-zmq \
'nlohmann_json>=3.9.1,<3.10' \
'cppzmq>=4.6.0,<5' \
'xtl>=0.7,<0.8' \
'openssl<4' \
ipykernel \
pugixml \
zlib \
libxml2 \
'cxxopts>=2.2.1,<2.3' \
libuuid \
pytest \
jupyter_kernel_test
mamba install -y jupyter
hash -r
pip install ipython
jupyter notebook --generate-config -y
mamba clean --all -f -y
npm cache clean --force
jupyter lab clean
- name: Build and Install CppInterOp on Unix Systems
if: runner.os != 'Windows'
run: |
source "${HOME}/conda/etc/profile.d/conda.sh"
source "${HOME}/conda/etc/profile.d/mamba.sh"
mamba activate .venv
# Build CppInterOp next to cling and llvm-project.
LLVM_DIR="$(pwd)/clang-dev"
cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]')
if [[ "${cling_on}" == "ON" ]]; then
LLVM_BUILD_DIR="$(pwd)/clang-dev/build"
CLING_DIR="$(pwd)/cling"
CLING_BUILD_DIR="$(pwd)/cling/build"
CPLUS_INCLUDE_PATH="${CLING_DIR}/tools/cling/include:${CLING_BUILD_DIR}/include:${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include:$PWD/include"
else
LLVM_BUILD_DIR="$(pwd)/clang-dev/inst"
CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include:$PWD/include"
fi
git clone --depth=1 https://github.com/compiler-research/CppInterOp.git
export CB_PYTHON_DIR="$PWD/cppyy-backend/python"
export CPPINTEROP_DIR=$CB_PYTHON_DIR/cppyy_backend/
mkdir CppInterOp/build
cd CppInterOp/build
export CPPINTEROP_BUILD_DIR=$PWD
cling_on=$(echo "${{ matrix.cling }}" | tr '[:lower:]' '[:upper:]')
if [[ "${cling_on}" == "ON" ]]; then
cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DUSE_CLING=ON \
-DUSE_REPL=OFF \
-DCling_DIR=$LLVM_BUILD_DIR/tools/cling \
-DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \
-DClang_DIR=$LLVM_BUILD_DIR/lib/cmake/clang \
-DBUILD_SHARED_LIBS=ON \
-DCODE_COVERAGE=${{ env.CODE_COVERAGE }} \
-DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \
../
else
cmake -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DUSE_CLING=OFF \
-DUSE_REPL=ON \
-DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \
-DClang_DIR=$LLVM_BUILD_DIR/lib/cmake/clang \
-DBUILD_SHARED_LIBS=ON \
-DCODE_COVERAGE=${{ env.CODE_COVERAGE }} \
-DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \
../
fi
cmake --build . --parallel ${{ env.ncpus }}
cd ../..
# We need CB_PYTHON_DIR later
echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV
# We need CPPINTEROP_DIR, LLVM_BUILD_DIR and CPLUS_INCLUDE_PATH later
echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV
echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV
echo "LLVM_BUILD_DIR=$LLVM_BUILD_DIR" >> $GITHUB_ENV
echo "CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
- name: Build xeus-clang-repl on Unix Systems
if: runner.os != 'Windows' #TODO: exclude on release
run: |
source "${HOME}/conda/etc/profile.d/conda.sh"
source "${HOME}/conda/etc/profile.d/mamba.sh"
mamba activate .venv
## Build and Install xeus-clang-repl
mkdir build
cd build
PP=$KERNEL_PYTHON_PREFIX
if [[ -z "$PP" ]]; then
PP=$(conda info --base)
fi
cmake -DCMAKE_PREFIX_PATH=$PP -DCMAKE_INSTALL_PREFIX=$PP \
-DCMAKE_INSTALL_LIBDIR=lib -DLLVM_DIR=$PATH_TO_LLVM_BUILD \
-DLLVM_CONFIG_EXTRA_PATH_HINTS=$PATH_TO_LLVM_BUILD \
-DCPPINTEROP_DIR=$CPPINTEROP_BUILD_DIR \
-DLLVM_USE_LINKER=lld ..
make install
- name: Setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
# When debugging increase to a suitable value!
timeout-minutes: ${{ github.event.pull_request && 1 || 20 }}