From 4f3691be0f85f7b313c887c3b770bc837fd11518 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Nov 2023 22:30:16 +0100 Subject: [PATCH] add CI workflow for software.eessi.io + use software.eessi.io in tests for eessi_container.sh script --- ...{test_eessi.yml => test_eessi-hpc.org.yml} | 2 +- .github/workflows/test_eessi.io.yml | 76 +++++++++++++++++++ .../workflows/test_eessi_container_script.yml | 12 +-- 3 files changed, 83 insertions(+), 7 deletions(-) rename .github/workflows/{test_eessi.yml => test_eessi-hpc.org.yml} (98%) create mode 100644 .github/workflows/test_eessi.io.yml diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi-hpc.org.yml similarity index 98% rename from .github/workflows/test_eessi.yml rename to .github/workflows/test_eessi-hpc.org.yml index 6dfd78e428..3e3d7cca7b 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi-hpc.org.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests relying on having EESSI pilot repo mounted +name: Tests relying on having pilot.eessi-hpc.org mounted on: [push, pull_request, workflow_dispatch] permissions: contents: read # to fetch code (actions/checkout) diff --git a/.github/workflows/test_eessi.io.yml b/.github/workflows/test_eessi.io.yml new file mode 100644 index 0000000000..e2ee713765 --- /dev/null +++ b/.github/workflows/test_eessi.io.yml @@ -0,0 +1,76 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests relying on having software.eessi.io mounted +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + pilot: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR: + - aarch64/generic + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + steps: + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Test check_missing_installations.sh script + run: | + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}*.yml)" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done + + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) + run: | + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + # create dummy easystack file with a single entry (something that is not installed in EESSI) + easystack_file="test.yml" + echo "easyconfigs:" > ${easystack_file} + echo " - GCC-8.3.0:" >> ${easystack_file} + echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):" + cat ${easystack_file} + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package + if ./check_missing_installations.sh ${easystack_file}; then + echo "did NOT capture missing package; test FAILED" + exit 1 + else + echo "captured missing package; test PASSED" + exit 0 + fi diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 929fb22cec..590aa4225b 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -90,15 +90,15 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then outfile=out_readwrite.txt fn="test_${RANDOM}.txt" - echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir - ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then @@ -120,12 +120,12 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then outfile=out_save.txt fn="test_${RANDOM}.txt" - test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + test_cmd="touch /cvmfs/software.eessi.io/${fn}" ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} rm -f ${outfile} - ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile} + grep "/cvmfs/software.eessi.io/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}"