From ff17223cee098ff9c87d09ca2a6321b46af03e7f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 27 Sep 2024 10:27:56 -0600 Subject: [PATCH 01/21] Configure oneapi@2024.2.1 for Narwhal --- configs/sites/tier1/narwhal/compilers.yaml | 32 +++++++++++++++++++ .../sites/tier1/narwhal/packages_oneapi.yaml | 32 +++++++++++++++++++ doc/source/PreConfiguredSites.rst | 30 ++++++++++++++++- 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 configs/sites/tier1/narwhal/packages_oneapi.yaml diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 1b7c23dcf..5e2e17c11 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -21,6 +21,38 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] + - compiler: + spec: oneapi@2024.2.1 + paths: + #cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx + #cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx + #f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + #fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-intel/8.3.3 + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + - ifort/2024.2.1 + - cray-libsci/23.05.1.4 + - libfabric/1.12.1.2.2.1 + environment: + prepend_path: + PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' + append_path: + CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml new file mode 100644 index 000000000..9b032dcf1 --- /dev/null +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -0,0 +1,32 @@ +packages: + all: + compiler:: [oneapi@2024.2.1,gcc@10.3.0] + providers: + mpi:: [cray-mpich@8.1.21] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers + modules: + - cray-mpich-ucx/8.1.21 + - craype-network-ucx + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mkl/2024.2 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + externals: + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + - compiler-rt/2024.2.1 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 97b532daa..0ea8e3a7b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -63,7 +63,7 @@ Pre-configured sites (tier 1) +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | | U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| | Nautilus | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -169,6 +169,34 @@ The following is required for building new spack environments with Intel on this module unload cray-libsci module load cray-libsci/23.05.1.4 +The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! + +.. code-block:: console + + umask 0022 + module unload PrgEnv-cray + module load PrgEnv-intel/8.3.3 + module unload intel + module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles + module load tbb/2021.13 + module load compiler-rt/2024.2.1 + module load compiler/2024.2.1 + module load ifort/2024.2.1 + module unload cray-mpich + module unload craype-network-ofi + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load craype-network-ucx + #module load cray-mpich-ucx/8.1.21 + module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 + The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console From e0576ca5139c54de2ef65d52a99938b1ebffb9eb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 30 Sep 2024 16:07:52 -0600 Subject: [PATCH 02/21] Update doc/source/PreConfiguredSites.rst: add note that oneAPI Narwhal setup currently does not compile Python packages --- doc/source/PreConfiguredSites.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 0ea8e3a7b..09ddec8d1 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -171,6 +171,8 @@ The following is required for building new spack environments with Intel on this The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! +.. note:: This particular setup of the oneAPI compilers is outside the Cray-blessed environment and there are problems installing Python packages with spack. + .. code-block:: console umask 0022 From 76361b06745500f2b1521589ec79ff291a7a5964 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 3 Oct 2024 15:46:18 -0600 Subject: [PATCH 03/21] Add configs/sites/tier1/atlantis/mirrors.yaml --- configs/sites/tier1/atlantis/mirrors.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 configs/sites/tier1/atlantis/mirrors.yaml diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml new file mode 100644 index 000000000..94e16cfaf --- /dev/null +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -0,0 +1,20 @@ +mirrors: + local-source: + fetch: + # TODO UPDATE? + url: file:///neptune_diagnostics/spack-stack/source-cache/ + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + # TODO UPDATE? + url: file:///neptune_diagnostics/spack-stack/source-cache/ + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null From 87067e943e5b51167ae139845f171b005bd31577 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Thu, 3 Oct 2024 22:45:20 -0400 Subject: [PATCH 04/21] Update site/packages.yaml and site/modules.yaml (#1322) Latest changes in site/packages.yaml and site/modules.yaml config files regarding ecFlow and QT. Removed exclude for creating ecflow modulefile, added path for qt and removed ecflow from package list. --- configs/sites/tier1/derecho/modules.yaml | 2 -- configs/sites/tier1/gaea-c5/modules.yaml | 2 -- configs/sites/tier1/gaea-c5/packages.yaml | 8 +------- configs/sites/tier1/gaea-c6/packages.yaml | 7 +------ configs/sites/tier1/hera/modules.yaml | 2 -- configs/sites/tier1/hera/packages.yaml | 11 ++++------- configs/sites/tier1/hercules/modules.yaml | 2 -- configs/sites/tier1/hercules/packages.yaml | 13 +++---------- configs/sites/tier1/jet/modules.yaml | 2 -- configs/sites/tier1/jet/packages.yaml | 12 +++++------- configs/sites/tier1/noaa-aws/modules.yaml | 2 -- configs/sites/tier1/noaa-aws/packages.yaml | 12 +++++------- configs/sites/tier1/noaa-azure/modules.yaml | 2 -- configs/sites/tier1/noaa-azure/packages.yaml | 12 +++++------- configs/sites/tier1/noaa-gcloud/modules.yaml | 2 -- configs/sites/tier1/noaa-gcloud/packages.yaml | 12 +++++------- configs/sites/tier1/orion/modules.yaml | 2 -- configs/sites/tier1/orion/packages.yaml | 7 ------- 18 files changed, 29 insertions(+), 83 deletions(-) diff --git a/configs/sites/tier1/derecho/modules.yaml b/configs/sites/tier1/derecho/modules.yaml index b9d5b4703..960198ae7 100644 --- a/configs/sites/tier1/derecho/modules.yaml +++ b/configs/sites/tier1/derecho/modules.yaml @@ -3,8 +3,6 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - cray-mpich - python diff --git a/configs/sites/tier1/gaea-c5/modules.yaml b/configs/sites/tier1/gaea-c5/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/gaea-c5/modules.yaml +++ b/configs/sites/tier1/gaea-c5/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml index 6e13f710e..e9f4fc586 100644 --- a/configs/sites/tier1/gaea-c5/packages.yaml +++ b/configs/sites/tier1/gaea-c5/packages.yaml @@ -81,12 +81,6 @@ packages: externals: - spec: dos2unix@7.4.0 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4 - modules: [ecflow/5.8.4] file: externals: - spec: file@5.32 @@ -197,7 +191,7 @@ packages: qt: externals: - spec: qt@5.15.2 - prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2/5.15.2/gcc_64 + prefix: /ncrc/proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64 rdma-core: externals: - spec: rdma-core@37.0 diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 75473846c..7e9663d31 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -70,11 +70,6 @@ packages: externals: - spec: dos2unix@7.4.0 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /autofs/ncrc-svm1_proj/epic/spack-stack/ecflow-5.8.4 file: externals: - spec: file@5.32 @@ -184,7 +179,7 @@ packages: qt: externals: - spec: qt@5.15.2 - prefix: /autofs/ncrc-svm1_proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64 + prefix: /ncrc/proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64 rdma-core: externals: - spec: rdma-core@42.0 diff --git a/configs/sites/tier1/hera/modules.yaml b/configs/sites/tier1/hera/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/hera/modules.yaml +++ b/configs/sites/tier1/hera/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/hera/packages.yaml b/configs/sites/tier1/hera/packages.yaml index 894af0f5e..36a0a3f68 100644 --- a/configs/sites/tier1/hera/packages.yaml +++ b/configs/sites/tier1/hera/packages.yaml @@ -34,13 +34,6 @@ packages: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.5.3+ui+static_boost - prefix: /apps/ecflow/5.5.3 - modules: - - ecflow/5.5.3 file: externals: - spec: file@5.11 @@ -120,6 +113,10 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr + qt: + externals: + - spec: qt@5.12.6 + prefix: /scratch1/NCEPDEV/nems/role.epic/installs/qt-5.12.6/qtbase rsync: externals: - spec: rsync@3.1.2 diff --git a/configs/sites/tier1/hercules/modules.yaml b/configs/sites/tier1/hercules/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/hercules/modules.yaml +++ b/configs/sites/tier1/hercules/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml index 9c2c95209..0128505f1 100644 --- a/configs/sites/tier1/hercules/packages.yaml +++ b/configs/sites/tier1/hercules/packages.yaml @@ -19,13 +19,6 @@ packages: externals: - spec: diffutils@3.7 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.8.0 @@ -73,10 +66,10 @@ packages: prefix: /usr qt: externals: - - spec: qt@5.15.8 - prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd + - spec: qt@5.15.14 + prefix: /apps/contrib/spack-stack-1.1/gcc-11.3.1/qt-5.15.14-mfeuvcidmyqoi2m5i2tfrk6yd7xtk6pt modules: - - qt/5.15.8 + - qt/5.15.14 sed: externals: - spec: sed@4.8 diff --git a/configs/sites/tier1/jet/modules.yaml b/configs/sites/tier1/jet/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/jet/modules.yaml +++ b/configs/sites/tier1/jet/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/jet/packages.yaml b/configs/sites/tier1/jet/packages.yaml index 04633817d..fa6a0d123 100644 --- a/configs/sites/tier1/jet/packages.yaml +++ b/configs/sites/tier1/jet/packages.yaml @@ -34,13 +34,6 @@ packages: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.5.3+ui+static_boost - prefix: /apps/ecflow/5.5.3 - modules: - - ecflow/5.5.3 file: externals: - spec: file@5.11 @@ -124,6 +117,11 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.12.6 + prefix: /contrib/spack-stack/installs/qt-5.12.6/qtbase rsync: externals: - spec: rsync@3.1.2 diff --git a/configs/sites/tier1/noaa-aws/modules.yaml b/configs/sites/tier1/noaa-aws/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/noaa-aws/modules.yaml +++ b/configs/sites/tier1/noaa-aws/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/noaa-aws/packages.yaml b/configs/sites/tier1/noaa-aws/packages.yaml index ee153e8a5..dda1f369b 100644 --- a/configs/sites/tier1/noaa-aws/packages.yaml +++ b/configs/sites/tier1/noaa-aws/packages.yaml @@ -15,13 +15,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.33 @@ -82,6 +75,11 @@ packages: externals: - spec: pkg-config@1.4.2 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 rsync: externals: - spec: rsync@3.1.3 diff --git a/configs/sites/tier1/noaa-azure/modules.yaml b/configs/sites/tier1/noaa-azure/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/noaa-azure/modules.yaml +++ b/configs/sites/tier1/noaa-azure/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/noaa-azure/packages.yaml b/configs/sites/tier1/noaa-azure/packages.yaml index 004aea213..397bc780b 100644 --- a/configs/sites/tier1/noaa-azure/packages.yaml +++ b/configs/sites/tier1/noaa-azure/packages.yaml @@ -15,13 +15,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.33 @@ -78,6 +71,11 @@ packages: externals: - spec: pkg-config@1.4.2 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 rsync: externals: - spec: rsync@3.1.3 diff --git a/configs/sites/tier1/noaa-gcloud/modules.yaml b/configs/sites/tier1/noaa-gcloud/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/noaa-gcloud/modules.yaml +++ b/configs/sites/tier1/noaa-gcloud/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/noaa-gcloud/packages.yaml b/configs/sites/tier1/noaa-gcloud/packages.yaml index 004aea213..397bc780b 100644 --- a/configs/sites/tier1/noaa-gcloud/packages.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages.yaml @@ -15,13 +15,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.33 @@ -78,6 +71,11 @@ packages: externals: - spec: pkg-config@1.4.2 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 rsync: externals: - spec: rsync@3.1.3 diff --git a/configs/sites/tier1/orion/modules.yaml b/configs/sites/tier1/orion/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/orion/modules.yaml +++ b/configs/sites/tier1/orion/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/orion/packages.yaml b/configs/sites/tier1/orion/packages.yaml index 3fcecb71c..bef7849a9 100644 --- a/configs/sites/tier1/orion/packages.yaml +++ b/configs/sites/tier1/orion/packages.yaml @@ -31,13 +31,6 @@ packages: externals: - spec: diffutils@3.7 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /work/noaa/epic/role-epic/spack-stack/orion/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.8.0 From 91fe139a3d8fce9393f84d9435a0732266ae1ed2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 4 Oct 2024 05:29:09 -0600 Subject: [PATCH 05/21] Add configs/sites/tier1/atlantis/packages_gcc.yaml --- .../sites/tier1/atlantis/packages_gcc.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 configs/sites/tier1/atlantis/packages_gcc.yaml diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml new file mode 100644 index 000000000..7d36ed0b4 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -0,0 +1,23 @@ +packages: + all: + compiler:: [gcc@11.2.0] + providers: + mpi:: [openmpi@4.1.6] + #blas:: [intel-oneapi-mkl] + #fftw-api:: [intel-oneapi-mkl] + #lapack:: [intel-oneapi-mkl] + slurm: + buildable: False + externals: + - spec: slurm@21.08.8 +pmix + modules: + - slurm/slurm/21.08.8 + openmpi: + require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers' +modules: + default: + lmod: + include: + - openmpi + exclude: + - slurm From 5469c560ce72ac2d27317389a00a04b3d53a9c1a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 4 Oct 2024 05:37:30 -0600 Subject: [PATCH 06/21] Remove site sandy --- configs/sites/tier1/sandy/compilers.yaml | 14 --- configs/sites/tier1/sandy/config.yaml | 2 - configs/sites/tier1/sandy/mirrors.yaml | 18 ---- configs/sites/tier1/sandy/modules.yaml | 8 -- configs/sites/tier1/sandy/packages.yaml | 113 ----------------------- 5 files changed, 155 deletions(-) delete mode 100644 configs/sites/tier1/sandy/compilers.yaml delete mode 100644 configs/sites/tier1/sandy/config.yaml delete mode 100644 configs/sites/tier1/sandy/mirrors.yaml delete mode 100644 configs/sites/tier1/sandy/modules.yaml delete mode 100644 configs/sites/tier1/sandy/packages.yaml diff --git a/configs/sites/tier1/sandy/compilers.yaml b/configs/sites/tier1/sandy/compilers.yaml deleted file mode 100644 index b86ff0a33..000000000 --- a/configs/sites/tier1/sandy/compilers.yaml +++ /dev/null @@ -1,14 +0,0 @@ -compilers: -- compiler: - spec: gcc@=9.3.0 - paths: - cc: /software/depot/gcc-9.3.0/bin/gcc - cxx: /software/depot/gcc-9.3.0/bin/g++ - f77: /software/depot/gcc-9.3.0/bin/gfortran - fc: /software/depot/gcc-9.3.0/bin/gfortran - flags: {} - operating_system: centos7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/configs/sites/tier1/sandy/config.yaml b/configs/sites/tier1/sandy/config.yaml deleted file mode 100644 index 84fe05ab9..000000000 --- a/configs/sites/tier1/sandy/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - build_jobs: 8 diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/sandy/mirrors.yaml deleted file mode 100644 index c48f33bf1..000000000 --- a/configs/sites/tier1/sandy/mirrors.yaml +++ /dev/null @@ -1,18 +0,0 @@ -mirrors: - local-source: - fetch: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml deleted file mode 100644 index aeb254c2e..000000000 --- a/configs/sites/tier1/sandy/modules.yaml +++ /dev/null @@ -1,8 +0,0 @@ -modules: - default: - enable:: - - tcl - tcl: - include: - # List of packages for which we need modules that are blacklisted by default - - python diff --git a/configs/sites/tier1/sandy/packages.yaml b/configs/sites/tier1/sandy/packages.yaml deleted file mode 100644 index 1af1ab029..000000000 --- a/configs/sites/tier1/sandy/packages.yaml +++ /dev/null @@ -1,113 +0,0 @@ -packages: - all: - compiler:: [gcc@9.3.0] - providers: - mpi:: [openmpi@4.0.5] - -### MPI, Python, MKL - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.0.5%gcc@9.3.0 - prefix: /software7/depot/openmpi-4.0.5 - -### All other external packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - binutils: - externals: - - spec: binutils@2.30.117 - prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - curl: - externals: - - spec: curl@7.61.1 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - git: - externals: - - spec: git@2.39.2+tcltk - prefix: /software8/depot/git-2.39.2 - git-lfs: - externals: - - spec: git-lfs@3.3.0 - prefix: /software8/depot/git-2.39.2 - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - groff: - externals: - - spec: groff@1.22.3 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - # Don't use external openssl, too old - #openssl: - # externals: - # - spec: openssl@1.1.1k - # prefix: /usr - # Can't use with py-xnrl - #perl: - # externals: - # - spec: perl@5.26.3~cpanm+shared+threads - # prefix: /usr - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - subversion: - externals: - - spec: subversion@1.10.2 - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr - texinfo: - externals: - - spec: texinfo@6.5 - prefix: /usr - wget: - externals: - - spec: wget@1.19.5 - prefix: /usr From ab722bb3cfa31a761aa78f25c6c2800e363467b1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 4 Oct 2024 10:21:35 -0600 Subject: [PATCH 07/21] More updates to atlantis site config for GNU --- configs/sites/tier1/atlantis/modules.yaml | 3 +++ configs/sites/tier1/atlantis/packages_gcc.yaml | 12 +++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml index 11c5514e2..a3a386a9c 100644 --- a/configs/sites/tier1/atlantis/modules.yaml +++ b/configs/sites/tier1/atlantis/modules.yaml @@ -6,3 +6,6 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python + - openmpi + exclude: + - slurm diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml index 7d36ed0b4..c5e189964 100644 --- a/configs/sites/tier1/atlantis/packages_gcc.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -10,14 +10,8 @@ packages: buildable: False externals: - spec: slurm@21.08.8 +pmix - modules: - - slurm/slurm/21.08.8 + prefix: /cm/shared/apps/slurm/21.08.8 + #modules: + #- slurm/slurm/21.08.8 openmpi: require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers' -modules: - default: - lmod: - include: - - openmpi - exclude: - - slurm From 46234a70f0c72f5e94c177773219eb8977a3ba9f Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:14:23 -0400 Subject: [PATCH 08/21] fix openmpi version on Hercules (#1326) --- configs/sites/tier1/hercules/packages_gcc.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/sites/tier1/hercules/packages_gcc.yaml b/configs/sites/tier1/hercules/packages_gcc.yaml index 5781224d8..37c496c45 100644 --- a/configs/sites/tier1/hercules/packages_gcc.yaml +++ b/configs/sites/tier1/hercules/packages_gcc.yaml @@ -2,13 +2,13 @@ packages: all: compiler:: [gcc@12.2.0] providers: - mpi:: [openmpi@4.1.6] + mpi:: [openmpi@4.1.4] mpi: buildable: False openmpi: externals: - - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath + - spec: openmpi@4.1.4%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath schedulers=slurm modules: - gcc/12.2.0 - - openmpi/4.1.6-gcc-12.2.0-spack + - openmpi/4.1.4 From 5101d55845ad79c591874367ec34262d80b22229 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 7 Oct 2024 15:50:04 -0600 Subject: [PATCH 09/21] For release/1.8.0: update S4 site config and documentation (#1328) * Update site config and documentation for S4 after Rocky8 update (Intel only so far with gcc@13 backend) --- configs/sites/tier1/s4/compilers.yaml | 48 +++++---- configs/sites/tier1/s4/packages.yaml | 141 +++++++++++++------------- doc/source/PreConfiguredSites.rst | 5 - 3 files changed, 102 insertions(+), 92 deletions(-) diff --git a/configs/sites/tier1/s4/compilers.yaml b/configs/sites/tier1/s4/compilers.yaml index d364dd768..dff2729fb 100644 --- a/configs/sites/tier1/s4/compilers.yaml +++ b/configs/sites/tier1/s4/compilers.yaml @@ -1,34 +1,46 @@ compilers: - compiler: - spec: intel@2021.5.0 + spec: intel@=2021.10.0 paths: - cc: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/icc - cxx: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/icpc - f77: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/ifort - fc: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/ifort + cc: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/ifort flags: {} - operating_system: centos7 + operating_system: rocky8 target: x86_64 modules: - - license_intel/S4 - - intel/2022.1 + - intel/2023.2 environment: prepend_path: - PATH: '/data/prod/hpc-stack/gnu/9.3.0/bin' - LD_LIBRARY_PATH: '/home/opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/compiler/lib/intel64_lin:/data/prod/hpc-stack/gnu/9.3.0/lib64' - CPATH: '/data/prod/hpc-stack/gnu/9.3.0/include' + PATH: /opt/gcc/13.3/bin + LD_LIBRARY_PATH: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/opt/gcc/13.3/lib64:/opt/gcc/13.3/lib + CPATH: /opt/gcc/13.3/include extra_rpaths: [] - compiler: - spec: gcc@9.3.0 + spec: gcc@=13.3.0 paths: - cc: /data/prod/hpc-stack/gnu/9.3.0/bin/gcc - cxx: /data/prod/hpc-stack/gnu/9.3.0/bin/g++ - f77: /data/prod/hpc-stack/gnu/9.3.0/bin/gfortran - fc: /data/prod/hpc-stack/gnu/9.3.0/bin/gfortran + cc: /opt/gcc/13.3/bin/gcc + cxx: /opt/gcc/13.3/bin/g++ + f77: /opt/gcc/13.3/bin/gfortran + fc: /opt/gcc/13.3/bin/gfortran flags: {} - operating_system: centos7 + operating_system: rocky8 target: x86_64 modules: - - gnu/9.3.0 + - gcc/13.3 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] environment: {} extra_rpaths: [] diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index e0312b799..a2457222e 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -1,10 +1,8 @@ packages: all: - compiler:: [intel@2021.5.0,gcc@9.3.0] + compiler:: [intel@2021.10.0,gcc@13.3.0] providers: - mpi:: [intel-oneapi-mpi@2021.5.0] - # https://github.com/JCSDA/spack-stack/issues/1055 - zlib-api:: [zlib] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -15,15 +13,18 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0 - prefix: /opt/intel/oneapi/2022.1 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + prefix: /opt/intel/oneapi/2023.2 + modules: + - intel/2023.2 intel-oneapi-mkl: # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl buildable: False #externals: - #- spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0 - # prefix: /opt/intel/oneapi/2022.1 - + #- spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0 + # prefix: /opt/intel/oneapi/2023.2 + # modules: + # - intel/2023.2 # Remove this section to switch to intel-oneapi-mkl ectrans: require:: @@ -42,47 +43,63 @@ packages: prefix: /usr automake: externals: - - spec: automake@1.15 + - spec: automake@1.16.1 prefix: /usr bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 + prefix: /usr + binutils: + externals: + - spec: binutils@2.30.123 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 prefix: /usr + cmake: + externals: + - spec: cmake@3.26.5 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr - diffutils: + curl: externals: - - spec: diffutils@3.3 + - spec: curl@7.61.1+gssapi+ldap+nghttp2 prefix: /usr - doxygen: + diffutils: externals: - - spec: doxygen@1.8.5+graphviz~mscgen + - spec: diffutils@3.6 prefix: /usr file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr flex: externals: - - spec: flex@2.5.37+lex + - spec: flex@2.6.1+lex prefix: /usr gawk: externals: - - spec: gawk@4.0.2 + - spec: gawk@4.2.1 prefix: /usr gettext: externals: @@ -90,105 +107,91 @@ packages: prefix: /usr ghostscript: externals: - - spec: ghostscript@9.25 + - spec: ghostscript@9.27 prefix: /usr git: externals: - - spec: git@2.30.0+tcltk - prefix: /opt/git/2.30.0 - modules: - - git/2.30.0 + - spec: git@2.43.5+tcltk + prefix: /usr git-lfs: externals: - - spec: git-lfs@2.10.0 + - spec: git-lfs@3.5.1 prefix: /usr gmake: externals: - - spec: gmake@3.82 + - spec: gmake@4.2.1 prefix: /usr go: externals: - - spec: go@1.11.5 + - spec: go@1.21.13 prefix: /usr groff: externals: - - spec: groff@1.22.2 - prefix: /usr - openjdk: - externals: - - spec: openjdk@1.8.0_262-b10 + - spec: groff@1.22.3 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - libfuse: - externals: - - spec: libfuse@2.9.2 + - spec: krb5@1.18.2 prefix: /usr libtool: externals: - - spec: libtool@2.4.2 + - spec: libtool@2.4.6 prefix: /usr - libxpm: + m4: externals: - - spec: libxpm@4.11.0 + - spec: m4@1.4.18 prefix: /usr - lustre: + openssh: externals: - - spec: lustre@2.10.5 + - spec: openssh@8.0p1 prefix: /usr - m4: + openssl: externals: - - spec: m4@1.4.16 + - spec: openssl@1.1.1k prefix: /usr - mysql: - buildable: False - externals: - - spec: mysql@8.0.31 - prefix: /data/prod/jedi/spack-stack/mysql-8.0.31 - modules: - - mysql/8.0.31 - pkg-config: + perl: externals: - - spec: pkg-config@0.27.1 + - spec: perl@5.26.3~cpanm+opcode+open+shared+threads prefix: /usr - qt: - buildable: False + pkgconf: externals: - - spec: qt@5.9.7 + - spec: pkgconf@1.4.2 prefix: /usr + # TODO - installing via spack-stack works, for example for + # spack-stack-1.8.0, but would be good if we didn't have to. + # https://github.com/JCSDA/spack-stack/issues/1329 + #qt: + # buildable: False + # externals: + # - spec: qt@5.9.7 + # prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 - prefix: /usr - texlive: - externals: - - spec: texlive@20130530 + - spec: texinfo@6.5 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr xz: externals: - - spec: xz@5.2.2 + - spec: xz@5.2.4 prefix: /usr zip: externals: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 2ea670ac2..bfcd808d3 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -343,14 +343,9 @@ UW (Univ. of Wisconsin) S4 The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge - module use /data/prod/jedi/spack-stack/modulefiles - module load miniconda/3.9.12 - module load ecflow/5.8.4 .. _Preconfigured_Sites_AWS_Parallelcluster: From cbf2bd3f66da87278b97b45af974d2a9339e4c1e Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Mon, 7 Oct 2024 21:55:36 -0400 Subject: [PATCH 10/21] Release 1.8.0 additional fixes for Jet, Hercules and Orion (after running ufs-weather-model) (#1331) * Update Jet packages_intel.yaml * Update Orion packages_intel.yaml * Update Orion packages_gcc.yaml * Update Hercules packages.yaml --- configs/sites/tier1/hercules/packages.yaml | 12 ++++++++++++ configs/sites/tier1/jet/packages_intel.yaml | 2 ++ configs/sites/tier1/orion/packages_gcc.yaml | 2 -- configs/sites/tier1/orion/packages_intel.yaml | 2 -- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml index 0128505f1..bb6e6222c 100644 --- a/configs/sites/tier1/hercules/packages.yaml +++ b/configs/sites/tier1/hercules/packages.yaml @@ -15,6 +15,10 @@ packages: externals: - spec: coreutils@8.32 prefix: /usr + curl: + externals: + - spec: curl@7.76.1+gssapi+ldap+nghttp2 + prefix: /usr diffutils: externals: - spec: diffutils@3.7 @@ -60,6 +64,14 @@ packages: prefix: /work/noaa/epic/role-epic/spack-stack/hercules/mysql-8.0.31 modules: - mysql/8.0.31 + openssh: + externals: + - spec: openssh@8.7p1 + prefix: /usr + openssl: + externals: + - spec: openssl@3.0.1 + prefix: /usr pkgconf: externals: - spec: pkgconf@1.7.3 diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index cc0cac045..5e0b82aaf 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -8,6 +8,8 @@ packages: intel-oneapi-mpi: externals: - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + modules: + - impi/2022.1.2 prefix: /apps/oneapi intel-oneapi-mkl: externals: diff --git a/configs/sites/tier1/orion/packages_gcc.yaml b/configs/sites/tier1/orion/packages_gcc.yaml index 8a8f8d641..aadfeb18c 100644 --- a/configs/sites/tier1/orion/packages_gcc.yaml +++ b/configs/sites/tier1/orion/packages_gcc.yaml @@ -3,8 +3,6 @@ packages: compiler:: [gcc@12.2.0] providers: mpi:: [openmpi@4.1.4] - # https://github.com/JCSDA/spack-stack/issues/1055 - zlib-api:: [zlib] mpi: buildable: False openmpi: diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index 8c1e33385..eb4a22dc7 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -3,8 +3,6 @@ packages: compiler:: [intel@2021.9.0,gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0] - # https://github.com/JCSDA/spack-stack/issues/1055 - zlib-api:: [zlib] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] From 4ecbdae943a6709d3221ad13bf3a8a1ba50ded71 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 11 Oct 2024 15:09:42 -0600 Subject: [PATCH 11/21] Next round of updates for Atlantis: use Bright OpenMPI --- configs/sites/tier1/atlantis/modules.yaml | 3 --- configs/sites/tier1/atlantis/packages_gcc.yaml | 18 +++++++----------- .../sites/tier1/atlantis/packages_intel.yaml | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml index a3a386a9c..11c5514e2 100644 --- a/configs/sites/tier1/atlantis/modules.yaml +++ b/configs/sites/tier1/atlantis/modules.yaml @@ -6,6 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - - openmpi - exclude: - - slurm diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml index c5e189964..bc10f8b75 100644 --- a/configs/sites/tier1/atlantis/packages_gcc.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -2,16 +2,12 @@ packages: all: compiler:: [gcc@11.2.0] providers: - mpi:: [openmpi@4.1.6] - #blas:: [intel-oneapi-mkl] - #fftw-api:: [intel-oneapi-mkl] - #lapack:: [intel-oneapi-mkl] - slurm: + mpi:: [openmpi@4.1.5] + mpi: buildable: False - externals: - - spec: slurm@21.08.8 +pmix - prefix: /cm/shared/apps/slurm/21.08.8 - #modules: - #- slurm/slurm/21.08.8 openmpi: - require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers' + buildable: False + externals: + - spec: openmpi@4.1.5%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm + modules: + - openmpi/mlnx/gcc/64/4.1.5a1 diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml index ea2f4de32..4271f3f10 100644 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ b/configs/sites/tier1/atlantis/packages_intel.yaml @@ -15,6 +15,6 @@ packages: intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.1.0 + prefix: /cm/shared/apps/intel/oneapi modules: - mkl/2022.1.0 - prefix: /cm/shared/apps/intel/oneapi From d6a41f55b18961db57997ef8bec38b1142b943db Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 11 Oct 2024 15:10:38 -0600 Subject: [PATCH 12/21] Update configs/common/packages.yaml: do not require, but prefer, the Python variant for ESMF, set accordingly in spack-ext/repos/spack-stack/packages/neptune-env/package.py --- configs/common/packages.yaml | 24 ++++++++++--------- .../packages/neptune-env/package.py | 3 ++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index fc7089741..1b48f823b 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -54,17 +54,19 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - '~xerces ~pnetcdf +shared +external-parallelio +python' - - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] - - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] - when: "%intel" - message: "Extra ESMF compile options for Intel" - #- any_of: [''] - # when: "%gcc" - # message: "Extra ESMF compile options for GCC" - #- any_of: [''] - # when: "%apple-clang" - # message: "Extra ESMF compile options for GCC" + - '~xerces ~pnetcdf +shared +external-parallelio' + - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] + - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] + when: "%intel" + message: "Extra ESMF compile options for Intel" + #- any_of: [''] + # when: "%gcc" + # message: "Extra ESMF compile options for GCC" + #- any_of: [''] + # when: "%apple-clang" + # message: "Extra ESMF compile options for GCC" + prefer: + - '+python' fckit: require: '@0.11.0 +eckit' fftw: diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index f79968eac..94c7baac7 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -39,7 +39,8 @@ class NeptuneEnv(BundlePackage): depends_on("ip@5:", type="run", when="%gcc") depends_on("ip@5:", type="run", when="%intel") depends_on("ip@5:", type="run", when="%oneapi") - depends_on("esmf", type="run") + depends_on("esmf +python", when="+python", type="run") + depends_on("esmf ~python", when="~python", type="run") depends_on("nco", type="run") depends_on("mct", type="run") From 28ea3f72cf0dbd1c15e0dba67fd3b3dac1ff32da Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 14 Oct 2024 10:00:50 -0600 Subject: [PATCH 13/21] Remove legacy environment variables for hdf5, p4est, libyaml from configs/common/modules_{lmod,tcl}.yaml --- configs/common/modules_lmod.yaml | 16 ---------------- configs/common/modules_tcl.yaml | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 41bfa24df..7c994ce45 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -112,20 +112,10 @@ modules: environment: set: 'ESMFMKFILE': '{prefix}/lib/esmf.mk' - hdf5: - environment: - set: - 'HDF5_DIR': '{prefix}' libpng: environment: set: 'PNG_ROOT': '{prefix}' - libyaml: - environment: - set: - 'YAML_DIR': '{prefix}' - 'YAML_LIB': '{prefix}/lib' - 'YAML_INC': '{prefix}/include' madis: environment: set: @@ -151,12 +141,6 @@ modules: set: 'OMPI_MCA_rmaps_base_oversubscribe': '1' 'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe' - p4est: - environment: - set: - 'P4EST_API_DIR': '{prefix}' - 'P4EST_API_LIB': '{prefix}/lib' - 'P4EST_API_INC': '{prefix}/include' bacio: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 5ce83e8b4..551c12e65 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -114,20 +114,10 @@ modules: environment: set: 'ESMFMKFILE': '{prefix}/lib/esmf.mk' - hdf5: - environment: - set: - 'HDF5_DIR': '{prefix}' libpng: environment: set: 'PNG_ROOT': '{prefix}' - libyaml: - environment: - set: - 'YAML_DIR': '{prefix}' - 'YAML_LIB': '{prefix}/lib' - 'YAML_INC': '{prefix}/include' madis: environment: set: @@ -153,12 +143,6 @@ modules: set: 'OMPI_MCA_rmaps_base_oversubscribe': '1' 'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe' - p4est: - environment: - set: - 'P4EST_API_DIR': '{prefix}' - 'P4EST_API_LIB': '{prefix}/lib' - 'P4EST_API_INC': '{prefix}/include' bacio: environment: set: From cd7c3208a04fc42339f03a4e7f004de4b47510ff Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 14 Oct 2024 10:02:09 -0600 Subject: [PATCH 14/21] Split neptune-env into neptune-env (with default espc false) and neptune-python-env --- configs/templates/neptune-dev/spack.yaml | 3 +- .../packages/neptune-env/package.py | 33 ++++----------- .../packages/neptune-python-env/package.py | 41 +++++++++++++++++++ 3 files changed, 50 insertions(+), 27 deletions(-) create mode 100644 spack-ext/repos/spack-stack/packages/neptune-python-env/package.py diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 5f3332f79..e0219fa0d 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,7 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc +python +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 + - neptune-python-env +xnrl specs: - matrix: diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index f79968eac..ddd732692 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,7 +9,7 @@ class NeptuneEnv(BundlePackage): - """Development environment for neptune standalone""" + """Development environment for NEPTUNE standalone""" # Fake URL homepage = "https://github.com/notavalidaccount/neptune" @@ -17,11 +17,9 @@ class NeptuneEnv(BundlePackage): maintainers("climbfuji", "areinecke") - version("1.4.0") + version("1.5.0") - variant("python", default=True, description="Build Python dependencies") - variant("espc", default=True, description="Build ESPC dependencies") - variant("xnrl", default=True, description="Build XNRL and its extra Python dependencies") + variant("espc", default=False, description="Build ESPC dependencies") depends_on("base-env", type="run") @@ -33,7 +31,6 @@ class NeptuneEnv(BundlePackage): depends_on("libyaml", type="run") depends_on("p4est", type="run") depends_on("w3emc", type="run") - depends_on("w3nco", type="run") depends_on("sp", type="run", when="%aocc") depends_on("ip@5:", type="run", when="%apple-clang") depends_on("ip@5:", type="run", when="%gcc") @@ -43,28 +40,12 @@ class NeptuneEnv(BundlePackage): depends_on("nco", type="run") depends_on("mct", type="run") - conflicts("+xnrl", when="~python", msg="Variant xnrl requires variant python") - with when("+espc"): depends_on("fftw", type="build") depends_on("netlib-lapack", type="build") - with when("+python"): - depends_on("py-f90nml", type="run") - depends_on("py-h5py", type="run") - depends_on("py-netcdf4", type="run") - depends_on("py-pandas", type="run") - depends_on("py-pycodestyle", type="run") - depends_on("py-pybind11", type="run") - depends_on("py-pyhdf", type="run") - depends_on("py-python-dateutil", type="run") - depends_on("py-pyyaml", type="run") - depends_on("py-scipy", type="run") - depends_on("py-xarray", type="run") - depends_on("py-pytest", type="run") - depends_on("py-fortranformat", type="run") - - with when("+xnrl"): - depends_on("py-xnrl", type="run") + # Basic Python dependencies that are always needed + depends_on("py-f90nml", type="run") + depends_on("py-python-dateutil", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py new file mode 100644 index 000000000..5b6f243b8 --- /dev/null +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -0,0 +1,41 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys + +from spack.package import * + + +class NeptunePythonEnv(BundlePackage): + """Development environment for NEPTUNE standalone with all Python dependencies""" + + # Fake URL + homepage = "https://github.com/notavalidaccount/neptune" + git = "https://github.com/notavalidaccount/neptune.git" + + maintainers("climbfuji", "areinecke") + + version("1.5.0") + + variant("xnrl", default=False, description="Build non-pulic XNRL") + + depends_on("neptune-env", type="run") + + depends_on("py-h5py", type="run") + depends_on("py-netcdf4", type="run") + depends_on("py-pandas", type="run") + depends_on("py-pycodestyle", type="run") + depends_on("py-pybind11", type="run") + depends_on("py-pyhdf", type="run") + depends_on("py-pyyaml", type="run") + depends_on("py-scipy", type="run") + depends_on("py-xarray", type="run") + depends_on("py-pytest", type="run") + depends_on("py-fortranformat", type="run") + + with when("+xnrl"): + depends_on("py-xnrl", type="run") + + # There is no need for install() since there is no code. From 886cf903d9116e2b05a47240a05efcde0e47b193 Mon Sep 17 00:00:00 2001 From: Nate Crossette Date: Mon, 14 Oct 2024 06:22:32 -0600 Subject: [PATCH 15/21] Add note to docs for selecting branch when cloning (#1341) --- doc/source/NewSiteConfigs.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 036fa41f9..65b59b6b0 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -211,11 +211,11 @@ Creating a new environment Remember to activate the ``lua`` module environment and have MacTeX in your search path, if applicable. It is also recommended to increase the stacksize limit to 65Kb using ``ulimit -S -s unlimited``. -1. You will need to clone spack-stack and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. +1. You will need to clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. .. code-block:: console - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git + git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} @@ -500,11 +500,11 @@ Creating a new environment It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimited``, and to test if the module environment functions correctly (``module available``). -1. You will need to clone spack-stack and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. +1. You will need to clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. .. code-block:: console - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git + git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} @@ -675,11 +675,11 @@ With all of that in mind, the following instructions were used on an Amazon Web module use /opt/nvidia/hpc_sdk/modulefiles module load nvhpc-openmpi3/24.3 -4. Clone spack-stack and its dependencies and activate the spack-stack tool. +4. Clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. .. code-block:: console - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git + git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} From 3d25bfb129406711d6a1067602116eb65dc9570b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 14 Oct 2024 10:55:30 -0600 Subject: [PATCH 16/21] Update configs/templates/{neptune-dev,skylab-dev,unified-dev} --- configs/templates/neptune-dev/spack.yaml | 4 ++-- configs/templates/skylab-dev/spack.yaml | 3 ++- configs/templates/unified-dev/spack.yaml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index e0219fa0d..564cf6d86 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,8 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 - - neptune-python-env +xnrl + - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 + - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 1ba272a63..4616138ca 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -17,7 +17,8 @@ spack: - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 - soca-env # Various crtm tags (list all to avoid duplicate packages) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index a05ed4a75..608b4462a 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -21,7 +21,8 @@ spack: - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 From d6641e4968207de1b83791fdd518e76660c488d8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 16 Oct 2024 10:40:34 -0600 Subject: [PATCH 17/21] Add py-pyyaml to neptune-env, remove invalid Python variant from neptune-env and add esmf+python to neptune-python-env --- spack-ext/repos/spack-stack/packages/neptune-env/package.py | 4 ++-- .../repos/spack-stack/packages/neptune-python-env/package.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index a8d3d5807..ff7aef9d4 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -36,8 +36,7 @@ class NeptuneEnv(BundlePackage): depends_on("ip@5:", type="run", when="%gcc") depends_on("ip@5:", type="run", when="%intel") depends_on("ip@5:", type="run", when="%oneapi") - depends_on("esmf +python", when="+python", type="run") - depends_on("esmf ~python", when="~python", type="run") + depends_on("esmf", type="run") depends_on("nco", type="run") depends_on("mct", type="run") @@ -48,5 +47,6 @@ class NeptuneEnv(BundlePackage): # Basic Python dependencies that are always needed depends_on("py-f90nml", type="run") depends_on("py-python-dateutil", type="run") + depends_on("py-pyyaml", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py index 5b6f243b8..2e20105b8 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -22,6 +22,8 @@ class NeptunePythonEnv(BundlePackage): variant("xnrl", default=False, description="Build non-pulic XNRL") depends_on("neptune-env", type="run") + # Enable the Python variant for ESMF + depends_on("esmf +python", type="run") depends_on("py-h5py", type="run") depends_on("py-netcdf4", type="run") From c6a067f7730e4554fd75f9685d603d10f60c55de Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 16 Oct 2024 14:33:29 -0600 Subject: [PATCH 18/21] Update site config and documentation for Narwhal to use new, official oneapi@2024.2 installation --- configs/sites/tier1/narwhal/compilers.yaml | 16 ++++------------ .../sites/tier1/narwhal/packages_oneapi.yaml | 19 +++++++------------ doc/source/PreConfiguredSites.rst | 10 ++-------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 5e2e17c11..838a8a1dd 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -22,12 +22,8 @@ compilers:: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] - compiler: - spec: oneapi@2024.2.1 + spec: oneapi@2024.2.0 paths: - #cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx - #cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx - #f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort - #fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort cc: cc cxx: CC f77: ftn @@ -35,11 +31,8 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 - - ifort/2024.2.1 + - PrgEnv-intel/8.4.0 + - intel/2024.2 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 environment: @@ -47,9 +40,8 @@ compilers:: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' - MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' + CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64' set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml index 9b032dcf1..4724e25a3 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -7,26 +7,21 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers + - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - mkl/2024.2 + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 intel-oneapi-tbb: externals: - - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 modules: - tbb/2021.13 intel-oneapi-runtime: externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - tbb/2021.13 - - compiler-rt/2024.2.1 + - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 79567859e..cd3b8e3bb 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -166,19 +166,13 @@ The following is required for building new spack environments with Intel on this The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! -.. note:: This particular setup of the oneAPI compilers is outside the Cray-blessed environment and there are problems installing Python packages with spack. - .. code-block:: console umask 0022 module unload PrgEnv-cray - module load PrgEnv-intel/8.3.3 + module load PrgEnv-intel/8.4.0 module unload intel - module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles - module load tbb/2021.13 - module load compiler-rt/2024.2.1 - module load compiler/2024.2.1 - module load ifort/2024.2.1 + module load intel/2024.2 module unload cray-mpich module unload craype-network-ofi # Warning. Do not load craype-network-ucx From 716932c9dc5e4597c60fc61c50877e13694fa742 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 18 Oct 2024 07:59:46 -0600 Subject: [PATCH 19/21] Fix typo in configs/sites/tier1/narwhal/packages_oneapi.yaml --- configs/sites/tier1/narwhal/packages_oneapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml index 4724e25a3..3bcf99f38 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.2.1,gcc@10.3.0] + compiler:: [oneapi@2024.2.0,gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.21] mpi: From 41b59e35d7358e2418f1bbad04a2208928ef7be5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 18 Oct 2024 18:37:17 -0600 Subject: [PATCH 20/21] Update submodule pointer for spack (esmf@8.7.0) and replace esmf@8.7.0b11 with esmf@8.7.0 in all configs --- configs/common/modules_lmod.yaml | 2 ++ configs/common/modules_tcl.yaml | 2 ++ configs/common/packages | 0 configs/common/packages.yaml | 2 +- configs/templates/neptune-dev/spack.yaml | 4 ++-- configs/templates/skylab-dev/spack.yaml | 8 ++++---- configs/templates/unified-dev/spack.yaml | 8 ++++---- spack | 2 +- 8 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 configs/common/packages diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 7c994ce45..8fe637fb6 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -136,6 +136,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 551c12e65..3275aee44 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -138,6 +138,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/packages b/configs/common/packages new file mode 100644 index 000000000..e69de29bb diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 1b48f823b..99f17b060 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -55,7 +55,7 @@ packages: esmf: require: - '~xerces ~pnetcdf +shared +external-parallelio' - - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] + - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: "%intel" message: "Extra ESMF compile options for Intel" diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 564cf6d86..9a9e3dfec 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,8 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 - - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0 + - neptune-python-env +xnrl ^esmf@8.7.0 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 4616138ca..be901ec7e 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -14,11 +14,11 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env # Various crtm tags (list all to avoid duplicate packages) @@ -27,7 +27,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none specs: - matrix: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 608b4462a..370a03e7c 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -17,12 +17,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 @@ -33,7 +33,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/spack b/spack index 6f9a6ba4e..b9f2dd225 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9 +Subproject commit b9f2dd2254cfb1086040f1a100746bd22cb12702 From 4b2afed39127320af04f3b1d80d2e98dc17db101 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 09:44:20 -0600 Subject: [PATCH 21/21] Change spack source-cache locations on atlantis, narwhal, nautilus --- configs/sites/tier1/atlantis/mirrors.yaml | 2 -- configs/sites/tier1/narwhal/mirrors.yaml | 4 ++-- configs/sites/tier1/nautilus/mirrors.yaml | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml index 94e16cfaf..4d363e009 100644 --- a/configs/sites/tier1/atlantis/mirrors.yaml +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -1,7 +1,6 @@ mirrors: local-source: fetch: - # TODO UPDATE? url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null @@ -10,7 +9,6 @@ mirrors: profile: null endpoint_url: null push: - # TODO UPDATE? url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null