From 63bb917ac05f67b08d8c74cb3e513f769eaa116e Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 6 Aug 2024 17:42:22 -0600
Subject: [PATCH 01/28] Create packages_{apple-clang,gcc,intel,oneapi}.yaml and
 move compiler-specific settings out of packages.yaml

---
 configs/common/packages.yaml             | 531 +++++++++++------------
 configs/common/packages_apple-clang.yaml |  18 +
 configs/common/packages_gcc.yaml         |  18 +
 configs/common/packages_intel.yaml       |  23 +
 configs/common/packages_oneapi.yaml      |  23 +
 5 files changed, 347 insertions(+), 266 deletions(-)
 create mode 100644 configs/common/packages_apple-clang.yaml
 create mode 100644 configs/common/packages_gcc.yaml
 create mode 100644 configs/common/packages_intel.yaml
 create mode 100644 configs/common/packages_oneapi.yaml

diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
index b26f9a038..4d6bfec9c 100644
--- a/configs/common/packages.yaml
+++ b/configs/common/packages.yaml
@@ -1,267 +1,266 @@
 # Pin versions and specs when building packages
-  packages:
-    #
-    all:
-      providers:
-        blas: [openblas]
-        fftw-api: [fftw]
-        gl: [opengl]
-        glu: [openglu]
-        jpeg: [libjpeg-turbo]
-        lapack: [openblas]
-        pkgconfig: [pkg-config]
-        yacc: [bison]
-        zlib-api: [zlib-ng]
-    #
-    awscli-v2:
-      require: '~examples'
-    bacio:
-      require: '@2.4.1'
-    bison:
-      require: '@3.4:'
-    # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version'
-    boost:
-      require: '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden'
-    bufr:
-      require: '@12.0.1 +python'
-    cairo:
-      require: '+pic'
-    cdo:
-      require: '@2.3.0 ~openmp'
-    cmake:
-      version: ['3.27.9']
-      require: '+ownlibs'
-    # Attention - when updating also check the various jcsda-emc-bundles env packages
-    crtm:
-      require: '+fix'
-    ecbuild:
-      require: '@3.7.2'
-    eccodes:
-      require: '@2.33.0 +png +tools'
-    ecflow:
-      require: '@5.11.4 +ui'
-    eckit:
-      require: '@1.24.5 linalg=eigen,lapack compression=lz4,bzip2'
-    ecmwf-atlas:
-      require: '@0.36.0 +fckit +trans +tesselation +fftw'
-    ectrans:
-      require: '@1.2.0 ~mkl +fftw'
-    eigen:
-      require: '@3.4.0'
-    # Attention - when updating the version also check the common modules.yaml
-    # config and update the projections for lmod/tcl.
-    # Also, check the acorn and derecho site configs which have esmf modifications.
-    esmf:
-      require:
-        - '~xerces ~pnetcdf +shared +external-parallelio'
-        - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b04 snapshot=b04']
-        - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
-          when: "%intel"
-          message: "Extra ESMF compile options for Intel"
-        #- any_of: ['']
-        #  when: "%gcc"
-        #  message: "Extra ESMF compile options for GCC"
-        #- any_of: ['']
-        #  when: "%apple-clang"
-        #  message: "Extra ESMF compile options for GCC"
-    fckit:
-      require: '@0.11.0 +eckit'
-    fftw:
-      require: '@3.3.10'
-    fiat:
-      require: '@1.2.0'
-    flex:
-      # Pin version to avoid duplicates
-      require: '@2.6.4'
-    fms:
-      require: '@2023.04'
-      variants: 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io'
-    fontconfig:
-      require: '+pic'
-    freetype:
-      require: '+pic'
-    g2:
-      require: '@3.4.9'
-    g2c:
-      require: '@1.6.4'
-    g2tmpl:
-      require:
-        - '@1.10.2'
-    gcc-runtime:
-      require: '%gcc'
-    gfsio:
-      require: '@1.4.1'
-    #git-lfs:
-      # Assume git-lfs is provided, hard to install
-      # because of dependencies on go/go-bootstrap.
-      # Note: Uncommenting this entry will break
-      # the container builds.
-      #require: '@2.11.0'
-    grib-util:
-      require: '@1.4.0'
-    gsibec:
-      require: '@1.2.1'
-    gsi-ncdiag:
-      require: '@1.1.2'
-    gsl-lite:
-      require: '@0.37.0'
-    hdf:
-      require: '@4.2.15 +external-xdr ~fortran ~netcdf'
-    hdf5:
-      require: '@1.14.3 +hl +fortran +mpi +threadsafe ~szip'
-    # Newer versions of hdf-eos2 require manual downloading, avoid
-    hdf-eos2:
-      require: '@2.20v1.00'
-    ip:
-      require: '@5.0.0 precision=4,d,8'
-    ip2:
-      require: '@1.1.2'
-    jasper:
-      require: '@2.0.32'
-    jedi-cmake:
-      require: '@1.4.0'
-    landsfcutil:
-      require: '@2.4.1'
-    libjpeg-turbo:
-      require: '@2.1.0'
-    libpng:
-      require: '@1.6.37 +pic'
-    # Newest version of magics needed for oneapi compilers
-    magics:
-      require: "@4.15.3:"
-    mapl:
-      require: '@2.46.2 +shared ~f2py'
-      variants: '+pflogger'
-    # If making changes here, also check the Discover site configs and the CI workflows
-    met:
-      require: '@11.1.0 +python +grib2'
-    metplus:
-      require: '@5.1.0'
-    metis:
-      require: '+int64 +real64'
-    mpich:
-      require: '~hwloc'
-    nco:
-      require: '@5.1.6 ~doc'
-    # ncview - when adding information here, also check Orion
-    # and Discover site configs
-    nemsio:
-      require: '@2.5.4'
-    nemsiogfs:
-      require: '@2.5.3'
-    nccmp:
-      require: '@1.9.0.1'
-    ncio:
-      require: '@1.1.2'
-    netcdf-c:
-      # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange
-      require: '@4.9.2 +dap +mpi ~parallel-netcdf ~szip build_system=autotools'
-    netcdf-cxx4:
-      require: '@4.3.1'
-    netcdf-fortran:
-      require: '@4.6.1'
-    # ninja - when adding information here, also check Discover site config
-    odc:
-      require: '@1.4.6 ~fortran'
-    openblas:
-      require: '@0.3.24 +noavx512'
-    openmpi:
-      require: '~internal-hwloc +two_level_namespace'
-    openssl:
-      require: '+shared'
-    p4est:
-      require: '@2.8'
-    parallelio:
-      require: '@2.6.2 +pnetcdf'
-    parallel-netcdf:
-      require: '@1.12.3'
-    pflogger:
-      require: '+mpi'
-    pixman:
-      require: '+pic'
-    prod-util:
-      require: '@2.1.1'
-    proj:
-      require: '~tiff'
-    python:
-      require: '@3.10.13'
-    py-attrs:
-      # https://github.com/JCSDA/spack-stack/issues/740
-      require: '@21.4.0'
-    py-cartopy:
-      require: '+plotting'
-    py-cryptography:
-      require: '+rust_bootstrap'
-    # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
-    # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
-    py-cython:
-      require: '@0.29.36'
-    # To avoid duplicate packages
-    py-flit-core:
-      require: '@3.8.0'
-    py-h5py:
-      require: '~mpi'
-    ## To avoid duplicate packages
-    #py-jinja2:
-    #  require: '@3.1.2'
-    py-meson-python:
-       require: '@0.15.0'
-    # Comment out for now until build problems are solved
-    # https://github.com/jcsda/spack-stack/issues/522
-    # see also ewok-env virtual package and container
-    # README.md
-    #py-mysql-connector-python:
-    #  require: '@8.0.32'
-    py-netcdf4:
-      require: '@1.5.8 ~mpi'
-    # py-numpy@1.26 causes many build problems with older Python packages
-    # also check Nautilus site config when making changes here
-    py-numpy:
-      require:
-      - '@:1.25'
-    py-pandas:
-      require: '+excel'
-    py-pybind11:
-      require: '@2.11.0'
-    # Pin the py-setuptools version to avoid duplicate Python packages
-    py-setuptools:
-      require: '@63.4.3'
-    py-setuptools-rust:
-      require: '+rust_bootstrap'
-    py-shapely:
-      require: '@1.8.0'
-    py-torch:
-      require: '+custom-protobuf ~mkldnn'
-    # To avoid duplicate packages
-    py-urllib3:
-      require: '@1.26.12'
-    qt:
-      require: '@5'
-    scotch:
-      require: '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch'
-    sfcio:
-      require: '@1.4.1'
-    shumlib:
-      require: '@macos_clang_linux_intel_port'
-    sigio:
-      require: '@2.3.2'
-    sp:
-      require: '@2.5.0 precision=4,d,8'
-    udunits:
-      require: '@2.2.28'
-    ufs-utils:
-      require: '@1.13.0'
-    # Note - we can remove upp from stack at some point?
-    upp:
-      require: '@10.0.10'
-    # Need extradeps for grib-utils, enable by default to avoid duplicate packages
-    w3emc:
-      require: '@2.10.0 precision=4,d,8 +extradeps'
-    w3nco:
-      require: '@2.4.1'
-    wgrib2:
-      require: '@3.1.1'
-    wrf-io:
-      require: '@1.2.0'
-    zstd:
-      require: '@1.5.2 +programs'
+packages:
+  #
+  all:
+    providers:
+      gl: [opengl]
+      glu: [openglu]
+      jpeg: [libjpeg-turbo]
+      pkgconfig: [pkg-config]
+      yacc: [bison]
+      zlib-api: [zlib-ng]
+  #
+  awscli-v2:
+    require: '~examples'
+  bacio:
+    require: '@2.4.1'
+  bison:
+    require: '@3.4:'
+  # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version'
+  boost:
+    require: '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden'
+  bufr:
+    require: '@12.0.1 +python'
+  cairo:
+    require: '+pic'
+  cdo:
+    require: '@2.3.0 ~openmp'
+  cmake:
+    version: ['3.27.9']
+    require: '+ownlibs'
+  # Attention - when updating also check the various jcsda-emc-bundles env packages
+  crtm:
+    require: '+fix'
+  ecbuild:
+    require: '@3.7.2'
+  eccodes:
+    require: '@2.33.0 +png +tools'
+  ecflow:
+    require: '@5.11.4 +ui'
+  eckit:
+    require: '@1.24.5 linalg=eigen,lapack compression=lz4,bzip2'
+  ecmwf-atlas:
+    require: '@0.36.0 +fckit +trans +tesselation +fftw'
+  ectrans:
+    require:
+    - '@1.2.0'
+  eigen:
+    require: '@3.4.0'
+  # Attention - when updating the version also check the common modules.yaml
+  # config and update the projections for lmod/tcl.
+  # Also, check the acorn and derecho site configs which have esmf modifications.
+  esmf:
+    require:
+      - '~xerces ~pnetcdf +shared +external-parallelio'
+      - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b04 snapshot=b04']
+      - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+        when: "%intel"
+        message: "Extra ESMF compile options for Intel"
+      #- any_of: ['']
+      #  when: "%gcc"
+      #  message: "Extra ESMF compile options for GCC"
+      #- any_of: ['']
+      #  when: "%apple-clang"
+      #  message: "Extra ESMF compile options for GCC"
+  fckit:
+    require: '@0.11.0 +eckit'
+  fftw:
+    require: '@3.3.10'
+  fiat:
+    require: '@1.2.0'
+  flex:
+    # Pin version to avoid duplicates
+    require: '@2.6.4'
+  fms:
+    require: '@2023.04'
+    variants: 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io'
+  fontconfig:
+    require: '+pic'
+  freetype:
+    require: '+pic'
+  g2:
+    require: '@3.4.9'
+  g2c:
+    require: '@1.6.4'
+  g2tmpl:
+    require:
+    - '@1.10.2'
+  gcc-runtime:
+    require: '%gcc'
+  gfsio:
+    require: '@1.4.1'
+  #git-lfs:
+    # Assume git-lfs is provided, hard to install
+    # because of dependencies on go/go-bootstrap.
+    # Note: Uncommenting this entry will break
+    # the container builds.
+    #require: '@2.11.0'
+  grib-util:
+    require: '@1.4.0'
+  gsibec:
+    require:
+    - '@1.2.1'
+  gsi-ncdiag:
+    require: '@1.1.2'
+  gsl-lite:
+    require: '@0.37.0'
+  hdf:
+    require: '@4.2.15 +external-xdr ~fortran ~netcdf'
+  hdf5:
+    require: '@1.14.3 +hl +fortran +mpi +threadsafe ~szip'
+  # Newer versions of hdf-eos2 require manual downloading, avoid
+  hdf-eos2:
+    require: '@2.20v1.00'
+  ip:
+    require: '@5.0.0 precision=4,d,8'
+  ip2:
+    require: '@1.1.2'
+  jasper:
+    require: '@2.0.32'
+  jedi-cmake:
+    require: '@1.4.0'
+  landsfcutil:
+    require: '@2.4.1'
+  libjpeg-turbo:
+    require: '@2.1.0'
+  libpng:
+    require: '@1.6.37 +pic'
+  # Newest version of magics needed for oneapi compilers
+  magics:
+    require: "@4.15.3:"
+  mapl:
+    require: '@2.46.2 +shared ~f2py'
+    variants: '+pflogger'
+  # If making changes here, also check the Discover site configs and the CI workflows
+  met:
+    require: '@11.1.0 +python +grib2'
+  metplus:
+    require: '@5.1.0'
+  metis:
+    require: '+int64 +real64'
+  mpich:
+    require: '~hwloc'
+  nco:
+    require: '@5.1.6 ~doc'
+  # ncview - when adding information here, also check Orion
+  # and Discover site configs
+  nemsio:
+    require: '@2.5.4'
+  nemsiogfs:
+    require: '@2.5.3'
+  nccmp:
+    require: '@1.9.0.1'
+  ncio:
+    require: '@1.1.2'
+  netcdf-c:
+    # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange
+    require: '@4.9.2 +dap +mpi ~parallel-netcdf ~szip build_system=autotools'
+  netcdf-cxx4:
+    require: '@4.3.1'
+  netcdf-fortran:
+    require: '@4.6.1'
+  # ninja - when adding information here, also check Discover site config
+  odc:
+    require: '@1.4.6 ~fortran'
+  openblas:
+    require: '@0.3.24 +noavx512'
+  openmpi:
+    require: '~internal-hwloc +two_level_namespace'
+  openssl:
+    require: '+shared'
+  p4est:
+    require: '@2.8'
+  parallelio:
+    require: '@2.6.2 +pnetcdf'
+  parallel-netcdf:
+    require: '@1.12.3'
+  pflogger:
+    require: '+mpi'
+  pixman:
+    require: '+pic'
+  prod-util:
+    require: '@2.1.1'
+  proj:
+    require: '~tiff'
+  python:
+    require: '@3.10.13'
+  py-attrs:
+    # https://github.com/JCSDA/spack-stack/issues/740
+    require: '@21.4.0'
+  py-cartopy:
+    require: '+plotting'
+  py-cryptography:
+    require: '+rust_bootstrap'
+  # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
+  # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
+  py-cython:
+    require: '@0.29.36'
+  # To avoid duplicate packages
+  py-flit-core:
+    require: '@3.8.0'
+  py-h5py:
+    require: '~mpi'
+  ## To avoid duplicate packages
+  #py-jinja2:
+  #  require: '@3.1.2'
+  py-meson-python:
+     require: '@0.15.0'
+  # Comment out for now until build problems are solved
+  # https://github.com/jcsda/spack-stack/issues/522
+  # see also ewok-env virtual package and container
+  # README.md
+  #py-mysql-connector-python:
+  #  require: '@8.0.32'
+  py-netcdf4:
+    require: '@1.5.8 ~mpi'
+  # py-numpy@1.26 causes many build problems with older Python packages
+  # also check Nautilus site config when making changes here
+  py-numpy:
+    require:
+    - '@:1.25'
+  py-pandas:
+    require: '+excel'
+  py-pybind11:
+    require: '@2.11.0'
+  # Pin the py-setuptools version to avoid duplicate Python packages
+  py-setuptools:
+    require: '@63.4.3'
+  py-setuptools-rust:
+    require: '+rust_bootstrap'
+  py-shapely:
+    require: '@1.8.0'
+  py-torch:
+    require: '+custom-protobuf ~mkldnn'
+  # To avoid duplicate packages
+  py-urllib3:
+    require: '@1.26.12'
+  qt:
+    require: '@5'
+  scotch:
+    require: '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch'
+  sfcio:
+    require: '@1.4.1'
+  shumlib:
+    require: '@macos_clang_linux_intel_port'
+  sigio:
+    require: '@2.3.2'
+  sp:
+    require: '@2.5.0 precision=4,d,8'
+  udunits:
+    require: '@2.2.28'
+  ufs-utils:
+    require: '@1.13.0'
+  # Note - we can remove upp from stack at some point?
+  upp:
+    require: '@10.0.10'
+  # Need extradeps for grib-utils, enable by default to avoid duplicate packages
+  w3emc:
+    require: '@2.10.0 precision=4,d,8 +extradeps'
+  w3nco:
+    require: '@2.4.1'
+  wgrib2:
+    require: '@3.1.1'
+  wrf-io:
+    require: '@1.2.0'
+  zstd:
+    require: '@1.5.2 +programs'
diff --git a/configs/common/packages_apple-clang.yaml b/configs/common/packages_apple-clang.yaml
new file mode 100644
index 000000000..cc7276436
--- /dev/null
+++ b/configs/common/packages_apple-clang.yaml
@@ -0,0 +1,18 @@
+# Pin versions and specs when building packages
+packages:
+  #
+  all:
+    providers:
+      blas: [openblas]
+      fftw-api: [fftw]
+      lapack: [openblas]
+  #
+  ectrans:
+    require:
+    - '~mkl +fftw'
+  gsibec:
+    require:
+    - '~mkl'
+  #py-numpy:
+  #  require:
+  #  - '^intel-oneapi-mkl' ^ ???
diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml
new file mode 100644
index 000000000..cc7276436
--- /dev/null
+++ b/configs/common/packages_gcc.yaml
@@ -0,0 +1,18 @@
+# Pin versions and specs when building packages
+packages:
+  #
+  all:
+    providers:
+      blas: [openblas]
+      fftw-api: [fftw]
+      lapack: [openblas]
+  #
+  ectrans:
+    require:
+    - '~mkl +fftw'
+  gsibec:
+    require:
+    - '~mkl'
+  #py-numpy:
+  #  require:
+  #  - '^intel-oneapi-mkl' ^ ???
diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml
new file mode 100644
index 000000000..6dec75f43
--- /dev/null
+++ b/configs/common/packages_intel.yaml
@@ -0,0 +1,23 @@
+packages:
+  all:
+    providers:
+      blas: [intel-oneapi-mkl]
+      fftw-api: [intel-oneapi-mkl]
+      lapack: [intel-oneapi-mkl]
+  ectrans:
+    require:
+    - '+mkl ~fftw'
+  gsibec:
+    require:
+    - '+mkl'
+  py-numpy:
+    require:
+    - '^intel-oneapi-mkl'
+  bison:
+    require: '%gcc'
+  gmake:
+    require: '%gcc'
+  libmd:
+    require: '%gcc'
+  libbsd:
+    require: '%gcc'
diff --git a/configs/common/packages_oneapi.yaml b/configs/common/packages_oneapi.yaml
new file mode 100644
index 000000000..6dec75f43
--- /dev/null
+++ b/configs/common/packages_oneapi.yaml
@@ -0,0 +1,23 @@
+packages:
+  all:
+    providers:
+      blas: [intel-oneapi-mkl]
+      fftw-api: [intel-oneapi-mkl]
+      lapack: [intel-oneapi-mkl]
+  ectrans:
+    require:
+    - '+mkl ~fftw'
+  gsibec:
+    require:
+    - '+mkl'
+  py-numpy:
+    require:
+    - '^intel-oneapi-mkl'
+  bison:
+    require: '%gcc'
+  gmake:
+    require: '%gcc'
+  libmd:
+    require: '%gcc'
+  libbsd:
+    require: '%gcc'

From b9be67492d2daa90efa2eb4b707026df63153841 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 6 Aug 2024 22:16:00 -0600
Subject: [PATCH 02/28] Update blackpearl site config: remove oneapi settings
 (now in common packages), add mirror

---
 configs/sites/tier2/blackpearl/mirrors.yaml    |  2 ++
 .../tier2/blackpearl/packages_oneapi.yaml      | 18 ------------------
 2 files changed, 2 insertions(+), 18 deletions(-)
 create mode 100644 configs/sites/tier2/blackpearl/mirrors.yaml

diff --git a/configs/sites/tier2/blackpearl/mirrors.yaml b/configs/sites/tier2/blackpearl/mirrors.yaml
new file mode 100644
index 000000000..95ab7092e
--- /dev/null
+++ b/configs/sites/tier2/blackpearl/mirrors.yaml
@@ -0,0 +1,2 @@
+mirrors:
+  local-source: file:///home/dom/prod/spack-source-cache
diff --git a/configs/sites/tier2/blackpearl/packages_oneapi.yaml b/configs/sites/tier2/blackpearl/packages_oneapi.yaml
index 385837c86..8466a209c 100644
--- a/configs/sites/tier2/blackpearl/packages_oneapi.yaml
+++ b/configs/sites/tier2/blackpearl/packages_oneapi.yaml
@@ -3,24 +3,6 @@ packages:
     compiler:: [oneapi@2024.1.2]
     providers:
       mpi:: [intel-oneapi-mpi]
-      blas:: [intel-oneapi-mkl]
-      fftw-api:: [intel-oneapi-mkl]
-      lapack:: [intel-oneapi-mkl]
-  ectrans:
-    require:: '+mkl ~fftw'
-  gsibec:
-    require:: '+mkl'
-  py-numpy:
-    require:
-    - '^intel-oneapi-mkl'
-  bison:
-    require: '%gcc'
-  gmake:
-    require: '%gcc'
-  libmd:
-    require: '%gcc'
-  libbsd:
-    require: '%gcc'
 
   mpi:
     buildable: False

From b69c4e01a591e0bf6487f2af93598a29ce7274d9 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:08:23 -0600
Subject: [PATCH 03/28] Update .gitmodules and submodule pointer for spack for
 code review and testing

---
 .gitmodules | 6 ++++--
 spack       | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index a69f3e4f1..df95b971e 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,7 +1,9 @@
 [submodule "spack"]
   path = spack
-  url = https://github.com/jcsda/spack
-  branch = spack-stack-dev
+  #url = https://github.com/jcsda/spack
+  #branch = spack-stack-dev
+  url = https://github.com/climbfuji/spack
+  branch = feature/update_odc_and_yafyaml_for_gcc13
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules
diff --git a/spack b/spack
index e77862a08..7b1808fc8 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit e77862a0839789e9ee5bb501aba3884b6b0e4119
+Subproject commit 7b1808fc8a344e551d6012a23b385571a354062d

From 8089b573537ffb4eaab33e2e6dec08a7330b2032 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:11:36 -0600
Subject: [PATCH 04/28] Temporarily remove wgrib2 requirement from various
 virtual packages for '%oneapi'

---
 .../spack-stack/packages/emc-verif-global-env/package.py    | 6 +++++-
 .../spack-stack/packages/global-workflow-env/package.py     | 6 +++++-
 .../repos/spack-stack/packages/nceplibs-env/package.py      | 6 +++++-
 .../repos/spack-stack/packages/ufs-srw-app-env/package.py   | 6 +++++-
 .../repos/spack-stack/packages/ufs-utils-env/package.py     | 6 +++++-
 spack-ext/repos/spack-stack/packages/upp-env/package.py     | 6 +++++-
 6 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py b/spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py
index 3632d9e29..e06c15e82 100644
--- a/spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/emc-verif-global-env/package.py
@@ -24,7 +24,11 @@ class EmcVerifGlobalEnv(BundlePackage):
     depends_on("grib-util")
     # Test grads
     # depends_on('grads')
-    depends_on("wgrib2")
+    # Currently, wgrib2 doesn't build with oneapi,
+    # but there isn't a "when not" option in spack yet
+    depends_on("wgrib2", when="%apple-clang")
+    depends_on("wgrib2", when="%gcc")
+    depends_on("wgrib2", when="%intel")
     depends_on("python")
     depends_on("prod-util")
     depends_on("met")
diff --git a/spack-ext/repos/spack-stack/packages/global-workflow-env/package.py b/spack-ext/repos/spack-stack/packages/global-workflow-env/package.py
index 309e75fcb..1da3f6238 100644
--- a/spack-ext/repos/spack-stack/packages/global-workflow-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/global-workflow-env/package.py
@@ -38,7 +38,11 @@ class GlobalWorkflowEnv(BundlePackage):
     depends_on("landsfcutil")
     depends_on("sigio")
     depends_on("bufr")
-    depends_on("wgrib2")
+    # Currently, wgrib2 doesn't build with oneapi,
+    # but there isn't a "when not" option in spack yet
+    depends_on("wgrib2", when="%apple-clang")
+    depends_on("wgrib2", when="%gcc")
+    depends_on("wgrib2", when="%intel")
     depends_on("met")
     depends_on("metplus")
     depends_on("gsi-ncdiag")
diff --git a/spack-ext/repos/spack-stack/packages/nceplibs-env/package.py b/spack-ext/repos/spack-stack/packages/nceplibs-env/package.py
index e5c25bfcb..a740bf2f4 100644
--- a/spack-ext/repos/spack-stack/packages/nceplibs-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/nceplibs-env/package.py
@@ -36,6 +36,10 @@ class NceplibsEnv(BundlePackage):
     depends_on("w3emc")
     depends_on("w3nco")
     depends_on("wrf-io")
-    depends_on("wgrib2")
+    # Currently, wgrib2 doesn't build with oneapi,
+    # but there isn't a "when not" option in spack yet
+    depends_on("wgrib2", when="%apple-clang")
+    depends_on("wgrib2", when="%gcc")
+    depends_on("wgrib2", when="%intel")
 
     # There is no need for install() since there is no code.
diff --git a/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py b/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py
index d4b53a699..7ce84d52d 100644
--- a/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py
@@ -41,7 +41,11 @@ class UfsSrwAppEnv(BundlePackage):
     depends_on("sigio")
     depends_on("wrf-io")
     depends_on("w3emc")
-    depends_on("wgrib2")
+    # Currently, wgrib2 doesn't build with oneapi,
+    # but there isn't a "when not" option in spack yet
+    depends_on("wgrib2", when="%apple-clang")
+    depends_on("wgrib2", when="%gcc")
+    depends_on("wgrib2", when="%intel")
     depends_on("gsi-ncdiag")
     depends_on("met")
     depends_on("metplus")
diff --git a/spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py b/spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py
index 26f02c38e..e7ba375db 100644
--- a/spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/ufs-utils-env/package.py
@@ -45,6 +45,10 @@ class UfsUtilsEnv(BundlePackage):
     depends_on("wrf-io")
     depends_on("ncio")
     depends_on("landsfcutil")
-    depends_on("wgrib2")
+    # Currently, wgrib2 doesn't build with oneapi,
+    # but there isn't a "when not" option in spack yet
+    depends_on("wgrib2", when="%apple-clang")
+    depends_on("wgrib2", when="%gcc")
+    depends_on("wgrib2", when="%intel")
 
     # There is no need for install() since there is no code.
diff --git a/spack-ext/repos/spack-stack/packages/upp-env/package.py b/spack-ext/repos/spack-stack/packages/upp-env/package.py
index e4fe72d09..c0ca64638 100644
--- a/spack-ext/repos/spack-stack/packages/upp-env/package.py
+++ b/spack-ext/repos/spack-stack/packages/upp-env/package.py
@@ -31,7 +31,11 @@ class UppEnv(BundlePackage):
     depends_on("wrf-io")
     depends_on("prod-util")
     # For testing:
-    depends_on("wgrib2")
+    # Currently, wgrib2 doesn't build with oneapi,
+    # but there isn't a "when not" option in spack yet
+    depends_on("wgrib2", when="%apple-clang")
+    depends_on("wgrib2", when="%gcc")
+    depends_on("wgrib2", when="%intel")
     depends_on("grib-util")
 
     # There is no need for install() since there is no code.

From 2b9ccd24ecd4c41b4f30756045deda2c99de6693 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:12:40 -0600
Subject: [PATCH 05/28] First pass to update configs/sites/tier1/narwhal

---
 configs/sites/tier1/narwhal/compilers.yaml    | 17 ++++++-----
 configs/sites/tier1/narwhal/packages.yaml     | 30 -------------------
 configs/sites/tier1/narwhal/packages_gcc.yaml | 13 ++++++++
 .../sites/tier1/narwhal/packages_intel.yaml   | 21 +++++++++++++
 4 files changed, 44 insertions(+), 37 deletions(-)
 create mode 100644 configs/sites/tier1/narwhal/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/narwhal/packages_intel.yaml

diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml
index 4102aa0cd..1f6aa9370 100644
--- a/configs/sites/tier1/narwhal/compilers.yaml
+++ b/configs/sites/tier1/narwhal/compilers.yaml
@@ -1,21 +1,24 @@
 compilers::
   - compiler:
-      spec: intel@2021.4.0
+      spec: intel@2021.10.0
       paths:
         cc: cc
         cxx: CC
         f77: ftn
         fc: ftn
-      flags: {}
+      flags:
+        cflags: -diag-disable=10441
+        cxxflags: -diag-disable=10441
+        fflags: -diag-disable=10448
       operating_system: sles15
       modules:
-      - PrgEnv-intel/8.3.2
-      - intel-classic/2021.4.0
+      - PrgEnv-intel/8.3.3
+      - intel-classic/2023.2.0
       environment:
         prepend_path:
           PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
           CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
-          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/INTEL/19.0/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
+          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.11.1.2/INTEL/19.0/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
@@ -29,11 +32,11 @@ compilers::
       flags: {}
       operating_system: sles15
       modules:
-      - PrgEnv-gnu/8.3.2
+      - PrgEnv-gnu/8.3.3
       - gcc/10.3.0
       environment:
         prepend_path:
-          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib'
+          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib'
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
diff --git a/configs/sites/tier1/narwhal/packages.yaml b/configs/sites/tier1/narwhal/packages.yaml
index 737163f0d..9351731bb 100644
--- a/configs/sites/tier1/narwhal/packages.yaml
+++ b/configs/sites/tier1/narwhal/packages.yaml
@@ -1,34 +1,4 @@
 packages:
-  all:
-    compiler:: [intel@2021.4.0, gcc@10.3.0]
-    providers:
-      mpi:: [cray-mpich@8.1.14]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  cray-mpich:
-    externals:
-    - spec: cray-mpich@8.1.14%intel@2021.4.0 ~wrappers
-      modules:
-      - cray-mpich/8.1.14
-    - spec: cray-mpich@8.1.14%gcc@10.3.0 ~wrappers
-      modules:
-      - cray-mpich/8.1.14
-  #intel-oneapi-tbb:
-  #  externals:
-  #  - spec: intel-oneapi-tbb@2021.3.0
-  #    prefix: /opt/intel/oneapi_2021.3.0.3219
-  #    modules:
-  #    - intel/2021.3.0
-  #intel-oneapi-mkl:
-  #  externals:
-  #  - spec: intel-oneapi-mkl@2021.3.0
-  #    prefix: /opt/intel/oneapi_2021.3.0.3219
-  #    modules:
-  #    - intel/2021.3.0
-
-### All other external packages listed alphabetically
   autoconf:
     externals:
     - spec: autoconf@2.69
diff --git a/configs/sites/tier1/narwhal/packages_gcc.yaml b/configs/sites/tier1/narwhal/packages_gcc.yaml
new file mode 100644
index 000000000..3261960c6
--- /dev/null
+++ b/configs/sites/tier1/narwhal/packages_gcc.yaml
@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [gcc@10.3.0]
+    providers:
+      mpi:: [cray-mpich@8.1.21]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: cray-mpich@8.1.21%gcc@10.3.0 ~wrappers
+      modules:
+      - cray-mpich-ucx/8.1.21
+      - craype-network-ucx
diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml
new file mode 100644
index 000000000..e829ae672
--- /dev/null
+++ b/configs/sites/tier1/narwhal/packages_intel.yaml
@@ -0,0 +1,21 @@
+packages:
+  all:
+    compiler:: [intel@2021.10.0]
+    providers:
+      mpi:: [cray-mpich@8.1.21]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers
+      modules:
+      - cray-mpich-ucx/8.1.21
+      - craype-network-ucx
+  intel-oneapi-tbb:
+    externals:
+    - spec: intel-oneapi-tbb@2021.10.0
+      prefix: /opt/intel/oneapi_2023.2.0.49397
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2023.2.0
+      prefix: /opt/intel/oneapi_2023.2.0.49397

From b586a262b703c3331d3abf72302cf60886eb2fbe Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:42:58 -0600
Subject: [PATCH 06/28] Update formatting in configs/common/packages_*.yaml

---
 configs/common/packages_apple-clang.yaml | 9 +++------
 configs/common/packages_gcc.yaml         | 9 +++------
 configs/common/packages_intel.yaml       | 8 --------
 3 files changed, 6 insertions(+), 20 deletions(-)

diff --git a/configs/common/packages_apple-clang.yaml b/configs/common/packages_apple-clang.yaml
index cc7276436..70012d1a0 100644
--- a/configs/common/packages_apple-clang.yaml
+++ b/configs/common/packages_apple-clang.yaml
@@ -1,18 +1,15 @@
-# Pin versions and specs when building packages
 packages:
-  #
   all:
     providers:
       blas: [openblas]
       fftw-api: [fftw]
       lapack: [openblas]
-  #
   ectrans:
     require:
     - '~mkl +fftw'
   gsibec:
     require:
     - '~mkl'
-  #py-numpy:
-  #  require:
-  #  - '^intel-oneapi-mkl' ^ ???
+  py-numpy:
+    require:
+    - '^openblas'
diff --git a/configs/common/packages_gcc.yaml b/configs/common/packages_gcc.yaml
index cc7276436..70012d1a0 100644
--- a/configs/common/packages_gcc.yaml
+++ b/configs/common/packages_gcc.yaml
@@ -1,18 +1,15 @@
-# Pin versions and specs when building packages
 packages:
-  #
   all:
     providers:
       blas: [openblas]
       fftw-api: [fftw]
       lapack: [openblas]
-  #
   ectrans:
     require:
     - '~mkl +fftw'
   gsibec:
     require:
     - '~mkl'
-  #py-numpy:
-  #  require:
-  #  - '^intel-oneapi-mkl' ^ ???
+  py-numpy:
+    require:
+    - '^openblas'
diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml
index 6dec75f43..f04aece5e 100644
--- a/configs/common/packages_intel.yaml
+++ b/configs/common/packages_intel.yaml
@@ -13,11 +13,3 @@ packages:
   py-numpy:
     require:
     - '^intel-oneapi-mkl'
-  bison:
-    require: '%gcc'
-  gmake:
-    require: '%gcc'
-  libmd:
-    require: '%gcc'
-  libbsd:
-    require: '%gcc'

From be60b114888174e9f8157968c58696ed343f60e1 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:57:16 -0600
Subject: [PATCH 07/28] Update nautilus site config: use packages_COMPILER.yaml

---
 configs/sites/tier1/nautilus/packages.yaml    | 60 -------------------
 .../sites/tier1/nautilus/packages_gcc.yaml    | 14 +++++
 .../sites/tier1/nautilus/packages_intel.yaml  | 18 ++++++
 .../sites/tier1/nautilus/packages_oneapi.yaml | 19 ++++++
 4 files changed, 51 insertions(+), 60 deletions(-)
 create mode 100644 configs/sites/tier1/nautilus/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/nautilus/packages_intel.yaml
 create mode 100644 configs/sites/tier1/nautilus/packages_oneapi.yaml

diff --git a/configs/sites/tier1/nautilus/packages.yaml b/configs/sites/tier1/nautilus/packages.yaml
index 0e2794885..f8dbc3501 100644
--- a/configs/sites/tier1/nautilus/packages.yaml
+++ b/configs/sites/tier1/nautilus/packages.yaml
@@ -1,64 +1,4 @@
 packages:
-  all:
-    compiler:: [intel@2021.5.0, aocc@4.0.0, gcc@12.2.1, oneapi@2024.1.2]
-    providers:
-      mpi:: [openmpi@4.1.6, openmpi@5.0.1, intel-oneapi-mpi@2021.12]
-      blas:: [intel-oneapi-mkl]
-      fftw-api:: [intel-oneapi-mkl]
-      lapack:: [intel-oneapi-mkl]
-  ectrans:
-    # do we need to set 1.2.0 here?
-    require:: '+mkl ~fftw'
-  gsibec:
-    # do we need to set the rest here?
-    require:: '+mkl'
-  py-numpy:
-    require: '@:1.25 ^intel-oneapi-mkl'
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.6%intel@2021.5.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
-      prefix: /p/app/penguin/openmpi/4.1.6/intel-classic-2022.0.2
-      modules:
-      - penguin/openmpi/4.1.6/intel-classic-2022.0.2
-      - slurm
-    - spec: openmpi@4.1.4%aocc@4.0.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
-      prefix: /p/app/penguin/openmpi/4.1.4/aoc
-      modules:
-      - penguin/openmpi/4.1.4/aocc
-      - slurm
-    - spec: openmpi@5.0.1%gcc@12.2.1~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm
-      prefix: /p/app/penguin/openmpi/5.0.1/gcc-8.5.0
-      modules:
-      - penguin/openmpi/5.0.1/gcc-8.5.0
-      - slurm
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.12%oneapi@2024.1.2
-      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
-  intel-oneapi-mkl:
-    externals:
-    - spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0
-      prefix: /p/app/compilers/intel/oneapi
-      modules:
-      - intel-oneapi-mkl@2022.0.2
-    - spec: intel-oneapi-mkl@2022.0.2%gcc@12.2.1
-      prefix: /p/app/compilers/intel/oneapi
-      modules:
-      - intel-oneapi-mkl@2022.0.2
-    - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2
-      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
-  intel-oneapi-runtime:
-    externals:
-    - spec: intel-oneapi-runtime@2024.1.2%oneapi@2024.1.2
-      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
-
-### Modifications of common packages
-
-### All other external packages listed alphabetically
   autoconf:
     externals:
     - spec: autoconf@2.69
diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml
new file mode 100644
index 000000000..08bb02aa5
--- /dev/null
+++ b/configs/sites/tier1/nautilus/packages_gcc.yaml
@@ -0,0 +1,14 @@
+packages:
+  all:
+    compiler:: [gcc@12.2.1]
+    providers:
+      mpi:: [openmpi@5.0.1]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@5.0.1%gcc@12.2.1~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      prefix: /p/app/penguin/openmpi/5.0.1/gcc-8.5.0
+      modules:
+      - penguin/openmpi/5.0.1/gcc-8.5.0
+      - slurm
diff --git a/configs/sites/tier1/nautilus/packages_intel.yaml b/configs/sites/tier1/nautilus/packages_intel.yaml
new file mode 100644
index 000000000..625387414
--- /dev/null
+++ b/configs/sites/tier1/nautilus/packages_intel.yaml
@@ -0,0 +1,18 @@
+packages:
+  all:
+    compiler:: [intel@2021.5.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6%intel@2021.5.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      prefix: /p/app/penguin/openmpi/4.1.6/intel-classic-2022.0.2
+      modules:
+      - penguin/openmpi/4.1.6/intel-classic-2022.0.2
+      - slurm
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0
+      prefix: /p/app/compilers/intel/oneapi
diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml
new file mode 100644
index 000000000..40d0378e7
--- /dev/null
+++ b/configs/sites/tier1/nautilus/packages_oneapi.yaml
@@ -0,0 +1,19 @@
+packages:
+  all:
+    compiler:: [oneapi@2024.1.2]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.12]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.12%oneapi@2024.1.2
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2
+  intel-oneapi-runtime:
+    externals:
+    - spec: intel-oneapi-runtime@2024.1.2%oneapi@2024.1.2
+      prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2

From 38ed6fbd9049639ade16072f4901f842d9e1cffc Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:57:46 -0600
Subject: [PATCH 08/28] Remove unused external intel-oneapi-tbb from Narwhal
 site config

---
 configs/sites/tier1/narwhal/packages_intel.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml
index e829ae672..b8effd96d 100644
--- a/configs/sites/tier1/narwhal/packages_intel.yaml
+++ b/configs/sites/tier1/narwhal/packages_intel.yaml
@@ -11,10 +11,6 @@ packages:
       modules:
       - cray-mpich-ucx/8.1.21
       - craype-network-ucx
-  intel-oneapi-tbb:
-    externals:
-    - spec: intel-oneapi-tbb@2021.10.0
-      prefix: /opt/intel/oneapi_2023.2.0.49397
   intel-oneapi-mkl:
     externals:
     - spec: intel-oneapi-mkl@2023.2.0

From 6e9b458a1d6ef823524ce11466a87e843fc4e51f Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 7 Aug 2024 15:58:13 -0600
Subject: [PATCH 09/28] Update CI workflows to use Intel MKL with Intel and
 OneAPI compilers

---
 .github/workflows/ubuntu-ci-x86_64-intel.yaml  | 11 +++++++++++
 .github/workflows/ubuntu-ci-x86_64-oneapi.yaml |  2 --
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
index a4a299b2b..406e51235 100644
--- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
@@ -96,6 +96,17 @@ jobs:
           echo "    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
 
+          # Add external Intel MKL and oneAPI runtime
+          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "  intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          #echo "  intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          #echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          #echo "    - spec: intel-oneapi-runtime@2023.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          #echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
           # Add external ecflow for Intel
           echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "  ecflow::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
index 585bcc773..d99fc079c 100644
--- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
@@ -84,8 +84,6 @@ jobs:
           echo "    target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
           echo "    modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
           echo "    environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      #prepend_path:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
-          echo "      #  LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
           echo "    extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
 
           # Need to find external Intel MPI and annotate with the

From 5b3c534106684054bd619f17e83731accadea430 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 09:53:24 -0600
Subject: [PATCH 10/28] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index 55db9fe05..668313a11 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 55db9fe05f6c0d5d63796d3261c4f1d09740a051
+Subproject commit 668313a11dc798e2a7d65349f69877d2e6377135

From 18ae9fcee46a900549034f68b8daf30c3817f2aa Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 11:16:47 -0600
Subject: [PATCH 11/28] Add libfabric to LD_LIBRARY_PATH in narwhal compiler
 config

---
 configs/sites/tier1/narwhal/compilers.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml
index 1f6aa9370..22ddd17b4 100644
--- a/configs/sites/tier1/narwhal/compilers.yaml
+++ b/configs/sites/tier1/narwhal/compilers.yaml
@@ -14,11 +14,12 @@ compilers::
       modules:
       - PrgEnv-intel/8.3.3
       - intel-classic/2023.2.0
+      - libfabric/1.12.1.2.2.1
       environment:
         prepend_path:
           PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
           CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
-          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.11.1.2/INTEL/19.0/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/INTEL/19.0/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
@@ -36,7 +37,7 @@ compilers::
       - gcc/10.3.0
       environment:
         prepend_path:
-          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib'
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []

From bc82b78c3dccf96a4fa690c95babc78910d33099 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 11:48:36 -0600
Subject: [PATCH 12/28] Bump odc to 1.5.2 in configs/common/packages.yaml and
 configs/containers/specs/jedi-ci.yaml

---
 configs/common/packages.yaml          | 2 +-
 configs/containers/specs/jedi-ci.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
index 0b637948e..9f12c6428 100644
--- a/configs/common/packages.yaml
+++ b/configs/common/packages.yaml
@@ -161,7 +161,7 @@ packages:
     require: '@4.6.1'
   # ninja - when adding information here, also check Discover site config
   odc:
-    require: '@1.4.6 ~fortran'
+    require: '@1.5.2 ~fortran'
   openblas:
     require: '@0.3.24 +noavx512'
   openmpi:
diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml
index 3a07273e7..044fe19a4 100644
--- a/configs/containers/specs/jedi-ci.yaml
+++ b/configs/containers/specs/jedi-ci.yaml
@@ -9,7 +9,7 @@
     py-gitpython@3.1.40, py-h5py@3.8.0, py-numpy@1.22.3,
     py-pandas@1.5.3, py-pip, py-pyyaml@6.0, py-scipy@1.11.4, py-shapely@1.8.0, py-xarray@2023.7.0,
     sp@2.5.0, udunits@2.2.28, w3emc@2.10.0, nco@5.1.6, esmf@8.6.1, mapl@2.46.2,
-    zlib-ng@2.1.5, zstd@1.5.2, odc@1.4.6, shumlib@macos_clang_linux_intel_port,
+    zlib-ng@2.1.5, zstd@1.5.2, odc@1.5.2, shumlib@macos_clang_linux_intel_port,
     awscli-v2@2.13.22, py-globus-cli@3.16.0,
     # Added for new CI system 2024/04/30
     py-ansi2html@1.6.0, py-pygithub@2.1.1, jq@1.6 ]

From 108ba70f4f82cf41b8aa497bc7c34a0be0a8ec12 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 20:49:08 -0600
Subject: [PATCH 13/28] For Intel Classic, pin gettext to 0.21.1 since 0.22.5
 doesn't build

---
 configs/common/packages_intel.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml
index f04aece5e..79fa77090 100644
--- a/configs/common/packages_intel.yaml
+++ b/configs/common/packages_intel.yaml
@@ -7,6 +7,11 @@ packages:
   ectrans:
     require:
     - '+mkl ~fftw'
+  # gettext 0.22.5 does not compile with Intel Classic compilers:
+  # malloca.c(49): error #3895: expected a comma (the one-argument version of static_assert is not enabled in this mode)
+  gettext:
+    require:
+    - '@:0.21'
   gsibec:
     require:
     - '+mkl'

From 7ab7ef014030702c02066d73d4581338fe0bcac0 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 21:19:11 -0600
Subject: [PATCH 14/28] Revert .gitmodules and update submodule pointer for
 spack

---
 .gitmodules | 6 ++----
 spack       | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index df95b971e..a69f3e4f1 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,7 @@
 [submodule "spack"]
   path = spack
-  #url = https://github.com/jcsda/spack
-  #branch = spack-stack-dev
-  url = https://github.com/climbfuji/spack
-  branch = feature/update_odc_and_yafyaml_for_gcc13
+  url = https://github.com/jcsda/spack
+  branch = spack-stack-dev
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules
diff --git a/spack b/spack
index 668313a11..eafca9b58 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 668313a11dc798e2a7d65349f69877d2e6377135
+Subproject commit eafca9b585a4fae55c3c269e02a2ab7c625a1ee8

From 7967004206cb792c4b3ebac98b3bc230d9985c43 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 21:43:59 -0600
Subject: [PATCH 15/28] Update atlantis site config

---
 configs/sites/tier1/atlantis/packages.yaml | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml
index 2030f4eb2..0487117dd 100644
--- a/configs/sites/tier1/atlantis/packages.yaml
+++ b/configs/sites/tier1/atlantis/packages.yaml
@@ -1,15 +1,8 @@
 packages:
   all:
     compiler:: [intel@2021.6.0]
-    providers: 
+    providers:
       mpi:: [intel-oneapi-mpi@2021.6.0]
-      blas:: [intel-oneapi-mkl]
-      fftw-api:: [intel-oneapi-mkl]
-      lapack:: [intel-oneapi-mkl]
-  ectrans:
-    variants:: +mkl ~fftw
-  gsibec:
-    variants:: +mkl
 
 ### MPI, Python, MKL
   mpi:
@@ -25,7 +18,7 @@ packages:
     externals:
     - spec: intel-oneapi-mkl@2022.1.0
       modules:
-      -  mkl/2022.1.0        
+      -  mkl/2022.1.0
       prefix: /cm/shared/apps/intel/oneapi
   intel-oneapi-ipp:
     externals:
@@ -41,7 +34,7 @@ packages:
       prefix: /cm/shared/apps/intel/oneapi
   #python:
   #  buildable: False
-  #  externals: 
+  #  externals:
   #  - spec: python@3.9.15
   #    modules:
   #    - python39

From 7c2def4b5fb86e993dbe90a468d97db42beac422 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 8 Aug 2024 21:56:56 -0600
Subject: [PATCH 16/28] Update S4 site config

---
 configs/sites/tier1/s4/packages.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml
index f5d41523b..7e72afbe2 100644
--- a/configs/sites/tier1/s4/packages.yaml
+++ b/configs/sites/tier1/s4/packages.yaml
@@ -17,6 +17,10 @@ packages:
     externals:
     - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0
       prefix: /opt/intel/oneapi/2022.1
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0
+      prefix: /opt/intel/oneapi/2022.1
   # Commenting this out for now, not working as expected
   #mpich:
   #  externals:

From ba3f7385ebf24e101d9ad7bb5c3f8bddf886f6d9 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 05:31:08 -0600
Subject: [PATCH 17/28] Update Orion site config

---
 configs/sites/tier1/orion/packages.yaml       | 25 -------------------
 configs/sites/tier1/orion/packages_gcc.yaml   | 15 +++++++++++
 configs/sites/tier1/orion/packages_intel.yaml | 19 ++++++++++++++
 3 files changed, 34 insertions(+), 25 deletions(-)
 create mode 100644 configs/sites/tier1/orion/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/orion/packages_intel.yaml

diff --git a/configs/sites/tier1/orion/packages.yaml b/configs/sites/tier1/orion/packages.yaml
index 771fb4a7d..ed60cff45 100644
--- a/configs/sites/tier1/orion/packages.yaml
+++ b/configs/sites/tier1/orion/packages.yaml
@@ -1,29 +1,4 @@
 packages:
-  all:
-    compiler:: [intel@2021.9.0, gcc@12.2.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.9.0, openmpi@4.1.4]
-      zlib-api:: [zlib]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0
-      #prefix: /apps/spack-managed/oneapi-2023.1.0/intel-oneapi-mpi-2021.9.0-a66eaipzsnyrdgaqzxmqmqz64qzvhkse
-      modules:
-      - intel-oneapi-mpi/2021.9.0
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.4%gcc@12.2.0 ~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~internal-libevent~internal-pmix~java~legacylaunchers~lustre~memchecker~openshmem~orterunprefix+pmi+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=none romio-filesystem=none schedulers=slurm
-      modules:
-      - gcc/12.2.0
-      - openmpi/4.1.4
-
-### Modifications of common packages
-
-### All other external packages listed alphabetically
   autoconf:
     externals:
     - spec: autoconf@2.69
diff --git a/configs/sites/tier1/orion/packages_gcc.yaml b/configs/sites/tier1/orion/packages_gcc.yaml
new file mode 100644
index 000000000..8a8f8d641
--- /dev/null
+++ b/configs/sites/tier1/orion/packages_gcc.yaml
@@ -0,0 +1,15 @@
+packages:
+  all:
+    compiler:: [gcc@12.2.0]
+    providers:
+      mpi:: [openmpi@4.1.4]
+      # https://github.com/JCSDA/spack-stack/issues/1055
+      zlib-api:: [zlib]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.4%gcc@12.2.0 ~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~internal-libevent~internal-pmix~java~legacylaunchers~lustre~memchecker~openshmem~orterunprefix+pmi+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=none romio-filesystem=none schedulers=slurm
+      modules:
+      - gcc/12.2.0
+      - openmpi/4.1.4
diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml
new file mode 100644
index 000000000..83d07589a
--- /dev/null
+++ b/configs/sites/tier1/orion/packages_intel.yaml
@@ -0,0 +1,19 @@
+packages:
+  all:
+    compiler:: [intel@2021.9.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.9.0]
+      # https://github.com/JCSDA/spack-stack/issues/1055
+      zlib-api:: [zlib]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0
+      modules:
+      - intel-oneapi-mpi/2021.9.0
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2023.1.0%intel@2021.9.0
+      modules:
+      - intel-oneapi-mkl/2023.1.0

From ba4445604d1c0a270b3087e9b02f9a29fe2c99c0 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 05:31:49 -0600
Subject: [PATCH 18/28] Work in progress: update doc/source/NewSiteConfigs.rst
 and doc/source/PreConfiguredSites.rst

---
 doc/source/NewSiteConfigs.rst     | 25 ++++++---------
 doc/source/PreConfiguredSites.rst | 51 ++++++++-----------------------
 2 files changed, 21 insertions(+), 55 deletions(-)

diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst
index 9de90ef9f..b4116538c 100644
--- a/doc/source/NewSiteConfigs.rst
+++ b/doc/source/NewSiteConfigs.rst
@@ -5,7 +5,7 @@ Generating new site configs
 
 The instructions here describe how to generate a new site config. In addition to configuring new production and testing systems, this is the recommended way for developers to use spack-stack locally on their Linux or MacOS workstations. In general, the recommended approach is to start with an empty/default site config (`linux.default` or `macos.default`). The instructions differ slightly for macOS and Linux and assume that the prerequisites for the platform have been installed as described in :numref:`Sections %s <NewSiteConfigs_macOS>` and :numref:`%s <NewSiteConfigs_Linux>`.
 
-It is also instructive to peruse the GitHub actions scripts in ``.github/workflows`` and ``.github/actions`` to see how automated spack-stack builds are configured for CI testing, as well as the existing site configs in ``configs/sites``.
+The instructions below are for GNU (`gcc`), since this is the easiest and best supported setup. Creating site configs for other compilers is more involved and not described here. It is recommended to peruse the GitHub actions scripts in ``.github/workflows`` and ``.github/actions`` to see how automated spack-stack builds are configured for CI testing, as well as the existing site configs in ``configs/sites``.
 
 .. note::
    We try to maintain compatibility with as many compilers and compiler versions as possible. The following table lists the compilers that are known to work. Please be aware that if you choose to use a different, older or newer compiler, spack-stack may not work as expected and we have limited resources available for support. Further note that Intel compiler versions are confusing, because the oneAPI version doesn't match the compiler version. We generally refer to the compiler version being the version string in the path to the compiler, e.g, `/apps/oneapi/compiler/2022.0.2/linux/bin/intel64/ifort`.
@@ -13,11 +13,11 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo
 +-------------------------------------------+----------------------------------------------------------------------+---------------------------+
 | Compiler                                  | Versions tested/in use in one or more site configs                   | Spack compiler identifier |
 +===========================================+======================================================================+===========================+
-| Intel classic (icc, icpc, ifort)          | 2021.3.0 to the final version in oneAPI 2023.2.3 [#fn1]_             | ``intel@``                |
+| Intel classic (icc, icpc, ifort)          | 2021.3.0 to the final version in oneAPI 2023.2.4 [#fn1]_             | ``intel@``                |
 +-------------------------------------------+----------------------------------------------------------------------+---------------------------+
-| Intel mixed (icx, icpx, ifort)            | 2024.1.2                                                             | ``oneapi@``               |
+| Intel mixed (icx, icpx, ifort)            | 2024.1.2 to 2024.2.0                                                 | ``oneapi@``               |
 +-------------------------------------------+----------------------------------------------------------------------+---------------------------+
-| GNU (gcc, g++, gfortran)                  | 9.2.0 to 12.2.0 (note: 13.x.y is **not** yet supported)              | ``gcc@``                  |
+| GNU (gcc, g++, gfortran)                  | 9.2.0 to 13.3.0 (note: 14.x.y is **not** yet supported)              | ``gcc@``                  |
 +-------------------------------------------+----------------------------------------------------------------------+---------------------------+
 | Apple clang (clang, clang++, w/ gfortran) | 13.1.6 to 15.0.0 [#fn2]_                                             | ``apple-clang@``          |
 +-------------------------------------------+----------------------------------------------------------------------+---------------------------+
@@ -581,14 +581,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
 
    sed -i 's/tcl/lmod/g' site/modules.yaml
 
-10. If applicable (depends on the environment), edit the main config file for the environment and adjust the compiler matrix to match the compilers for Linux, as above:
-
-.. code-block:: console
-
-   definitions:
-   - compilers: ['%gcc']
-
-11. Edit site config files and common config files, for example to remove duplicate versions of external packages that are unwanted, add specs in ``spack.yaml``, etc.
+10. Edit site config files and common config files, for example to remove duplicate versions of external packages that are unwanted, add specs in ``spack.yaml``, etc.
 
 .. code-block:: console
 
@@ -596,7 +589,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
    vi common/*.yaml
    vi site/*.yaml
 
-12. Process the specs and install
+11. Process the specs and install
 
 It is recommended to save the output of concretize in a log file and inspect that log file manually and also using the :ref:`show_duplicate_packages.py <Duplicate_Checker>` utility.
 The former is to ensure that the correct compiler and MPI libraries are being used. The latter is done to find and eliminate duplicate package specifications which can cause issues at the module creation step below.
@@ -609,19 +602,19 @@ See the :ref:`documentation <Duplicate_Checker>` for usage information including
    ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d [-c] log.concretize
    spack install [--verbose] [--fail-fast] 2>&1 | tee log.install
 
-13. Create tcl module files (replace ``tcl`` with ``lmod`` if you have manually installed lmod)
+12. Create tcl module files (replace ``tcl`` with ``lmod`` if you have manually installed lmod)
 
 .. code-block:: console
 
    spack module tcl refresh
 
-14. Create meta-modules for compiler, mpi, python
+13. Create meta-modules for compiler, mpi, python
 
 .. code-block:: console
 
    spack stack setup-meta-modules
 
-15. You now have a spack-stack environment that can be accessed by running ``module use ${SPACK_STACK_DIR}/envs/unified-env.mylinux/install/modulefiles/Core``. The modules defined here can be loaded to build and run code as described in :numref:`Section %s <UsingSpackEnvironments>`.
+14. You now have a spack-stack environment that can be accessed by running ``module use ${SPACK_STACK_DIR}/envs/unified-env.mylinux/install/modulefiles/Core``. The modules defined here can be loaded to build and run code as described in :numref:`Section %s <UsingSpackEnvironments>`.
 
 
 ..  _NewSiteConfigs_Linux_CreateEnv_Nvidia:
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index 0a8366246..c434e140b 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -249,20 +249,14 @@ With Intel, the following is required for building new spack environments and fo
 
    umask 0022
    module unload PrgEnv-cray
-   module load PrgEnv-intel/8.3.2
+   module load PrgEnv-intel/8.3.3
    module unload intel
-   module load intel-classic/2021.4.0
+   module load intel-classic/2023.2.0
    module unload cray-mpich
-   module load cray-mpich/8.1.14
-   module unload cray-python
-   module load cray-python/3.9.7.1
-   module unload cray-libsci
-   module load cray-libsci/22.08.1.1
-
-   module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
-   module load ecflow/5.8.4
+   module load cray-mpich/8.1.21
+   module load libfabric/1.12.1.2.2.1
 
-For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules:
+TODO UPDATE - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules:
 
 .. code-block:: console
 
@@ -271,8 +265,8 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules
    module unload craype-network-ofi
    module load craype-network-ucx
    module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.4.0/install/modulefiles/Core
-   module load stack-intel/2021.4.0
-   module load stack-cray-mpich/8.1.14
+   module load stack-intel/2021.10.0
+   module load stack-cray-mpich/8.1.21
    module load stack-python/3.10.13
 
 With GNU, the following is required for building new spack environments and for using spack to build and run software.  Don't use ``module purge`` on Narwhal!
@@ -281,20 +275,14 @@ With GNU, the following is required for building new spack environments and for
 
    umask 0022
    module unload PrgEnv-cray
-   module load PrgEnv-gnu/8.3.2
+   module load PrgEnv-gnu/8.3.3
    module unload gcc
    module load gcc/10.3.0
    module unload cray-mpich
-   module load cray-mpich/8.1.14
-   module unload cray-python
-   module load cray-python/3.9.7.1
-   module unload cray-libsci
-   module load cray-libsci/22.08.1.1
-
-   module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
-   module load ecflow/5.8.4
+   module load cray-mpich/8.1.21
+   module load libfabric/1.12.1.2.2.1
 
-For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules:
+TODO UPDATE - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules:
 
 .. code-block:: console
 
@@ -304,7 +292,7 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules:
    module load craype-network-ucx
    module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-gcc-10.3.0/install/modulefiles/Core
    module load stack-gcc/10.3.0
-   module load stack-cray-mpich/8.1.14
+   module load stack-cray-mpich/8.1.21
    module load stack-python/3.10.13
 
 .. _Preconfigured_Sites_Nautilus:
@@ -320,13 +308,6 @@ With Intel, the following is required for building new spack environments and fo
    umask 0022
    module purge
 
-   module load slurm
-   module load intel/compiler/2022.0.2
-   module load penguin/openmpi/4.1.6/intel-classic-2022.0.2
-
-   module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
-   module load ecflow/5.8.4
-
 For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules:
 
 .. code-block:: console
@@ -343,14 +324,6 @@ With AMD clang/flang (aocc), the following is required for building new spack en
    umask 0022
    module purge
 
-   module load slurm
-   module load amd/aocc/4.0.0
-   module load amd/aocl/aocc/4.0
-   module load penguin/openmpi/4.1.4/aocc
-
-   module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
-   module load ecflow/5.8.4
-
 .. note::
 
    ``spack-stack-1.7.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead.

From 6f08eada085f3cf7a4fdaefd303dad88955dc388 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 05:38:15 -0600
Subject: [PATCH 19/28] Update Hercules site config

---
 configs/sites/tier1/hercules/packages.yaml    | 26 -------------------
 .../sites/tier1/hercules/packages_gcc.yaml    | 14 ++++++++++
 .../sites/tier1/hercules/packages_intel.yaml  | 17 ++++++++++++
 3 files changed, 31 insertions(+), 26 deletions(-)
 create mode 100644 configs/sites/tier1/hercules/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/hercules/packages_intel.yaml

diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml
index 76c4672ef..9c2c95209 100644
--- a/configs/sites/tier1/hercules/packages.yaml
+++ b/configs/sites/tier1/hercules/packages.yaml
@@ -1,30 +1,4 @@
 packages:
-  all:
-    compiler:: [intel@2021.9.0, gcc@12.2.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.9.0, openmpi@4.1.6]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0
-      prefix: /apps/spack-managed/oneapi-2023.1.0/intel-oneapi-mpi-2021.9.0-a66eaipzsnyrdgaqzxmqmqz64qzvhkse 
-      modules:
-      - intel-oneapi-mpi/2021.9.0
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
-        schedulers=slurm
-      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack
-      modules:
-      - gcc/12.2.0
-      - openmpi/4.1.6-gcc-12.2.0-spack
-
-### Modifications of common packages
-
-### All other external packages listed alphabetically
   autoconf:
     externals:
     - spec: autoconf@2.69
diff --git a/configs/sites/tier1/hercules/packages_gcc.yaml b/configs/sites/tier1/hercules/packages_gcc.yaml
new file mode 100644
index 000000000..5781224d8
--- /dev/null
+++ b/configs/sites/tier1/hercules/packages_gcc.yaml
@@ -0,0 +1,14 @@
+packages:
+  all:
+    compiler:: [gcc@12.2.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
+        schedulers=slurm
+      modules:
+      - gcc/12.2.0
+      - openmpi/4.1.6-gcc-12.2.0-spack
diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml
new file mode 100644
index 000000000..f53f05a13
--- /dev/null
+++ b/configs/sites/tier1/hercules/packages_intel.yaml
@@ -0,0 +1,17 @@
+packages:
+  all:
+    compiler:: [intel@2021.9.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.9.0]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0
+      modules:
+      - intel-oneapi-mpi/2021.9.0
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2023.1.0%intel@2021.9.0
+      modules:
+      - intel-oneapi-mkl/2023.1.0

From 632b281239db9937aa3bc06253f2c6ce5b20cf96 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 15:03:51 -0600
Subject: [PATCH 20/28] [skip ci] TEST forcing orion intel config to use
 openblas and fftw instead of intel-oneapi-mkl

---
 configs/sites/tier1/orion/packages_intel.yaml | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml
index 83d07589a..d6d482561 100644
--- a/configs/sites/tier1/orion/packages_intel.yaml
+++ b/configs/sites/tier1/orion/packages_intel.yaml
@@ -13,7 +13,24 @@ packages:
       modules:
       - intel-oneapi-mpi/2021.9.0
   intel-oneapi-mkl:
-    externals:
-    - spec: intel-oneapi-mkl@2023.1.0%intel@2021.9.0
-      modules:
-      - intel-oneapi-mkl/2023.1.0
+    # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2023.1.0%intel@2021.9.0
+    #  modules:
+    #  - intel-oneapi-mkl/2023.1.0
+  # Remove this section to switch to intel-oneapi-mkl
+  all:
+    providers:
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'

From 9b11c9eb28227951cd7f58dfb81252d0f03ac924 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 15:04:35 -0600
Subject: [PATCH 21/28] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index eafca9b58..e422f4293 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit eafca9b585a4fae55c3c269e02a2ab7c625a1ee8
+Subproject commit e422f42938148d69e310c85a4c15b4795c9c000b

From e1a055dca167ad4b85081ff4f6492800ed70fbda Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 15:06:41 -0600
Subject: [PATCH 22/28] [skip ci] TEST forcing orion intel config to use
 openblas and fftw instead of intel-oneapi-mkl part 2

---
 configs/sites/tier1/orion/packages_intel.yaml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml
index d6d482561..193d5aac0 100644
--- a/configs/sites/tier1/orion/packages_intel.yaml
+++ b/configs/sites/tier1/orion/packages_intel.yaml
@@ -5,6 +5,10 @@ packages:
       mpi:: [intel-oneapi-mpi@2021.9.0]
       # https://github.com/JCSDA/spack-stack/issues/1055
       zlib-api:: [zlib]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
   mpi:
     buildable: False
   intel-oneapi-mpi:
@@ -20,11 +24,6 @@ packages:
     #  modules:
     #  - intel-oneapi-mkl/2023.1.0
   # Remove this section to switch to intel-oneapi-mkl
-  all:
-    providers:
-      blas:: [openblas]
-      fftw-api:: [fftw]
-      lapack:: [openblas]
   ectrans:
     require::
     - '@1.2.0 ~mkl +fftw'

From 1c74f3093677efc8c0107218733a4029a9ca7516 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 9 Aug 2024 15:53:52 -0600
Subject: [PATCH 23/28] Update S4 and Hercules site configs to use openblas and
 fftw with Intel

---
 .../sites/tier1/hercules/packages_intel.yaml  | 24 ++++++++++---
 configs/sites/tier1/s4/packages.yaml          | 34 +++++++++++--------
 2 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml
index f53f05a13..1919c5d1d 100644
--- a/configs/sites/tier1/hercules/packages_intel.yaml
+++ b/configs/sites/tier1/hercules/packages_intel.yaml
@@ -3,6 +3,10 @@ packages:
     compiler:: [intel@2021.9.0]
     providers:
       mpi:: [intel-oneapi-mpi@2021.9.0]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
   mpi:
     buildable: False
   intel-oneapi-mpi:
@@ -11,7 +15,19 @@ packages:
       modules:
       - intel-oneapi-mpi/2021.9.0
   intel-oneapi-mkl:
-    externals:
-    - spec: intel-oneapi-mkl@2023.1.0%intel@2021.9.0
-      modules:
-      - intel-oneapi-mkl/2023.1.0
+    # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2023.1.0%intel@2021.9.0
+    #  modules:
+    #  - intel-oneapi-mkl/2023.1.0
+  # Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml
index 7e72afbe2..a5e542ffd 100644
--- a/configs/sites/tier1/s4/packages.yaml
+++ b/configs/sites/tier1/s4/packages.yaml
@@ -1,14 +1,14 @@
 packages:
   all:
-    # Removing gcc/mpich for now, not working as expected
-    #compiler:: [intel@2021.5.0, gcc@9.3.0]
-    #providers:
-    #  mpi:: [intel-oneapi-mpi@2021.5.0, mpich@4.0.1]
     compiler:: [intel@2021.5.0]
     providers:
       mpi:: [intel-oneapi-mpi@2021.5.0]
       # https://github.com/JCSDA/spack-stack/issues/1055
       zlib-api:: [zlib]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
 
 ### MPI, Python, MKL
   mpi:
@@ -18,16 +18,22 @@ packages:
     - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0
       prefix: /opt/intel/oneapi/2022.1
   intel-oneapi-mkl:
-    externals:
-    - spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0
-      prefix: /opt/intel/oneapi/2022.1
-  # Commenting this out for now, not working as expected
-  #mpich:
-  #  externals:
-  #  - spec: mpich@4.0.1%gcc@9.3.0 ~hydra device=ch4
-  #    prefix: /data/prod/hpc-stack/gnu-9.3.0/mpich/4.0.1
-  #    modules:
-  #    - mpich/4.0.1
+    # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0
+    #  prefix: /opt/intel/oneapi/2022.1
+
+  # Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
 
 ### All other external packages listed alphabetically
   autoconf:

From 10ea19d31f6a94ed1865482d7518071081f49f73 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 13 Aug 2024 07:16:20 -0600
Subject: [PATCH 24/28] Update site configs for Hera and Jet (use openblas for
 now with Intel)

---
 configs/sites/tier1/hera/compilers.yaml      | 11 ------
 configs/sites/tier1/hera/packages.yaml       | 30 ----------------
 configs/sites/tier1/hera/packages_gcc.yaml   | 13 +++++++
 configs/sites/tier1/hera/packages_intel.yaml | 36 ++++++++++++++++++++
 configs/sites/tier1/jet/compilers.yaml       | 11 ------
 configs/sites/tier1/jet/packages.yaml        | 32 -----------------
 configs/sites/tier1/jet/packages_gcc.yaml    | 14 ++++++++
 configs/sites/tier1/jet/packages_intel.yaml  | 19 +++++++++++
 8 files changed, 82 insertions(+), 84 deletions(-)
 create mode 100644 configs/sites/tier1/hera/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/hera/packages_intel.yaml
 create mode 100644 configs/sites/tier1/jet/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/jet/packages_intel.yaml

diff --git a/configs/sites/tier1/hera/compilers.yaml b/configs/sites/tier1/hera/compilers.yaml
index 8150d7afc..d2e98f057 100644
--- a/configs/sites/tier1/hera/compilers.yaml
+++ b/configs/sites/tier1/hera/compilers.yaml
@@ -16,17 +16,6 @@ compilers:
         LD_LIBRARY_PATH: '/apps/gnu/gcc-9.2.0/lib64'
         CPATH: '/apps/gnu/gcc-9.2.0/include'
     extra_rpaths: []
-- compiler:
-    spec: intel@18.0.5.274
-    paths:
-      cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc
-      cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc
-      f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort
-      fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort
-    flags: {}
-    operating_system: rocky8
-    modules:
-    - intel/18.0.5.274
 - compiler:
     spec: gcc@9.2.0
     paths:
diff --git a/configs/sites/tier1/hera/packages.yaml b/configs/sites/tier1/hera/packages.yaml
index 443f88e99..741c1cb31 100644
--- a/configs/sites/tier1/hera/packages.yaml
+++ b/configs/sites/tier1/hera/packages.yaml
@@ -1,37 +1,7 @@
 packages:
   all:
-    compiler:: [intel@2021.5.0, gcc@9.2.0]
-    #compiler:: [intel@18.0.5.274]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.5.1, openmpi@4.1.6]
-      #mpi:: [intel-mpi@2018.0.4]
     # To support hecflow01
     target: [haswell]
-  mpi:
-    buildable: False
-  intel-mpi:
-    externals:
-    - spec: intel-mpi@2018.0.4%intel@18.0.5.274
-      modules:
-      - impi/2018.0.4
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0
-      modules:
-      - impi/2022.1.2
-      prefix: /apps/oneapi
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.6~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~internal-libevent~internal-pmix~java+legacylaunchers~lustre~memchecker~openshmem~orterunprefix+pmi+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=ucx schedulers=slurm
-      modules:
-      - gnu/9.2.0
-      - openmpi/4.1.6_gnu9.2.0
-  zlib-ng:
-    require:
-      - any_of: ['~shared']
-        when: "%intel"
-
-### All other external packages listed alphabetically
   autoconf:
     externals:
     - spec: autoconf@2.69
diff --git a/configs/sites/tier1/hera/packages_gcc.yaml b/configs/sites/tier1/hera/packages_gcc.yaml
new file mode 100644
index 000000000..3d2b4d6cb
--- /dev/null
+++ b/configs/sites/tier1/hera/packages_gcc.yaml
@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [gcc@9.2.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~internal-libevent~internal-pmix~java+legacylaunchers~lustre~memchecker~openshmem~orterunprefix+pmi+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=ucx schedulers=slurm
+      modules:
+      - gnu/9.2.0
+      - openmpi/4.1.6_gnu9.2.0
diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml
new file mode 100644
index 000000000..b80a5e850
--- /dev/null
+++ b/configs/sites/tier1/hera/packages_intel.yaml
@@ -0,0 +1,36 @@
+packages:
+  all:
+    compiler:: [intel@2021.5.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.5.1]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0
+      modules:
+      - impi/2022.1.2
+  intel-oneapi-mkl:
+    # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0
+    #  prefix: /apps/oneapi
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
+  zlib-ng:
+    require:
+      - '~shared'
diff --git a/configs/sites/tier1/jet/compilers.yaml b/configs/sites/tier1/jet/compilers.yaml
index 3105b2da7..c8a914103 100644
--- a/configs/sites/tier1/jet/compilers.yaml
+++ b/configs/sites/tier1/jet/compilers.yaml
@@ -16,17 +16,6 @@ compilers:
         LD_LIBRARY_PATH: '/apps/gnu/gcc-9.2.0b/lib64'
         CPATH: '/apps/gnu/gcc-9.2.0b/include'
     extra_rpaths: []
-- compiler:
-    spec: intel@18.0.5.274
-    paths:
-      cc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icc
-      cxx: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/icpc
-      f77: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort
-      fc: /apps/intel/parallel_studio_xe_2018.4.057/compilers_and_libraries_2018/linux/bin/intel64/ifort
-    flags: {}
-    operating_system: rocky8
-    modules:
-    - intel/18.0.5.274
 - compiler:
     spec: gcc@9.2.0
     paths:
diff --git a/configs/sites/tier1/jet/packages.yaml b/configs/sites/tier1/jet/packages.yaml
index 80e2c3a23..665e659ac 100644
--- a/configs/sites/tier1/jet/packages.yaml
+++ b/configs/sites/tier1/jet/packages.yaml
@@ -1,39 +1,7 @@
 packages:
   all:
-    compiler:: [intel@2021.5.0, gcc@9.2.0]
-    #compiler:: [intel@18.0.5.274]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.5.1, openmpi@4.1.6]
-      #mpi:: [intel-mpi@2018.4.274]
     # To support all generations of jet
     target: [core2]
-  mpi:
-    buildable: False
-  intel-mpi:
-    externals:
-    - spec: intel-mpi@2018.4.274%intel@18.0.5.274
-      prefix: /apps/intel/compilers_and_libraries_2018.5.274/linux/mpi
-      modules:
-      - impi/2018.4.274
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0
-      prefix: /apps/oneapi
-      modules:
-      - impi/2022.1.2
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.6%gcc@9.2.0
-      prefix: /apps/openmpi/4.1.6/gnu9.2.0/
-      modules:
-      - gnu/9.2.0b
-      - openmpi/4.1.6_gnu9.2.0
-  zlib-ng:
-    require:
-      - any_of: ['~shared']
-        when: "%intel"
-
-### All other external packages listed alphabetically
   autoconf:
     externals:
     - spec: autoconf@2.69
diff --git a/configs/sites/tier1/jet/packages_gcc.yaml b/configs/sites/tier1/jet/packages_gcc.yaml
new file mode 100644
index 000000000..f810d8345
--- /dev/null
+++ b/configs/sites/tier1/jet/packages_gcc.yaml
@@ -0,0 +1,14 @@
+packages:
+  all:
+    compiler:: [gcc@9.2.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6%gcc@9.2.0
+      prefix: /apps/openmpi/4.1.6/gnu9.2.0/
+      modules:
+      - gnu/9.2.0b
+      - openmpi/4.1.6_gnu9.2.0
diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml
new file mode 100644
index 000000000..a31345813
--- /dev/null
+++ b/configs/sites/tier1/jet/packages_intel.yaml
@@ -0,0 +1,19 @@
+packages:
+  all:
+    compiler:: [intel@2021.5.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.5.1]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0
+      modules:
+      - impi/2022.1.2
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0
+      prefix: /apps/oneapi
+  zlib-ng:
+    require:
+      - '~shared'

From 139dd432611ead7091e19c17e1e58223ea35ece3 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 14 Aug 2024 07:12:40 -0600
Subject: [PATCH 25/28] Update site configs for aws-pcluster, derecho,
 discover-scu16, discover-scu17

---
 .../sites/tier1/aws-pcluster/packages.yaml    | 32 --------------
 .../tier1/aws-pcluster/packages_gcc.yaml      | 15 +++++++
 .../tier1/aws-pcluster/packages_intel.yaml    | 35 ++++++++++++++++
 configs/sites/tier1/derecho/packages.yaml     | 26 ------------
 configs/sites/tier1/derecho/packages_gcc.yaml | 15 +++++++
 .../sites/tier1/derecho/packages_intel.yaml   | 42 +++++++++++++++++++
 .../sites/tier1/discover-scu16/packages.yaml  | 24 -----------
 .../tier1/discover-scu16/packages_gcc.yaml    | 14 +++++++
 .../tier1/discover-scu16/packages_intel.yaml  | 17 ++++++++
 .../sites/tier1/discover-scu17/packages.yaml  | 24 -----------
 .../tier1/discover-scu17/packages_gcc.yaml    | 14 +++++++
 .../tier1/discover-scu17/packages_intel.yaml  | 17 ++++++++
 12 files changed, 169 insertions(+), 106 deletions(-)
 create mode 100644 configs/sites/tier1/aws-pcluster/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/aws-pcluster/packages_intel.yaml
 create mode 100644 configs/sites/tier1/derecho/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/derecho/packages_intel.yaml
 create mode 100644 configs/sites/tier1/discover-scu16/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/discover-scu16/packages_intel.yaml
 create mode 100644 configs/sites/tier1/discover-scu17/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/discover-scu17/packages_intel.yaml

diff --git a/configs/sites/tier1/aws-pcluster/packages.yaml b/configs/sites/tier1/aws-pcluster/packages.yaml
index 6b7387393..17d5b64a6 100644
--- a/configs/sites/tier1/aws-pcluster/packages.yaml
+++ b/configs/sites/tier1/aws-pcluster/packages.yaml
@@ -7,38 +7,6 @@ packages:
     externals:
     - spec: wget@1.21.2
       prefix: /usr
-  all:
-    compiler: [intel@2021.10.0, gcc@11.4.0]
-    #compiler: [oneapi@2024.0.2]
-    providers:
-      mpi: [intel-oneapi-mpi@2021.10.0, openmpi@4.1.6]
-      #mpi: [intel-oneapi-mpi@2021.11]
-
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0
-      prefix: /opt/intel/oneapi
-      modules:
-      - libfabric-aws/1.19.0amzn4.0
-      - mpi
-    #externals:
-    #- spec: intel-oneapi-mpi@2021.11%oneapi@2024.0.2
-    #  prefix: /mnt/experiments-efs/intel-oneapi-2024.0.2
-    #  modules:
-    #  - libfabric-aws/1.16.0~amzn4.0
-    #  - mpi/2021.11
-  mpi:
-    buildable: false
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.6%gcc@11.4.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
-        fabrics=ofi schedulers=slurm
-      prefix: /opt/amazon/openmpi
-      modules:
-      - libfabric-aws/1.19.0amzn4.0
-      - openmpi/4.1.6
-
-
   binutils:
     externals:
     - spec: binutils@2.38
diff --git a/configs/sites/tier1/aws-pcluster/packages_gcc.yaml b/configs/sites/tier1/aws-pcluster/packages_gcc.yaml
new file mode 100644
index 000000000..a1f93559e
--- /dev/null
+++ b/configs/sites/tier1/aws-pcluster/packages_gcc.yaml
@@ -0,0 +1,15 @@
+packages:
+  all:
+    compiler:: [gcc@11.4.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6%gcc@11.4.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
+        fabrics=ofi schedulers=slurm
+      prefix: /opt/amazon/openmpi
+      modules:
+      - libfabric-aws/1.19.0amzn4.0
+      - openmpi/4.1.6
diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml
new file mode 100644
index 000000000..bc5f464d4
--- /dev/null
+++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml
@@ -0,0 +1,35 @@
+packages:
+  all:
+    compiler:: [intel@2021.10.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.10.0]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0
+      prefix: /opt/intel/oneapi
+      modules:
+      - libfabric-aws/1.19.0amzn4.0
+      - mpi
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure version+uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.10.0
+    #  prefix: /opt/intel/oneapi
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
diff --git a/configs/sites/tier1/derecho/packages.yaml b/configs/sites/tier1/derecho/packages.yaml
index e862ae4f2..8ea313601 100644
--- a/configs/sites/tier1/derecho/packages.yaml
+++ b/configs/sites/tier1/derecho/packages.yaml
@@ -1,38 +1,12 @@
 packages:
   all:
-    compiler:: [intel@2021.10.0, gcc@12.2.0]
-    providers:
-      mpi:: [cray-mpich@8.1.25]
     target: [core2]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  cray-mpich:
-    externals:
-    - spec: cray-mpich@8.1.25%intel@2021.10.0 +wrappers
-      modules:
-      - craype/2.7.20
-      - cray-mpich/8.1.25
-      - libfabric/1.15.2.0
-      - cray-pals/1.2.11
-    - spec: cray-mpich@8.1.25%gcc@12.2.0 +wrappers
-      modules:
-      - craype/2.7.20
-      - cray-mpich/8.1.25
-      - libfabric/1.15.2.0
-      - cray-pals/1.2.11
-
 ### Modification of common packages
   esmf:
     variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio esmf_os=Linux esmf_comm=mpich
   # Need to force spack to build python with ssl variant when using older openssl from OS
   python:
     require: "+ssl"
-  # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho
-  patchelf:
-    version:: ['0.17.2']
-
 ### All other external packages listed alphabetically
   autoconf:
     externals:
diff --git a/configs/sites/tier1/derecho/packages_gcc.yaml b/configs/sites/tier1/derecho/packages_gcc.yaml
new file mode 100644
index 000000000..f84016fe8
--- /dev/null
+++ b/configs/sites/tier1/derecho/packages_gcc.yaml
@@ -0,0 +1,15 @@
+packages:
+  all:
+    compiler:: [gcc@12.2.0]
+    providers:
+      mpi:: [cray-mpich@8.1.25]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: cray-mpich@8.1.25%gcc@12.2.0 +wrappers
+      modules:
+      - craype/2.7.20
+      - cray-mpich/8.1.25
+      - libfabric/1.15.2.0
+      - cray-pals/1.2.11
diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml
new file mode 100644
index 000000000..edf995c8f
--- /dev/null
+++ b/configs/sites/tier1/derecho/packages_intel.yaml
@@ -0,0 +1,42 @@
+packages:
+  all:
+    compiler:: [intel@2021.10.0]
+    providers:
+      mpi:: [cray-mpich@8.1.25]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: cray-mpich@8.1.25%intel@2021.10.0 +wrappers
+      modules:
+      - craype/2.7.20
+      - cray-mpich/8.1.25
+      - libfabric/1.15.2.0
+      - cray-pals/1.2.11
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+    # Note that on 2024/08/14, Dom H couldn't find any MKL installation on Derecho - keep using fftw
+    # and openblas, or find another provider (Cray stuff)?
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0
+    #  prefix: /apps/oneapi
+  # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho
+  patchelf:
+    version:: ['0.17.2']
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
+
diff --git a/configs/sites/tier1/discover-scu16/packages.yaml b/configs/sites/tier1/discover-scu16/packages.yaml
index 21f3d7ad2..4cfc9f64e 100644
--- a/configs/sites/tier1/discover-scu16/packages.yaml
+++ b/configs/sites/tier1/discover-scu16/packages.yaml
@@ -1,28 +1,5 @@
 packages:
-  all:
-    compiler:: [intel@2021.6.0, gcc@12.1.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.6.0, openmpi@4.1.3]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0
-      prefix: /usr/local/intel/oneapi/2021
-      modules:
-      - mpi/impi/2021.6.0
-  openmpi::
-    externals:
-    - spec: openmpi@4.1.3%gcc@12.1.0 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
-        fabrics=ucx schedulers=slurm
-      prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.3/gcc-12.1.0
-      modules:
-      - mpi/openmpi/4.1.3/gcc-12.1.0
-
 ### Modification of common packages
-
   # Problems building shared hdf-eos2 with Intel, not needed
   hdf-eos2:
     variants: ~shared
@@ -32,7 +9,6 @@ packages:
   # xnnpack option doesn't build on this system
   py-torch:
     variants: ~xnnpack
-
 ### All other external packages listed alphabetically
   autoconf:
     externals:
diff --git a/configs/sites/tier1/discover-scu16/packages_gcc.yaml b/configs/sites/tier1/discover-scu16/packages_gcc.yaml
new file mode 100644
index 000000000..f3f07103d
--- /dev/null
+++ b/configs/sites/tier1/discover-scu16/packages_gcc.yaml
@@ -0,0 +1,14 @@
+packages:
+  all:
+    compiler:: [gcc@12.1.0]
+    providers:
+      mpi:: [openmpi@4.1.3]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.3%gcc@12.1.0 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
+        fabrics=ucx schedulers=slurm
+      prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.3/gcc-12.1.0
+      modules:
+      - mpi/openmpi/4.1.3/gcc-12.1.0
diff --git a/configs/sites/tier1/discover-scu16/packages_intel.yaml b/configs/sites/tier1/discover-scu16/packages_intel.yaml
new file mode 100644
index 000000000..139fd869c
--- /dev/null
+++ b/configs/sites/tier1/discover-scu16/packages_intel.yaml
@@ -0,0 +1,17 @@
+packages:
+  all:
+    compiler:: [intel@2021.6.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.6.0]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0
+      prefix: /usr/local/intel/oneapi/2021
+      modules:
+      - mpi/impi/2021.6.0
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2022.2.0%intel@2021.6.0
+      prefix: /usr/local/intel/oneapi/2021
diff --git a/configs/sites/tier1/discover-scu17/packages.yaml b/configs/sites/tier1/discover-scu17/packages.yaml
index b2fccf181..086f342a5 100644
--- a/configs/sites/tier1/discover-scu17/packages.yaml
+++ b/configs/sites/tier1/discover-scu17/packages.yaml
@@ -1,34 +1,10 @@
 packages:
-  all:
-    compiler:: [intel@2021.10.0, gcc@12.3.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.10.0, openmpi@4.1.6]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.10.0%intel@=2021.10.0
-      prefix: /usr/local/intel/oneapi/2021/
-      modules:
-      - mpi/impi/2021.10.0
-  openmpi:
-    externals:
-    - spec: openmpi@4.1.6%gcc@=12.3.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
-        fabrics=ucx schedulers=slurm
-      prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-12.3.0
-      modules:
-      - mpi/openmpi/4.1.6/gcc-12.3.0
-
 ### Modification of common packages
-
   # Problems building shared hdf-eos2 with Intel, not needed
   hdf-eos2:
     variants: ~shared
   met:
     variants: +python +grib2 +graphics +lidar2nc +modis
-
 ### All other external packages listed alphabetically
   autoconf:
     externals:
diff --git a/configs/sites/tier1/discover-scu17/packages_gcc.yaml b/configs/sites/tier1/discover-scu17/packages_gcc.yaml
new file mode 100644
index 000000000..0e25fe18c
--- /dev/null
+++ b/configs/sites/tier1/discover-scu17/packages_gcc.yaml
@@ -0,0 +1,14 @@
+packages:
+  all:
+    compiler:: [gcc@12.3.0]
+    providers:
+      mpi:: [openmpi@4.1.6]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@4.1.6%gcc@=12.3.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
+        fabrics=ucx schedulers=slurm
+      prefix: /discover/swdev/gmao_SIteam/MPI/openmpi/4.1.6-SLES15/gcc-12.3.0
+      modules:
+      - mpi/openmpi/4.1.6/gcc-12.3.0
diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml
new file mode 100644
index 000000000..0595c84a7
--- /dev/null
+++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml
@@ -0,0 +1,17 @@
+packages:
+  all:
+    compiler:: [intel@2021.10.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.10.0]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.10.0%intel@=2021.10.0
+      prefix: /usr/local/intel/oneapi/2021
+      modules:
+      - mpi/impi/2021.10.0
+  intel-oneapi-mkl:
+    externals:
+    - spec: intel-oneapi-mkl@2032.2.0%intel@2021.10.0
+      prefix: /usr/local/intel/oneapi/2021

From 9b0d8cf9f233e4fa0c7779c71df7d55e215c376a Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 14 Aug 2024 07:32:53 -0600
Subject: [PATCH 26/28] Update site configs for acorn, gaea-c5, gaea-c6

---
 configs/sites/tier1/acorn/packages.yaml       | 27 ----------
 configs/sites/tier1/acorn/packages_gcc.yaml   | 14 +++++
 configs/sites/tier1/acorn/packages_intel.yaml | 51 +++++++++++++++++++
 configs/sites/tier1/gaea-c5/packages.yaml     | 25 +++++++++
 configs/sites/tier1/gaea-c6/packages.yaml     | 27 +++++++++-
 5 files changed, 116 insertions(+), 28 deletions(-)
 create mode 100644 configs/sites/tier1/acorn/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/acorn/packages_intel.yaml

diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml
index 7114e61e2..fcd79f254 100644
--- a/configs/sites/tier1/acorn/packages.yaml
+++ b/configs/sites/tier1/acorn/packages.yaml
@@ -1,15 +1,4 @@
   packages:
-    all:
-      compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@11.2.0]
-      providers:
-        mpi:: [cray-mpich@8.1.9]
-    cray-mpich:
-      externals:
-      - spec: cray-mpich@8.1.9~wrappers
-        modules:
-        - libfabric
-        - craype-network-ofi
-        - cray-mpich/8.1.9
     git:
       buildable: false
       externals:
@@ -52,19 +41,3 @@
       externals:
       - spec: flex@2.6.4+lex
         prefix: /usr
-    eckit:
-      require:
-      - any_of: ["@1.23.1"]
-        when: "%intel@19.1.3.304"
-    eccodes:
-      require:
-      - any_of: ["@2.27.0"]
-        when: "%intel@19.1.3.304"
-    py-scipy:
-      require:
-      - any_of: ["@1.10.1"]
-        when: "%intel@19.1.3.304"
-    cdo:
-      require:
-      - any_of: ["@2.0.5"]
-        when: "%intel@19.1.3.304"
diff --git a/configs/sites/tier1/acorn/packages_gcc.yaml b/configs/sites/tier1/acorn/packages_gcc.yaml
new file mode 100644
index 000000000..b801a9758
--- /dev/null
+++ b/configs/sites/tier1/acorn/packages_gcc.yaml
@@ -0,0 +1,14 @@
+  packages:
+    all:
+      compiler:: [gcc@11.2.0]
+      providers:
+        mpi:: [cray-mpich@8.1.9]
+    mpi:
+      buildable: False
+    cray-mpich:
+      externals:
+      - spec: cray-mpich@8.1.9~wrappers
+        modules:
+        - libfabric
+        - craype-network-ofi
+        - cray-mpich/8.1.9
diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml
new file mode 100644
index 000000000..13c5246fd
--- /dev/null
+++ b/configs/sites/tier1/acorn/packages_intel.yaml
@@ -0,0 +1,51 @@
+  packages:
+    all:
+      compiler:: [intel@19.1.3.304,intel@2022.0.2.262]
+      providers:
+        mpi:: [cray-mpich@8.1.9]
+        # Remove the next three lines to switch to intel-oneapi-mkl
+        blas:: [openblas]
+        fftw-api:: [fftw]
+        lapack:: [openblas]
+    mpi:
+      buildable: False
+    cray-mpich:
+      externals:
+      - spec: cray-mpich@8.1.9~wrappers
+        modules:
+        - libfabric
+        - craype-network-ofi
+        - cray-mpich/8.1.9
+    intel-oneapi-mkl:
+      # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+      buildable: False
+      #externals:
+      #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0
+      #  prefix: /apps/oneapi
+    # DH* Remove this section to switch to intel-oneapi-mkl
+    ectrans:
+      require::
+      - '@1.2.0 ~mkl +fftw'
+    gsibec:
+      require::
+      - '@1.2.1 ~mkl'
+    py-numpy:
+      require::
+      - '@:1.25 ^openblas'
+    # *DH
+    eckit:
+      require:
+      - any_of: ["@1.23.1"]
+        when: "%intel@19.1.3.304"
+    eccodes:
+      require:
+      - any_of: ["@2.27.0"]
+        when: "%intel@19.1.3.304"
+    py-scipy:
+      require:
+      - any_of: ["@1.10.1"]
+        when: "%intel@19.1.3.304"
+    cdo:
+      require:
+      - any_of: ["@2.0.5"]
+        when: "%intel@19.1.3.304"
diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml
index 7d9ff5395..99b4bae6a 100644
--- a/configs/sites/tier1/gaea-c5/packages.yaml
+++ b/configs/sites/tier1/gaea-c5/packages.yaml
@@ -3,8 +3,14 @@ packages:
     compiler:: [intel@2023.1.0]
     providers:
       mpi:: [cray-mpich@8.1.25]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
 
 ### MPI, Python, MKL
+  mpi:
+    buildable: False
   cray-mpich:
     externals:
     - spec: cray-mpich@8.1.25%intel@2023.1.0~wrappers
@@ -12,6 +18,25 @@ packages:
       modules:
       - craype-network-ofi
       - cray-mpich/8.1.25
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2023.1.0
+    #  prefix: /apps/oneapi
+
+### Modification of common packages
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
 
 ### All other external packages listed alphabetically
   autoconf:
diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml
index 3cab899f1..321b33a7b 100644
--- a/configs/sites/tier1/gaea-c6/packages.yaml
+++ b/configs/sites/tier1/gaea-c6/packages.yaml
@@ -3,14 +3,39 @@ packages:
     compiler:: [intel@2023.2.0]
     providers:
       mpi:: [cray-mpich@8.1.29]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
 
-### MPI
+### MPI, Python, MKL
+  mpi:
+    buildable: False
   cray-mpich:
     externals:
     - spec: cray-mpich@8.1.29%intel@2023.2.0~wrappers
       modules:
       - craype-network-ofi
       - cray-mpich/8.1.29
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2023.2.0
+    #  prefix: /apps/oneapi
+
+### Modification of common packages
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
 
 ### All other external packages listed alphabetically
   autoconf:

From 75061922306f735e77a6a0b8e745bbc22319569f Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 14 Aug 2024 07:40:23 -0600
Subject: [PATCH 27/28] Update site configs for noaa-{aws,azure,gcloud}

---
 configs/sites/tier1/noaa-aws/packages.yaml    | 22 ------------
 .../sites/tier1/noaa-aws/packages_gcc.yaml    | 13 +++++++
 .../sites/tier1/noaa-aws/packages_intel.yaml  | 34 +++++++++++++++++++
 configs/sites/tier1/noaa-azure/packages.yaml  | 22 ------------
 .../sites/tier1/noaa-azure/packages_gcc.yaml  | 13 +++++++
 .../tier1/noaa-azure/packages_intel.yaml      | 34 +++++++++++++++++++
 configs/sites/tier1/noaa-gcloud/packages.yaml | 22 ------------
 .../sites/tier1/noaa-gcloud/packages_gcc.yaml | 13 +++++++
 .../tier1/noaa-gcloud/packages_intel.yaml     | 34 +++++++++++++++++++
 9 files changed, 141 insertions(+), 66 deletions(-)
 create mode 100644 configs/sites/tier1/noaa-aws/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/noaa-aws/packages_intel.yaml
 create mode 100644 configs/sites/tier1/noaa-azure/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/noaa-azure/packages_intel.yaml
 create mode 100644 configs/sites/tier1/noaa-gcloud/packages_gcc.yaml
 create mode 100644 configs/sites/tier1/noaa-gcloud/packages_intel.yaml

diff --git a/configs/sites/tier1/noaa-aws/packages.yaml b/configs/sites/tier1/noaa-aws/packages.yaml
index 9e5363a55..a16a87b90 100644
--- a/configs/sites/tier1/noaa-aws/packages.yaml
+++ b/configs/sites/tier1/noaa-aws/packages.yaml
@@ -1,30 +1,8 @@
 packages:
-  all:
-    compiler:: [intel@2021.3.0, gcc@9.2.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0
-      prefix: /apps/oneapi
-      modules:
-      - impi/2021.3.0
-  openmpi:
-    externals:
-    - spec: openmpi@3.1.4%gcc@9.2.0
-      prefix: /apps/openmpi/3.1.4/gnu/gcc-9.2.0
-      modules:
-      - openmpi/3.1.4
-
 ### Modifications of common packages
   # Pin flex to avoid duplicate packages
   flex:
     version: ['2.6.4']
-
 ### All other external packages listed alphabetically
   bash:
     externals:
diff --git a/configs/sites/tier1/noaa-aws/packages_gcc.yaml b/configs/sites/tier1/noaa-aws/packages_gcc.yaml
new file mode 100644
index 000000000..6e04fbe3f
--- /dev/null
+++ b/configs/sites/tier1/noaa-aws/packages_gcc.yaml
@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [gcc@9.2.0]
+    providers:
+      mpi:: [openmpi@3.1.4]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@3.1.4%gcc@9.2.0
+      prefix: /apps/openmpi/3.1.4/gnu/gcc-9.2.0
+      modules:
+      - openmpi/3.1.4
diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml
new file mode 100644
index 000000000..c125f2911
--- /dev/null
+++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml
@@ -0,0 +1,34 @@
+packages:
+  all:
+    compiler:: [intel@2021.3.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.3.0]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0
+      prefix: /apps/oneapi
+      modules:
+      - impi/2021.3.0
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.3.0
+    #  prefix: /apps/oneapi
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
diff --git a/configs/sites/tier1/noaa-azure/packages.yaml b/configs/sites/tier1/noaa-azure/packages.yaml
index 2b7b5bd78..c3522a095 100644
--- a/configs/sites/tier1/noaa-azure/packages.yaml
+++ b/configs/sites/tier1/noaa-azure/packages.yaml
@@ -1,30 +1,8 @@
 packages:
-  all:
-    compiler:: [intel@2021.3.0, gcc@9.2.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0
-      prefix: /apps/oneapi
-      modules:
-      - impi/2021.3.0
-  openmpi:
-    externals:
-    - spec: openmpi@3.1.4%gcc@9.2.0
-      prefix: /apps/openmpi/3.1.4/gnu/gcc-9.2.0
-      modules:
-      - openmpi/3.1.4
-
 ### Modifications of common packages
   # Pin flex to avoid duplicate packages
   flex:
     version: ['2.6.4']
-
 ### All other external packages listed alphabetically
   autoconf:
     externals:
diff --git a/configs/sites/tier1/noaa-azure/packages_gcc.yaml b/configs/sites/tier1/noaa-azure/packages_gcc.yaml
new file mode 100644
index 000000000..6e04fbe3f
--- /dev/null
+++ b/configs/sites/tier1/noaa-azure/packages_gcc.yaml
@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [gcc@9.2.0]
+    providers:
+      mpi:: [openmpi@3.1.4]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@3.1.4%gcc@9.2.0
+      prefix: /apps/openmpi/3.1.4/gnu/gcc-9.2.0
+      modules:
+      - openmpi/3.1.4
diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml
new file mode 100644
index 000000000..c125f2911
--- /dev/null
+++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml
@@ -0,0 +1,34 @@
+packages:
+  all:
+    compiler:: [intel@2021.3.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.3.0]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0
+      prefix: /apps/oneapi
+      modules:
+      - impi/2021.3.0
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.3.0
+    #  prefix: /apps/oneapi
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH
diff --git a/configs/sites/tier1/noaa-gcloud/packages.yaml b/configs/sites/tier1/noaa-gcloud/packages.yaml
index d1dc19f93..b86026f11 100644
--- a/configs/sites/tier1/noaa-gcloud/packages.yaml
+++ b/configs/sites/tier1/noaa-gcloud/packages.yaml
@@ -1,30 +1,8 @@
 packages:
-  all:
-    compiler:: [intel@2021.3.0, gcc@9.2.0]
-    providers:
-      mpi:: [intel-oneapi-mpi@2021.3.0, openmpi@3.1.4]
-
-### MPI, Python, MKL
-  mpi:
-    buildable: False
-  intel-oneapi-mpi:
-    externals:
-    - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0
-      prefix: /apps/oneapi
-      modules:
-      - impi/2021.3.0
-  openmpi:
-    externals:
-    - spec: openmpi@3.1.4%gcc@9.2.0
-      prefix: /apps/openmpi/3.1.4/gnu/gcc-9.2.0
-      modules:
-      - openmpi/3.1.4
-
 ### Modifications of common packages
   # Pin flex to avoid duplicate packages
   flex:
     version: ['2.6.4']
-
 ### All other external packages listed alphabetically
   bash:
     externals:
diff --git a/configs/sites/tier1/noaa-gcloud/packages_gcc.yaml b/configs/sites/tier1/noaa-gcloud/packages_gcc.yaml
new file mode 100644
index 000000000..6e04fbe3f
--- /dev/null
+++ b/configs/sites/tier1/noaa-gcloud/packages_gcc.yaml
@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [gcc@9.2.0]
+    providers:
+      mpi:: [openmpi@3.1.4]
+  mpi:
+    buildable: False
+  openmpi:
+    externals:
+    - spec: openmpi@3.1.4%gcc@9.2.0
+      prefix: /apps/openmpi/3.1.4/gnu/gcc-9.2.0
+      modules:
+      - openmpi/3.1.4
diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml
new file mode 100644
index 000000000..c125f2911
--- /dev/null
+++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml
@@ -0,0 +1,34 @@
+packages:
+  all:
+    compiler:: [intel@2021.3.0]
+    providers:
+      mpi:: [intel-oneapi-mpi@2021.3.0]
+      # Remove the next three lines to switch to intel-oneapi-mkl
+      blas:: [openblas]
+      fftw-api:: [fftw]
+      lapack:: [openblas]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0
+      prefix: /apps/oneapi
+      modules:
+      - impi/2021.3.0
+  intel-oneapi-mkl:
+    # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl
+    buildable: False
+    #externals:
+    #- spec: intel-oneapi-mkl@2022.0.2%intel@2021.3.0
+    #  prefix: /apps/oneapi
+  # DH* Remove this section to switch to intel-oneapi-mkl
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  py-numpy:
+    require::
+    - '@:1.25 ^openblas'
+  # *DH

From 30c70278e7bd1b5a90651c4a4d89c09b8a6b3dd2 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 14 Aug 2024 07:42:54 -0600
Subject: [PATCH 28/28] Final update of doc/source/PreConfiguredSites.rst

---
 doc/source/PreConfiguredSites.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index c434e140b..4084281e0 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -256,7 +256,7 @@ With Intel, the following is required for building new spack environments and fo
    module load cray-mpich/8.1.21
    module load libfabric/1.12.1.2.2.1
 
-TODO UPDATE - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules:
+THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION -  - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules:
 
 .. code-block:: console
 
@@ -282,7 +282,7 @@ With GNU, the following is required for building new spack environments and for
    module load cray-mpich/8.1.21
    module load libfabric/1.12.1.2.2.1
 
-TODO UPDATE - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules:
+THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules:
 
 .. code-block:: console