Forpy cime

config/cesm/machines/config_batch.xml

@@ -370,6 +370,21 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="derecho-gpu" type="pbs" >
+    <batch_submit>qsub</batch_submit>
+    <submit_args>
+      <arg flag="-l job_priority=" name="$JOB_PRIORITY"/>
+    </submit_args>
+    <directives>
+      <directive default="/bin/bash" > -S {{ shell }}  </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:ngpus=4</directive>
+    </directives>
+    <queues>
+      <queue walltimemax="12:00:00" nodemin="1" nodemax="82" default="true">main</queue>
+      <queue walltimemax="1:00:00" nodemin="1" nodemax="2">develop</queue>
+    </queues>
+  </batch_system>
+
 
   <batch_system MACH="eastwind" type="slurm" >
     <batch_submit>sbatch</batch_submit>

config/cesm/machines/config_compilers.xml

@@ -125,10 +125,10 @@ using a fortran linker.
     <base>  -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none </base>
     <append compile_threaded="true"> -fopenmp </append>
     <!-- Ideally, we would also have 'invalid' in the ffpe-trap list. But at
-	 least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and
-	 7.1.0), gfortran's isnan (which is called in cime via the
-	 CPRGNU-specific shr_infnan_isnan) causes a floating point exception
-	 when called on a signaling NaN. -->
+         least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and
+         7.1.0), gfortran's isnan (which is called in cime via the
+         CPRGNU-specific shr_infnan_isnan) causes a floating point exception
+         when called on a signaling NaN. -->
     <append DEBUG="TRUE"> -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds </append>
     <append DEBUG="FALSE"> -O </append>
   </FFLAGS>
@@ -745,19 +745,36 @@ using a fortran linker.
 </compiler>
 
 <compiler MACH="derecho">
+  <NETCDF_PATH>$ENV{NETCDF}</NETCDF_PATH>
+  <PNETCDF_PATH>$ENV{PNETCDF}</PNETCDF_PATH>
+  <SLIBS>
+    <append> -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf </append>
+  </SLIBS>
+</compiler>
+
+<compiler MACH="derecho" COMPILER="intel">
   <CFLAGS>
     <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
     <append MODEL="mpi-serial"> -std=gnu89 </append>
+    <append> -march=core-avx2 -no-fma</append>
   </CFLAGS>
+  <FFLAGS>
+    <append> -march=core-avx2 -no-fma</append>
+  </FFLAGS>
+</compiler>
+
+<compiler MACH="derecho-gpu">
   <NETCDF_PATH>$ENV{NETCDF}</NETCDF_PATH>
   <PNETCDF_PATH>$ENV{PNETCDF}</PNETCDF_PATH>
   <SLIBS>
-    <append> -lnetcdff -lnetcdf </append>
+    <append> -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf </append>
   </SLIBS>
 </compiler>
 
-<compiler MACH="derecho" COMPILER="intel">
+<compiler MACH="derecho-gpu" COMPILER="intel">
   <CFLAGS>
+    <base>  -qno-opt-dynamic-align -fp-model precise -std=gnu99 </base>
+    <append MODEL="mpi-serial"> -std=gnu89 </append>
     <append> -march=core-avx2 -no-fma</append>
   </CFLAGS>
   <FFLAGS>
@@ -767,7 +784,7 @@ using a fortran linker.
 
 <compiler MACH="perlmutter" >
   <FFLAGS>
-    <append> -march=core-avx2 -no-fma</append>
+    <append> -march=core-avx2 -no-fma </append>
   </FFLAGS>
   <CFLAGS>
     <append> -march=core-avx2 -no-fma -qno-opt-dynamic-align -fp-model precise -std=gnu99 </append>
@@ -781,6 +798,7 @@ using a fortran linker.
 
 
 
+
 <compiler MACH="eastwind" COMPILER="intel">
   <CFLAGS>
     <append DEBUG="FALSE"> -O2 </append>
@@ -1050,7 +1068,7 @@ using a fortran linker.
 <compiler MACH="homebrew" COMPILER="gnu">
   <LDFLAGS>
     <!-- These LDFLAGS provide lapack and blas support on a Mac. This
-	 may require installation of the Apple Developer Tools. -->
+         may require installation of the Apple Developer Tools. -->
     <append> -framework Accelerate </append>
   </LDFLAGS>
 </compiler>

config/cesm/machines/config_machines.xml

@@ -1093,6 +1093,107 @@ This allows using a different mpirun command to launch unit tests
 
   </machine>
 
+  <machine MACH="derecho-gpu">
+    <DESC>NCAR AMD EPYC </DESC>
+    <OS>CNL</OS>
+    <COMPILERS>intel</COMPILERS>
+<!--    <COMPILERS>intel,gnu,cray,nvhpc,intel-oneapi,intel-classic</COMPILERS> -->
+    <MPILIBS>mpich</MPILIBS>
+    <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>$ENV{CESMDATAROOT}/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/cprnc/cprnc</CCSM_CPRNC>
+    <GMAKE_J>16</GMAKE_J>
+    <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
+    <SUPPORTED_BY>cseg</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>64</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+	<arg name="label"> --label</arg>
+	<arg name="buffer"> --line-buffer</arg>
+	<arg name="num_tasks" > -n {{ total_tasks }}</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module" allow_error="true">
+      <init_path lang="perl">$LMOD_ROOT/lmod/init/perl</init_path>
+      <init_path lang="python">$LMOD_ROOT/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">$LMOD_ROOT/lmod/init/sh</init_path>
+      <init_path lang="csh">$LMOD_ROOT/lmod/init/csh</init_path>
+      <cmd_path lang="perl">$LMOD_ROOT/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">$LMOD_ROOT/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="load">cesmdev/1.0</command>
+	<command name="load">ncarenv/23.09</command>
+	<command name="purge"/>
+	<command name="load">craype</command>
+        <command name="load">conda/latest</command>
+      </modules>
+      <modules compiler="intel">
+	<command name="load">intel/2023.2.1</command>
+	<command name="load">mkl</command>
+	<command name="load">spherepack/3.2</command>
+      </modules>
+      <modules compiler="intel-oneapi">
+	<command name="load">intel-oneapi/2023.2.1</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="intel-classic">
+	<command name="load">intel-classic/2023.2.1</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="cray">
+	<command name="load">cce/15.0.1</command>
+	<command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="gnu">
+	<command name="load">gcc/12.2.0</command>
+	<command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="nvhpc">
+	<command name="load">nvhpc/23.7</command>
+      </modules>
+      <modules>
+	<command name="load">ncarcompilers/1.0.0</command>
+	<command name="load">cmake</command>
+      </modules>
+      <modules mpilib="mpich">
+	<command name="load">cray-mpich/8.1.27</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+	<command name="load">mpi-serial/2.3.0</command>
+      </modules>
+
+      <modules mpilib="mpi-serial">
+	<command name="load">netcdf/4.9.2</command>
+      </modules>
+
+      <modules mpilib="!mpi-serial">
+	<command name="load">netcdf-mpi/4.9.2</command>
+	<command name="load">parallel-netcdf/1.12.3</command>
+      </modules>
+
+      <modules>
+        <command name="load">esmf/8.6.0</command>
+      </modules>
+
+    </module_system>
+
+    <environment_variables>
+      <env name="OMP_STACKSIZE">64M</env>
+      <env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
+      <env name="FI_MR_CACHE_MONITOR">memhooks</env>
+      <env name="SPHEREPACK_LIBDIR">$ENV{NCAR_ROOT_SPHEREPACK}/lib</env>
+    </environment_variables>
+
+  </machine>
+
   <machine MACH="eastwind">
     <DESC>PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM</DESC>
     <OS>LINUX</OS>

scripts/Tools/Makefile

@@ -217,6 +217,7 @@ else
     endif
   endif
 endif
+
 # Set PETSc info if it is being used
 ifeq ($(strip $(USE_PETSC)), TRUE)
   ifdef PETSC_PATH
@@ -546,6 +547,16 @@ ifdef LIB_MPI
    endif
 endif
 
+# Set python info if it is being used
+ifeq ($(strip $(USE_CONDA)), True)
+  ifdef CONDA_PREFIX
+    SLIBS += -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib
+  else
+    $(error CONDA_PREFIX must be defined when USE_CONDA is TRUE)
+  endif
+endif
+
+
 ifdef XIOS_PATH
   SLIBS += -L$(XIOS_PATH)/lib -lxios -lstdc++
 endif

src/build_scripts/buildlib.csm_share

@@ -45,6 +45,7 @@ def buildlib(bldroot, installpath, caseroot):
 ###############################################################################
     with Case(caseroot, read_only=False) as case:
         cimeroot = case.get_value("CIMEROOT")
+        srcroot = case.get_value("SRCROOT")
         filepath = [os.path.join(caseroot,"SourceMods","src.share"),
                     os.path.join(cimeroot,"src","drivers","mct","shr"),
                     os.path.join(cimeroot,"src","share","streams"),
@@ -59,6 +60,10 @@ def buildlib(bldroot, installpath, caseroot):
             use_esmf = "noesmf"
             filepath.append(os.path.join(cimeroot, "src", "share", "esmf_wrf_timemgr"))
 
+        if case.get_value("USE_CONDA"):
+            filepath.append(os.path.join(srcroot, "forpy"))
+
+
         comp_interface = case.get_value("COMP_INTERFACE")
         ninst_value = case.get_value("NINST_VALUE")
 

src/drivers/mct/cime_config/buildexe

@@ -33,7 +33,8 @@ def _main_func():
         model     = case.get_value("MODEL")
         num_esp   = case.get_value("NUM_COMP_INST_ESP")
         os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION"))
-
+        os.environ["USE_CONDA"] = str(case.get_value("USE_CONDA"))
+    print(f'USE_CONDA = {os.environ["USE_CONDA"]} CONDA_PREFIX = {os.environ["CONDA_PREFIX"]}')
     expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance")
 
 

src/drivers/mct/cime_config/config_component.xml

@@ -722,6 +722,19 @@
     CLM. This is currently only supported for certain machines.</desc>
   </entry>
 
+  <entry id="USE_CONDA">
+    <type>logical</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>build_def</group>
+    <file>env_build.xml</file>
+    <desc> TRUE implies linking to a python3 library located in CONDA_PREFIX/lib
+    and using the file forpy_mod.F90 to allow calls between fortran and python
+    typical python usage might include pytorch, tensorflow, and/or numpy
+    </desc>
+  </entry>
+
+
   <entry id="USE_ALBANY">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>