Merge pull request #1844 from GEOS-ESM/develop

GitFlow: Merge Develop into Main for 2.32.0 Release

.circleci/config.yml

@@ -16,7 +16,7 @@ parameters:
 
 # Anchors to prevent forgetting to update a version
 os_version: &os_version ubuntu20
-baselibs_version: &baselibs_version v7.6.1
+baselibs_version: &baselibs_version v7.7.0
 bcs_version: &bcs_version v10.23.0
 tag_build_arg_name: &tag_build_arg_name maplversion
 
@@ -131,7 +131,7 @@ workflows:
           fixture_branch: feature/mathomp4/mapldevelop
           checkout_mapl_branch: true
           mepodevelop: false
-          rebuild_procs: 8
+          rebuild_procs: 1
 
       # Run GCM (1 hour, no ExtData)
       - ci/run_gcm:

.github/workflows/enforce-labels.yml

@@ -0,0 +1,29 @@
+name: Enforce PR Labels
+
+on:
+  pull_request:
+    types: [opened, labeled, unlabeled, edited, synchronize]
+
+jobs:
+  require-label:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mheap/github-action-required-labels@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          mode: minimum
+          count: 1
+          labels: "0 diff,0 diff trivial,Non 0-diff,0 diff structural,0-diff trivial,Not 0-diff,0-diff,automatic,0-diff uncoupled"
+          add_comment: true
+  blocking-label:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: mheap/github-action-required-labels@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          mode: exactly
+          count: 0
+          labels: "Contingent - DNA,Needs Lead Approval,Contingent -- Do Not Approve"
+          add_comment: true

.github/workflows/workflow.yml

@@ -17,7 +17,7 @@ jobs:
     name: Build and Test MAPL GNU
     runs-on: ubuntu-latest
     container:
-      image: gmao/ubuntu20-geos-env-mkl:v7.6.1-openmpi_4.1.4-gcc_12.1.0
+      image: gmao/ubuntu20-geos-env-mkl:v7.7.0-openmpi_4.1.4-gcc_12.1.0
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests
@@ -77,7 +77,7 @@ jobs:
     name: Build and Test MAPL Intel
     runs-on: ubuntu-latest
     container:
-      image: gmao/ubuntu20-geos-env:v7.6.1-intelmpi_2021.6.0-intel_2022.1.0
+      image: gmao/ubuntu20-geos-env:v7.7.0-intelmpi_2021.6.0-intel_2022.1.0
       # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495
       # It seems like we might not need secrets on GitHub Actions which is good for forked
       # pull requests

CHANGELOG.md

@@ -9,12 +9,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+### Changed
+
 ### Fixed
 
 ### Removed
 
 ### Deprecated
 
+## [2.32.0] - 2022-12-02
+
+### Added
+
+- Add fArgParse CLI argument parser
+- Added subroutines for reading 4d integers in NetCDF4_FileFormatter
+- Added new option to allow for gaps in datasets ingested by ExtDataNG
+
+### Changed
+
+- Updated to ESMA_cmake v3.21.0
+  - Adds support for a generic `x86_64` processor for GNU
+- Updated to ESMA_env v4.8.0
+   - Baselibs v7.7.0
+      - fArgParse v1.4.1
+      - pFUnit v4.6.1
+- Move to use GitHub Actions for label enforcement
+
 ## [2.31.0] - 2022-11-28
 
 ### Fixed

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW)
 
 project (
   MAPL
-  VERSION 2.31.0
+  VERSION 2.32.0
   LANGUAGES Fortran CXX C)  # Note - CXX is required for ESMF
 
 # Set the default build type to release
@@ -82,8 +82,16 @@ endif()
 if(NOT TARGET GFTL_SHARED::gftl-shared)
   find_package(GFTL_SHARED REQUIRED)
 endif()
-if(NOT TARGET FARGPARSE::fargparse)
-  find_package(FARGPARSE QUIET)
+
+option(BUILD_WITH_FARGPARSE "Use fArgParse for command line processing" ON)
+if(BUILD_WITH_FARGPARSE)
+  if(NOT TARGET FARGPARSE::fargparse)
+    find_package(FARGPARSE 1.4.1 REQUIRED)
+  else()
+    if (FARGPARSE_VERSION VERSION_LESS 1.4.1)
+      message(FATAL_ERROR "fArgParse must be at least 1.4.1")
+    endif ()
+  endif()
 endif()
 
 option(USE_EXTDATA2G "Use ExtData2G" ON)

Tests/CMakeLists.txt

@@ -18,6 +18,7 @@ if (BUILD_WITH_FLAP)
   endif ()
   set_target_properties(ExtDataDriver.x PROPERTIES Fortran_MODULE_DIRECTORY ${MODULE_DIRECTORY})
   target_compile_definitions (ExtDataDriver.x PRIVATE $<$<BOOL:${USE_EXTDATA2G}>:BUILD_WITH_EXTDATA2G>)
+  add_subdirectory(ExtData_Testing_Framework EXCLUDE_FROM_ALL)
 
   ecbuild_add_executable (TARGET pfio_MAPL_demo.x SOURCES pfio_MAPL_demo.F90)
   target_link_libraries (pfio_MAPL_demo.x PRIVATE MAPL FLAP::FLAP esmf)
@@ -26,6 +27,17 @@ if (BUILD_WITH_FLAP)
     target_link_libraries(pfio_MAPL_demo.x PRIVATE OpenMP::OpenMP_Fortran)
   endif ()
   set_target_properties(pfio_MAPL_demo.x PROPERTIES Fortran_MODULE_DIRECTORY ${MODULE_DIRECTORY})
-  add_subdirectory(ExtData_Testing_Framework EXCLUDE_FROM_ALL)
+
+endif ()
+
+if (BUILD_WITH_FARGPARSE)
+
+  ecbuild_add_executable (TARGET MAPL_demo_fargparse.x SOURCES MAPL_demo_fargparse.F90)
+  target_link_libraries (MAPL_demo_fargparse.x PRIVATE MAPL FARGPARSE::fargparse esmf)
+  # CMake has an OpenMP issue with NAG Fortran: https://gitlab.kitware.com/cmake/cmake/-/issues/21280
+  if (NOT CMAKE_Fortran_COMPILER_ID MATCHES "NAG")
+    target_link_libraries(MAPL_demo_fargparse.x PRIVATE OpenMP::OpenMP_Fortran)
+  endif ()
+  set_target_properties(MAPL_demo_fargparse.x PROPERTIES Fortran_MODULE_DIRECTORY ${MODULE_DIRECTORY})
 
 endif ()

Tests/ExtDataRoot_GridComp.F90

@@ -626,57 +626,49 @@ subroutine CompareState(State1,State2,tol,rc)
 
       integer :: status
       character(len=*), parameter :: Iam=__FILE__//"::CompareState"
-      integer                             :: ii,i,j,k
-      real, pointer                       :: ptr3_1(:,:,:) => null()
-      real, pointer                       :: ptr3_2(:,:,:) => null()
-      real, pointer                       :: ptr2_1(:,:) => null()
-      real, pointer                       :: ptr2_2(:,:) => null()
-      integer :: itemcount,rank1,rank2,lb(3),ub(3)
+      integer                             :: i
+      real, pointer                       :: ptr3_1(:,:,:)
+      real, pointer                       :: ptr3_2(:,:,:)
+      real, pointer                       :: ptr2_1(:,:)
+      real, pointer                       :: ptr2_2(:,:)
+      integer :: itemcount,rank1,rank2
       character(len=ESMF_MAXSTR), allocatable :: NameList(:)
       logical, allocatable :: foundDiff(:)
       type(ESMF_Field) :: Field1,Field2
+      logical :: all_undef1, all_undef2
 
       call ESMF_StateGet(State1,itemcount=itemCount,_RC)
          allocate(NameList(itemCount),stat=status)
          _VERIFY(status)
-         allocate(foundDiff(itemCount),stat=status)
+         allocate(foundDiff(itemCount),stat=status,source=.false.)
          _VERIFY(status)
          call ESMF_StateGet(State1,itemNameList=NameList,_RC)
-         do ii=1,itemCount
-            call ESMF_StateGet(State1,trim(nameList(ii)),field1,_RC)
-            call ESMF_StateGet(State2,trim(nameList(ii)),field2,_RC)
+         do i=1,itemCount
+            call ESMF_StateGet(State1,trim(nameList(i)),field1,_RC)
+            call ESMF_StateGet(State2,trim(nameList(i)),field2,_RC)
             call ESMF_FieldGet(field1,rank=rank1,_RC)
-            call ESMF_FieldGet(field1,rank=rank2,_RC)
+            call ESMF_FieldGet(field2,rank=rank2,_RC)
+            all_undef1 = is_field_undef(field1,_RC)
+            all_undef2 = is_field_undef(field2,_RC)
+            if (all_undef1 .or. all_undef2) then
+               exit
+            end if
             _ASSERT(rank1==rank2,'needs informative message')
-            foundDiff(ii)=.false.
+            foundDiff(i)=.false.
             if (rank1==2) then
-               call MAPL_GetPointer(state1,ptr2_1,trim(nameList(ii)),_RC)
-               call MAPL_GetPointer(state2,ptr2_2,trim(nameList(ii)),_RC)
-               do i=1,size(ptr2_1,1) 
-                  do j=1,size(ptr2_1,2)
-                     if (abs(ptr2_1(i,j)-ptr2_2(i,j)) .gt. tol) then
-                        foundDiff(ii)=.true.
-                        exit
-                     end if
-                  enddo
-               enddo
+               call MAPL_GetPointer(state1,ptr2_1,trim(nameList(i)),_RC)
+               call MAPL_GetPointer(state2,ptr2_2,trim(nameList(i)),_RC)
+               if (any((ptr2_1-ptr2_2) > tol)) then
+                   foundDiff(i) = .true.
+               end if
             else if (rank1==3) then
-               call MAPL_GetPointer(state1,ptr3_1,trim(nameList(ii)),_RC)
-               call MAPL_GetPointer(state2,ptr3_2,trim(nameList(ii)),_RC)
-               lb=lbound(ptr3_1)
-               ub=ubound(ptr3_1) 
-               do i=1,size(ptr3_1,1) 
-                  do j=1,size(ptr3_1,2) 
-                     do k=lb(3),ub(3)
-                        if (abs(ptr3_1(i,j,k)-ptr3_2(i,j,k)) .gt. tol) then
-                           foundDiff(ii)=.true.
-                           exit
-                        end if
-                     enddo
-                  enddo
-               enddo
+               call MAPL_GetPointer(state1,ptr3_1,trim(nameList(i)),_RC)
+               call MAPL_GetPointer(state2,ptr3_2,trim(nameList(i)),_RC)
+               if (any((ptr3_1-ptr3_2) > tol)) then
+                   foundDiff(i) = .true.
+               end if
             end if
-            if (foundDiff(ii)) then 
+            if (foundDiff(i)) then 
                _FAIL('found difference when compare state')
             end if
          enddo

Tests/ExtData_Testing_Framework/CMakeLists.txt

@@ -5,11 +5,11 @@ if(MPI_Fortran_LIBRARY_VERSION_FIRSTWORD MATCHES "Open")
    list(APPEND MPIEXEC_PREFLAGS "-oversubscribe")
 endif()
 
-file(STRINGS "test_cases/cases.txt" TEST_CASES)
+file(STRINGS "test_cases/extdata_1g_cases.txt" TEST_CASES_1G)
 
 set(cutoff "7")
 
-foreach(TEST_CASE ${TEST_CASES})
+foreach(TEST_CASE ${TEST_CASES_1G})
   if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
      file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
   else()
@@ -31,7 +31,18 @@ foreach(TEST_CASE ${TEST_CASES})
   else()
      set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_BIG_TESTS")
   endif()
+
+endforeach()
+
+file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G)
+
+foreach(TEST_CASE ${TEST_CASES_2G})
 
+  if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc)
+     file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs)
+  else()
+     set(num_procs "1")
+  endif()
   add_test(
      NAME "ExtData2G_${TEST_CASE}"
      COMMAND ${CMAKE_COMMAND} 

Tests/ExtData_Testing_Framework/run_extdata.cmake

@@ -13,6 +13,12 @@ macro(run_case CASE)
     if (${IS_EXTDATA2G} STREQUAL "YES")
        file(APPEND "${tempdir}/CAP1.rc" "USE_EXTDATA2G: .true.")
        file(APPEND "${tempdir}/CAP2.rc" "USE_EXTDATA2G: .true.")
+       if (EXISTS  "${tempdir}/CAP3.rc")
+          file(APPEND "${tempdir}/CAP3.rc" "USE_EXTDATA2G: .true.")
+       endif()
+       if (EXISTS  "${tempdir}/CAP4.rc")
+          file(APPEND "${tempdir}/CAP4.rc" "USE_EXTDATA2G: .true.")
+       endif()
     endif()
     execute_process(
       COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} ${MPIEXEC_PREFLAGS} ${MY_BINARY_DIR}/ExtDataDriver.x

Tests/ExtData_Testing_Framework/test_cases/case18/CAP1.rc

@@ -22,5 +22,3 @@ RUN_TIMES::
 20041115 210000
 20041215 210000
 ::
-
-USE_EXTDATA2G: .true.

Tests/ExtData_Testing_Framework/test_cases/case18/CAP2.rc

@@ -11,5 +11,3 @@ HEARTBEAT_DT:   3600
 RUN_TIMES::
 20041126 210000
 ::
-
-USE_EXTDATA2G: .true.

Tests/ExtData_Testing_Framework/test_cases/case22/CAP1.rc

@@ -6,7 +6,3 @@ BEG_DATE:     20071231 230000
 
 JOB_SGMT:     00001201 000000
 HEARTBEAT_DT:   3600
-
-
-USE_EXTDATA2G: .true.
-USE_EXTDATA2G: .true.

Tests/ExtData_Testing_Framework/test_cases/case22/CAP2.rc

@@ -6,7 +6,3 @@ BEG_DATE:     20081231 230000
 
 JOB_SGMT:     00001200 000000
 HEARTBEAT_DT:   3600
-
-
-USE_EXTDATA2G: .true.
-USE_EXTDATA2G: .true.

Tests/ExtData_Testing_Framework/test_cases/case22/CAP3.rc

@@ -7,5 +7,3 @@ BEG_DATE:     20081229 000000
 JOB_SGMT:     00000010 000000
 HEARTBEAT_DT:   3600
 
-
-USE_EXTDATA2G: .true.

Tests/ExtData_Testing_Framework/test_cases/case23/CAP1.rc

@@ -6,6 +6,3 @@ BEG_DATE:     20151231 000000
 
 JOB_SGMT:     00001200 000000
 HEARTBEAT_DT:   3600
-
-
-USE_EXTDATA2G: .true.

Tests/ExtData_Testing_Framework/test_cases/case23/CAP2.rc

@@ -6,6 +6,3 @@ BEG_DATE:     20191225 000000
 
 JOB_SGMT:     00000020 000000
 HEARTBEAT_DT:   3600
-
-
-USE_EXTDATA2G: .true.