diff --git a/.circleci/config.yml b/.circleci/config.yml index a86c5a2a6278..618303d4d1e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ parameters: # Anchors to prevent forgetting to update a version os_version: &os_version ubuntu20 -baselibs_version: &baselibs_version v7.6.1 +baselibs_version: &baselibs_version v7.7.0 bcs_version: &bcs_version v10.23.0 tag_build_arg_name: &tag_build_arg_name maplversion @@ -131,7 +131,7 @@ workflows: fixture_branch: feature/mathomp4/mapldevelop checkout_mapl_branch: true mepodevelop: false - rebuild_procs: 8 + rebuild_procs: 1 # Run GCM (1 hour, no ExtData) - ci/run_gcm: diff --git a/.github/workflows/enforce-labels.yml b/.github/workflows/enforce-labels.yml new file mode 100644 index 000000000000..6e1720ef6adc --- /dev/null +++ b/.github/workflows/enforce-labels.yml @@ -0,0 +1,29 @@ +name: Enforce PR Labels + +on: + pull_request: + types: [opened, labeled, unlabeled, edited, synchronize] + +jobs: + require-label: + runs-on: ubuntu-latest + steps: + - uses: mheap/github-action-required-labels@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + mode: minimum + count: 1 + labels: "0 diff,0 diff trivial,Non 0-diff,0 diff structural,0-diff trivial,Not 0-diff,0-diff,automatic,0-diff uncoupled" + add_comment: true + blocking-label: + runs-on: ubuntu-latest + steps: + - uses: mheap/github-action-required-labels@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + mode: exactly + count: 0 + labels: "Contingent - DNA,Needs Lead Approval,Contingent -- Do Not Approve" + add_comment: true diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 01649c4a2f9a..e8daa4d8320a 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -17,7 +17,7 @@ jobs: name: Build and Test MAPL GNU runs-on: ubuntu-latest container: - image: gmao/ubuntu20-geos-env-mkl:v7.6.1-openmpi_4.1.4-gcc_12.1.0 + image: gmao/ubuntu20-geos-env-mkl:v7.7.0-openmpi_4.1.4-gcc_12.1.0 # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495 # It seems like we might not need secrets on GitHub Actions which is good for forked # pull requests @@ -77,7 +77,7 @@ jobs: name: Build and Test MAPL Intel runs-on: ubuntu-latest container: - image: gmao/ubuntu20-geos-env:v7.6.1-intelmpi_2021.6.0-intel_2022.1.0 + image: gmao/ubuntu20-geos-env:v7.7.0-intelmpi_2021.6.0-intel_2022.1.0 # Per https://github.com/actions/virtual-environments/issues/1445#issuecomment-713861495 # It seems like we might not need secrets on GitHub Actions which is good for forked # pull requests diff --git a/CHANGELOG.md b/CHANGELOG.md index 30c1f648f02c..0561cc7780d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +### Changed + ### Fixed ### Removed ### Deprecated +## [2.32.0] - 2022-12-02 + +### Added + +- Add fArgParse CLI argument parser +- Added subroutines for reading 4d integers in NetCDF4_FileFormatter +- Added new option to allow for gaps in datasets ingested by ExtDataNG + +### Changed + +- Updated to ESMA_cmake v3.21.0 + - Adds support for a generic `x86_64` processor for GNU +- Updated to ESMA_env v4.8.0 + - Baselibs v7.7.0 + - fArgParse v1.4.1 + - pFUnit v4.6.1 +- Move to use GitHub Actions for label enforcement + ## [2.31.0] - 2022-11-28 ### Fixed diff --git a/CMakeLists.txt b/CMakeLists.txt index 90fd1893fb07..78e3355ad408 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_policy (SET CMP0054 NEW) project ( MAPL - VERSION 2.31.0 + VERSION 2.32.0 LANGUAGES Fortran CXX C) # Note - CXX is required for ESMF # Set the default build type to release @@ -82,8 +82,16 @@ endif() if(NOT TARGET GFTL_SHARED::gftl-shared) find_package(GFTL_SHARED REQUIRED) endif() -if(NOT TARGET FARGPARSE::fargparse) - find_package(FARGPARSE QUIET) + +option(BUILD_WITH_FARGPARSE "Use fArgParse for command line processing" ON) +if(BUILD_WITH_FARGPARSE) + if(NOT TARGET FARGPARSE::fargparse) + find_package(FARGPARSE 1.4.1 REQUIRED) + else() + if (FARGPARSE_VERSION VERSION_LESS 1.4.1) + message(FATAL_ERROR "fArgParse must be at least 1.4.1") + endif () + endif() endif() option(USE_EXTDATA2G "Use ExtData2G" ON) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 216bcf16855f..89bfe936b11b 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -18,6 +18,7 @@ if (BUILD_WITH_FLAP) endif () set_target_properties(ExtDataDriver.x PROPERTIES Fortran_MODULE_DIRECTORY ${MODULE_DIRECTORY}) target_compile_definitions (ExtDataDriver.x PRIVATE $<$:BUILD_WITH_EXTDATA2G>) + add_subdirectory(ExtData_Testing_Framework EXCLUDE_FROM_ALL) ecbuild_add_executable (TARGET pfio_MAPL_demo.x SOURCES pfio_MAPL_demo.F90) target_link_libraries (pfio_MAPL_demo.x PRIVATE MAPL FLAP::FLAP esmf) @@ -26,6 +27,17 @@ if (BUILD_WITH_FLAP) target_link_libraries(pfio_MAPL_demo.x PRIVATE OpenMP::OpenMP_Fortran) endif () set_target_properties(pfio_MAPL_demo.x PROPERTIES Fortran_MODULE_DIRECTORY ${MODULE_DIRECTORY}) - add_subdirectory(ExtData_Testing_Framework EXCLUDE_FROM_ALL) + +endif () + +if (BUILD_WITH_FARGPARSE) + + ecbuild_add_executable (TARGET MAPL_demo_fargparse.x SOURCES MAPL_demo_fargparse.F90) + target_link_libraries (MAPL_demo_fargparse.x PRIVATE MAPL FARGPARSE::fargparse esmf) + # CMake has an OpenMP issue with NAG Fortran: https://gitlab.kitware.com/cmake/cmake/-/issues/21280 + if (NOT CMAKE_Fortran_COMPILER_ID MATCHES "NAG") + target_link_libraries(MAPL_demo_fargparse.x PRIVATE OpenMP::OpenMP_Fortran) + endif () + set_target_properties(MAPL_demo_fargparse.x PROPERTIES Fortran_MODULE_DIRECTORY ${MODULE_DIRECTORY}) endif () diff --git a/Tests/ExtDataRoot_GridComp.F90 b/Tests/ExtDataRoot_GridComp.F90 index 0173db163cb9..0a2274acc8e5 100644 --- a/Tests/ExtDataRoot_GridComp.F90 +++ b/Tests/ExtDataRoot_GridComp.F90 @@ -626,57 +626,49 @@ subroutine CompareState(State1,State2,tol,rc) integer :: status character(len=*), parameter :: Iam=__FILE__//"::CompareState" - integer :: ii,i,j,k - real, pointer :: ptr3_1(:,:,:) => null() - real, pointer :: ptr3_2(:,:,:) => null() - real, pointer :: ptr2_1(:,:) => null() - real, pointer :: ptr2_2(:,:) => null() - integer :: itemcount,rank1,rank2,lb(3),ub(3) + integer :: i + real, pointer :: ptr3_1(:,:,:) + real, pointer :: ptr3_2(:,:,:) + real, pointer :: ptr2_1(:,:) + real, pointer :: ptr2_2(:,:) + integer :: itemcount,rank1,rank2 character(len=ESMF_MAXSTR), allocatable :: NameList(:) logical, allocatable :: foundDiff(:) type(ESMF_Field) :: Field1,Field2 + logical :: all_undef1, all_undef2 call ESMF_StateGet(State1,itemcount=itemCount,_RC) allocate(NameList(itemCount),stat=status) _VERIFY(status) - allocate(foundDiff(itemCount),stat=status) + allocate(foundDiff(itemCount),stat=status,source=.false.) _VERIFY(status) call ESMF_StateGet(State1,itemNameList=NameList,_RC) - do ii=1,itemCount - call ESMF_StateGet(State1,trim(nameList(ii)),field1,_RC) - call ESMF_StateGet(State2,trim(nameList(ii)),field2,_RC) + do i=1,itemCount + call ESMF_StateGet(State1,trim(nameList(i)),field1,_RC) + call ESMF_StateGet(State2,trim(nameList(i)),field2,_RC) call ESMF_FieldGet(field1,rank=rank1,_RC) - call ESMF_FieldGet(field1,rank=rank2,_RC) + call ESMF_FieldGet(field2,rank=rank2,_RC) + all_undef1 = is_field_undef(field1,_RC) + all_undef2 = is_field_undef(field2,_RC) + if (all_undef1 .or. all_undef2) then + exit + end if _ASSERT(rank1==rank2,'needs informative message') - foundDiff(ii)=.false. + foundDiff(i)=.false. if (rank1==2) then - call MAPL_GetPointer(state1,ptr2_1,trim(nameList(ii)),_RC) - call MAPL_GetPointer(state2,ptr2_2,trim(nameList(ii)),_RC) - do i=1,size(ptr2_1,1) - do j=1,size(ptr2_1,2) - if (abs(ptr2_1(i,j)-ptr2_2(i,j)) .gt. tol) then - foundDiff(ii)=.true. - exit - end if - enddo - enddo + call MAPL_GetPointer(state1,ptr2_1,trim(nameList(i)),_RC) + call MAPL_GetPointer(state2,ptr2_2,trim(nameList(i)),_RC) + if (any((ptr2_1-ptr2_2) > tol)) then + foundDiff(i) = .true. + end if else if (rank1==3) then - call MAPL_GetPointer(state1,ptr3_1,trim(nameList(ii)),_RC) - call MAPL_GetPointer(state2,ptr3_2,trim(nameList(ii)),_RC) - lb=lbound(ptr3_1) - ub=ubound(ptr3_1) - do i=1,size(ptr3_1,1) - do j=1,size(ptr3_1,2) - do k=lb(3),ub(3) - if (abs(ptr3_1(i,j,k)-ptr3_2(i,j,k)) .gt. tol) then - foundDiff(ii)=.true. - exit - end if - enddo - enddo - enddo + call MAPL_GetPointer(state1,ptr3_1,trim(nameList(i)),_RC) + call MAPL_GetPointer(state2,ptr3_2,trim(nameList(i)),_RC) + if (any((ptr3_1-ptr3_2) > tol)) then + foundDiff(i) = .true. + end if end if - if (foundDiff(ii)) then + if (foundDiff(i)) then _FAIL('found difference when compare state') end if enddo diff --git a/Tests/ExtData_Testing_Framework/CMakeLists.txt b/Tests/ExtData_Testing_Framework/CMakeLists.txt index a91051be6019..257555524a4b 100644 --- a/Tests/ExtData_Testing_Framework/CMakeLists.txt +++ b/Tests/ExtData_Testing_Framework/CMakeLists.txt @@ -5,11 +5,11 @@ if(MPI_Fortran_LIBRARY_VERSION_FIRSTWORD MATCHES "Open") list(APPEND MPIEXEC_PREFLAGS "-oversubscribe") endif() -file(STRINGS "test_cases/cases.txt" TEST_CASES) +file(STRINGS "test_cases/extdata_1g_cases.txt" TEST_CASES_1G) set(cutoff "7") -foreach(TEST_CASE ${TEST_CASES}) +foreach(TEST_CASE ${TEST_CASES_1G}) if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) else() @@ -31,7 +31,18 @@ foreach(TEST_CASE ${TEST_CASES}) else() set_tests_properties ("ExtData1G_${TEST_CASE}" PROPERTIES LABELS "EXTDATA1G_BIG_TESTS") endif() + +endforeach() + +file(STRINGS "test_cases/extdata_2g_cases.txt" TEST_CASES_2G) + +foreach(TEST_CASE ${TEST_CASES_2G}) + if (EXISTS ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc) + file(READ ${CMAKE_CURRENT_LIST_DIR}/test_cases/${TEST_CASE}/nproc.rc num_procs) + else() + set(num_procs "1") + endif() add_test( NAME "ExtData2G_${TEST_CASE}" COMMAND ${CMAKE_COMMAND} diff --git a/Tests/ExtData_Testing_Framework/run_extdata.cmake b/Tests/ExtData_Testing_Framework/run_extdata.cmake index 48986e47d042..e5b8af69d042 100644 --- a/Tests/ExtData_Testing_Framework/run_extdata.cmake +++ b/Tests/ExtData_Testing_Framework/run_extdata.cmake @@ -13,6 +13,12 @@ macro(run_case CASE) if (${IS_EXTDATA2G} STREQUAL "YES") file(APPEND "${tempdir}/CAP1.rc" "USE_EXTDATA2G: .true.") file(APPEND "${tempdir}/CAP2.rc" "USE_EXTDATA2G: .true.") + if (EXISTS "${tempdir}/CAP3.rc") + file(APPEND "${tempdir}/CAP3.rc" "USE_EXTDATA2G: .true.") + endif() + if (EXISTS "${tempdir}/CAP4.rc") + file(APPEND "${tempdir}/CAP4.rc" "USE_EXTDATA2G: .true.") + endif() endif() execute_process( COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} ${MPIEXEC_PREFLAGS} ${MY_BINARY_DIR}/ExtDataDriver.x diff --git a/Tests/ExtData_Testing_Framework/test_cases/case18/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case18/CAP1.rc index 11e0e36bd675..7af21c5fc0f9 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case18/CAP1.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case18/CAP1.rc @@ -22,5 +22,3 @@ RUN_TIMES:: 20041115 210000 20041215 210000 :: - -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case18/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case18/CAP2.rc index 4a0f45af02e4..58dd40e4dd6f 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case18/CAP2.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case18/CAP2.rc @@ -11,5 +11,3 @@ HEARTBEAT_DT: 3600 RUN_TIMES:: 20041126 210000 :: - -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case22/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case22/CAP1.rc index 2dd970d08f1b..f3d012508e4a 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case22/CAP1.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case22/CAP1.rc @@ -6,7 +6,3 @@ BEG_DATE: 20071231 230000 JOB_SGMT: 00001201 000000 HEARTBEAT_DT: 3600 - - -USE_EXTDATA2G: .true. -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case22/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case22/CAP2.rc index 18c901f66ed9..cf4842a07145 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case22/CAP2.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case22/CAP2.rc @@ -6,7 +6,3 @@ BEG_DATE: 20081231 230000 JOB_SGMT: 00001200 000000 HEARTBEAT_DT: 3600 - - -USE_EXTDATA2G: .true. -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case22/CAP3.rc b/Tests/ExtData_Testing_Framework/test_cases/case22/CAP3.rc index cbb7b325f460..5a5a490a6ae8 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case22/CAP3.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case22/CAP3.rc @@ -7,5 +7,3 @@ BEG_DATE: 20081229 000000 JOB_SGMT: 00000010 000000 HEARTBEAT_DT: 3600 - -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case22/warnings_and_errors.log b/Tests/ExtData_Testing_Framework/test_cases/case22/warnings_and_errors.log deleted file mode 100644 index 9144b2f54291..000000000000 --- a/Tests/ExtData_Testing_Framework/test_cases/case22/warnings_and_errors.log +++ /dev/null @@ -1,60 +0,0 @@ -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: EXTDATA: In ExtData resource file, could not find: VAR2D -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: EXTDATA: In ExtData resource file, could not find: VAR2D -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: EXTDATA: In ExtData resource file, could not find: VAR2D -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage -pe=00000: MAPL: No configure file specified for logging layer. Using defaults. -pe=00000: EXTDATA: Using ExtData2G, note this is still in BETA stage diff --git a/Tests/ExtData_Testing_Framework/test_cases/case23/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case23/CAP1.rc index cd699d9b294a..b039ede25bed 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case23/CAP1.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case23/CAP1.rc @@ -6,6 +6,3 @@ BEG_DATE: 20151231 000000 JOB_SGMT: 00001200 000000 HEARTBEAT_DT: 3600 - - -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case23/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case23/CAP2.rc index 70bafc168604..10fe766675a9 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case23/CAP2.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case23/CAP2.rc @@ -6,6 +6,3 @@ BEG_DATE: 20191225 000000 JOB_SGMT: 00000020 000000 HEARTBEAT_DT: 3600 - - -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case23/CAP3.rc b/Tests/ExtData_Testing_Framework/test_cases/case23/CAP3.rc index 177c24b23153..b1c282250fee 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/case23/CAP3.rc +++ b/Tests/ExtData_Testing_Framework/test_cases/case23/CAP3.rc @@ -6,6 +6,3 @@ BEG_DATE: 20191227 000000 JOB_SGMT: 00000011 000000 HEARTBEAT_DT: 3600 - - -USE_EXTDATA2G: .true. diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM1.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM1.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM1.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM2.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM2.rc new file mode 100644 index 000000000000..83ad27a2c551 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM2.rc @@ -0,0 +1,24 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: GenerateExports + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM3.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM3.rc new file mode 100644 index 000000000000..f4eb4fa95d52 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM3.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: FillImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM4.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM4.rc new file mode 100644 index 000000000000..f4eb4fa95d52 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/AGCM4.rc @@ -0,0 +1,29 @@ +NX: 1 +NY: 1 + +Root.GRID_TYPE: LatLon +Root.GRIDNAME: DC90x45-PC +Root.LM: 3 +Root.IM_WORLD: 90 +Root.JM_WORLD: 45 +Root.POLE: 'PC' +Root.DATELINE: 'DC' + +RUN_MODE: FillImports + +IMPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +EXPORT_STATE:: +VAR2D , time , days , xy , c +VAR3D , time , days , xyz , c +:: + +FILL_DEF:: +VAR2D time +VAR3D time +:: + +REF_TIME: 20040701 000000 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/CAP.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP.rc new file mode 100644 index 000000000000..b7edb691288c --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP.rc @@ -0,0 +1,4 @@ +CASES:: +CAP3.rc +CAP4.rc +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/CAP1.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP1.rc new file mode 100644 index 000000000000..195f28b58e5f --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP1.rc @@ -0,0 +1,11 @@ +USE_EXTDATA2G: .true. +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040111 210000 + +JOB_SGMT: 00000003 000000 +HEARTBEAT_DT: 3600 + + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/CAP2.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP2.rc new file mode 100644 index 000000000000..d1b51d205efd --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP2.rc @@ -0,0 +1,11 @@ +USE_EXTDATA2G: .true. +ROOT_NAME: Root +ROOT_CF: AGCM1.rc +HIST_CF: HISTORY1.rc + +BEG_DATE: 20040115 210000 + +JOB_SGMT: 00000003 000000 +HEARTBEAT_DT: 3600 + + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/CAP3.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP3.rc new file mode 100644 index 000000000000..b035979f0176 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP3.rc @@ -0,0 +1,11 @@ +USE_EXTDATA2G: .true. + +ROOT_NAME: Root +ROOT_CF: AGCM3.rc +HIST_CF: HISTORY3.rc + +BEG_DATE: 20040114 030000 + +JOB_SGMT: 00000003 000000 +HEARTBEAT_DT: 3600 + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/CAP4.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP4.rc new file mode 100644 index 000000000000..d3664e1a8b15 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/CAP4.rc @@ -0,0 +1,10 @@ +USE_EXTDATA2G: .true. +ROOT_NAME: Root +ROOT_CF: AGCM4.rc +HIST_CF: HISTORY4.rc + +BEG_DATE: 20040115 030000 + +JOB_SGMT: 00000003 000000 +HEARTBEAT_DT: 3600 + diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/ExtData.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/ExtData.rc new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY1.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY1.rc new file mode 100644 index 000000000000..a3043dfb878d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY1.rc @@ -0,0 +1,12 @@ +GRID_LABELS: +:: + +COLLECTIONS: testcase +:: + + testcase.template: '%y4%m2%d2_%h2%n2z.nc4', + testcase.format: 'CFIO', + testcase.frequency: 030000, + testcase.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY2.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY2.rc new file mode 100644 index 000000000000..a3043dfb878d --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY2.rc @@ -0,0 +1,12 @@ +GRID_LABELS: +:: + +COLLECTIONS: testcase +:: + + testcase.template: '%y4%m2%d2_%h2%n2z.nc4', + testcase.format: 'CFIO', + testcase.frequency: 030000, + testcase.fields: 'VAR2D', 'Root', + 'VAR3D', 'Root', + :: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY3.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY3.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY3.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY4.rc b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY4.rc new file mode 100644 index 000000000000..2895432e995a --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/HISTORY4.rc @@ -0,0 +1,5 @@ +GRID_LABELS: +:: + +COLLECTIONS: +:: diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/README b/Tests/ExtData_Testing_Framework/test_cases/case27/README new file mode 100644 index 000000000000..0e3e44d136b5 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/README @@ -0,0 +1 @@ +allow for gaps in the dataset diff --git a/Tests/ExtData_Testing_Framework/test_cases/case27/extdata.yaml b/Tests/ExtData_Testing_Framework/test_cases/case27/extdata.yaml new file mode 100644 index 000000000000..57b020646864 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/case27/extdata.yaml @@ -0,0 +1,7 @@ +Collections: + fstream1: {template: testcase.%y4%m2%d2_%h2%n2z.nc4, freq: PT3H } +Exports: + VAR2D: {variable: VAR2D, collection: fstream1, fail_on_missing_file: false, sample: no_interp} + VAR3D: {variable: VAR3D, collection: fstream1, fail_on_missing_file: false, sample: no_interp} +Samplings: + no_interp: {time_interpolation: false} diff --git a/Tests/ExtData_Testing_Framework/test_cases/extdata_1g_cases.txt b/Tests/ExtData_Testing_Framework/test_cases/extdata_1g_cases.txt new file mode 100644 index 000000000000..ce213d266d74 --- /dev/null +++ b/Tests/ExtData_Testing_Framework/test_cases/extdata_1g_cases.txt @@ -0,0 +1,24 @@ +case1 +case2 +case3 +case4 +case5 +case6 +case7 +case8 +case9 +case10 +case11 +case12 +case13 +case14 +case15 +case16 +case18 +case19 +case20 +case21 +case23 +case24 +case25 +case26 diff --git a/Tests/ExtData_Testing_Framework/test_cases/cases.txt b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt similarity index 95% rename from Tests/ExtData_Testing_Framework/test_cases/cases.txt rename to Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt index 9753d563eb9a..02026c1d3f40 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/cases.txt +++ b/Tests/ExtData_Testing_Framework/test_cases/extdata_2g_cases.txt @@ -23,3 +23,4 @@ case23 case24 case25 case26 +case27 diff --git a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md index 2b8b02b42681..f6859c535e8e 100644 --- a/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md +++ b/Tests/ExtData_Testing_Framework/test_cases/test_case_descriptions.md @@ -30,4 +30,5 @@ path_to_script/run_extdatadriver_cases.py --builddir path_to_geos_install/bin -- 23. Test multiple datasets and treat Climatology in the first and a real-time in the 2nd 24. Test reading cubed-sphere input 25. Test reading edge variable -26. Test reading edge + cetner variables +26. Test reading edge + center variables +27. Case with a "gap" in the data diff --git a/Tests/MAPL_demo_fargparse.F90 b/Tests/MAPL_demo_fargparse.F90 new file mode 100755 index 000000000000..082d936b9e95 --- /dev/null +++ b/Tests/MAPL_demo_fargparse.F90 @@ -0,0 +1,80 @@ +!------------------------------------------------------------------------------ +!># Standalone Program for Testing fargparse +! +!------------------------------------------------------------------------------ +#define I_AM_MAIN +#include "MAPL_ErrLog.h" +#include "unused_dummy.H" + +program main + use MAPL + use mpi + use fargparse + + implicit none + + type(MAPL_FargparseCLI) :: cli + type(MAPL_CapOptions) :: cap_options + integer :: status + +!------------------------------------------------------------------------------ + + call run(_RC) + + contains + +#undef I_AM_MAIN +#include "MAPL_ErrLog.h" + subroutine run(rc) + + integer, intent(out), optional :: rc + + integer :: status + character(len=:), allocatable :: input_file + + call MPI_Init(status) + _VERIFY(status) + + ! Read and parse the command line, and set parameters + ! If you have extra options you make a procedure as seen below and add arguments + ! there and pass in here + cli = MAPL_FargparseCLI(extra=extra_options) + + ! This does the casting of arguments into cap_options for CAP + cap_options = MAPL_CapOptions(cli, _RC) + + write(*,*) "done with MAPL_FargparseCLI" + write(*,*) " cap_options%with_esmf_moab = ", cap_options%with_esmf_moab + write(*,*) " cap_options%npes_input_server = ", cap_options%npes_input_server + write(*,*) " cap_options%nodes_input_server = ", cap_options%nodes_input_server + write(*,*) " cap_options%npes_output_server = ", cap_options%npes_output_server + write(*,*) " cap_options%nodes_output_server = ", cap_options%nodes_output_server + write(*,*) " cap_options%egress_file = ", cap_options%egress_file + + ! For our extra options we have to explicitly cast them + call cast(cli%options%at('file'), input_file, _RC) + + write(*,*) "" + write(*,*) "Extra arguments" + write(*,*) " input file = ", input_file + + _RETURN(_SUCCESS) + + end subroutine run + + subroutine extra_options(parser, rc) + type (ArgParser), intent(inout) :: parser + integer, intent(out), optional :: rc + + call parser%add_argument('-f', '--file', & + help='A file to read', & + type='string', & + default='default.config', & + action='store') + + !_RETURN(_SUCCESS) + if (present(rc)) rc = 0 + + end subroutine extra_options + +end program main diff --git a/base/Base.F90 b/base/Base.F90 index 5b7adc230164..012aedc8b79c 100644 --- a/base/Base.F90 +++ b/base/Base.F90 @@ -51,6 +51,7 @@ module MAPLBase_Mod use MAPL_ServerManager use MAPL_FileMetadataUtilsMod use MAPL_VerticalDataMod + use MAPL_FieldUtilities logical, save, private :: mapl_is_initialized = .false. end module MAPLBase_Mod diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index 1a427456eb71..091582d1e1a5 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -52,6 +52,7 @@ set (srcs Base/Base_Base.F90 Base/Base_Base_implementation.F90 TimeStringConversion.F90 MAPL_ISO8601_DateTime_ESMF.F90 + FieldUtilities.F90 # Orphaned program: should not be in this library. # tstqsat.F90 ) diff --git a/base/ESMFL_Mod.F90 b/base/ESMFL_Mod.F90 index 5a8e4c0e55c0..7f86a45f7932 100644 --- a/base/ESMFL_Mod.F90 +++ b/base/ESMFL_Mod.F90 @@ -61,6 +61,7 @@ module ESMFL_MOD public MAPL_AreaMean public ESMFL_Diff public ESMFL_statistics + public ESMFL_field_is_undefined !EOP ! regridding @@ -4472,5 +4473,39 @@ logical function check_list_ (this,list) endif end function check_list_ + function ESMFL_field_is_undefined(field,rc) result(field_is_undefined) + use mpi + logical :: field_is_undefined + type(ESMF_Field), intent(in) :: field + integer, optional, intent(out) :: rc + + integer :: status + type(ESMF_Grid) :: grid + type(ESMF_VM) :: vm + integer :: my_mpi_comm, local_undef, global_undef,grid_size(3),rank + real, pointer :: ptr2d(:,:), ptr3d(:,:,:) + + call ESMF_VMGetCurrent(VM,_RC) + call ESMF_VMGet(VM,mpiCommunicator=my_mpi_comm,_RC) + call ESMF_FieldGet(field,rank=rank,grid=grid,_RC) + call MAPL_GridGet(grid,globalcellcountperdim=grid_size,_RC) + if (rank ==2) then + call ESMF_FieldGet(field,0,farrayPtr=ptr2d,_RC) + local_undef = count(ptr2d==MAPL_UNDEF) + call MPI_AllReduce(local_undef,global_undef,1,MPI_INTEGER,MPI_SUM,my_mpi_comm,status) + _VERIFY(status) + field_is_undefined = global_undef == (grid_size(1)*grid_size(2)) + else if (rank ==3) then + call ESMF_FieldGet(field,0,farrayPtr=ptr3d,_RC) + local_undef = count(ptr3d==MAPL_UNDEF) + call MPI_AllReduce(local_undef,global_undef,1,MPI_INTEGER,MPI_SUM,my_mpi_comm,status) + _VERIFY(status) + field_is_undefined = global_undef == (grid_size(1)*grid_size(2)*grid_size(3)) + else + _FAIL("Unsupported rank when checking for undef") + end if + + _RETURN(_SUCCESS) + end function end module ESMFL_MOD diff --git a/base/FieldUtilities.F90 b/base/FieldUtilities.F90 new file mode 100644 index 000000000000..6a83c6e6dd8a --- /dev/null +++ b/base/FieldUtilities.F90 @@ -0,0 +1,53 @@ +#include "MAPL_Generic.h" + +module MAPL_FieldUtilities +use ESMF +use MAPL_BaseMod, only: MAPL_Undef +use MAPL_ErrorHandlingMod + +implicit none +private + +public is_field_undef + +contains + +function is_field_undef(field,rc) result(field_is_undef) + logical :: field_is_undef + type(ESMF_Field), intent(in) :: field + integer, optional, intent(out) :: rc + + integer :: status + + real(ESMF_KIND_R4), pointer :: ptr_1d_r4(:), ptr_2d_r4(:,:), ptr_3d_r4(:,:,:), ptr_4d_r4(:,:,:,:) + + integer :: rank + type(ESMF_TypeKind_Flag) :: typekind + + call ESMF_FieldGet(field,rank=rank,typekind=typekind,_RC) + + if (typekind == ESMF_TYPEKIND_R4) then + select case(rank) + case(1) + call ESMF_FieldGet(field,0,farrayPtr=ptr_1d_r4,_RC) + field_is_undef = all(ptr_1d_r4 == MAPL_UNDEF) + case(2) + call ESMF_FieldGet(field,0,farrayPtr=ptr_2d_r4,_RC) + field_is_undef = all(ptr_2d_r4 == MAPL_UNDEF) + case(3) + call ESMF_FieldGet(field,0,farrayPtr=ptr_3d_r4,_RC) + field_is_undef = all(ptr_3d_r4 == MAPL_UNDEF) + case(4) + call ESMF_FieldGet(field,0,farrayPtr=ptr_4d_r4,_RC) + field_is_undef = all(ptr_4d_r4 == MAPL_UNDEF) + end select + else + _FAIL("MAPL_UNDEF is single precision so you can not check if it is all undef for an R8") + end if + + _RETURN(_SUCCESS) + +end function + +end module + diff --git a/base/tests/test_sort.pf b/base/tests/test_sort.pf index e468f36dd284..97a4cee8eb33 100644 --- a/base/tests/test_sort.pf +++ b/base/tests/test_sort.pf @@ -1,34 +1,39 @@ -@test -subroutine test_mapl_sort +module test_sort use pfunit use mapl_sortMod use iso_fortran_env, only: INT32, INT64 - implicit none - - integer(kind=INT64) :: a(5), c(5) - integer(kind=INT32) :: a4(5) - integer(kind=INT32) :: b1(5,3), b2(3,5) - integer(kind=INT32) :: d1(5,3) - - integer :: i, j - - a = [5,3,8,6,-7] - do j=1,3 - b1(:,j) = a - b2(j,:) = a - enddo - a4 = a - - - C=A - d1=b1 - call MAPL_SORT(C,d1,dim=1) - - do j=1,3 - do i = 2, size(d1,dim=1) - @assertTrue(d1(i-1,j) <= d1(i,j)) + +contains + + @test + subroutine test_mapl_sort + implicit none + + integer(kind=INT64) :: a(5), c(5) + integer(kind=INT32) :: a4(5) + integer(kind=INT32) :: b1(5,3), b2(3,5) + integer(kind=INT32) :: d1(5,3) + + integer :: i, j + + a = [5,3,8,6,-7] + do j=1,3 + b1(:,j) = a + b2(j,:) = a + enddo + a4 = a + + + C=A + d1=b1 + call MAPL_SORT(C,d1,dim=1) + + do j=1,3 + do i = 2, size(d1,dim=1) + @assertTrue(d1(i-1,j) <= d1(i,j)) + end do end do - end do -end subroutine test_mapl_sort + end subroutine test_mapl_sort +end module test_sort diff --git a/components.yaml b/components.yaml index 48775ff10933..23a0d6a37033 100644 --- a/components.yaml +++ b/components.yaml @@ -5,16 +5,16 @@ MAPL: ESMA_env: local: ./ESMA_env remote: ../ESMA_env.git - tag: v4.7.0 + tag: v4.8.0 develop: main ESMA_cmake: local: ./ESMA_cmake remote: ../ESMA_cmake.git - tag: v3.20.0 + tag: v3.21.0 develop: develop ecbuild: local: ./ESMA_cmake/ecbuild remote: ../ecbuild.git - tag: geos/v1.2.0 + tag: geos/v1.3.0 diff --git a/gridcomps/CMakeLists.txt b/gridcomps/CMakeLists.txt index 6493a3ad2de6..a477ebb529e3 100644 --- a/gridcomps/CMakeLists.txt +++ b/gridcomps/CMakeLists.txt @@ -3,7 +3,8 @@ esma_add_library (${this} SRCS MAPL_GridComps.F90 DEPENDENCIES MAPL.base MAPL.pfio MAPL_cfio_r4 MAPL.cap $<$:FLAP::FLAP> - TYPE ${MAPL_LIBRARY_TYPE} + $<$:FARGPARSE::fargparse> + TYPE ${MAPL_LIBRARY_TYPE} ) target_include_directories (${this} PUBLIC @@ -12,6 +13,10 @@ if (BUILD_WITH_FLAP) target_link_libraries(${this} PRIVATE FLAP::FLAP) target_compile_definitions (${this} PRIVATE USE_FLAP) endif() +if (BUILD_WITH_FARGPARSE) + target_link_libraries(${this} PRIVATE FARGPARSE::fargparse) + target_compile_definitions (${this} PRIVATE USE_FARGPARSE) +endif() add_subdirectory(Cap) add_subdirectory(History) diff --git a/gridcomps/Cap/CMakeLists.txt b/gridcomps/Cap/CMakeLists.txt index 07a2fe92b3cb..194c41300aaa 100644 --- a/gridcomps/Cap/CMakeLists.txt +++ b/gridcomps/Cap/CMakeLists.txt @@ -9,12 +9,17 @@ set (srcs if (BUILD_WITH_FLAP) list (APPEND srcs FlapCLI.F90) endif() +if (BUILD_WITH_FARGPARSE) + list (APPEND srcs FargparseCLI.F90) +endif() esma_add_library (${this} SRCS ${srcs} DEPENDENCIES MAPL.shared MAPL.constants MAPL.base MAPL.profiler MAPL.history MAPL.ExtData ${EXTDATA2G_TARGET} TYPE ${MAPL_LIBRARY_TYPE}) target_link_libraries (${this} PUBLIC GFTL::gftl GFTL_SHARED::gftl-shared esmf NetCDF::NetCDF_Fortran - PRIVATE MPI::MPI_Fortran $<$:FLAP::FLAP>) + PRIVATE MPI::MPI_Fortran + $<$:FLAP::FLAP> + $<$:FARGPARSE::fargparse>) target_compile_definitions (${this} PRIVATE $<$:BUILD_WITH_EXTDATA2G>) @@ -24,7 +29,6 @@ if (NOT CMAKE_Fortran_COMPILER_ID MATCHES "NAG") endif () target_include_directories (${this} PUBLIC $) if (BUILD_WITH_FLAP) - target_link_libraries(${this} PRIVATE FLAP::FLAP) target_compile_definitions (${this} PRIVATE USE_FLAP) endif() diff --git a/gridcomps/Cap/CapOptions.F90 b/gridcomps/Cap/CapOptions.F90 index 661dedc5593e..f1aea9ab186f 100644 --- a/gridcomps/Cap/CapOptions.F90 +++ b/gridcomps/Cap/CapOptions.F90 @@ -28,7 +28,7 @@ module mapl_CapOptionsMod ! whether or not the nodes are padding with idle when mod(model total npes , each node npes) /=0 logical :: isolate_nodes = .true. ! whether or not copy the data before isend to the oserver - ! it is faster but demands more memory if it is true + ! it is faster but demands more memory if it is true logical :: fast_oclient = .false. ! whether or not turn on the io profiler logical :: with_io_profiler = .false. @@ -59,7 +59,7 @@ function new_CapOptions(unusable, cap_rc_file, egress_file, ensemble_subdir_pref class (KeywordEnforcer), optional, intent(in) :: unusable character(*), optional, intent(in) :: cap_rc_file character(*), optional, intent(in) :: egress_file - character(*), optional, intent(in) :: ensemble_subdir_prefix + character(*), optional, intent(in) :: ensemble_subdir_prefix type(ESMF_LogKind_Flag), optional, intent(in) :: esmf_logging_mode integer, optional, intent(out) :: rc diff --git a/gridcomps/Cap/FargparseCLI.F90 b/gridcomps/Cap/FargparseCLI.F90 new file mode 100644 index 000000000000..68360a1b0a5f --- /dev/null +++ b/gridcomps/Cap/FargparseCLI.F90 @@ -0,0 +1,604 @@ +#include "MAPL_ErrLog.h" +#include "unused_dummy.H" + +module MAPL_FargparseCLIMod + use MPI + use ESMF + use fArgParse + use gFTL2_IntegerVector + use mapl_KeywordEnforcerMod + use mapl_ExceptionHandling + use mapl_CapOptionsMod, only: MAPL_CapOptions !Rename is for backward compatibility. Remove renaming for 3.0 + implicit none + private + + public :: MAPL_FargparseCLI + public :: MAPL_CapOptions !Needed for backward compatibility. Remove for 3.0 + + type :: MAPL_FargparseCLI + type(ArgParser) :: parser + type(StringUnlimitedMap) :: options + contains + procedure, nopass :: add_command_line_options + procedure :: fill_cap_options + end type MAPL_FargparseCLI + + interface MAPL_FargparseCLI + module procedure new_CapOptions_from_fargparse + module procedure new_CapOptions_from_fargparse_back_comp + end interface MAPL_FargparseCLI + + interface MAPL_CapOptions !Needed for backward compatibility. Remove for 3.0 + module procedure old_CapOptions_from_fargparse + end interface MAPL_CapOptions + + integer, parameter :: NO_VALUE_PASSED_IN = -999 + + abstract interface + subroutine I_extraoptions(parser, rc) + import ArgParser + type(ArgParser), intent(inout) :: parser + integer, optional, intent(out) :: rc + end subroutine + end interface +contains + + function new_CapOptions_from_fargparse(unusable, dummy, extra, rc) result (cap_options) + class(KeywordEnforcer), optional, intent(in) :: unusable + type (MAPL_CapOptions) :: cap_options + character(*), intent(in) :: dummy !Needed for backward compatibility. Remove after 3.0 + procedure(I_extraoptions), optional :: extra + integer, optional, intent(out) :: rc + integer :: status + + type(MAPL_FargparseCLI) :: fargparse_cli + + fargparse_cli%parser = ArgParser() + + call fargparse_cli%add_command_line_options(fargparse_cli%parser, _RC) + + if (present(extra)) then + call extra(fargparse_cli%parser, _RC) + end if + + fargparse_cli%options = fargparse_cli%parser%parse_args() + + call fargparse_cli%fill_cap_options(cap_options, _RC) + + _RETURN(_SUCCESS) + _UNUSED_DUMMY(unusable) + _UNUSED_DUMMY(dummy) + end function new_CapOptions_from_fargparse + + function new_CapOptions_from_fargparse_back_comp(unusable, extra, rc) result (fargparsecap) + class(KeywordEnforcer), optional, intent(in) :: unusable + type (MAPL_FargparseCLI) :: fargparsecap + procedure(I_extraoptions), optional :: extra + integer, optional, intent(out) :: rc + integer :: status + + fargparsecap%parser = ArgParser() + + call fargparsecap%add_command_line_options(fargparsecap%parser, _RC) + + if (present(extra)) then + call extra(fargparsecap%parser, _RC) + end if + + fargparsecap%options = fargparsecap%parser%parse_args() + + _RETURN(_SUCCESS) + _UNUSED_DUMMY(unusable) + end function new_CapOptions_from_fargparse_back_comp + + ! Static method + subroutine add_command_line_options(parser, unusable, rc) + type (ArgParser), intent(inout) :: parser + class (KeywordEnforcer), optional, intent(in) :: unusable + integer, optional, intent(out) :: rc + + type(IntegerVector) :: intvec + + call parser%add_argument('--root_dso', & + help='name of root dso to use', & + type='string', & + default='none', & + action='store') + + call parser%add_argument('--esmf_logtype', & + help='ESMF Logging type (allowed: none, single, multi, multi_on_error)', & + !choices='none,single,multi,multi_on_error', & + type='string', & + default='none', & + action='store') + + call parser%add_argument('--egress_file', & + help='Egress file name', & + type='string', & + default='EGRESS', & + action='store') + + call parser%add_argument('--cap_rc', & + help='CAP resource file name', & + type='string', & + default='CAP.rc', & + action='store') + + call parser%add_argument('--npes_model', & + help='Number of MPI processes used by model CapGridComp', & + type='integer', & + action='store', & + default=-1) + + call parser%add_argument('--n_members', & + help='Number of MPI processes used by model CapGridComp1', & + type='integer', & + action='store', & + default=1) + + call parser%add_argument('--use_sub_comm', & + help='The model by default is using MPI_COMM_WORLD : .true. or .false.', & + action='store_true') + + call parser%add_argument('--comm_model', & + help='The model will use the communicator passed in', & + type='string', & + action='store', & + default='*') + + call parser%add_argument('--prefix', & + help='prefix for ensemble subdirectories', & + type='string', & + action='store', & + default='mem') + + ! We create an IntegerVector with a bad value to test if the user + ! passed in anything + + call intvec%push_back(NO_VALUE_PASSED_IN) + + call parser%add_argument('--npes_input_server', & + help='Number of MPI processes used by input server', & + type='integer', & + n_arguments ='+', & + default = intvec, & + action='store') + + call parser%add_argument('--npes_output_server', & + help='Number of MPI processes used by output server', & + type='integer', & + n_arguments ='+', & + default = intvec, & + action='store') + + call parser%add_argument('--nodes_input_server', & + help='Number of nodes used by input server', & + type='integer', & + n_arguments ='+', & + default = intvec, & + action='store') + + call parser%add_argument('--nodes_output_server', & + help='Number of nodes used by output server', & + type='integer', & + n_arguments ='+', & + default = intvec, & + action='store') + + call parser%add_argument('--logging_config', & + help='Configuration file for logging', & + type='string', & + default='', & + action='store') + + call parser%add_argument('--oserver_type', & + help='Output Server Type', & + type='string', & + default='single', & + action='store') + + call parser%add_argument('--npes_backend_pernode', & + help='Number of MPI processes used by the backend output', & + type='integer', & + default=0, & + action='store') + + call parser%add_argument('--compress_nodes', & + help='MPI processes continue on the nodes even MPI communicator is divided', & + action='store_true') + + call parser%add_argument('--fast_oclient', & + help='Copying data before isend. Client would wait until it is re-used', & + action='store_true') + + call parser%add_argument('--one_node_output', & + help='Specify if each output server has only one nodes', & + action='store_true') + + call parser%add_argument('--with_io_profiler', & + help='Turning on io_profler', & + action='store_true') + + call parser%add_argument('--with_esmf_moab', & + help='Enables use of MOAB library for ESMF meshes', & + action='store_true') + + _RETURN(_SUCCESS) + _UNUSED_DUMMY(unusable) + + end subroutine add_command_line_options + + subroutine fill_cap_options(fargparseCLI, cap_options, unusable, rc) + class(MAPL_FargparseCLI), intent(inout) :: fargparseCLI + type(MAPL_CapOptions), intent(out) :: cap_options + class(KeywordEnforcer), optional, intent(in) :: unusable + integer, optional, intent(out) :: rc + integer :: status + character(:), allocatable :: buffer + logical :: one_node_output, compress_nodes, use_sub_comm + + integer, allocatable :: nodes_output_server(:) + class(*), pointer :: option, option_npes, option_nodes + type (IntegerVector) :: tmp_int_vector, tmp_npes_vector, tmp_nodes_vector + + option => fargparseCLI%options%at('root_dso') + if (associated(option)) then + call cast(option, cap_options%root_dso, _RC) + end if + + option => fargparseCLI%options%at('egress_file') + if (associated(option)) then + call cast(option, cap_options%egress_file, _RC) + end if + + option => fargparseCLI%options%at('use_sub_comm') + if (associated(option)) then + call cast(option, use_sub_comm, _RC) + cap_options%use_comm_world = .not. use_sub_comm + end if + + if ( .not. cap_options%use_comm_world) then + option => fargparseCLI%options%at('comm_model') + if (associated(option)) then + call cast(option, buffer, _RC) + _ASSERT(trim(buffer) /= '*', "Should provide comm for model") + call cast(option, cap_options%comm, _RC) + end if + else + ! comm will be set to MPI_COMM_WORLD later on in initialize_mpi + ! npes will be set to npes_world later on in initialize_mpi + endif + + option => fargparseCLI%options%at('npes_model') + if (associated(option)) then + call cast(option, cap_options%npes_model, _RC) + end if + + option => fargparseCLI%options%at('compress_nodes') + if (associated(option)) then + call cast(option, compress_nodes, _RC) + cap_options%isolate_nodes = .not. compress_nodes + end if + + option => fargparseCLI%options%at('fast_oclient') + if (associated(option)) then + call cast(option, cap_options%fast_oclient, _RC) + end if + + option => fargparseCLI%options%at('with_io_profiler') + if (associated(option)) then + call cast(option, cap_options%with_io_profiler, _RC) + end if + + option => fargparseCLI%options%at('with_esmf_moab') + if (associated(option)) then + call cast(option, cap_options%with_esmf_moab, _RC) + end if + + ! We only allow one of npes_input_server or nodes_input_server + option_npes => fargparseCLI%options%at('npes_input_server') + call cast(option_npes, tmp_npes_vector, _RC) + option_nodes => fargparseCLI%options%at('nodes_input_server') + call cast(option_nodes, tmp_nodes_vector, _RC) + _ASSERT(.not.(tmp_npes_vector%of(1) /= NO_VALUE_PASSED_IN .and. tmp_nodes_vector%of(1) /= NO_VALUE_PASSED_IN), 'Cannot specify both --npes_input_server and --nodes_input_server') + + ! npes_input_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('npes_input_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + cap_options%npes_input_server = tmp_int_vector%data() + else + cap_options%npes_input_server = [0] + end if + + ! nodes_input_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('nodes_input_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + cap_options%nodes_input_server = tmp_int_vector%data() + else + cap_options%nodes_input_server = [0] + end if + + ! We only allow one of npes_output_server or nodes_output_server + option_npes => fargparseCLI%options%at('npes_output_server') + call cast(option_npes, tmp_npes_vector, _RC) + option_nodes => fargparseCLI%options%at('nodes_output_server') + call cast(option_nodes, tmp_nodes_vector, _RC) + _ASSERT(.not.(tmp_npes_vector%of(1) /= NO_VALUE_PASSED_IN .and. tmp_nodes_vector%of(1) /= NO_VALUE_PASSED_IN), 'Cannot specify both --npes_output_server and --nodes_output_server') + + ! npes_output_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('npes_output_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + cap_options%npes_output_server = tmp_int_vector%data() + else + cap_options%npes_output_server = [0] + end if + + ! nodes_output_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('nodes_output_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + nodes_output_server = tmp_int_vector%data() + else + nodes_output_server = [0] + end if + + option => fargparseCLI%options%at('one_node_output') + if (associated(option)) then + call cast(option, one_node_output, _RC) + else + one_node_output = .false. + end if + if (one_node_output) then + allocate(cap_options%nodes_output_server(sum(nodes_output_server)), source =1) + else + cap_options%nodes_output_server = nodes_output_server + endif + + cap_options%n_iserver_group = max(size(cap_options%npes_input_server),size(cap_options%nodes_input_server)) + cap_options%n_oserver_group = max(size(cap_options%npes_output_server),size(cap_options%nodes_output_server)) + + option => fargparseCLI%options%at('esmf_logtype') + if (associated(option)) then + call cast(option, buffer, _RC) + end if + ! set_esmf_logging_mode + select case (trim(buffer)) + case ('none') + cap_options%esmf_logging_mode = ESMF_LOGKIND_NONE + case ('single') + cap_options%esmf_logging_mode = ESMF_LOGKIND_SINGLE + case ('multi') + cap_options%esmf_logging_mode = ESMF_LOGKIND_MULTI + case ('multi_on_error') + cap_options%esmf_logging_mode = ESMF_LOGKIND_MULTI_ON_ERROR + case default + _FAIL("Unsupported ESMF logging option: "//trim(buffer)) + end select + + ! Ensemble specific options + option => fargparseCLI%options%at('prefix') + if (associated(option)) then + call cast(option, cap_options%ensemble_subdir_prefix, _RC) + end if + + option => fargparseCLI%options%at('n_members') + if (associated(option)) then + call cast(option, cap_options%n_members, _RC) + end if + + option => fargparseCLI%options%at('cap_rc') + if (associated(option)) then + call cast(option, cap_options%cap_rc_file, _RC) + end if + + ! Logging options + option => fargparseCLI%options%at('logging_config') + if (associated(option)) then + call cast(option, cap_options%logging_config, _RC) + end if + + option => fargparseCLI%options%at('oserver_type') + if (associated(option)) then + call cast(option, cap_options%oserver_type, _RC) + end if + + option => fargparseCLI%options%at('npes_backend_pernode') + if (associated(option)) then + call cast(option, cap_options%npes_backend_pernode, _RC) + end if + + _RETURN(_SUCCESS) + _UNUSED_DUMMY(unusable) + end subroutine fill_cap_options + + !Function for backward compatibility. Remove for 3.0 + function old_CapOptions_from_Fargparse( fargparseCLI, unusable, rc) result (cap_options) + type (MAPL_CapOptions) :: cap_options + type (MAPL_FargparseCLI), intent(inout) :: fargparseCLI + class (KeywordEnforcer), optional, intent(in) :: unusable + integer, optional, intent(out) :: rc + integer :: status + character(:), allocatable :: buffer + logical :: one_node_output, compress_nodes, use_sub_comm + + integer, allocatable :: nodes_output_server(:) + class(*), pointer :: option, option_npes, option_nodes + type (IntegerVector) :: tmp_int_vector, tmp_npes_vector, tmp_nodes_vector + + option => fargparseCLI%options%at('root_dso') + if (associated(option)) then + call cast(option, cap_options%root_dso, _RC) + end if + + option => fargparseCLI%options%at('egress_file') + if (associated(option)) then + call cast(option, cap_options%egress_file, _RC) + end if + + option => fargparseCLI%options%at('use_sub_comm') + if (associated(option)) then + call cast(option, use_sub_comm, _RC) + cap_options%use_comm_world = .not. use_sub_comm + end if + + if ( .not. cap_options%use_comm_world) then + option => fargparseCLI%options%at('comm_model') + if (associated(option)) then + call cast(option, buffer, _RC) + _ASSERT(trim(buffer) /= '*', "Should provide comm for model") + call cast(option, cap_options%comm, _RC) + end if + else + ! comm will be set to MPI_COMM_WORLD later on in initialize_mpi + ! npes will be set to npes_world later on in initialize_mpi + endif + + option => fargparseCLI%options%at('npes_model') + if (associated(option)) then + call cast(option, cap_options%npes_model, _RC) + end if + + option => fargparseCLI%options%at('compress_nodes') + if (associated(option)) then + call cast(option, compress_nodes, _RC) + cap_options%isolate_nodes = .not. compress_nodes + end if + + option => fargparseCLI%options%at('fast_oclient') + if (associated(option)) then + call cast(option, cap_options%fast_oclient, _RC) + end if + + option => fargparseCLI%options%at('with_io_profiler') + if (associated(option)) then + call cast(option, cap_options%with_io_profiler, _RC) + end if + + option => fargparseCLI%options%at('with_esmf_moab') + if (associated(option)) then + call cast(option, cap_options%with_esmf_moab, _RC) + end if + + ! We only allow one of npes_input_server or nodes_input_server + option_npes => fargparseCLI%options%at('npes_input_server') + call cast(option_npes, tmp_npes_vector, _RC) + option_nodes => fargparseCLI%options%at('nodes_input_server') + call cast(option_nodes, tmp_nodes_vector, _RC) + _ASSERT(.not.(tmp_npes_vector%of(1) /= NO_VALUE_PASSED_IN .and. tmp_nodes_vector%of(1) /= NO_VALUE_PASSED_IN), 'Cannot specify both --npes_input_server and --nodes_input_server') + + ! npes_input_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('npes_input_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + cap_options%npes_input_server = tmp_int_vector%data() + else + cap_options%npes_input_server = [0] + end if + + ! nodes_input_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('nodes_input_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + cap_options%nodes_input_server = tmp_int_vector%data() + else + cap_options%nodes_input_server = [0] + end if + + ! We only allow one of npes_output_server or nodes_output_server + option_npes => fargparseCLI%options%at('npes_output_server') + call cast(option_npes, tmp_npes_vector, _RC) + option_nodes => fargparseCLI%options%at('nodes_output_server') + call cast(option_nodes, tmp_nodes_vector, _RC) + _ASSERT(.not.(tmp_npes_vector%of(1) /= NO_VALUE_PASSED_IN .and. tmp_nodes_vector%of(1) /= NO_VALUE_PASSED_IN), 'Cannot specify both --npes_output_server and --nodes_output_server') + + ! npes_output_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('npes_output_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + cap_options%npes_output_server = tmp_int_vector%data() + else + cap_options%npes_output_server = [0] + end if + + ! nodes_output_server is a gFTL IntegerVector that we need to convert to an integer array + option => fargparseCLI%options%at('nodes_output_server') + call cast(option, tmp_int_vector, _RC) + if (tmp_int_vector%of(1) /= NO_VALUE_PASSED_IN) then + nodes_output_server = tmp_int_vector%data() + else + nodes_output_server = [0] + end if + + option => fargparseCLI%options%at('one_node_output') + if (associated(option)) then + call cast(option, one_node_output, _RC) + else + one_node_output = .false. + end if + if (one_node_output) then + allocate(cap_options%nodes_output_server(sum(nodes_output_server)), source =1) + else + cap_options%nodes_output_server = nodes_output_server + endif + + cap_options%n_iserver_group = max(size(cap_options%npes_input_server),size(cap_options%nodes_input_server)) + cap_options%n_oserver_group = max(size(cap_options%npes_output_server),size(cap_options%nodes_output_server)) + + option => fargparseCLI%options%at('esmf_logtype') + if (associated(option)) then + call cast(option, buffer, _RC) + end if + ! set_esmf_logging_mode + select case (trim(buffer)) + case ('none') + cap_options%esmf_logging_mode = ESMF_LOGKIND_NONE + case ('single') + cap_options%esmf_logging_mode = ESMF_LOGKIND_SINGLE + case ('multi') + cap_options%esmf_logging_mode = ESMF_LOGKIND_MULTI + case ('multi_on_error') + cap_options%esmf_logging_mode = ESMF_LOGKIND_MULTI_ON_ERROR + case default + _FAIL("Unsupported ESMF logging option: "//trim(buffer)) + end select + + ! Ensemble specific options + option => fargparseCLI%options%at('prefix') + if (associated(option)) then + call cast(option, cap_options%ensemble_subdir_prefix, _RC) + end if + + option => fargparseCLI%options%at('n_members') + if (associated(option)) then + call cast(option, cap_options%n_members, _RC) + end if + + option => fargparseCLI%options%at('cap_rc') + if (associated(option)) then + call cast(option, cap_options%cap_rc_file, _RC) + end if + + ! Logging options + option => fargparseCLI%options%at('logging_config') + if (associated(option)) then + call cast(option, cap_options%logging_config, _RC) + end if + + option => fargparseCLI%options%at('oserver_type') + if (associated(option)) then + call cast(option, cap_options%oserver_type, _RC) + end if + + option => fargparseCLI%options%at('npes_backend_pernode') + if (associated(option)) then + call cast(option, cap_options%npes_backend_pernode, _RC) + end if + + _RETURN(_SUCCESS) + _UNUSED_DUMMY(unusable) + end function old_CapOptions_from_Fargparse + +end module MAPL_FargparseCLIMod diff --git a/gridcomps/Cap/MAPL_CapGridComp.F90 b/gridcomps/Cap/MAPL_CapGridComp.F90 index 92752e98ec4f..9bad95652230 100644 --- a/gridcomps/Cap/MAPL_CapGridComp.F90 +++ b/gridcomps/Cap/MAPL_CapGridComp.F90 @@ -608,6 +608,8 @@ subroutine initialize_gc(gc, import_state, export_state, clock, rc) _VERIFY(STATUS) call MAPL_ConfigSetAttribute(cap%cf_ext, value=EXTDATA_CF, Label="CF_EXTDATA:", rc=status) _VERIFY(STATUS) + call MAPL_ConfigSetAttribute(cap%cf_ext, value=EXPID, Label="EXPID:", rc=status) + _VERIFY(STATUS) ! Query MAPL for the the children's for GCS, IMPORTS, EXPORTS !------------------------------------------------------------- diff --git a/gridcomps/ExtData2G/ExtDataAbstractFileHandler.F90 b/gridcomps/ExtData2G/ExtDataAbstractFileHandler.F90 index 4016e6640ee6..2ade22e98162 100644 --- a/gridcomps/ExtData2G/ExtDataAbstractFileHandler.F90 +++ b/gridcomps/ExtData2G/ExtDataAbstractFileHandler.F90 @@ -35,7 +35,7 @@ module MAPL_ExtdataAbstractFileHandler end type abstract interface - subroutine get_file_bracket(this, input_time, source_time, bracket, rc) + subroutine get_file_bracket(this, input_time, source_time, bracket, fail_on_missing_file, rc) use ESMF use MAPL_ExtDataBracket import ExtDataAbstractFileHandler @@ -43,6 +43,7 @@ subroutine get_file_bracket(this, input_time, source_time, bracket, rc) type(ESMF_Time), intent(in) :: input_time type(ESMF_Time), intent(in) :: source_time(:) type(ExtDataBracket), intent(inout) :: bracket + logical, intent(in) :: fail_on_missing_file integer, optional, intent(out) :: rc end subroutine get_file_bracket @@ -100,6 +101,9 @@ subroutine get_time_on_file(this,filename,target_time,bracketside,time_index,out wrap_=.false. end if time_index=time_not_found + if (trim(filename) == file_not_found) then + _RETURN(_SUCCESS) + end if call this%make_metadata(filename,file_metadata,_RC) call file_metadata%get_time_info(timeVector=time_series,_RC) @@ -164,5 +168,4 @@ subroutine make_metadata(this,file,metadata,rc) end subroutine make_metadata - end module MAPL_ExtdataAbstractFileHandler diff --git a/gridcomps/ExtData2G/ExtDataBracket.F90 b/gridcomps/ExtData2G/ExtDataBracket.F90 index 9dc61b77b576..f6fd5dc964f9 100644 --- a/gridcomps/ExtData2G/ExtDataBracket.F90 +++ b/gridcomps/ExtData2G/ExtDataBracket.F90 @@ -6,6 +6,8 @@ module MAPL_ExtDataBracket use MAPL_ExceptionHandling use MAPL_BaseMod, only: MAPL_UNDEF use MAPL_ExtDataNode + use MAPL_ExtDataConstants + use MAPL_CommsMod implicit none private @@ -20,6 +22,7 @@ module MAPL_ExtDataBracket logical :: intermittent_disable = .false. logical :: new_file_right logical :: new_file_left + logical :: exact = .false. contains procedure :: interpolate_to_time procedure :: time_in_bracket @@ -111,7 +114,7 @@ subroutine get_node(this, bracketside, unusable, field, file, time, time_index, end subroutine get_node - subroutine set_parameters(this, unusable, linear_trans, disable_interpolation, left_field, right_field, intermittent_disable, rc) + subroutine set_parameters(this, unusable, linear_trans, disable_interpolation, left_field, right_field, intermittent_disable, exact, rc) class(ExtDataBracket), intent(inout) :: this class(KeywordEnforcer), optional, intent(in) :: unusable real, optional, intent(in) :: linear_trans(2) @@ -119,6 +122,7 @@ subroutine set_parameters(this, unusable, linear_trans, disable_interpolation, l type(ESMF_Field), optional, intent(in) :: left_field type(ESMF_Field), optional, intent(in) :: right_field logical, optional, intent(in) :: intermittent_disable + logical, optional, intent(in) :: exact integer, optional, intent(out) :: rc _UNUSED_DUMMY(unusable) @@ -130,6 +134,7 @@ subroutine set_parameters(this, unusable, linear_trans, disable_interpolation, l if (present(left_field)) this%left_node%field=left_field if (present(right_field)) this%right_node%field=right_field if (present(intermittent_disable)) this%intermittent_disable = intermittent_disable + if (present(exact)) this%exact = exact _RETURN(_SUCCESS) end subroutine set_parameters @@ -181,28 +186,45 @@ subroutine interpolate_to_time(this,field,time,rc) real, pointer :: var3d_right(:,:,:) => null() integer :: field_rank integer :: status + logical :: right_node_set, left_node_set + character(len=ESMF_MAXPATHLEN) :: left_file, right_file + + right_node_set = this%right_node%check_if_initialized(_RC) + left_node_set = this%left_node%check_if_initialized(_RC) + call this%right_node%get(file=right_file) + call this%left_node%get(file=left_file) + right_node_set = right_file /= file_not_found + left_node_set = left_file /= file_not_found + call ESMF_FieldGet(field,dimCount=field_rank,_RC) alpha = 0.0 - if ( (.not.this%disable_interpolation) .and. (.not.this%intermittent_disable)) then + if ( (.not.this%disable_interpolation) .and. (.not.this%intermittent_disable) .and. right_node_set .and. left_node_set) then tinv1 = time - this%left_node%time tinv2 = this%right_node%time - this%left_node%time alpha = tinv1/tinv2 end if if (field_rank==2) then - call ESMF_FieldGet(field,localDE=0,farrayPtr=var2d,_RC) - call ESMF_FieldGet(this%right_node%field,localDE=0,farrayPtr=var2d_right,_RC) - call ESMF_FieldGet(this%left_node%field,localDE=0,farrayPtr=var2d_left,_RC) - if (time == this%left_node%time .or. this%disable_interpolation) then + + call esmf_fieldget(field,localde=0,farrayptr=var2d,_RC) + if (right_node_set) then + call esmf_fieldget(this%right_node%field,localde=0,farrayptr=var2d_right,_RC) + end if + if (left_node_set) then + call esmf_fieldget(this%left_node%field,localde=0,farrayptr=var2d_left,_RC) + end if + if ( left_node_set .and. (time == this%left_node%time .or. this%disable_interpolation)) then var2d = var2d_left - else if (time == this%right_node%time) then + else if (right_node_set .and. (time == this%right_node%time)) then var2d = var2d_right - else - where( (var2d_left /= MAPL_UNDEF) .and. (var2d_right /= MAPL_UNDEF)) + else if ( (left_node_set .and. right_node_set) .and. (.not.this%exact) ) then + where( (var2d_left /= mapl_undef) .and. (var2d_right /= mapl_undef)) var2d = var2d_left + alpha*(var2d_right-var2d_left) elsewhere - var2d = MAPL_UNDEF + var2d = mapl_undef endwhere + else + var2d = mapl_undef end if if (this%scale_factor == 0.0 .and. this%offset /= 0.0) then @@ -216,19 +238,27 @@ subroutine interpolate_to_time(this,field,time,rc) end if else if (field_rank==3) then - call ESMF_FieldGet(field,localDE=0,farrayPtr=var3d,_RC) - call ESMF_FieldGet(this%right_node%field,localDE=0,farrayPtr=var3d_right,_RC) - call ESMF_FieldGet(this%left_node%field,localDE=0,farrayPtr=var3d_left,_RC) - if (time == this%left_node%time .or. this%disable_interpolation) then + call esmf_fieldget(field,localde=0,farrayptr=var3d,_RC) + if (right_node_set) then + call esmf_fieldget(this%right_node%field,localde=0,farrayptr=var3d_right,_RC) + end if + if (left_node_set) then + call esmf_fieldget(this%left_node%field,localde=0,farrayptr=var3d_left,_RC) + end if + if ( left_node_set .and. (time == this%left_node%time .or. this%disable_interpolation) ) then var3d = var3d_left - else if (time == this%right_node%time) then + else if ( right_node_set .and. (time == this%right_node%time) ) then var3d = var3d_right - else - where( (var3d_left /= MAPL_UNDEF) .and. (var3d_right /= MAPL_UNDEF)) + else if (right_node_set .and. (time == this%right_node%time)) then + var3d = var3d_right + else if ( (left_node_set .and. right_node_set) .and. (.not.this%exact) )then + where( (var3d_left /= mapl_undef) .and. (var3d_right /= mapl_undef)) var3d = var3d_left + alpha*(var3d_right-var3d_left) elsewhere - var3d = MAPL_UNDEF + var3d = mapl_undef endwhere + else + var3d = mapl_undef end if if (this%scale_factor == 0.0 .and. this%offset /= 0.0) then @@ -242,6 +272,7 @@ subroutine interpolate_to_time(this,field,time,rc) end if end if + _RETURN(_SUCCESS) end subroutine interpolate_to_time @@ -253,16 +284,21 @@ subroutine swap_node_fields(this,rc) integer :: field_rank real, pointer :: var3d_left(:,:,:),var3d_right(:,:,:) real, pointer :: var2d_left(:,:),var2d_right(:,:) + logical :: left_created, right_created - call ESMF_FieldGet(this%left_node%field,dimCount=field_rank,_RC) - if (field_rank == 2) then - call ESMF_FieldGet(this%right_node%field,localDE=0,farrayPtr=var2d_right,_RC) - call ESMF_FieldGet(this%left_node%field,localDE=0,farrayPtr=var2d_left,_RC) - var2d_left = var2d_right - else if (field_rank ==3) then - call ESMF_FieldGet(this%right_node%field,localDE=0,farrayPtr=var3d_right,_RC) - call ESMF_FieldGet(this%left_node%field,localDE=0,farrayPtr=var3d_left,_RC) - var3d_left = var3d_right + left_created = ESMF_FieldIsCreated(this%left_node%field,_RC) + right_created = ESMF_FieldIsCreated(this%right_node%field,_RC) + if (left_created .and. right_created) then + call ESMF_FieldGet(this%left_node%field,dimCount=field_rank,_RC) + if (field_rank == 2) then + call ESMF_FieldGet(this%right_node%field,localDE=0,farrayPtr=var2d_right,_RC) + call ESMF_FieldGet(this%left_node%field,localDE=0,farrayPtr=var2d_left,_RC) + var2d_left = var2d_right + else if (field_rank ==3) then + call ESMF_FieldGet(this%right_node%field,localDE=0,farrayPtr=var3d_right,_RC) + call ESMF_FieldGet(this%left_node%field,localDE=0,farrayPtr=var3d_left,_RC) + var3d_left = var3d_right + end if end if _RETURN(_SUCCESS) end subroutine swap_node_fields diff --git a/gridcomps/ExtData2G/ExtDataClimFileHandler.F90 b/gridcomps/ExtData2G/ExtDataClimFileHandler.F90 index 36a53ca4ef6c..96703be2cda3 100644 --- a/gridcomps/ExtData2G/ExtDataClimFileHandler.F90 +++ b/gridcomps/ExtData2G/ExtDataClimFileHandler.F90 @@ -27,11 +27,12 @@ module MAPL_ExtdataClimFileHandler contains - subroutine get_file_bracket(this, input_time, source_time, bracket, rc) + subroutine get_file_bracket(this, input_time, source_time, bracket, fail_on_missing_file, rc) class(ExtdataClimFileHandler), intent(inout) :: this type(ESMF_Time), intent(in) :: input_time type(ESMF_Time), intent(in) :: source_time(:) type(ExtDataBracket), intent(inout) :: bracket + logical, intent(in) :: fail_on_missing_file integer, optional, intent(out) :: rc type(ESMF_Time) :: time @@ -44,7 +45,8 @@ subroutine get_file_bracket(this, input_time, source_time, bracket, rc) integer :: target_year, original_year,clim_shift,valid_years(2) integer, allocatable :: source_years(:) - + + _ASSERT(fail_on_missing_file,"Failure on missing file not allowed when rule is climatology") if (bracket%time_in_bracket(input_time)) then _RETURN(_SUCCESS) end if diff --git a/gridcomps/ExtData2G/ExtDataConstants.F90 b/gridcomps/ExtData2G/ExtDataConstants.F90 index dd711bf12b74..54ea2249ec51 100644 --- a/gridcomps/ExtData2G/ExtDataConstants.F90 +++ b/gridcomps/ExtData2G/ExtDataConstants.F90 @@ -8,5 +8,6 @@ module MAPL_ExtDataConstants integer, parameter, public :: Primary_Type_Vector_comp2 = 3 integer, parameter, public :: Derived_TYpe = 4 integer, parameter, public :: time_not_found = -1 + character(len=14), parameter, public :: file_not_found = "file_not_found" end module MAPL_ExtDataConstants diff --git a/gridcomps/ExtData2G/ExtDataFileStream.F90 b/gridcomps/ExtData2G/ExtDataFileStream.F90 index fe1d85857957..2178b8f3443b 100644 --- a/gridcomps/ExtData2G/ExtDataFileStream.F90 +++ b/gridcomps/ExtData2G/ExtDataFileStream.F90 @@ -152,7 +152,6 @@ subroutine detect_metadata(this,metadata_out,time,multi_rule,get_range,rc) type(FileMetadataUtils), pointer :: metadata type(ESMF_Time), allocatable :: time_series(:) integer :: status - character(len=ESMF_MAXPATHLEN) :: filename if (multi_rule) then _ASSERT(allocated(this%valid_range),"must use a collection with valid range") @@ -175,13 +174,6 @@ subroutine detect_metadata(this,metadata_out,time,multi_rule,get_range,rc) end if end if - if (get_range_ .or. multi_rule) then - call fill_grads_template(filename,this%file_template,time=this%valid_range(1),_RC) - else - call fill_grads_template(filename,this%file_template,time=time,_RC) - end if - metadata => collection%find(filename,_RC) - metadata_out = metadata _RETURN(_SUCCESS) end subroutine detect_metadata diff --git a/gridcomps/ExtData2G/ExtDataGridCompNG.F90 b/gridcomps/ExtData2G/ExtDataGridCompNG.F90 index 6d6cece47d77..d068cf26cf70 100644 --- a/gridcomps/ExtData2G/ExtDataGridCompNG.F90 +++ b/gridcomps/ExtData2G/ExtDataGridCompNG.F90 @@ -358,8 +358,6 @@ SUBROUTINE Initialize_ ( GC, IMPORT, EXPORT, CLOCK, rc ) ! ----------------------- call MAPL_GenericInitialize ( GC, IMPORT, EXPORT, clock, _RC ) - call extdata_lgr%info("Using ExtData2G, note this is still in BETA stage") - ! --------------------------- ! Parse ExtData Resource File ! --------------------------- @@ -456,16 +454,9 @@ SUBROUTINE Initialize_ ( GC, IMPORT, EXPORT, CLOCK, rc ) current_base_name => self%primary%import_names%at(i) idx = self%primary%get_item_index(current_base_name,time,_RC) item => self%primary%item(idx) - item%initialized = .true. - - item%pfioCollection_id = MAPL_DataAddCollection(item%file_template) - call create_primary_field(item,self%ExtDataState,_RC) - if (item%isConst) then - call set_constant_field(item,self%extDataState,_RC) - cycle - end if - call create_bracketing_fields(item,self%ExtDataState,cf_master,_RC) + item%pfioCOllection_id = MAPL_DataAddCollection(item%file_template) + call create_primary_field(item,self%ExtDataState,time,_RC) end do PrimaryLoop ! Check if we have any files that would need to be vertically interpolated @@ -559,12 +550,12 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) type(DerivedExport), pointer :: derivedItem integer :: i - type(ESMF_Time) :: time, time0 + type(ESMF_Time) :: time, current_time type(MAPL_MetaComp), pointer :: MAPLSTATE logical :: doUpdate_ character(len=ESMF_MAXPATHLEN) :: file_processed - logical, allocatable :: doUpdate(:) + logical, allocatable :: do_pointer_update(:) type(ESMF_Time), allocatable :: useTime(:) integer :: bracket_side @@ -575,6 +566,7 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) character(len=:), pointer :: current_base_name integer :: idx,nitems type(ESMF_Config) :: cf_master + type(ESMF_Time) :: adjusted_time _UNUSED_DUMMY(IMPORT) _UNUSED_DUMMY(EXPORT) @@ -596,14 +588,14 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) call MAPL_TimerOn(MAPLSTATE,"TOTAL") call MAPL_TimerOn(MAPLSTATE,"Run") - call ESMF_ClockGet(CLOCK, currTIME=time0, _RC) + call ESMF_ClockGet(CLOCK, currTIME=current_time, _RC) ! Fill in the internal state with data from the files ! --------------------------------------------------- - allocate(doUpdate(self%primary%nitems),stat=status) + allocate(do_pointer_update(self%primary%nitems),stat=status) _VERIFY(STATUS) - doUpdate = .false. + do_pointer_update = .false. allocate(useTime(self%primary%nitems),stat=status) _VERIFY(STATUS) @@ -615,7 +607,7 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) READ_LOOP: do i=1,self%primary%import_names%size() current_base_name => self%primary%import_names%at(i) - idx = self%primary%get_item_index(current_base_name,time0,_RC) + idx = self%primary%get_item_index(current_base_name,current_time,_RC) item => self%primary%item(idx) if (.not.item%initialized) then @@ -624,7 +616,6 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) call set_constant_field(item,self%extDataState,_RC) cycle end if - call create_bracketing_fields(item,self%ExtDataState,cf_master, _RC) item%initialized=.true. end if @@ -640,21 +631,19 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) call MAPL_TimerOn(MAPLSTATE,"--CheckUpd") - call item%update_freq%check_update(doUpdate(i),time,time0,.not.hasRun,_RC) + call item%update_freq%check_update(do_pointer_update(i),time,current_time,.not.hasRun,_RC) + adjusted_time = item%update_freq%get_adjusted_time(current_time) call MAPL_TimerOff(MAPLSTATE,"--CheckUpd") - DO_UPDATE: if (doUpdate(i)) then - - !call extdata_lgr%info('Going to update %a with file template: %a ',current_base_name, item%file_template) - call item%modelGridFields%comp1%reset() - call item%filestream%get_file_bracket(time,item%source_time, item%modelGridFields%comp1,_RC) - if (item%vartype == MAPL_VectorField) then - call item%filestream%get_file_bracket(time,item%source_time, item%modelGridFields%comp2,_RC) - end if - call IOBundle_Add_Entry(IOBundles,item,idx) - useTime(i)=time - - end if DO_UPDATE + !call extdata_lgr%info('Going to update %a with file template: %a ',current_base_name, item%file_template) + call item%modelGridFields%comp1%reset() + call item%filestream%get_file_bracket(time,item%source_time, item%modelGridFields%comp1,item%fail_on_missing_file, _RC) + if (item%vartype == MAPL_VectorField) then + call item%filestream%get_file_bracket(time,item%source_time, item%modelGridFields%comp2, item%fail_on_missing_file,_RC) + end if + call create_bracketing_fields(item,self%ExtDataState,cf_master, _RC) + call IOBundle_Add_Entry(IOBundles,item,idx) + useTime(i)=time end do READ_LOOP @@ -707,7 +696,7 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) bracket_side = io_bundle%bracket_side entry_num = io_bundle%entry_index item => self%primary%item(entry_num) - call MAPL_ExtDataVerticalInterpolate(self,item,bracket_side,time0,rc=status) + call MAPL_ExtDataVerticalInterpolate(self,item,bracket_side,current_time,rc=status) _VERIFY(status) call bundle_iter%next() enddo @@ -723,10 +712,10 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) INTERP_LOOP: do i=1,self%primary%import_names%size() current_base_name => self%primary%import_names%at(i) - idx = self%primary%get_item_index(current_base_name,time0,_RC) + idx = self%primary%get_item_index(current_base_name,current_time,_RC) item => self%primary%item(idx) - if (doUpdate(i)) then + if (do_pointer_update(i)) then call extdata_lgr%debug('ExtData Run_: INTERP_LOOP: interpolating between bracket times, variable: %a, file: %a', & & trim(current_base_name), trim(item%file_template)) @@ -748,7 +737,7 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) derivedItem => self%derived%item(i) - call derivedItem%update_freq%check_update(doUpdate_,time,time0,.not.hasRun,_RC) + call derivedItem%update_freq%check_update(doUpdate_,time,current_time,.not.hasRun,_RC) if (doUpdate_) then @@ -762,7 +751,6 @@ SUBROUTINE Run_ ( GC, IMPORT, EXPORT, CLOCK, rc ) ! All done ! -------- - deallocate(doUpdate) deallocate(useTime) if (hasRun .eqv. .false.) hasRun = .true. @@ -992,11 +980,6 @@ subroutine MAPL_ExtDataInterpField(item,state,time,rc) call ESMF_StateGet(state,item%vcomp1,field,_RC) call item%modelGridFields%comp1%interpolate_to_time(field,time,_RC) - block - character(len=1024) :: fname - integer :: rank - call ESMF_FieldGet(field,name=fname,rank=rank,_RC) - end block if (item%vartype == MAPL_VectorField) then call ESMF_StateGet(state,item%vcomp2,field,_RC) call item%modelGridFields%comp2%interpolate_to_time(field,time,_RC) @@ -1546,21 +1529,25 @@ subroutine IOBundle_Add_Entry(IOBundles,item,entry_num,rc) call item%modelGridFields%comp1%get_parameters('L',update=update,file=current_file,time_index=time_index) if (update) then - call itemsL%push_back(item%fileVars) - io_bundle = ExtDataNG_IOBundle(MAPL_ExtDataLeft, entry_num, current_file, time_index, item%trans, item%fracval, item%file_template, & - item%pfioCollection_id,item%iclient_collection_id,itemsL,rc=status) - _VERIFY(status) - call IOBundles%push_back(io_bundle) - call extdata_lgr%info('%a updated L bracket with: %a at time index %i3 ',item%name, current_file, time_index) + if (trim(current_file)/=file_not_found) then + call itemsL%push_back(item%fileVars) + io_bundle = ExtDataNG_IOBundle(MAPL_ExtDataLeft, entry_num, current_file, time_index, item%trans, item%fracval, item%file_template, & + item%pfioCollection_id,item%iclient_collection_id,itemsL,rc=status) + _VERIFY(status) + call IOBundles%push_back(io_bundle) + call extdata_lgr%info('%a updated L bracket with: %a at time index %i3 ',item%name, current_file, time_index) + end if end if call item%modelGridFields%comp1%get_parameters('R',update=update,file=current_file,time_index=time_index) if (update) then - call itemsR%push_back(item%fileVars) - io_bundle = ExtDataNG_IOBundle(MAPL_ExtDataRight, entry_num, current_file, time_index, item%trans, item%fracval, item%file_template, & - item%pfioCollection_id,item%iclient_collection_id,itemsR,rc=status) - _VERIFY(status) - call IOBundles%push_back(io_bundle) - call extdata_lgr%info('%a updated R bracket with: %a at time index %i3 ',item%name,current_file, time_index) + if (trim(current_file)/=file_not_found) then + call itemsR%push_back(item%fileVars) + io_bundle = ExtDataNG_IOBundle(MAPL_ExtDataRight, entry_num, current_file, time_index, item%trans, item%fracval, item%file_template, & + item%pfioCollection_id,item%iclient_collection_id,itemsR,rc=status) + _VERIFY(status) + call IOBundles%push_back(io_bundle) + call extdata_lgr%info('%a updated R bracket with: %a at time index %i3 ',item%name,current_file, time_index) + end if end if _RETURN(ESMF_SUCCESS) @@ -1620,61 +1607,91 @@ subroutine create_bracketing_fields(item,ExtDataState,cf,rc) type(ESMF_Field) :: field,left_field,right_field type(ESMF_Grid) :: grid real(kind=REAL32), pointer :: ptr3d(:,:,:) + character(len=ESMF_MAXPATHLEN) :: file_left, file_right, filename + logical :: found_file + type(FileMetadataUtils), pointer :: metadata + type(MAPLDataCollection), pointer :: collection - call GetLevs(item,_RC) - item%iclient_collection_id=i_clients%add_ext_collection(trim(item%file_template)) - if (item%vartype == MAPL_FieldItem) then - - call ESMF_StateGet(ExtDataState, trim(item%name), field,_RC) - call ESMF_FieldGet(field,grid=grid,rank=fieldRank,_RC) - - lm=0 - if (fieldRank==3) then - call ESMF_FieldGet(field,0,farrayPtr=ptr3d,_RC) - lm = size(ptr3d,3) - end if - if (item%lm /= lm .and. lm /= 0 .and. item%havePressure) then - item%do_VertInterp = .true. - else if (item%lm /= lm .and. lm /= 0 .and. item%lm /= 0) then - item%do_Fill = .true. + if (item%modelGridFields%initialized) then + _RETURN(_SUCCESS) + else + found_file = .false. + call item%modelGridFields%comp1%get_parameters('L',file=file_left) + if (trim(file_left) /= file_not_found) then + filename = file_left + found_file = .true. + else + call item%modelGridFields%comp1%get_parameters('R',file=file_right) + if (trim(file_right) /= file_not_found) then + filename = file_right + found_file = .true. + end if end if - left_field = MAPL_FieldCreate(field,item%var,doCopy=.true.,_RC) - right_field = MAPL_FieldCreate(field,item%var,doCopy=.true.,_RC) - call item%modelGridFields%comp1%set_parameters(left_field=left_field,right_field=right_field, _RC) - if (item%do_fill .or. item%do_vertInterp) then - call createFileLevBracket(item,cf,_RC) + if (found_file) then + collection => DataCollections%at(item%pfioCollection_id) + metadata => collection%find(filename,_RC) + item%file_metadata = metadata + item%modelGridFields%initialized = .true. end if + end if - else if (item%vartype == MAPL_VectorField) then + if (found_file) then + call GetLevs(item,_RC) + item%iclient_collection_id=i_clients%add_ext_collection(trim(item%file_template)) + if (item%vartype == MAPL_FieldItem) then - if (item%Trans /= REGRID_METHOD_BILINEAR) then - _FAIL('No conservative re-gridding with vectors') - end if + call ESMF_StateGet(ExtDataState, trim(item%name), field,_RC) + call ESMF_FieldGet(field,grid=grid,rank=fieldRank,_RC) - call ESMF_StateGet(ExtDataState, trim(item%vcomp1), field,_RC) - call ESMF_FieldGet(field,grid=grid,rank=fieldRank,_RC) + lm=0 + if (fieldRank==3) then + call ESMF_FieldGet(field,0,farrayPtr=ptr3d,_RC) + lm = size(ptr3d,3) + end if + if (item%lm /= lm .and. lm /= 0 .and. item%havePressure) then + item%do_VertInterp = .true. + else if (item%lm /= lm .and. lm /= 0 .and. item%lm /= 0) then + item%do_Fill = .true. + end if + left_field = MAPL_FieldCreate(field,item%var,doCopy=.true.,_RC) + right_field = MAPL_FieldCreate(field,item%var,doCopy=.true.,_RC) + call item%modelGridFields%comp1%set_parameters(left_field=left_field,right_field=right_field, _RC) + if (item%do_fill .or. item%do_vertInterp) then + call createFileLevBracket(item,cf,_RC) + end if - lm = 0 - if (fieldRank==3) then - call ESMF_FieldGet(field,0,farrayPtr=ptr3d,_RC) - lm = size(ptr3d,3) - end if - if (item%lm /= lm .and. item%havePressure) then - item%do_VertInterp = .true. - else if (item%lm /= lm .and. lm /= 0) then - item%do_Fill = .true. - end if + else if (item%vartype == MAPL_VectorField) then + + if (item%Trans /= REGRID_METHOD_BILINEAR) then + _FAIL('No conservative re-gridding with vectors') + end if + + call ESMF_StateGet(ExtDataState, trim(item%vcomp1), field,_RC) + call ESMF_FieldGet(field,grid=grid,rank=fieldRank,_RC) + + lm = 0 + if (fieldRank==3) then + call ESMF_FieldGet(field,0,farrayPtr=ptr3d,_RC) + lm = size(ptr3d,3) + end if + if (item%lm /= lm .and. item%havePressure) then + item%do_VertInterp = .true. + else if (item%lm /= lm .and. lm /= 0) then + item%do_Fill = .true. + end if - left_field = MAPL_FieldCreate(field,item%fcomp1,doCopy=.true.,_RC) - right_field = MAPL_FieldCreate(field,item%fcomp1,doCopy=.true.,_RC) - call item%modelGridFields%comp1%set_parameters(left_field=left_field,right_field=right_field, _RC) - call ESMF_StateGet(ExtDataState, trim(item%vcomp2), field,_RC) - left_field = MAPL_FieldCreate(field,item%fcomp2,doCopy=.true.,_RC) - right_field = MAPL_FieldCreate(field,item%fcomp2,doCopy=.true.,_RC) - call item%modelGridFields%comp2%set_parameters(left_field=left_field,right_field=right_field, _RC) + left_field = MAPL_FieldCreate(field,item%fcomp1,doCopy=.true.,_RC) + right_field = MAPL_FieldCreate(field,item%fcomp1,doCopy=.true.,_RC) + call item%modelGridFields%comp1%set_parameters(left_field=left_field,right_field=right_field, _RC) + call ESMF_StateGet(ExtDataState, trim(item%vcomp2), field,_RC) + left_field = MAPL_FieldCreate(field,item%fcomp2,doCopy=.true.,_RC) + right_field = MAPL_FieldCreate(field,item%fcomp2,doCopy=.true.,_RC) + call item%modelGridFields%comp2%set_parameters(left_field=left_field,right_field=right_field, _RC) + + if (item%do_fill .or. item%do_vertInterp) then + call createFileLevBracket(item,cf,_RC) + end if - if (item%do_fill .or. item%do_vertInterp) then - call createFileLevBracket(item,cf,_RC) end if end if @@ -1698,9 +1715,10 @@ subroutine create_holding_field(state,primary_name,derived_name,rc) _RETURN(_SUCCESS) end subroutine - subroutine create_primary_field(item,ExtDataState,rc) + subroutine create_primary_field(item,ExtDataState,current_time,rc) type(PrimaryExport), intent(inout) :: item type(ESMF_State), intent(inout) :: extDataState + type(ESMF_Time), intent(in) :: current_time integer, intent(out), optional :: rc integer :: status @@ -1708,6 +1726,10 @@ subroutine create_primary_field(item,ExtDataState,rc) type(ESMF_Grid) :: grid logical :: must_create character(len=ESMF_MAXSTR) :: derived_field_name + type(FileMetadataUtils), pointer :: metadata + type(MAPLDataCollection), pointer :: collection + character(len=ESMF_MAXPATHLEN) :: filename + logical :: file_found call ESMF_StateGet(ExtDataState,trim(item%name),field,_RC) call ESMF_FieldValidate(field,rc=status) @@ -1727,6 +1749,13 @@ subroutine create_primary_field(item,ExtDataState,rc) call ESMF_StateRemove(ExtDataState,[trim(item%name)],_RC) call ESMF_FieldDestroy(field,noGarbage=.true.,_RC) + call fill_grads_template(filename,item%file_template,time=current_time,_RC ) + inquire(file=trim(filename),exist=file_found) + _ASSERT(file_found,"Forcing extdata to allocate primary field but have gaps in data, not implemented currently") + collection => DataCollections%at(item%pfioCollection_id) + metadata => collection%find(filename,_RC) + item%file_metadata = metadata + call GetLevs(item,_RC) if (item%vartype == MAPL_FieldItem) then field = create_simple_field(item%name,grid,item%lm,_RC) @@ -1750,10 +1779,15 @@ function create_simple_field(field_name,grid,num_levels,rc) result(new_field) integer, optional, intent(out) :: rc integer :: status + real, pointer :: ptr2d(:,:), ptr3d(:,:,:) if (num_levels ==0) then new_field=ESMF_FieldCreate(grid,name=field_name,typekind=ESMF_TYPEKIND_R4,_RC) + call ESMF_FieldGet(new_field,0,farrayPtr=ptr2d,_RC) + ptr2d=0.0 else new_field=ESMF_FieldCreate(grid,name=field_name,typekind=ESMF_TYPEKIND_R4,ungriddedLBound=[1],ungriddedUBound=[num_levels],_RC) + call ESMF_FieldGet(new_field,0,farrayPtr=ptr3d,_RC) + ptr3d=0.0 end if _RETURN(_SUCCESS) end function diff --git a/gridcomps/ExtData2G/ExtDataNode.F90 b/gridcomps/ExtData2G/ExtDataNode.F90 index 3270f9868f9c..37d9610d467d 100644 --- a/gridcomps/ExtData2G/ExtDataNode.F90 +++ b/gridcomps/ExtData2G/ExtDataNode.F90 @@ -15,6 +15,7 @@ module MAPL_ExtDataNode integer :: time_index logical :: was_set = .false. contains + procedure :: check_if_initialized procedure :: set procedure :: get procedure :: equals @@ -23,6 +24,15 @@ module MAPL_ExtDataNode contains + function check_if_initialized(this,rc) result(field_initialized) + logical :: field_initialized + class(ExtDataNode), intent(inout) :: this + integer, intent(out), optional :: rc + integer :: status + field_initialized = ESMF_FieldIsCreated(this%field,_RC) + _RETURN(_SUCCESS) + end function + subroutine set(this, unusable, field, time, file, time_index, was_set, rc) class(ExtDataNode), intent(inout) :: this class(KeywordEnforcer), optional, intent(in) :: unusable diff --git a/gridcomps/ExtData2G/ExtDataOldTypesCreator.F90 b/gridcomps/ExtData2G/ExtDataOldTypesCreator.F90 index 7379fc2d9d61..6c78d49c06fe 100644 --- a/gridcomps/ExtData2G/ExtDataOldTypesCreator.F90 +++ b/gridcomps/ExtData2G/ExtDataOldTypesCreator.F90 @@ -72,7 +72,7 @@ subroutine fillin_primary(this,item_name,base_name,primary_item,time,clock,unusa type(ExtDataSimpleFileHandler) :: simple_handler type(ExtDataClimFileHandler) :: clim_handler integer :: status, semi_pos - logical :: disable_interpolation, get_range + logical :: disable_interpolation, get_range, exact _UNUSED_DUMMY(unusable) rule => this%rule_map%at(trim(item_name)) @@ -132,11 +132,12 @@ subroutine fillin_primary(this,item_name,base_name,primary_item,time,clock,unusa time_sample%refresh_frequency, time_sample%refresh_offset, time, clock, _RC) disable_interpolation = .not.time_sample%time_interpolation + exact = time_sample%exact - call primary_item%modelGridFields%comp1%set_parameters(linear_trans=rule%linear_trans,disable_interpolation=disable_interpolation) - call primary_item%modelGridFields%comp2%set_parameters(linear_trans=rule%linear_trans,disable_interpolation=disable_interpolation) - call primary_item%modelGridFields%auxiliary1%set_parameters(linear_trans=rule%linear_trans, disable_interpolation=disable_interpolation) - call primary_item%modelGridFields%auxiliary2%set_parameters(linear_trans=rule%linear_trans, disable_interpolation=disable_interpolation) + call primary_item%modelGridFields%comp1%set_parameters(linear_trans=rule%linear_trans,disable_interpolation=disable_interpolation,exact=exact) + call primary_item%modelGridFields%comp2%set_parameters(linear_trans=rule%linear_trans,disable_interpolation=disable_interpolation,exact=exact) + call primary_item%modelGridFields%auxiliary1%set_parameters(linear_trans=rule%linear_trans, disable_interpolation=disable_interpolation,exact=exact) + call primary_item%modelGridFields%auxiliary2%set_parameters(linear_trans=rule%linear_trans, disable_interpolation=disable_interpolation,exact=exact) ! file_template primary_item%isConst = .false. @@ -162,6 +163,8 @@ subroutine fillin_primary(this,item_name,base_name,primary_item,time,clock,unusa end if end if + primary_item%fail_on_missing_file = rule%fail_on_missing_file + _RETURN(_SUCCESS) end subroutine fillin_primary diff --git a/gridcomps/ExtData2G/ExtDataRule.F90 b/gridcomps/ExtData2G/ExtDataRule.F90 index 720a81f32d50..82af19610df5 100644 --- a/gridcomps/ExtData2G/ExtDataRule.F90 +++ b/gridcomps/ExtData2G/ExtDataRule.F90 @@ -21,6 +21,7 @@ module MAPL_ExtDataRule character(:), allocatable :: vector_component character(:), allocatable :: vector_file_partner logical :: multi_rule + logical :: fail_on_missing_file = .true. contains procedure :: set_defaults procedure :: split_vector @@ -106,6 +107,10 @@ function new_ExtDataRule(config,sample_map,key,unusable,multi_rule,rc) result(ru tempc = config%of("starting") rule%start_time = tempc end if + + if (config%has("fail_on_missing_file")) then + rule%fail_on_missing_file = config%of("fail_on_missing_file") + end if rule%multi_rule=usable_multi_rule diff --git a/gridcomps/ExtData2G/ExtDataSample.F90 b/gridcomps/ExtData2G/ExtDataSample.F90 index 8a7629e235c4..5e19e22fc9ff 100644 --- a/gridcomps/ExtData2G/ExtDataSample.F90 +++ b/gridcomps/ExtData2G/ExtDataSample.F90 @@ -10,6 +10,7 @@ module MAPL_ExtDataTimeSample type, public :: ExtDataTimeSample logical :: time_interpolation + logical :: exact type(ESMF_Time), allocatable :: source_time(:) character(:), allocatable :: extrap_outside character(:), allocatable :: refresh_time @@ -45,6 +46,11 @@ function new_ExtDataTimeSample(config,unusable,rc) result(TimeSample) else TimeSample%time_interpolation = .true. end if + if (config%has("exact")) then + TimeSample%exact = config%of("exact") + else + TimeSample%exact = .false. + end if if (config%has("update_reference_time")) TimeSample%refresh_time=config%of("update_reference_time") diff --git a/gridcomps/ExtData2G/ExtDataSimpleFileHandler.F90 b/gridcomps/ExtData2G/ExtDataSimpleFileHandler.F90 index aa07c4e2c922..1a80d9df315c 100644 --- a/gridcomps/ExtData2G/ExtDataSimpleFileHandler.F90 +++ b/gridcomps/ExtData2G/ExtDataSimpleFileHandler.F90 @@ -26,11 +26,12 @@ module MAPL_ExtdataSimpleFileHandler contains - subroutine get_file_bracket(this, input_time, source_time, bracket, rc) + subroutine get_file_bracket(this, input_time, source_time, bracket, fail_on_missing_file, rc) class(ExtdataSimpleFileHandler), intent(inout) :: this type(ESMF_Time), intent(in) :: input_time type(ESMF_Time), intent(in) :: source_time(:) type(ExtDataBracket), intent(inout) :: bracket + logical, intent(in) :: fail_on_missing_file integer, optional, intent(out) :: rc integer :: status type(ESMF_TimeInterval) :: zero @@ -38,96 +39,122 @@ subroutine get_file_bracket(this, input_time, source_time, bracket, rc) type(ESMF_Time) :: time integer :: time_index character(len=ESMF_MAXPATHLEN) :: current_file - logical :: get_left, get_right,in_range,was_set - type(ESMF_Time) :: target_time + logical :: get_left, get_right,in_range,left_was_set,right_was_set + type(ESMF_Time) :: target_time,ghost_time + logical :: allow_missing_file get_left=.true. get_right=.true. in_range=.true. target_time=input_time + + allow_missing_file = .not.fail_on_missing_file + + call bracket%get_node('L',was_set=left_was_set) + call bracket%get_node('R',was_set=right_was_set) + call bracket%set_parameters(intermittent_disable=.false.) if (this%persist_closest) then if (input_time <= this%valid_range(1)) then target_time = this%valid_range(1) get_right = .false. in_range = .false. - call bracket%get_node('L',was_set=was_set) - if (was_set) get_left=.false. + if (left_was_set) get_left=.false. call bracket%set_parameters(intermittent_disable=.true.) else if (input_time >= this%valid_range(2)) then target_time = this%valid_range(2) get_right = .false. in_range = .false. - call bracket%get_node('L',was_set=was_set) - if (was_set) get_left=.false. + if (left_was_set) get_left=.false. call bracket%set_parameters(intermittent_disable=.true.) end if + else + _ASSERT(left_was_set.eqv.right_was_set,"You should not be here") end if - if (bracket%time_in_bracket(target_time) .and. in_range) then - _RETURN(_SUCCESS) + if (in_range) then + if (bracket%time_in_bracket(target_time)) then + _RETURN(_SUCCESS) + end if end if + call ESMF_TimeIntervalSet(zero,_RC) if (this%frequency == zero) then current_file = this%file_template if (get_left) then call this%get_time_on_file(current_file,target_time,'L',time_index,time,_RC) _ASSERT(time_index/=time_not_found,"Time not found in file") - call bracket%set_node('L',file=current_file,time_index=time_index,time=time,_RC) + call bracket%set_node('L',file=current_file,time_index=time_index,time=time,was_set=.true.,_RC) if (in_range .and. (bracket%left_node == bracket%right_node)) then call bracket%swap_node_fields(rc=status) _VERIFY(status) else bracket%new_file_left=.true. - call bracket%set_node('L',was_set=.true.) end if end if if (get_right) then call this%get_time_on_file(current_file,target_time,'R',time_index,time,_RC) _ASSERT(time_index/=time_not_found,"Time not found in file") - call bracket%set_node('R',file=current_file,time_index=time_index,time=time,_RC) + call bracket%set_node('R',file=current_file,time_index=time_index,time=time,was_set=.true.,_RC) bracket%new_file_right=.true. end if else if (get_left) then - call this%get_file(current_file,target_time,0,_RC) + call this%get_file(current_file,target_time,0,allow_missing_file,ghost_time=ghost_time,_RC) call this%get_time_on_file(current_file,target_time,'L',time_index,time,_RC) - if (time_index == time_not_found) then - call this%get_file(current_file,target_time,-1,_RC) - call this%get_time_on_file(current_file,target_time,'L',time_index,time,_RC) - _ASSERT(time_index/=time_not_found,"Time not found in file") - end if - call bracket%set_node('L',file=current_file,time_index=time_index,time=time,_RC) + if (current_file == file_not_found) time=ghost_time + + call bracket%set_node('L',file=current_file,time_index=time_index,time=time,was_set=.true.,_RC) if (in_range .and. (bracket%left_node == bracket%right_node)) then - call bracket%swap_node_fields(rc=status) - _VERIFY(status) + call bracket%swap_node_fields(_RC) + bracket%new_file_left = .false. else + if (time_index == time_not_found ) then + call this%get_file(current_file,target_time,-1,allow_missing_file,_RC) + call this%get_time_on_file(current_file,target_time,'L',time_index,time,_RC) + if (time_index == time_not_found) then + if (allow_missing_file) then + time = ghost_time + else + _FAIL("Time not found in file") + end if + end if + end if + call bracket%set_node('L',file=current_file,time_index=time_index,time=time,was_set=.true.,_RC) bracket%new_file_left=.true. - call bracket%set_node('L',was_set=.true.) end if end if if (get_right) then - call this%get_file(current_file,target_time,0,_RC) + call this%get_file(current_file,target_time,0,allow_missing_file,_RC) call this%get_time_on_file(current_file,target_time,'R',time_index,time,_RC) if (time_index == time_not_found) then - call this%get_file(current_file,target_time,1,_RC) + call this%get_file(current_file,target_time,1,allow_missing_file,ghost_time,_RC) call this%get_time_on_file(current_file,target_time,'R',time_index,time,_RC) - _ASSERT(time_index /= time_not_found,"Time not found in file") + if (time_index == time_not_found) then + if (allow_missing_file) then + time = ghost_time + else + _FAIL("Time not found in file") + end if + end if end if - call bracket%set_node('R',file=current_file,time_index=time_index,time=time,_RC) + call bracket%set_node('R',file=current_file,time_index=time_index,time=time,was_set=.true.,_RC) bracket%new_file_right=.true. end if end if + _RETURN(_SUCCESS) end subroutine get_file_bracket - subroutine get_file(this,filename,input_time,shift,rc) + subroutine get_file(this,filename,input_time,shift,allow_missing_file,ghost_time,rc) class(ExtdataSimpleFileHandler), intent(inout) :: this character(len=*), intent(out) :: filename type(ESMF_Time) :: input_time integer, intent(in) :: shift + logical, intent(in) :: allow_missing_file + type(ESMF_Time), intent(out), optional :: ghost_time integer, intent(out), optional :: rc type(ESMF_Time) :: ftime @@ -150,7 +177,14 @@ subroutine get_file(this,filename,input_time,shift,rc) end if call fill_grads_template(filename,this%file_template,time=ftime,_RC) inquire(file=trim(filename),exist=file_found) - _ASSERT(file_found,"get_file did not file a file using: "//trim(this%file_template)) + if (.not.file_found) then + if (allow_Missing_file) then + filename = file_not_found + if (present(ghost_time)) ghost_time = ftime + else + _FAIL("get_file did not file a file using: "//trim(this%file_template)) + end if + end if _RETURN(_SUCCESS) end subroutine get_file diff --git a/gridcomps/ExtData2G/ExtDataTypeDef.F90 b/gridcomps/ExtData2G/ExtDataTypeDef.F90 index f7f7ec75ded3..e34d9c1a2907 100644 --- a/gridcomps/ExtData2G/ExtDataTypeDef.F90 +++ b/gridcomps/ExtData2G/ExtDataTypeDef.F90 @@ -23,6 +23,7 @@ module MAPL_ExtDataTypeDef ! if vertically interpolating vector fields type(ExtDataBracket) :: auxiliary1 type(ExtDataBracket) :: auxiliary2 + logical :: initialized = .false. end type BracketingFields type PrimaryExport @@ -73,6 +74,7 @@ module MAPL_ExtDataTypeDef ! for multiple collections type(ESMF_Time), allocatable :: start_end_time(:) logical :: initialized = .false. + logical :: fail_on_missing_file = .true. end type PrimaryExport type DerivedExport diff --git a/gridcomps/ExtData2G/ExtDataUpdatePointer.F90 b/gridcomps/ExtData2G/ExtDataUpdatePointer.F90 index d625327567ef..b42caa933b4e 100644 --- a/gridcomps/ExtData2G/ExtDataUpdatePointer.F90 +++ b/gridcomps/ExtData2G/ExtDataUpdatePointer.F90 @@ -29,10 +29,24 @@ module MAPL_ExtDataPointerUpdate procedure :: is_disabled procedure :: is_single_shot procedure :: disable + procedure :: get_adjusted_time end type contains + function get_adjusted_time(this,time,rc) result(adjusted_time) + type(ESMF_Time) :: adjusted_time + class(ExtDataPointerUpdate), intent(inout) :: this + type(ESMF_Time), intent(in) :: time + integer, optional, intent(out) :: rc + + integer :: status + + adjusted_time = time+this%offset + + _RETURN(_SUCCESS) + end function + subroutine create_from_parameters(this,update_time,update_freq,update_offset,time,clock,rc) class(ExtDataPointerUpdate), intent(inout) :: this character(len=*), intent(in) :: update_time diff --git a/gridcomps/MAPL_GridComps.F90 b/gridcomps/MAPL_GridComps.F90 index daedebb7f624..1e413ef4296a 100644 --- a/gridcomps/MAPL_GridComps.F90 +++ b/gridcomps/MAPL_GridComps.F90 @@ -4,6 +4,9 @@ module MAPL_GridCompsMod use mapl_externalGCStorage #ifdef USE_FLAP use mapl_FlapCLIMod +#endif +#ifdef USE_FARGPARSE + use mapl_FargParseCLIMod #endif implicit none end module MAPL_GridCompsMod diff --git a/pfio/NetCDF4_FileFormatter.F90 b/pfio/NetCDF4_FileFormatter.F90 index d9a64a5b8331..b7442380403f 100644 --- a/pfio/NetCDF4_FileFormatter.F90 +++ b/pfio/NetCDF4_FileFormatter.F90 @@ -45,6 +45,7 @@ module pFIO_NetCDF4_FileFormatterMod procedure :: ___SUB(get_var,int32,1) procedure :: ___SUB(get_var,int32,2) procedure :: ___SUB(get_var,int32,3) + procedure :: ___SUB(get_var,int32,4) procedure :: ___SUB(get_var,int64,0) procedure :: ___SUB(get_var,int64,1) procedure :: ___SUB(get_var,int64,2) @@ -65,6 +66,7 @@ module pFIO_NetCDF4_FileFormatterMod procedure :: ___SUB(put_var,int32,1) procedure :: ___SUB(put_var,int32,2) procedure :: ___SUB(put_var,int32,3) + procedure :: ___SUB(put_var,int32,4) procedure :: ___SUB(put_var,int64,0) procedure :: ___SUB(put_var,int64,1) procedure :: ___SUB(put_var,int64,2) @@ -86,6 +88,7 @@ module pFIO_NetCDF4_FileFormatterMod generic :: get_var => ___SUB(get_var,int32,1) generic :: get_var => ___SUB(get_var,int32,2) generic :: get_var => ___SUB(get_var,int32,3) + generic :: get_var => ___SUB(get_var,int32,4) generic :: get_var => ___SUB(get_var,int64,0) generic :: get_var => ___SUB(get_var,int64,1) generic :: get_var => ___SUB(get_var,int64,2) @@ -106,6 +109,7 @@ module pFIO_NetCDF4_FileFormatterMod generic :: put_var => ___SUB(put_var,int32,1) generic :: put_var => ___SUB(put_var,int32,2) generic :: put_var => ___SUB(put_var,int32,3) + generic :: put_var => ___SUB(put_var,int32,4) generic :: put_var => ___SUB(put_var,int64,0) generic :: put_var => ___SUB(put_var,int64,1) generic :: put_var => ___SUB(put_var,int64,2) @@ -1196,6 +1200,10 @@ end subroutine inq_variables # include "NetCDF4_get_var.H" # include "NetCDF4_put_var.H" # undef _RANK +# define _RANK 4 +# include "NetCDF4_get_var.H" +# include "NetCDF4_put_var.H" +# undef _RANK #undef _VARTYPE ! INT64