Skip to content

Commit

Permalink
Merge pull request #272 from GEOS-ESM/feature/mmanyin/chem_barriers
Browse files Browse the repository at this point in the history 
Option for timing tests
  • Loading branch information
@mmanyin
mmanyin authored Dec 29, 2023
2 parents 6436e3a + 63ff970 commit 21c6b6e
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 19 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

- Connectivity from GMI to ACHEM (4 fields), requires GMI v1.2.0 or later to run
- Slight improvement for lightning flash rate calculation (LOPEZ and MOIST schemes). See the option UsePreconCape in ChemEnv.rc . This involves new imports from MOIST: CAPE, BYNCY and INHB. **NOTE** THIS REQUIRES GEOSgcm_GridComp develop branch (as of 12/12/23).
- Added a flag for 'strict' child timing, intended to reduce the timing bias against child GC's that employ 'gather' calls. Such calls are occasionally necessary, but can cause timers to attribute excessive time to a child, time that is actually the synchronization lag time that would eventually be spent -somewhere- in the program, but which gets attributed to the child with a 'gather' or barrier call. The new flag is for timing tests only.

### Removed

Expand Down
84 changes: 65 additions & 19 deletions GEOS_ChemGridComp.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ module GEOS_ChemGridCompMod
LOGICAL :: enable_TR
LOGICAL :: enable_DNA
LOGICAL :: enable_HEMCO
LOGICAL :: strict_child_timing ! Call a barrier before and after each child is run
! Only use this to test timings, not operationally
INTEGER :: AERO_PROVIDER
INTEGER :: RATS_PROVIDER ! WARNING: May be multiple RATS_PROVIDERs
END TYPE GEOS_ChemGridComp
Expand Down Expand Up @@ -130,7 +132,8 @@ subroutine SetServices ( GC, RC )
! ErrLog Variables

__Iam__('SetServices') ! NOTE: this macro declares STATUS
! ALSO: Never set Iam = TRIM(Iam) // suffix
! ALSO: Never set Iam to a string containing Iam
! in any routine that is run more than once
! because Iam is a SAVED varaible
character(len=ESMF_MAXSTR) :: COMP_NAME

Expand Down Expand Up @@ -230,6 +233,8 @@ subroutine SetServices ( GC, RC )
call ESMF_ConfigGetAttribute(myCF, myState%enable_DNA, Default=.FALSE., Label="ENABLE_DNA:", __RC__ )
call ESMF_ConfigGetAttribute(myCF, myState%enable_HEMCO, Default=.FALSE., Label="ENABLE_HEMCO:", __RC__ )

call ESMF_ConfigGetAttribute(myCF, myState%strict_child_timing, Default=.FALSE., Label="strict_child_timing:", __RC__ )

!ALT: valgrind flagged a memory leak. myState%CF => myCF ! save for later
call ESMF_ConfigDestroy(myCF, __RC__)

Expand All @@ -243,23 +248,23 @@ subroutine SetServices ( GC, RC )
! -----------------------------------------------------------------
CHEMENV = MAPL_AddChild(GC, NAME='CHEMENV', SS=ChemEnv_SetServices, __RC__)

if ( myState%enable_HEMCO) HEMCO = MAPL_AddChild(GC, NAME= 'HEMCO', SS=HEMCO_SetServices, __RC__)
if ( myState%enable_PCHEM) PCHEM = MAPL_AddChild(GC, NAME= 'PCHEM', SS=PChem_SetServices, __RC__)
if ( myState%enable_ACHEM) ACHEM = MAPL_AddChild(GC, NAME= 'ACHEM', SS=AChem_SetServices, __RC__)
if ( myState%enable_GOCART) GOCART = MAPL_AddChild(GC, NAME= 'GOCART', SS=GOCART_SetServices, __RC__)
if (myState%enable_HEMCO ) HEMCO = MAPL_AddChild(GC, NAME= 'HEMCO', SS=HEMCO_SetServices, __RC__)
if (myState%enable_PCHEM ) PCHEM = MAPL_AddChild(GC, NAME= 'PCHEM', SS=PChem_SetServices, __RC__)
if (myState%enable_ACHEM ) ACHEM = MAPL_AddChild(GC, NAME= 'ACHEM', SS=AChem_SetServices, __RC__)
if (myState%enable_GOCART ) GOCART = MAPL_AddChild(GC, NAME= 'GOCART', SS=GOCART_SetServices, __RC__)
if (myState%enable_GOCARTdata) GOCARTdata = MAPL_AddChild(GC, NAME= 'GOCART.data', SS=GOCART_SetServices, __RC__)
if ( myState%enable_GOCART2G) GOCART2G = MAPL_AddChild(GC, NAME= 'GOCART2G', SS=GOCART2G_SetServices, __RC__)
if ( myState%enable_GAAS) GAAS = MAPL_AddChild(GC, NAME= 'GAAS', SS=GAAS_SetServices, __RC__)
if ( myState%enable_H2O) H2O = MAPL_AddChild(GC, NAME= 'H2O', SS=H2O_SetServices, __RC__)
if ( myState%enable_STRATCHEM) STRATCHEM = MAPL_AddChild(GC, NAME= 'STRATCHEM', SS=StratChem_SetServices, __RC__)
if ( myState%enable_GMICHEM) GMICHEM = MAPL_AddChild(GC, NAME= 'GMICHEM', SS=GMI_SetServices, __RC__)
if ( myState%enable_CARMA) CARMA = MAPL_AddChild(GC, NAME= 'CARMA', SS=CARMA_SetServices, __RC__)
if ( myState%enable_GEOSCHEM) GEOSCHEM = MAPL_AddChild(GC, NAME='GEOSCHEMCHEM', SS=GCChem_SetServices, __RC__)
if ( myState%enable_MATRIX) MATRIX = MAPL_AddChild(GC, NAME= 'MATRIX', SS=MATRIX_SetServices, __RC__)
if ( myState%enable_MAM) MAM = MAPL_AddChild(GC, NAME= 'MAM', SS=MAM_SetServices, __RC__)
if ( myState%enable_MAMdata) MAMdata = MAPL_AddChild(GC, NAME= 'MAM.data', SS=MAM_SetServices, __RC__)
if ( myState%enable_TR) TR = MAPL_AddChild(GC, NAME= 'TR', SS=TR_SetServices, __RC__)
if ( myState%enable_DNA) DNA = MAPL_AddChild(GC, NAME= 'DNA', SS=DNA_SetServices, __RC__)
if (myState%enable_GOCART2G ) GOCART2G = MAPL_AddChild(GC, NAME= 'GOCART2G', SS=GOCART2G_SetServices, __RC__)
if (myState%enable_GAAS ) GAAS = MAPL_AddChild(GC, NAME= 'GAAS', SS=GAAS_SetServices, __RC__)
if (myState%enable_H2O ) H2O = MAPL_AddChild(GC, NAME= 'H2O', SS=H2O_SetServices, __RC__)
if (myState%enable_STRATCHEM ) STRATCHEM = MAPL_AddChild(GC, NAME= 'STRATCHEM', SS=StratChem_SetServices, __RC__)
if (myState%enable_GMICHEM ) GMICHEM = MAPL_AddChild(GC, NAME= 'GMICHEM', SS=GMI_SetServices, __RC__)
if (myState%enable_CARMA ) CARMA = MAPL_AddChild(GC, NAME= 'CARMA', SS=CARMA_SetServices, __RC__)
if (myState%enable_GEOSCHEM ) GEOSCHEM = MAPL_AddChild(GC, NAME='GEOSCHEMCHEM', SS=GCChem_SetServices, __RC__)
if (myState%enable_MATRIX ) MATRIX = MAPL_AddChild(GC, NAME= 'MATRIX', SS=MATRIX_SetServices, __RC__)
if (myState%enable_MAM ) MAM = MAPL_AddChild(GC, NAME= 'MAM', SS=MAM_SetServices, __RC__)
if (myState%enable_MAMdata ) MAMdata = MAPL_AddChild(GC, NAME= 'MAM.data', SS=MAM_SetServices, __RC__)
if (myState%enable_TR ) TR = MAPL_AddChild(GC, NAME= 'TR', SS=TR_SetServices, __RC__)
if (myState%enable_DNA ) DNA = MAPL_AddChild(GC, NAME= 'DNA', SS=DNA_SetServices, __RC__)


! A container for the friendly tracers
Expand Down Expand Up @@ -320,6 +325,7 @@ subroutine SetServices ( GC, RC )

IF(MAPL_AM_I_ROOT()) THEN
PRINT *," "
PRINT *, TRIM(Iam)//": strict_child_timing =", myState%strict_child_timing
PRINT *, TRIM(Iam)//": RATs Provider List"
DO i = 1, numRATs
PRINT *," "//TRIM(speciesName(i))//": "//TRIM(RATsProviderName(i))
Expand Down Expand Up @@ -1095,6 +1101,9 @@ subroutine Run1 ( GC, IMPORT, EXPORT, CLOCK, RC )
integer :: userRC
character(len=ESMF_MAXSTR) :: CHILD_NAME
real, pointer :: th(:,:,:) => NULL()
type (GEOS_ChemGridComp), pointer :: myState ! private, that is
type (GEOS_ChemGridComp_Wrap) :: wrap
type (ESMF_VM) :: VM

!=============================================================================

Expand All @@ -1109,6 +1118,12 @@ subroutine Run1 ( GC, IMPORT, EXPORT, CLOCK, RC )
call MAPL_Get(MAPL, RUNALARM = ALARM, RC=STATUS )
VERIFY_(STATUS)

! Get my internal state
! ---------------------
call ESMF_UserCompGetInternalState(GC, 'GEOSchem_GridComp_State', WRAP, STATUS)
VERIFY_(STATUS)
myState => wrap%ptr

! Start timers
! ------------
call MAPL_TimerOn( MAPL, "TOTAL")
Expand Down Expand Up @@ -1158,6 +1173,12 @@ subroutine Run1 ( GC, IMPORT, EXPORT, CLOCK, RC )
! exists. Also updated MAPL_Get to accept the output
! argument NumRunPhases (ckeller, 09/10/2014)
! --------------------------------------------------------

IF ( myState%strict_child_timing ) THEN
call ESMF_VMGetCurrent ( VM=VM, __RC__ )
call ESMF_VMBarrier(VM, __RC__ )
END IF

do I=1,NCHLD
call ESMF_GridCompGet( GCS(I), NAME=CHILD_NAME, __RC__ )
call MAPL_GetObjectFromGC(GCS(I), CHLD, __RC__ )
Expand All @@ -1171,7 +1192,12 @@ subroutine Run1 ( GC, IMPORT, EXPORT, CLOCK, RC )
phase = IPHASE, &
userRC = userRC, &
__RC__ )
_ASSERT(userRC==ESMF_SUCCESS,'needs informative message')
_ASSERT(userRC==ESMF_SUCCESS,'Failed running the CHEM child '//trim(CHILD_NAME))

IF ( myState%strict_child_timing ) THEN
call ESMF_VMBarrier(VM, __RC__ )
END IF

call MAPL_TimerOff(MAPL,trim(CHILD_NAME))
endif
enddo !I
Expand Down Expand Up @@ -1243,6 +1269,9 @@ subroutine Run2 ( GC, IMPORT, EXPORT, CLOCK, RC )
integer :: NPHASE, IPHASE
integer :: userRC
character(len=ESMF_MAXSTR) :: CHILD_NAME
type (GEOS_ChemGridComp), pointer :: myState ! private, that is
type (GEOS_ChemGridComp_Wrap) :: wrap
type (ESMF_VM) :: VM
!-------------------------------------------------------------------
! Begin...

Expand All @@ -1254,6 +1283,12 @@ subroutine Run2 ( GC, IMPORT, EXPORT, CLOCK, RC )
VERIFY_(STATUS)
call MAPL_Get(MAPL, RUNALARM = ALARM, __RC__ )

! Get my internal state
! ---------------------
call ESMF_UserCompGetInternalState(GC, 'GEOSchem_GridComp_State', WRAP, STATUS)
VERIFY_(STATUS)
myState => wrap%ptr

! Start timers
! ------------
call MAPL_TimerOn( MAPL, "TOTAL")
Expand Down Expand Up @@ -1289,6 +1324,12 @@ subroutine Run2 ( GC, IMPORT, EXPORT, CLOCK, RC )
! do for every child: get child state, determine number of
! run phases and phase to call, execute.
! --------------------------------------------------------

IF ( myState%strict_child_timing ) THEN
call ESMF_VMGetCurrent ( VM=VM, __RC__ )
call ESMF_VMBarrier(VM, __RC__ )
END IF

do I=1,NCHLD
call MAPL_GetObjectFromGC(GCS(I), CHLD, __RC__ )
call ESMF_GridCompGet( GCS(I), NAME=CHILD_NAME, __RC__ )
Expand All @@ -1307,7 +1348,12 @@ subroutine Run2 ( GC, IMPORT, EXPORT, CLOCK, RC )
phase = IPHASE, &
userRC = userRC, &
__RC__ )
_ASSERT(userRC==ESMF_SUCCESS,'needs informative message')
_ASSERT(userRC==ESMF_SUCCESS,'Failed running the CHEM child '//trim(CHILD_NAME))

IF ( myState%strict_child_timing ) THEN
call ESMF_VMBarrier(VM, __RC__ )
END IF

call MAPL_TimerOff(MAPL,trim(CHILD_NAME))
enddo !I
endif
Expand Down
4 changes: 4 additions & 0 deletions GEOS_ChemGridComp.rc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,7 @@
ENABLE_TR: .TRUE.
ENABLE_DNA: .FALSE.
ENABLE_HEMCO: .TRUE.

# Use only for diagnostic timing tests:
# --------------------------------------------------
strict_child_timing: .FALSE.

0 comments on commit 21c6b6e

Please sign in to comment.