From e2d192e2c3914200d561d188abf548fb37320301 Mon Sep 17 00:00:00 2001 From: Peter Lawrence Date: Thu, 1 Mar 2018 15:04:31 -0700 Subject: [PATCH 001/332] Added new files to allow Gross Unrepresented Land Use transition --- src/biogeochem/CNBalanceCheckMod.F90 | 10 +- src/biogeochem/CNCIsoFluxMod.F90 | 279 +++++++++ src/biogeochem/CNCStateUpdate2Mod.F90 | 115 ++++ src/biogeochem/CNDriverMod.F90 | 54 +- src/biogeochem/CNNStateUpdate2Mod.F90 | 108 ++++ src/biogeochem/CNProductsMod.F90 | 86 ++- src/biogeochem/CNVegCarbonFluxType.F90 | 302 +++++++++- src/biogeochem/CNVegNitrogenFluxType.F90 | 151 ++++- src/biogeochem/CNVegetationFacade.F90 | 9 + src/biogeochem/dynConsBiogeochemMod.F90 | 70 ++- src/biogeochem/dynGrossUnrepMod.F90 | 528 ++++++++++++++++++ src/dyn_subgrid/dynSubgridControlMod.F90 | 35 ++ src/dyn_subgrid/dynSubgridDriverMod.F90 | 11 + .../SoilBiogeochemCarbonStateType.F90 | 27 +- .../SoilBiogeochemNitrogenStateType.F90 | 27 +- 15 files changed, 1719 insertions(+), 93 deletions(-) create mode 100644 src/biogeochem/dynGrossUnrepMod.F90 diff --git a/src/biogeochem/CNBalanceCheckMod.F90 b/src/biogeochem/CNBalanceCheckMod.F90 index f2811d290a..4a34f95f9c 100644 --- a/src/biogeochem/CNBalanceCheckMod.F90 +++ b/src/biogeochem/CNBalanceCheckMod.F90 @@ -141,6 +141,8 @@ subroutine CBalanceCheck(this, bounds, num_soilc, filter_soilc, & col_begcb => this%begcb_col , & ! Input: [real(r8) (:) ] (gC/m2) carbon mass, beginning of time step col_endcb => this%endcb_col , & ! Output: [real(r8) (:) ] (gC/m2) carbon mass, end of time step wood_harvestc => cnveg_carbonflux_inst%wood_harvestc_col , & ! Input: [real(r8) (:) ] (gC/m2/s) wood harvest (to product pools) + gru_conv_cflux => cnveg_carbonflux_inst%gru_conv_cflux_col , & ! Input: [real(r8) (:) ] (gC/m2/s) wood harvest (to product pools) + gru_wood_productc_gain => cnveg_carbonflux_inst%gru_wood_productc_gain_col , & ! Input: [real(r8) (:) ] (gC/m2/s) wood harvest (to product pools) grainc_to_cropprodc => cnveg_carbonflux_inst%grainc_to_cropprodc_col , & ! Input: [real(r8) (:) ] (gC/m2/s) grain C to 1-year crop product pool gpp => cnveg_carbonflux_inst%gpp_col , & ! Input: [real(r8) (:) ] (gC/m2/s) gross primary production er => cnveg_carbonflux_inst%er_col , & ! Input: [real(r8) (:) ] (gC/m2/s) total ecosystem respiration, autotrophic + heterotrophic @@ -167,7 +169,7 @@ subroutine CBalanceCheck(this, bounds, num_soilc, filter_soilc, & ! calculate total column-level outputs ! er = ar + hr, col_fire_closs includes patch-level fire losses - col_coutputs = er(c) + col_fire_closs(c) + col_hrv_xsmrpool_to_atm(c) + col_coutputs = er(c) + col_fire_closs(c) + col_hrv_xsmrpool_to_atm(c) + gru_conv_cflux(c) ! Fluxes to product pools are included in column-level outputs: the product ! pools are not included in totcolc, so are outside the system with respect to @@ -176,6 +178,7 @@ subroutine CBalanceCheck(this, bounds, num_soilc, filter_soilc, & ! after the dwt term has already been taken out.) col_coutputs = col_coutputs + & wood_harvestc(c) + & + gru_wood_productc_gain(c) + & grainc_to_cropprodc(c) ! subtract leaching flux @@ -268,6 +271,8 @@ subroutine NBalanceCheck(this, bounds, num_soilc, filter_soilc, & col_fire_nloss => cnveg_nitrogenflux_inst%fire_nloss_col , & ! Input: [real(r8) (:) ] (gN/m2/s) total column-level fire N loss wood_harvestn => cnveg_nitrogenflux_inst%wood_harvestn_col , & ! Input: [real(r8) (:) ] (gN/m2/s) wood harvest (to product pools) + gru_conv_nflux => cnveg_nitrogenflux_inst%gru_conv_nflux_col , & ! Input: [real(r8) (:) ] (gC/m2/s) wood harvest (to product pools) + gru_wood_productn_gain => cnveg_nitrogenflux_inst%gru_wood_productn_gain_col , & ! Input: [real(r8) (:) ] (gC/m2/s) wood harvest (to product pools) grainn_to_cropprodn => cnveg_nitrogenflux_inst%grainn_to_cropprodn_col , & ! Input: [real(r8) (:) ] (gN/m2/s) grain N to 1-year crop product pool totcoln => cnveg_nitrogenstate_inst%totn_col & ! Input: [real(r8) (:) ] (gN/m2) total column nitrogen, incl veg @@ -295,7 +300,7 @@ subroutine NBalanceCheck(this, bounds, num_soilc, filter_soilc, & end if ! calculate total column-level outputs - col_noutputs(c) = denit(c) + col_fire_nloss(c) + col_noutputs(c) = denit(c) + col_fire_nloss(c) + gru_conv_nflux(c) ! Fluxes to product pools are included in column-level outputs: the product ! pools are not included in totcoln, so are outside the system with respect to @@ -304,6 +309,7 @@ subroutine NBalanceCheck(this, bounds, num_soilc, filter_soilc, & ! after the dwt term has already been taken out.) col_noutputs(c) = col_noutputs(c) + & wood_harvestn(c) + & + gru_wood_productn_gain(c) + & grainn_to_cropprodn(c) if (.not. use_nitrif_denitrif) then diff --git a/src/biogeochem/CNCIsoFluxMod.F90 b/src/biogeochem/CNCIsoFluxMod.F90 index a92ab44e47..4d5e88a247 100644 --- a/src/biogeochem/CNCIsoFluxMod.F90 +++ b/src/biogeochem/CNCIsoFluxMod.F90 @@ -28,12 +28,14 @@ module CNCIsoFluxMod public :: CIsoFlux1 public :: CIsoFlux2 public :: CIsoFlux2h + public :: CIsoFlux2g public :: CIsoFlux3 ! ! !PRIVATE MEMBER FUNCTIONS: private :: CNCIsoLitterToColumn private :: CNCIsoGapPftToColumn private :: CNCIsoHarvestPftToColumn + private :: CNCIsoGrossUnrepPftToColumn private :: CIsoFluxCalc character(len=*), parameter, private :: sourcefile = & @@ -803,6 +805,157 @@ subroutine CIsoFlux2h(num_soilc , filter_soilc, num_soilp , filter_soilp, & end subroutine CIsoFlux2h + !----------------------------------------------------------------------- + subroutine CIsoFlux2g(num_soilc , filter_soilc, num_soilp , filter_soilp, & + soilbiogeochem_state_inst, & + cnveg_carbonflux_inst, cnveg_carbonstate_inst, & + iso_cnveg_carbonflux_inst, iso_cnveg_carbonstate_inst, isotope) + ! + ! !DESCRIPTION: + ! set the carbon isotopic fluxes for gross unrepresented landcover change mortality + ! + ! !ARGUMENTS: + integer , intent(in) :: num_soilc ! number of soil columns filter + integer , intent(in) :: filter_soilc(:) ! filter for soil columns + integer , intent(in) :: num_soilp ! number of soil patches in filter + integer , intent(in) :: filter_soilp(:) ! filter for soil patches + type(soilbiogeochem_state_type) , intent(in) :: soilbiogeochem_state_inst + type(cnveg_carbonflux_type) , intent(in) :: cnveg_carbonflux_inst + type(cnveg_carbonstate_type) , intent(in) :: cnveg_carbonstate_inst + type(cnveg_carbonflux_type) , intent(inout) :: iso_cnveg_carbonflux_inst + type(cnveg_carbonstate_type) , intent(in) :: iso_cnveg_carbonstate_inst + character(len=*) , intent(in) :: isotope ! 'c13' or 'c14' + + !----------------------------------------------------------------------- + + associate( & + cnveg_cf => cnveg_carbonflux_inst , & + cnveg_cs => cnveg_carbonstate_inst , & + iso_cnveg_cf => iso_cnveg_carbonflux_inst , & + iso_cnveg_cs => iso_cnveg_carbonstate_inst & + ) + + ! patch-level gap mortality fluxes + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_leafc_to_litter_patch , cnveg_cf%gru_leafc_to_litter_patch, & + iso_cnveg_cs%leafc_patch , cnveg_cs%leafc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_leafc_storage_to_atm_patch , cnveg_cf%gru_leafc_storage_to_atm_patch, & + iso_cnveg_cs%leafc_storage_patch , cnveg_cs%leafc_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_leafc_xfer_to_atm_patch , cnveg_cf%gru_leafc_xfer_to_atm_patch, & + iso_cnveg_cs%leafc_xfer_patch , cnveg_cs%leafc_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_frootc_to_litter_patch , cnveg_cf%gru_frootc_to_litter_patch, & + iso_cnveg_cs%frootc_patch , cnveg_cs%frootc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_frootc_storage_to_atm_patch , cnveg_cf%gru_frootc_storage_to_atm_patch, & + iso_cnveg_cs%frootc_storage_patch , cnveg_cs%frootc_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_frootc_xfer_to_atm_patch , cnveg_cf%gru_frootc_xfer_to_atm_patch, & + iso_cnveg_cs%frootc_xfer_patch , cnveg_cs%frootc_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_livestemc_to_atm_patch , cnveg_cf%gru_livestemc_to_atm_patch, & + iso_cnveg_cs%livestemc_patch , cnveg_cs%livestemc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_livestemc_storage_to_atm_patch , cnveg_cf%gru_livestemc_storage_to_atm_patch, & + iso_cnveg_cs%livestemc_storage_patch , cnveg_cs%livestemc_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_livestemc_xfer_to_atm_patch , cnveg_cf%gru_livestemc_xfer_to_atm_patch, & + iso_cnveg_cs%livestemc_xfer_patch , cnveg_cs%livestemc_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_deadstemc_to_atm_patch , cnveg_cf%gru_deadstemc_to_atm_patch, & + iso_cnveg_cs%deadstemc_patch , cnveg_cs%deadstemc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_wood_productc_gain_patch , cnveg_cf%gru_wood_productc_gain_patch, & + iso_cnveg_cs%deadstemc_patch , cnveg_cs%deadstemc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_deadstemc_storage_to_atm_patch , cnveg_cf%gru_deadstemc_storage_to_atm_patch, & + iso_cnveg_cs%deadstemc_storage_patch , cnveg_cs%deadstemc_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_deadstemc_xfer_to_atm_patch , cnveg_cf%gru_deadstemc_xfer_to_atm_patch, & + iso_cnveg_cs%deadstemc_xfer_patch , cnveg_cs%deadstemc_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_livecrootc_to_litter_patch , cnveg_cf%gru_livecrootc_to_litter_patch, & + iso_cnveg_cs%livecrootc_patch , cnveg_cs%livecrootc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_livecrootc_storage_to_atm_patch , cnveg_cf%gru_livecrootc_storage_to_atm_patch, & + iso_cnveg_cs%livecrootc_storage_patch , cnveg_cs%livecrootc_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_livecrootc_xfer_to_atm_patch , cnveg_cf%gru_livecrootc_xfer_to_atm_patch, & + iso_cnveg_cs%livecrootc_xfer_patch , cnveg_cs%livecrootc_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_deadcrootc_to_litter_patch , cnveg_cf%gru_deadcrootc_to_litter_patch, & + iso_cnveg_cs%deadcrootc_patch , cnveg_cs%deadcrootc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_deadcrootc_storage_to_atm_patch , cnveg_cf%gru_deadcrootc_storage_to_atm_patch, & + iso_cnveg_cs%deadcrootc_storage_patch , cnveg_cs%deadcrootc_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_deadcrootc_xfer_to_atm_patch , cnveg_cf%gru_deadcrootc_xfer_to_atm_patch, & + iso_cnveg_cs%deadcrootc_xfer_patch , cnveg_cs%deadcrootc_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_gresp_storage_to_atm_patch , cnveg_cf%gru_gresp_storage_to_atm_patch, & + iso_cnveg_cs%gresp_storage_patch , cnveg_cs%gresp_storage_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_gresp_xfer_to_atm_patch , cnveg_cf%gru_gresp_xfer_to_atm_patch, & + iso_cnveg_cs%gresp_xfer_patch , cnveg_cs%gresp_xfer_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + call CIsoFluxCalc(& + iso_cnveg_cf%gru_xsmrpool_to_atm_patch , cnveg_cf%gru_xsmrpool_to_atm_patch, & + iso_cnveg_cs%totvegc_patch , cnveg_cs%totvegc_patch, & + num_soilp , filter_soilp, 1._r8, 0, isotope) + + ! call routine to shift patch-level gap mortality fluxes to column, + ! for isotopes the non-isotope version of this routine is in CNGapMortalityMod.F90. + + call CNCIsoGrossUnrepPftToColumn(num_soilc, filter_soilc, soilbiogeochem_state_inst, iso_cnveg_carbonflux_inst) + + end associate + + end subroutine CIsoFlux2g + !----------------------------------------------------------------------- subroutine CIsoFlux3(num_soilc , filter_soilc, num_soilp , filter_soilp, & soilbiogeochem_state_inst , soilbiogeochem_carbonstate_inst, & @@ -1550,6 +1703,132 @@ subroutine CNCIsoHarvestPftToColumn (num_soilc, filter_soilc, & end subroutine CNCIsoHarvestPftToColumn + !----------------------------------------------------------------------- + subroutine CNCIsoGrossUnrepPftToColumn (num_soilc, filter_soilc, & + soilbiogeochem_state_inst, iso_cnveg_carbonflux_inst) + ! + ! !DESCRIPTION: + ! gather all patch-level gross unrepresented landcover change mortality fluxes + ! to the column level and assign them to the litter, cwd, and wood product pools + ! + ! !ARGUMENTS: + integer , intent(in) :: num_soilc ! number of soil columns in filter + integer , intent(in) :: filter_soilc(:) ! soil column filter + type(soilbiogeochem_state_type) , intent(in) :: soilbiogeochem_state_inst + type(cnveg_carbonflux_type) , intent(inout) :: iso_cnveg_carbonflux_inst + ! + ! !LOCAL VARIABLES: + integer :: fc,c,pi,p,j ! indices + !----------------------------------------------------------------------- + + associate( & + ivt => patch%itype , & ! Input: [integer (:) ] patch vegetation type + wtcol => patch%wtcol , & ! Input: [real(r8) (:) ] patch weight relative to column (0-1) + + lf_flab => pftcon%lf_flab , & ! Input: leaf litter labile fraction + lf_fcel => pftcon%lf_fcel , & ! Input: leaf litter cellulose fraction + lf_flig => pftcon%lf_flig , & ! Input: leaf litter lignin fraction + fr_flab => pftcon%fr_flab , & ! Input: fine root litter labile fraction + fr_fcel => pftcon%fr_fcel , & ! Input: fine root litter cellulose fraction + fr_flig => pftcon%fr_flig , & ! Input: fine root litter lignin fraction + + leaf_prof => soilbiogeochem_state_inst%leaf_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of leaves + froot_prof => soilbiogeochem_state_inst%froot_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of fine roots + croot_prof => soilbiogeochem_state_inst%croot_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of coarse roots + stem_prof => soilbiogeochem_state_inst%stem_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of stems + + gru_leafc_to_litter => iso_cnveg_carbonflux_inst%gru_leafc_to_litter_patch , & ! Input: [real(r8) (:) ] + gru_frootc_to_litter => iso_cnveg_carbonflux_inst%gru_frootc_to_litter_patch , & ! Input: [real(r8) (:) ] + gru_livestemc_to_atm => iso_cnveg_carbonflux_inst%gru_livestemc_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_deadstemc_to_atm => iso_cnveg_carbonflux_inst%gru_deadstemc_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_wood_productc_gain => iso_cnveg_carbonflux_inst%gru_wood_productc_gain_patch , & ! Input: [real(r8) (:) ] + gru_livecrootc_to_litter => iso_cnveg_carbonflux_inst%gru_livecrootc_to_litter_patch , & ! Input: [real(r8) (:) ] + gru_deadcrootc_to_litter => iso_cnveg_carbonflux_inst%gru_deadcrootc_to_litter_patch , & ! Input: [real(r8) (:) ] + gru_leafc_storage_to_atm => iso_cnveg_carbonflux_inst%gru_leafc_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_frootc_storage_to_atm => iso_cnveg_carbonflux_inst%gru_frootc_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_livestemc_storage_to_atm => iso_cnveg_carbonflux_inst%gru_livestemc_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_deadstemc_storage_to_atm => iso_cnveg_carbonflux_inst%gru_deadstemc_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_livecrootc_storage_to_atm => iso_cnveg_carbonflux_inst%gru_livecrootc_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_deadcrootc_storage_to_atm => iso_cnveg_carbonflux_inst%gru_deadcrootc_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_gresp_storage_to_atm => iso_cnveg_carbonflux_inst%gru_gresp_storage_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_leafc_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_leafc_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_frootc_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_frootc_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_livestemc_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_livestemc_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_deadstemc_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_deadstemc_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_livecrootc_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_livecrootc_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_deadcrootc_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_deadcrootc_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + gru_gresp_xfer_to_atm => iso_cnveg_carbonflux_inst%gru_gresp_xfer_to_atm_patch , & ! Input: [real(r8) (:) ] + cwood_harvestc => iso_cnveg_carbonflux_inst%wood_harvestc_col , & ! Output: [real(r8) (:) ] + gru_c_to_litr_met_c => iso_cnveg_carbonflux_inst%gru_c_to_litr_met_c_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with harvest to litter metabolic pool (gC/m3/s) + gru_c_to_litr_cel_c => iso_cnveg_carbonflux_inst%gru_c_to_litr_cel_c_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with harvest to litter cellulose pool (gC/m3/s) + gru_c_to_litr_lig_c => iso_cnveg_carbonflux_inst%gru_c_to_litr_lig_c_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with harvest to litter lignin pool (gC/m3/s) + gru_c_to_cwdc_c => iso_cnveg_carbonflux_inst%gru_c_to_cwdc_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with harvest to CWD pool (gC/m3/s) + gru_wood_productc_gain_c => iso_cnveg_carbonflux_inst%gru_wood_productc_gain_col & ! Input: [real(r8) (:) ] + ) + + do j = 1, nlevdecomp + do pi = 1,maxpatch_pft + do fc = 1,num_soilc + c = filter_soilc(fc) + + if (pi <= col%npatches(c)) then + p = col%patchi(c) + pi - 1 + + if (patch%active(p)) then + + ! leaf gross unrepresented landcover change mortality carbon fluxes + gru_c_to_litr_met_c(c,j) = gru_c_to_litr_met_c(c,j) + & + gru_leafc_to_litter(p) * lf_flab(ivt(p)) * wtcol(p) * leaf_prof(p,j) + gru_c_to_litr_cel_c(c,j) = gru_c_to_litr_cel_c(c,j) + & + gru_leafc_to_litter(p) * lf_fcel(ivt(p)) * wtcol(p) * leaf_prof(p,j) + gru_c_to_litr_lig_c(c,j) = gru_c_to_litr_lig_c(c,j) + & + gru_leafc_to_litter(p) * lf_flig(ivt(p)) * wtcol(p) * leaf_prof(p,j) + + ! fine root gross unrepresented landcover change mortality carbon fluxes + gru_c_to_litr_met_c(c,j) = gru_c_to_litr_met_c(c,j) + & + gru_frootc_to_litter(p) * fr_flab(ivt(p)) * wtcol(p) * froot_prof(p,j) + gru_c_to_litr_cel_c(c,j) = gru_c_to_litr_cel_c(c,j) + & + gru_frootc_to_litter(p) * fr_fcel(ivt(p)) * wtcol(p) * froot_prof(p,j) + gru_c_to_litr_lig_c(c,j) = gru_c_to_litr_lig_c(c,j) + & + gru_frootc_to_litter(p) * fr_flig(ivt(p)) * wtcol(p) * froot_prof(p,j) + + ! coarse root gross unrepresented landcover change mortality carbon fluxes + gru_c_to_cwdc_c(c,j) = gru_c_to_cwdc_c(c,j) + & + gru_livecrootc_to_litter(p) * wtcol(p) * croot_prof(p,j) + gru_c_to_cwdc_c(c,j) = gru_c_to_cwdc_c(c,j) + & + gru_deadcrootc_to_litter(p) * wtcol(p) * croot_prof(p,j) + + end if + end if + + end do + + end do + end do + + do pi = 1,maxpatch_pft + do fc = 1,num_soilc + c = filter_soilc(fc) + + if (pi <= col%npatches(c)) then + p = col%patchi(c) + pi - 1 + + if (patch%active(p)) then + ! wood gross unrepresented landcover change mortality carbon fluxes to product pools + gru_wood_productc_gain_c(c) = gru_wood_productc_gain_c(c) + & + gru_wood_productc_gain(p) * wtcol(p) + + end if + end if + + end do + + end do + + end associate + + end subroutine CNCIsoGrossUnrepPftToColumn + !----------------------------------------------------------------------- subroutine CIsoFluxCalc(& ciso_flux, ctot_flux, & diff --git a/src/biogeochem/CNCStateUpdate2Mod.F90 b/src/biogeochem/CNCStateUpdate2Mod.F90 index ef1f6f9345..38bf5bcfc5 100644 --- a/src/biogeochem/CNCStateUpdate2Mod.F90 +++ b/src/biogeochem/CNCStateUpdate2Mod.F90 @@ -20,6 +20,7 @@ module CNCStateUpdate2Mod ! !PUBLIC MEMBER FUNCTIONS: public:: CStateUpdate2 public:: CStateUpdate2h + public:: CStateUpdate2g !----------------------------------------------------------------------- contains @@ -244,4 +245,118 @@ subroutine CStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, & end subroutine CStateUpdate2h + !----------------------------------------------------------------------- + subroutine CStateUpdate2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + cnveg_carbonflux_inst, cnveg_carbonstate_inst, soilbiogeochem_carbonstate_inst) + ! + ! !DESCRIPTION: + ! Update all the prognostic carbon state + ! variables affected by gross unrepresented landcover change mortality fluxes + ! + ! !ARGUMENTS: + integer , intent(in) :: num_soilc ! number of soil columns in filter + integer , intent(in) :: filter_soilc(:) ! filter for soil columns + integer , intent(in) :: num_soilp ! number of soil patches in filter + integer , intent(in) :: filter_soilp(:) ! filter for soil patches + type(cnveg_carbonflux_type) , intent(in) :: cnveg_carbonflux_inst + type(cnveg_carbonstate_type) , intent(inout) :: cnveg_carbonstate_inst + type(soilbiogeochem_carbonstate_type) , intent(inout) :: soilbiogeochem_carbonstate_inst + ! + ! !LOCAL VARIABLES: + integer :: c,p,j,k,l ! indices + integer :: fp,fc ! lake filter indices + real(r8):: dt ! radiation time step (seconds) + !----------------------------------------------------------------------- + + associate( & + cf_veg => cnveg_carbonflux_inst , & + cs_veg => cnveg_carbonstate_inst , & + cs_soil => soilbiogeochem_carbonstate_inst & + ) + + ! set time steps + dt = real( get_step_size(), r8 ) + + ! column level carbon fluxes from gross unrepresented landcover change mortality + do j = 1, nlevdecomp + do fc = 1,num_soilc + c = filter_soilc(fc) + + ! column gross unrepresented landcover change fluxes + cs_soil%decomp_cpools_vr_col(c,j,i_met_lit) = & + cs_soil%decomp_cpools_vr_col(c,j,i_met_lit) + cf_veg%gru_c_to_litr_met_c_col(c,j) * dt + cs_soil%decomp_cpools_vr_col(c,j,i_cel_lit) = & + cs_soil%decomp_cpools_vr_col(c,j,i_cel_lit) + cf_veg%gru_c_to_litr_cel_c_col(c,j) * dt + cs_soil%decomp_cpools_vr_col(c,j,i_lig_lit) = & + cs_soil%decomp_cpools_vr_col(c,j,i_lig_lit) + cf_veg%gru_c_to_litr_lig_c_col(c,j) * dt + cs_soil%decomp_cpools_vr_col(c,j,i_cwd) = & + cs_soil%decomp_cpools_vr_col(c,j,i_cwd) + cf_veg%gru_c_to_cwdc_col(c,j) * dt + + ! wood to product pools - states updated in CNProducts + end do + end do + + ! patch loop + do fp = 1,num_soilp + p = filter_soilp(fp) + + ! patch-level carbon fluxes from gross unrepresented landcover change mortality + ! displayed pools + cs_veg%leafc_patch(p) = cs_veg%leafc_patch(p) & + - cf_veg%gru_leafc_to_litter_patch(p) * dt + cs_veg%frootc_patch(p) = cs_veg%frootc_patch(p) & + - cf_veg%gru_frootc_to_litter_patch(p) * dt + cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) & + - cf_veg%gru_livestemc_to_atm_patch(p) * dt + cs_veg%deadstemc_patch(p) = cs_veg%deadstemc_patch(p) & + - cf_veg%gru_deadstemc_to_atm_patch(p) * dt + cs_veg%deadstemc_patch(p) = cs_veg%deadstemc_patch(p) & + - cf_veg%gru_wood_productc_gain_patch(p) * dt + cs_veg%livecrootc_patch(p) = cs_veg%livecrootc_patch(p) & + - cf_veg%gru_livecrootc_to_litter_patch(p) * dt + cs_veg%deadcrootc_patch(p) = cs_veg%deadcrootc_patch(p) & + - cf_veg%gru_deadcrootc_to_litter_patch(p) * dt + + ! xsmrpool + cs_veg%xsmrpool_patch(p) = cs_veg%xsmrpool_patch(p) & + - cf_veg%gru_xsmrpool_to_atm_patch(p) * dt + + ! storage pools + cs_veg%leafc_storage_patch(p) = cs_veg%leafc_storage_patch(p) & + - cf_veg%gru_leafc_storage_to_atm_patch(p) * dt + cs_veg%frootc_storage_patch(p) = cs_veg%frootc_storage_patch(p) & + - cf_veg%gru_frootc_storage_to_atm_patch(p) * dt + cs_veg%livestemc_storage_patch(p) = cs_veg%livestemc_storage_patch(p) & + - cf_veg%gru_livestemc_storage_to_atm_patch(p) * dt + cs_veg%deadstemc_storage_patch(p) = cs_veg%deadstemc_storage_patch(p) & + - cf_veg%gru_deadstemc_storage_to_atm_patch(p) * dt + cs_veg%livecrootc_storage_patch(p) = cs_veg%livecrootc_storage_patch(p) & + - cf_veg%gru_livecrootc_storage_to_atm_patch(p) * dt + cs_veg%deadcrootc_storage_patch(p) = cs_veg%deadcrootc_storage_patch(p) & + - cf_veg%gru_deadcrootc_storage_to_atm_patch(p) * dt + cs_veg%gresp_storage_patch(p) = cs_veg%gresp_storage_patch(p) & + - cf_veg%gru_gresp_storage_to_atm_patch(p) * dt + + ! transfer pools + cs_veg%leafc_xfer_patch(p) = cs_veg%leafc_xfer_patch(p) & + - cf_veg%gru_leafc_xfer_to_atm_patch(p) * dt + cs_veg%frootc_xfer_patch(p) = cs_veg%frootc_xfer_patch(p) & + - cf_veg%gru_frootc_xfer_to_atm_patch(p) * dt + cs_veg%livestemc_xfer_patch(p) = cs_veg%livestemc_xfer_patch(p) & + - cf_veg%gru_livestemc_xfer_to_atm_patch(p) * dt + cs_veg%deadstemc_xfer_patch(p) = cs_veg%deadstemc_xfer_patch(p) & + - cf_veg%gru_deadstemc_xfer_to_atm_patch(p) * dt + cs_veg%livecrootc_xfer_patch(p) = cs_veg%livecrootc_xfer_patch(p) & + - cf_veg%gru_livecrootc_xfer_to_atm_patch(p) * dt + cs_veg%deadcrootc_xfer_patch(p) = cs_veg%deadcrootc_xfer_patch(p) & + - cf_veg%gru_deadcrootc_xfer_to_atm_patch(p) * dt + cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) & + - cf_veg%gru_gresp_xfer_to_atm_patch(p) * dt + + end do ! end of patch loop + + end associate + + end subroutine CStateUpdate2g + end module CNCStateUpdate2Mod diff --git a/src/biogeochem/CNDriverMod.F90 b/src/biogeochem/CNDriverMod.F90 index a8ed44cd97..31804bbf0a 100644 --- a/src/biogeochem/CNDriverMod.F90 +++ b/src/biogeochem/CNDriverMod.F90 @@ -7,7 +7,7 @@ module CNDriverMod ! !USES: use shr_kind_mod , only : r8 => shr_kind_r8 use clm_varctl , only : use_c13, use_c14, use_fates, use_dynroot - use dynSubgridControlMod , only : get_do_harvest + use dynSubgridControlMod , only : get_do_harvest, get_do_grossunrep use decompMod , only : bounds_type use perf_mod , only : t_startf, t_stopf use clm_varctl , only : use_century_decomp, use_nitrif_denitrif, use_nguardrail @@ -111,16 +111,17 @@ subroutine CNDriverNoLeaching(bounds, use CNPhenologyMod , only: CNPhenology use CNGRespMod , only: CNGResp use CNFireMethodMod , only: cnfire_method_type - use CNCIsoFluxMod , only: CIsoFlux1, CIsoFlux2, CIsoFlux2h, CIsoFlux3 + use CNCIsoFluxMod , only: CIsoFlux1, CIsoFlux2, CIsoFlux2h, CIsoFlux2g, CIsoFlux3 use CNC14DecayMod , only: C14Decay use CNCStateUpdate1Mod , only: CStateUpdate1,CStateUpdate0 - use CNCStateUpdate2Mod , only: CStateUpdate2, CStateUpdate2h + use CNCStateUpdate2Mod , only: CStateUpdate2, CStateUpdate2h, CStateUpdate2g use CNCStateUpdate3Mod , only: CStateUpdate3 use CNNStateUpdate1Mod , only: NStateUpdate1 - use CNNStateUpdate2Mod , only: NStateUpdate2, NStateUpdate2h + use CNNStateUpdate2Mod , only: NStateUpdate2, NStateUpdate2h, NStateUpdate2g use CNGapMortalityMod , only: CNGapMortality use CNSharedParamsMod , only: use_fun use dynHarvestMod , only: CNHarvest + use dynGrossUnrepMod , only: CNGrossUnrep use SoilBiogeochemDecompCascadeBGCMod , only: decomp_rate_constants_bgc use SoilBiogeochemDecompCascadeCNMod , only: decomp_rate_constants_cn use SoilBiogeochemCompetitionMod , only: SoilBiogeochemCompetition @@ -690,6 +691,47 @@ subroutine CNDriverNoLeaching(bounds, call NStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, & cnveg_nitrogenflux_inst, cnveg_nitrogenstate_inst, soilbiogeochem_nitrogenstate_inst) + + !-------------------------------------------- + ! Update2g (gross unrepresented landcover change) + !-------------------------------------------- + + ! Set gross unrepresented landcover change mortality routine + if (get_do_grossunrep()) then + call CNGrossUnrep(num_soilc, filter_soilc, num_soilp, filter_soilp, & + soilbiogeochem_state_inst, cnveg_carbonstate_inst, cnveg_nitrogenstate_inst, & + cnveg_carbonflux_inst, cnveg_nitrogenflux_inst) + end if + + if ( use_c13 ) then + call CIsoFlux2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + soilbiogeochem_state_inst, & + cnveg_carbonflux_inst, cnveg_carbonstate_inst, & + c13_cnveg_carbonflux_inst, c13_cnveg_carbonstate_inst, & + isotope='c13') + end if + if ( use_c14 ) then + call CIsoFlux2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + soilbiogeochem_state_inst, & + cnveg_carbonflux_inst, cnveg_carbonstate_inst, & + c14_cnveg_carbonflux_inst, c14_cnveg_carbonstate_inst, & + isotope='c14') + end if + + call CStateUpdate2g( num_soilc, filter_soilc, num_soilp, filter_soilp, & + cnveg_carbonflux_inst, cnveg_carbonstate_inst, soilbiogeochem_carbonstate_inst) + if ( use_c13 ) then + call CStateUpdate2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + c13_cnveg_carbonflux_inst, c13_cnveg_carbonstate_inst, c13_soilbiogeochem_carbonstate_inst) + end if + if ( use_c14 ) then + call CStateUpdate2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + c14_cnveg_carbonflux_inst, c14_cnveg_carbonstate_inst, c14_soilbiogeochem_carbonstate_inst) + end if + + call NStateUpdate2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + cnveg_nitrogenflux_inst, cnveg_nitrogenstate_inst, soilbiogeochem_nitrogenstate_inst) + call t_stopf('CNUpdate2') if ( use_nguardrail ) then @@ -708,6 +750,7 @@ subroutine CNDriverNoLeaching(bounds, call c_products_inst%UpdateProducts(bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch = cnveg_carbonflux_inst%dwt_wood_productc_gain_patch(begp:endp), & + gru_wood_product_gain_patch = cnveg_carbonflux_inst%gru_wood_productc_gain_patch(begp:endp), & wood_harvest_patch = cnveg_carbonflux_inst%wood_harvestc_patch(begp:endp), & dwt_crop_product_gain_patch = cnveg_carbonflux_inst%dwt_crop_productc_gain_patch(begp:endp), & grain_to_cropprod_patch = cnveg_carbonflux_inst%grainc_to_cropprodc_patch(begp:endp)) @@ -717,6 +760,7 @@ subroutine CNDriverNoLeaching(bounds, call c13_products_inst%UpdateProducts(bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch = c13_cnveg_carbonflux_inst%dwt_wood_productc_gain_patch(begp:endp), & + gru_wood_product_gain_patch = c13_cnveg_carbonflux_inst%gru_wood_productc_gain_patch(begp:endp), & wood_harvest_patch = c13_cnveg_carbonflux_inst%wood_harvestc_patch(begp:endp), & dwt_crop_product_gain_patch = c13_cnveg_carbonflux_inst%dwt_crop_productc_gain_patch(begp:endp), & grain_to_cropprod_patch = c13_cnveg_carbonflux_inst%grainc_to_cropprodc_patch(begp:endp)) @@ -726,6 +770,7 @@ subroutine CNDriverNoLeaching(bounds, call c14_products_inst%UpdateProducts(bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch = c14_cnveg_carbonflux_inst%dwt_wood_productc_gain_patch(begp:endp), & + gru_wood_product_gain_patch = c14_cnveg_carbonflux_inst%gru_wood_productc_gain_patch(begp:endp), & wood_harvest_patch = c14_cnveg_carbonflux_inst%wood_harvestc_patch(begp:endp), & dwt_crop_product_gain_patch = c14_cnveg_carbonflux_inst%dwt_crop_productc_gain_patch(begp:endp), & grain_to_cropprod_patch = c14_cnveg_carbonflux_inst%grainc_to_cropprodc_patch(begp:endp)) @@ -734,6 +779,7 @@ subroutine CNDriverNoLeaching(bounds, call n_products_inst%UpdateProducts(bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch = cnveg_nitrogenflux_inst%dwt_wood_productn_gain_patch(begp:endp), & + gru_wood_product_gain_patch = cnveg_nitrogenflux_inst%gru_wood_productn_gain_patch(begp:endp), & wood_harvest_patch = cnveg_nitrogenflux_inst%wood_harvestn_patch(begp:endp), & dwt_crop_product_gain_patch = cnveg_nitrogenflux_inst%dwt_crop_productn_gain_patch(begp:endp), & grain_to_cropprod_patch = cnveg_nitrogenflux_inst%grainn_to_cropprodn_patch(begp:endp)) diff --git a/src/biogeochem/CNNStateUpdate2Mod.F90 b/src/biogeochem/CNNStateUpdate2Mod.F90 index 24070f6890..e3548694c6 100644 --- a/src/biogeochem/CNNStateUpdate2Mod.F90 +++ b/src/biogeochem/CNNStateUpdate2Mod.F90 @@ -20,6 +20,7 @@ module CNNStateUpdate2Mod ! !PUBLIC MEMBER FUNCTIONS: public:: NStateUpdate2 public:: NStateUpdate2h + public:: NStateUpdate2g !----------------------------------------------------------------------- contains @@ -235,4 +236,111 @@ subroutine NStateUpdate2h(num_soilc, filter_soilc, num_soilp, filter_soilp, & end subroutine NStateUpdate2h + !----------------------------------------------------------------------- + subroutine NStateUpdate2g(num_soilc, filter_soilc, num_soilp, filter_soilp, & + cnveg_nitrogenflux_inst, cnveg_nitrogenstate_inst, soilbiogeochem_nitrogenstate_inst) + ! + ! !DESCRIPTION: + ! Update all the prognostic nitrogen state + ! variables affected by gross unrepresented landcover change mortality fluxes + ! NOTE - associate statements have been removed where there are + ! no science equations. This increases readability and maintainability + ! + ! !ARGUMENTS: + integer , intent(in) :: num_soilc ! number of soil columns in filter + integer , intent(in) :: filter_soilc(:) ! filter for soil columns + integer , intent(in) :: num_soilp ! number of soil patches in filter + integer , intent(in) :: filter_soilp(:) ! filter for soil patches + type(cnveg_nitrogenflux_type) , intent(in) :: cnveg_nitrogenflux_inst + type(cnveg_nitrogenstate_type) , intent(inout) :: cnveg_nitrogenstate_inst + type(soilbiogeochem_nitrogenstate_type) , intent(inout) :: soilbiogeochem_nitrogenstate_inst + ! + ! !LOCAL VARIABLES: + integer :: c,p,j,l ! indices + integer :: fp,fc ! lake filter indices + real(r8):: dt ! radiation time step (seconds) + !----------------------------------------------------------------------- + + associate( & + nf_veg => cnveg_nitrogenflux_inst , & + ns_veg => cnveg_nitrogenstate_inst , & + ns_soil => soilbiogeochem_nitrogenstate_inst & + ) + + ! set time steps + dt = real( get_step_size(), r8 ) + + ! column-level nitrogen fluxes from gross unrepresented landcover change mortality + + do j = 1,nlevdecomp + do fc = 1,num_soilc + c = filter_soilc(fc) + ns_soil%decomp_npools_vr_col(c,j,i_met_lit) = & + ns_soil%decomp_npools_vr_col(c,j,i_met_lit) + nf_veg%gru_n_to_litr_met_n_col(c,j) * dt + ns_soil%decomp_npools_vr_col(c,j,i_cel_lit) = & + ns_soil%decomp_npools_vr_col(c,j,i_cel_lit) + nf_veg%gru_n_to_litr_cel_n_col(c,j) * dt + ns_soil%decomp_npools_vr_col(c,j,i_lig_lit) = & + ns_soil%decomp_npools_vr_col(c,j,i_lig_lit) + nf_veg%gru_n_to_litr_lig_n_col(c,j) * dt + ns_soil%decomp_npools_vr_col(c,j,i_cwd) = & + ns_soil%decomp_npools_vr_col(c,j,i_cwd) + nf_veg%gru_n_to_cwdn_col(c,j) * dt + end do + end do + + ! patch-level nitrogen fluxes from gross unrepresented landcover change mortality + + do fp = 1,num_soilp + p = filter_soilp(fp) + + ! displayed pools + ns_veg%leafn_patch(p) = ns_veg%leafn_patch(p) & + - nf_veg%gru_leafn_to_litter_patch(p) * dt + ns_veg%frootn_patch(p) = ns_veg%frootn_patch(p) & + - nf_veg%gru_frootn_to_litter_patch(p) * dt + ns_veg%livestemn_patch(p) = ns_veg%livestemn_patch(p) & + - nf_veg%gru_livestemn_to_atm_patch(p) * dt + ns_veg%deadstemn_patch(p) = ns_veg%deadstemn_patch(p) & + - nf_veg%gru_deadstemn_to_atm_patch(p) * dt + ns_veg%deadstemn_patch(p) = ns_veg%deadstemn_patch(p) & + - nf_veg%gru_wood_productn_gain_patch(p) * dt + ns_veg%livecrootn_patch(p) = ns_veg%livecrootn_patch(p) & + - nf_veg%gru_livecrootn_to_litter_patch(p) * dt + ns_veg%deadcrootn_patch(p) = ns_veg%deadcrootn_patch(p) & + - nf_veg%gru_deadcrootn_to_litter_patch(p) * dt + ns_veg%retransn_patch(p) = ns_veg%retransn_patch(p) & + - nf_veg%gru_retransn_to_litter_patch(p) * dt + + ! storage pools + ns_veg%leafn_storage_patch(p) = ns_veg%leafn_storage_patch(p) & + - nf_veg%gru_leafn_storage_to_atm_patch(p) * dt + ns_veg%frootn_storage_patch(p) = ns_veg%frootn_storage_patch(p) & + - nf_veg%gru_frootn_storage_to_atm_patch(p) * dt + ns_veg%livestemn_storage_patch(p) = ns_veg%livestemn_storage_patch(p) & + - nf_veg%gru_livestemn_storage_to_atm_patch(p) * dt + ns_veg%deadstemn_storage_patch(p) = ns_veg%deadstemn_storage_patch(p) & + - nf_veg%gru_deadstemn_storage_to_atm_patch(p) * dt + ns_veg%livecrootn_storage_patch(p) = ns_veg%livecrootn_storage_patch(p) & + - nf_veg%gru_livecrootn_storage_to_atm_patch(p) * dt + ns_veg%deadcrootn_storage_patch(p) = ns_veg%deadcrootn_storage_patch(p) & + - nf_veg%gru_deadcrootn_storage_to_atm_patch(p) * dt + + ! transfer pools + ns_veg%leafn_xfer_patch(p) = ns_veg%leafn_xfer_patch(p) & + - nf_veg%gru_leafn_xfer_to_atm_patch(p) *dt + ns_veg%frootn_xfer_patch(p) = ns_veg%frootn_xfer_patch(p) & + - nf_veg%gru_frootn_xfer_to_atm_patch(p) *dt + ns_veg%livestemn_xfer_patch(p) = ns_veg%livestemn_xfer_patch(p) & + - nf_veg%gru_livestemn_xfer_to_atm_patch(p) *dt + ns_veg%deadstemn_xfer_patch(p) = ns_veg%deadstemn_xfer_patch(p) & + - nf_veg%gru_deadstemn_xfer_to_atm_patch(p) *dt + ns_veg%livecrootn_xfer_patch(p) = ns_veg%livecrootn_xfer_patch(p) & + - nf_veg%gru_livecrootn_xfer_to_atm_patch(p) *dt + ns_veg%deadcrootn_xfer_patch(p) = ns_veg%deadcrootn_xfer_patch(p) & + - nf_veg%gru_deadcrootn_xfer_to_atm_patch(p) *dt + + end do + + end associate + + end subroutine NStateUpdate2g + end module CNNStateUpdate2Mod diff --git a/src/biogeochem/CNProductsMod.F90 b/src/biogeochem/CNProductsMod.F90 index 59891ba330..046a01d482 100644 --- a/src/biogeochem/CNProductsMod.F90 +++ b/src/biogeochem/CNProductsMod.F90 @@ -43,6 +43,11 @@ module CNProductsMod real(r8), pointer :: dwt_prod100_gain_grc(:) ! (g[C or N]/m2/s) dynamic landcover addition to 100-year wood product pool real(r8), pointer :: dwt_woodprod_gain_grc(:) ! (g[C or N]/m2/s) dynamic landcover addition to wood product pools real(r8), pointer :: dwt_cropprod1_gain_grc(:) ! (g[C or N]/m2/s) dynamic landcover addition to 1-year crop product pool + real(r8), pointer :: gru_prod10_gain_patch(:) ! (g[C or N]/m2/s) gross unrepresented landcover addition to 10-year wood product pool + real(r8), pointer :: gru_prod10_gain_grc(:) ! (g[C or N]/m2/s) gross unrepresented landcover addition to 10-year wood product pool + real(r8), pointer :: gru_prod100_gain_patch(:) ! (g[C or N]/m2/s) gross unrepresented landcover addition to 100-year wood product pool + real(r8), pointer :: gru_prod100_gain_grc(:) ! (g[C or N]/m2/s) gross unrepresented landcover addition to 100-year wood product pool + real(r8), pointer :: gru_woodprod_gain_grc(:) ! (g[C or N]/m2/s) gross unrepresented landcover addition to wood product pools real(r8), pointer :: hrv_deadstem_to_prod10_patch(:) ! (g[C or N]/m2/s) dead stem harvest to 10-year wood product pool real(r8), pointer :: hrv_deadstem_to_prod10_grc(:) ! (g[C or N]/m2/s) dead stem harvest to 10-year wood product pool real(r8), pointer :: hrv_deadstem_to_prod100_patch(:) ! (g[C or N]/m2/s) dead stem harvest to 100-year wood product pool @@ -133,9 +138,14 @@ subroutine InitAllocate(this, bounds) allocate(this%dwt_cropprod1_gain_grc(begg:endg)) ; this%dwt_cropprod1_gain_grc(:) = nan + allocate(this%gru_prod10_gain_patch(begp:endp)) ; this%gru_prod10_gain_patch(:) = nan + allocate(this%gru_prod10_gain_grc(begg:endg)) ; this%gru_prod10_gain_grc(:) = nan + allocate(this%gru_prod100_gain_patch(begp:endp)) ; this%gru_prod100_gain_patch(:) = nan + allocate(this%gru_prod100_gain_grc(begg:endg)) ; this%gru_prod100_gain_grc(:) = nan + allocate(this%gru_woodprod_gain_grc(begg:endg)) ; this%gru_woodprod_gain_grc(:) = nan + allocate(this%hrv_deadstem_to_prod10_patch(begp:endp)) ; this%hrv_deadstem_to_prod10_patch(:) = nan allocate(this%hrv_deadstem_to_prod10_grc(begg:endg)) ; this%hrv_deadstem_to_prod10_grc(:) = nan - allocate(this%hrv_deadstem_to_prod100_patch(begp:endp)) ; this%hrv_deadstem_to_prod100_patch(:) = nan allocate(this%hrv_deadstem_to_prod100_grc(begg:endg)) ; this%hrv_deadstem_to_prod100_grc(:) = nan @@ -233,6 +243,22 @@ subroutine InitHistory(this, bounds) long_name = 'landcover change-driven addition to 1-year crop product pool', & ptr_gcell = this%dwt_cropprod1_gain_grc) + this%gru_prod10_gain_grc(begg:endg) = spval + call hist_addfld1d( & + fname = this%species%hist_fname('GRU_PROD10', suffix='_GAIN'), & + units = 'g' // this%species%get_species() // '/m^2/s', & + avgflag = 'A', & + long_name = 'gross unrepresented landcover change addition to 10-yr wood product pool', & + ptr_gcell = this%gru_prod10_gain_grc, default='inactive') + + this%gru_prod100_gain_grc(begg:endg) = spval + call hist_addfld1d( & + fname = this%species%hist_fname('GRU_PROD100', suffix='_GAIN'), & + units = 'g' // this%species%get_species() // '/m^2/s', & + avgflag = 'A', & + long_name = 'gross unrepresented landcover change addition to 100-yr wood product pool', & + ptr_gcell = this%gru_prod100_gain_grc, default='inactive') + this%cropprod1_loss_grc(begg:endg) = spval call hist_addfld1d( & fname = this%species%hist_fname('CROPPROD1', suffix='_LOSS'), & @@ -291,6 +317,8 @@ subroutine InitCold(this, bounds) do p = bounds%begp, bounds%endp this%hrv_deadstem_to_prod10_patch(p) = 0._r8 this%hrv_deadstem_to_prod100_patch(p) = 0._r8 + this%gru_prod10_gain_patch(p) = 0._r8 + this%gru_prod100_gain_patch(p) = 0._r8 this%grain_to_cropprod1_patch(p) = 0._r8 end do @@ -428,6 +456,7 @@ end subroutine Restart subroutine UpdateProducts(this, bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch, & + gru_wood_product_gain_patch, & wood_harvest_patch, & dwt_crop_product_gain_patch, & grain_to_cropprod_patch) @@ -446,6 +475,9 @@ subroutine UpdateProducts(this, bounds, & ! a patch-level flux, it is expressed per unit GRIDCELL area real(r8), intent(in) :: dwt_wood_product_gain_patch( bounds%begp: ) + ! gross unrepresented landcover addition to wood product pools (g/m2/s) [patch] + real(r8), intent(in) :: gru_wood_product_gain_patch( bounds%begp: ) + ! wood harvest addition to wood product pools (g/m2/s) [patch] real(r8), intent(in) :: wood_harvest_patch( bounds%begp: ) @@ -465,6 +497,7 @@ subroutine UpdateProducts(this, bounds, & !----------------------------------------------------------------------- SHR_ASSERT_ALL((ubound(dwt_wood_product_gain_patch) == (/bounds%endp/)), errMsg(sourcefile, __LINE__)) + SHR_ASSERT_ALL((ubound(gru_wood_product_gain_patch) == (/bounds%endp/)), errMsg(sourcefile, __LINE__)) SHR_ASSERT_ALL((ubound(wood_harvest_patch) == (/bounds%endp/)), errMsg(sourcefile, __LINE__)) SHR_ASSERT_ALL((ubound(dwt_crop_product_gain_patch) == (/bounds%endp/)), errMsg(sourcefile, __LINE__)) SHR_ASSERT_ALL((ubound(grain_to_cropprod_patch) == (/bounds%endp/)), errMsg(sourcefile, __LINE__)) @@ -472,6 +505,7 @@ subroutine UpdateProducts(this, bounds, & call this%PartitionWoodFluxes(bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch(bounds%begp:bounds%endp), & + gru_wood_product_gain_patch(bounds%begp:bounds%endp), & wood_harvest_patch(bounds%begp:bounds%endp)) call this%PartitionGrainFluxes(bounds, & @@ -504,6 +538,10 @@ subroutine UpdateProducts(this, bounds, & this%prod10_grc(g) = this%prod10_grc(g) + this%dwt_prod10_gain_grc(g)*dt this%prod100_grc(g) = this%prod100_grc(g) + this%dwt_prod100_gain_grc(g)*dt + ! fluxes into wood & grain product pools, from gross unrepresented landcover change + this%prod10_grc(g) = this%prod10_grc(g) + this%gru_prod10_gain_grc(g)*dt + this%prod100_grc(g) = this%prod100_grc(g) + this%gru_prod100_gain_grc(g)*dt + ! fluxes into wood & grain product pools, from harvest this%cropprod1_grc(g) = this%cropprod1_grc(g) + this%grain_to_cropprod1_grc(g)*dt this%prod10_grc(g) = this%prod10_grc(g) + this%hrv_deadstem_to_prod10_grc(g)*dt @@ -524,6 +562,7 @@ end subroutine UpdateProducts subroutine PartitionWoodFluxes(this, bounds, & num_soilp, filter_soilp, & dwt_wood_product_gain_patch, & + gru_wood_product_gain_patch, & wood_harvest_patch) ! ! !DESCRIPTION: @@ -543,6 +582,9 @@ subroutine PartitionWoodFluxes(this, bounds, & ! a patch-level flux, it is expressed per unit GRIDCELL area real(r8), intent(in) :: dwt_wood_product_gain_patch( bounds%begp: ) + ! gross unrepresented landcover addition to wood product pools (g/m2/s) [patch] + real(r8), intent(in) :: gru_wood_product_gain_patch( bounds%begp: ) + ! wood harvest addition to wood product pools (g/m2/s) [patch] real(r8), intent(in) :: wood_harvest_patch( bounds%begp: ) @@ -560,6 +602,44 @@ subroutine PartitionWoodFluxes(this, bounds, & character(len=*), parameter :: subname = 'PartitionWoodFluxes' !----------------------------------------------------------------------- + ! Partition patch-level gross unrepresented fluxes to 10 and 100-year product pools + do fp = 1, num_soilp + p = filter_soilp(fp) + + pprod10 = pftcon%pprod10(patch%itype(p)) + pprod100 = pftcon%pprod100(patch%itype(p)) + pprod_tot = pprod10 + pprod100 + if (pprod_tot > 0) then + pprod10_frac = pprod10 / pprod_tot + pprod100_frac = pprod100 / pprod_tot + else + ! Avoid divide by 0 + pprod10_frac = 0._r8 + pprod100_frac = 0._r8 + end if + + this%gru_prod10_gain_patch(p) = & + gru_wood_product_gain_patch(p) * pprod10_frac + this%gru_prod100_gain_patch(p) = & + gru_wood_product_gain_patch(p) * pprod100_frac + + end do + + ! Average gross unrepresented fluxes from patch to gridcell + call p2g(bounds, & + this%gru_prod10_gain_patch(bounds%begp:bounds%endp), & + this%gru_prod10_gain_grc(bounds%begg:bounds%endg), & + p2c_scale_type = 'unity', & + c2l_scale_type = 'unity', & + l2g_scale_type = 'unity') + + call p2g(bounds, & + this%gru_prod100_gain_patch(bounds%begp:bounds%endp), & + this%gru_prod100_gain_grc(bounds%begg:bounds%endg), & + p2c_scale_type = 'unity', & + c2l_scale_type = 'unity', & + l2g_scale_type = 'unity') + ! Partition patch-level harvest fluxes to 10 and 100-year product pools do fp = 1, num_soilp p = filter_soilp(fp) @@ -733,6 +813,10 @@ subroutine ComputeSummaryVars(this, bounds) this%dwt_woodprod_gain_grc(g) = & this%dwt_prod100_gain_grc(g) + & this%dwt_prod10_gain_grc(g) + + this%gru_woodprod_gain_grc(g) = & + this%gru_prod100_gain_grc(g) + & + this%gru_prod10_gain_grc(g) end do end subroutine ComputeSummaryVars diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index 57cee383d5..22f57ed9da 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -263,13 +263,49 @@ module CNVegCarbonFluxType real(r8), pointer :: dwt_conv_cflux_dribbled_grc (:) ! (gC/m2/s) dwt_conv_cflux_grc dribbled evenly throughout the year real(r8), pointer :: dwt_wood_productc_gain_patch (:) ! (gC/m2/s) addition to wood product pools from landcover change; although this is a patch-level flux, it is expressed per unit GRIDCELL area real(r8), pointer :: dwt_crop_productc_gain_patch (:) ! (gC/m2/s) addition to crop product pools from landcover change; although this is a patch-level flux, it is expressed per unit GRIDCELL area - real(r8), pointer :: dwt_slash_cflux_col (:) ! (gC/m2/s) conversion slash flux due to landcover change + real(r8), pointer :: dwt_slash_cflux_patch (:) ! (gC/m2/s) conversion slash flux due to landcover change + real(r8), pointer :: dwt_slash_cflux_grc (:) ! (gC/m2/s) dwt_slash_cflux_patch summed to the gridcell-level real(r8), pointer :: dwt_frootc_to_litr_met_c_col (:,:) ! (gC/m3/s) fine root to litter due to landcover change real(r8), pointer :: dwt_frootc_to_litr_cel_c_col (:,:) ! (gC/m3/s) fine root to litter due to landcover change real(r8), pointer :: dwt_frootc_to_litr_lig_c_col (:,:) ! (gC/m3/s) fine root to litter due to landcover change real(r8), pointer :: dwt_livecrootc_to_cwdc_col (:,:) ! (gC/m3/s) live coarse root to CWD due to landcover change real(r8), pointer :: dwt_deadcrootc_to_cwdc_col (:,:) ! (gC/m3/s) dead coarse root to CWD due to landcover change + ! gross unrepresented landcover fluxes + real(r8), pointer :: gru_leafc_to_litter_patch (:) ! leaf C gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_leafc_storage_to_atm_patch (:) ! leaf C storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_leafc_xfer_to_atm_patch (:) ! leaf C transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_frootc_to_litter_patch (:) ! fine root C gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_frootc_storage_to_atm_patch (:) ! fine root C storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_frootc_xfer_to_atm_patch (:) ! fine root C transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_livestemc_to_atm_patch (:) ! live stem C gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_livestemc_storage_to_atm_patch (:) ! live stem C storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_livestemc_xfer_to_atm_patch (:) ! live stem C transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_deadstemc_to_atm_patch (:) ! dead stem C gross unrepresented landcover change mortality to the atmosphere (gC/m2/s) + real(r8), pointer :: gru_deadstemc_storage_to_atm_patch (:) ! dead stem C storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_deadstemc_xfer_to_atm_patch (:) ! dead stem C transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_livecrootc_to_litter_patch (:) ! live coarse root C gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_livecrootc_storage_to_atm_patch (:) ! live coarse root C storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_livecrootc_xfer_to_atm_patch (:) ! live coarse root C transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_deadcrootc_to_litter_patch (:) ! dead coarse root C gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_deadcrootc_storage_to_atm_patch (:) ! dead coarse root C storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_deadcrootc_xfer_to_atm_patch (:) ! dead coarse root C transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_gresp_storage_to_atm_patch (:) ! growth respiration storage gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_gresp_xfer_to_atm_patch (:) ! growth respiration transfer gross unrepresented landcover change mortality (gC/m2/s) + real(r8), pointer :: gru_xsmrpool_to_atm_patch (:) ! excess MR pool gross unrepresented landcover change mortality (gC/m2/s) + + real(r8), pointer :: gru_conv_cflux_patch (:) ! (gC/m2/s) conversion C flux (immediate loss to atm) + real(r8), pointer :: gru_conv_cflux_col (:) ! (gC/m2/s) gru_conv_cflux_patch summed to the column-level + real(r8), pointer :: gru_conv_cflux_grc (:) ! (gC/m2/s) gru_conv_cflux_patch summed to the gridcell-level + real(r8), pointer :: gru_conv_cflux_dribbled_grc (:) ! (gC/m2/s) gru_conv_cflux_grc dribbled evenly throughout the year + real(r8), pointer :: gru_wood_productc_gain_patch (:) ! (gC/m2/s) addition to wood product pools from gross unrepresented landcover change + real(r8), pointer :: gru_wood_productc_gain_col (:) ! (gC/m2/s) gru_wood_productc_gain_patch summed to the column-level + real(r8), pointer :: gru_slash_cflux_patch (:) ! (gC/m2/s) conversion slash flux due to gross unrepresented landcover change + real(r8), pointer :: gru_c_to_litr_met_c_col (:,:) ! (gC/m3/s) C to litter due to gross unrepresented landcover change + real(r8), pointer :: gru_c_to_litr_cel_c_col (:,:) ! (gC/m3/s) C to litter due to gross unrepresented landcover change + real(r8), pointer :: gru_c_to_litr_lig_c_col (:,:) ! (gC/m3/s) C to litter due to gross unrepresented landcover change + real(r8), pointer :: gru_c_to_cwdc_col (:,:) ! (gC/m3/s) C to CWD due to gross unrepresented landcover change + ! crop fluxes real(r8), pointer :: crop_seedc_to_leaf_patch (:) ! (gC/m2/s) seed source to leaf, for crops @@ -340,7 +376,7 @@ module CNVegCarbonFluxType real(r8), pointer :: nee_grc (:) ! (gC/m2/s) net ecosystem exchange of carbon, includes fire and hrv_xsmrpool, excludes landuse and harvest flux, positive for source ! Dynamic landcover fluxnes - real(r8), pointer :: landuseflux_grc(:) ! (gC/m2/s) dwt_conv_cflux+product_closs + real(r8), pointer :: landuseflux_grc(:) ! (gC/m2/s) dwt_conv_cflux+gru_conv_cflux+product_closs real(r8), pointer :: npp_Nactive_patch (:) ! C used by mycorrhizal uptake (gC/m2/s) real(r8), pointer :: npp_burnedoff_patch (:) ! C that cannot be used for N uptake (gC/m2/s) real(r8), pointer :: npp_Nnonmyc_patch (:) ! C used by non-myc uptake (gC/m2/s) @@ -366,6 +402,7 @@ module CNVegCarbonFluxType ! Objects that help convert once-per-year dynamic land cover changes into fluxes ! that are dribbled throughout the year type(annual_flux_dribbler_type) :: dwt_conv_cflux_dribbler + type(annual_flux_dribbler_type) :: gru_conv_cflux_dribbler type(annual_flux_dribbler_type) :: hrv_xsmrpool_to_atm_dribbler contains @@ -378,6 +415,7 @@ module CNVegCarbonFluxType procedure , private :: RestartAllIsotopes ! Handle restart fields present for both bulk C and isotopes procedure , public :: SetValues procedure , public :: ZeroDWT + procedure , public :: ZeroGRU procedure , public :: Summary => Summary_carbonflux end type cnveg_carbonflux_type @@ -629,7 +667,8 @@ subroutine InitAllocate(this, bounds, carbon_type) allocate(this%harvest_c_to_litr_lig_c_col (begc:endc,1:nlevdecomp_full)); this%harvest_c_to_litr_lig_c_col (:,:)=nan allocate(this%harvest_c_to_cwdc_col (begc:endc,1:nlevdecomp_full)); this%harvest_c_to_cwdc_col (:,:)=nan - allocate(this%dwt_slash_cflux_col (begc:endc)) ; this%dwt_slash_cflux_col (:) =nan + allocate(this%dwt_slash_cflux_patch (begp:endp)) ; this%dwt_slash_cflux_patch (:) =nan + allocate(this%dwt_slash_cflux_grc (begg:endg)) ; this%dwt_slash_cflux_grc (:) =nan allocate(this%dwt_frootc_to_litr_met_c_col (begc:endc,1:nlevdecomp_full)); this%dwt_frootc_to_litr_met_c_col (:,:)=nan allocate(this%dwt_frootc_to_litr_cel_c_col (begc:endc,1:nlevdecomp_full)); this%dwt_frootc_to_litr_cel_c_col (:,:)=nan allocate(this%dwt_frootc_to_litr_lig_c_col (begc:endc,1:nlevdecomp_full)); this%dwt_frootc_to_litr_lig_c_col (:,:)=nan @@ -646,6 +685,40 @@ subroutine InitAllocate(this, bounds, carbon_type) allocate(this%dwt_wood_productc_gain_patch (begp:endp)) ; this%dwt_wood_productc_gain_patch(:) =nan allocate(this%dwt_crop_productc_gain_patch (begp:endp)) ; this%dwt_crop_productc_gain_patch(:) =nan + allocate(this%gru_leafc_to_litter_patch (begp:endp)) ; this%gru_leafc_to_litter_patch (:) = nan + allocate(this%gru_leafc_storage_to_atm_patch (begp:endp)) ; this%gru_leafc_storage_to_atm_patch (:) = nan + allocate(this%gru_leafc_xfer_to_atm_patch (begp:endp)) ; this%gru_leafc_xfer_to_atm_patch (:) = nan + allocate(this%gru_frootc_to_litter_patch (begp:endp)) ; this%gru_frootc_to_litter_patch (:) = nan + allocate(this%gru_frootc_storage_to_atm_patch (begp:endp)) ; this%gru_frootc_storage_to_atm_patch (:) = nan + allocate(this%gru_frootc_xfer_to_atm_patch (begp:endp)) ; this%gru_frootc_xfer_to_atm_patch (:) = nan + allocate(this%gru_livestemc_to_atm_patch (begp:endp)) ; this%gru_livestemc_to_atm_patch (:) = nan + allocate(this%gru_livestemc_storage_to_atm_patch (begp:endp)) ; this%gru_livestemc_storage_to_atm_patch (:) = nan + allocate(this%gru_livestemc_xfer_to_atm_patch (begp:endp)) ; this%gru_livestemc_xfer_to_atm_patch (:) = nan + allocate(this%gru_deadstemc_to_atm_patch (begp:endp)) ; this%gru_deadstemc_to_atm_patch (:) = nan + allocate(this%gru_deadstemc_storage_to_atm_patch (begp:endp)) ; this%gru_deadstemc_storage_to_atm_patch (:) = nan + allocate(this%gru_deadstemc_xfer_to_atm_patch (begp:endp)) ; this%gru_deadstemc_xfer_to_atm_patch (:) = nan + allocate(this%gru_livecrootc_to_litter_patch (begp:endp)) ; this%gru_livecrootc_to_litter_patch (:) = nan + allocate(this%gru_livecrootc_storage_to_atm_patch (begp:endp)) ; this%gru_livecrootc_storage_to_atm_patch (:) = nan + allocate(this%gru_livecrootc_xfer_to_atm_patch (begp:endp)) ; this%gru_livecrootc_xfer_to_atm_patch (:) = nan + allocate(this%gru_deadcrootc_to_litter_patch (begp:endp)) ; this%gru_deadcrootc_to_litter_patch (:) = nan + allocate(this%gru_deadcrootc_storage_to_atm_patch (begp:endp)) ; this%gru_deadcrootc_storage_to_atm_patch (:) = nan + allocate(this%gru_deadcrootc_xfer_to_atm_patch (begp:endp)) ; this%gru_deadcrootc_xfer_to_atm_patch (:) = nan + allocate(this%gru_gresp_storage_to_atm_patch (begp:endp)) ; this%gru_gresp_storage_to_atm_patch (:) = nan + allocate(this%gru_gresp_xfer_to_atm_patch (begp:endp)) ; this%gru_gresp_xfer_to_atm_patch (:) = nan + allocate(this%gru_xsmrpool_to_atm_patch (begp:endp)) ; this%gru_xsmrpool_to_atm_patch (:) = nan + + allocate(this%gru_conv_cflux_patch (begp:endp)) ; this%gru_conv_cflux_patch (:) =nan + allocate(this%gru_conv_cflux_col (begc:endc)) ; this%gru_conv_cflux_col (:) =nan + allocate(this%gru_conv_cflux_grc (begg:endg)) ; this%gru_conv_cflux_grc (:) =nan + allocate(this%gru_conv_cflux_dribbled_grc (begg:endg)) ; this%gru_conv_cflux_dribbled_grc (:) =nan + allocate(this%gru_wood_productc_gain_patch (begp:endp)) ; this%gru_wood_productc_gain_patch (:) =nan + allocate(this%gru_wood_productc_gain_col (begc:endc)) ; this%gru_wood_productc_gain_col (:) =nan + allocate(this%gru_slash_cflux_patch (begp:endp)) ; this%gru_slash_cflux_patch (:) =nan + allocate(this%gru_c_to_litr_met_c_col (begc:endc,1:nlevdecomp_full)); this%gru_c_to_litr_met_c_col (:,:)=nan + allocate(this%gru_c_to_litr_cel_c_col (begc:endc,1:nlevdecomp_full)); this%gru_c_to_litr_cel_c_col (:,:)=nan + allocate(this%gru_c_to_litr_lig_c_col (begc:endc,1:nlevdecomp_full)); this%gru_c_to_litr_lig_c_col (:,:)=nan + allocate(this%gru_c_to_cwdc_col (begc:endc,1:nlevdecomp_full)); this%gru_c_to_cwdc_col (:,:)=nan + allocate(this%crop_seedc_to_leaf_patch (begp:endp)) ; this%crop_seedc_to_leaf_patch (:) =nan allocate(this%cwdc_hr_col (begc:endc)) ; this%cwdc_hr_col (:) =nan @@ -763,6 +836,11 @@ subroutine InitAllocate(this, bounds, carbon_type) name = 'dwt_conv_flux_' // carbon_type_suffix, & units = 'gC/m^2', & allows_non_annual_delta = allows_non_annual_delta) + this%gru_conv_cflux_dribbler = annual_flux_dribbler_gridcell( & + bounds = bounds, & + name = 'gru_conv_flux_' // carbon_type_suffix, & + units = 'gC/m^2', & + allows_non_annual_delta = allows_non_annual_delta) this%hrv_xsmrpool_to_atm_dribbler = annual_flux_dribbler_gridcell( & bounds = bounds, & name = 'hrv_xsmrpool_to_atm_' // carbon_type_suffix, & @@ -2878,10 +2956,19 @@ subroutine InitHistory(this, bounds, carbon_type) '(per-area-gridcell; only makes sense with dov2xy=.false.)', & ptr_patch=this%dwt_wood_productc_gain_patch, default='inactive') - this%dwt_slash_cflux_col(begc:endc) = spval - call hist_addfld1d (fname='DWT_SLASH_CFLUX', units='gC/m^2/s', & - avgflag='A', long_name='slash C flux to litter and CWD due to land use', & - ptr_col=this%dwt_slash_cflux_col) + this%dwt_slash_cflux_grc(begg:endg) = spval + call hist_addfld1d (fname='DWT_SLASH_CFLUX', units='gC/m^2/s', & + avgflag='A', & + long_name='slash C flux (to litter diagnostic only) (0 at all times except first timestep of year)', & + ptr_gcell=this%dwt_slash_cflux_grc) + + this%dwt_slash_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='DWT_SLASH_CFLUX_PATCH', units='gC/m^2/s', & + avgflag='A', & + long_name='patch-level slash C flux (to litter diagnostic only) ' // & + '(0 at all times except first timestep of year) ' // & + '(per-area-gridcell; only makes sense with dov2xy=.false.)', & + ptr_patch=this%dwt_slash_cflux_patch, default='inactive') this%dwt_frootc_to_litr_met_c_col(begc:endc,:) = spval call hist_addfld_decomp (fname='DWT_FROOTC_TO_LITR_MET_C', units='gC/m^2/s', type2d='levdcmp', & @@ -2908,6 +2995,27 @@ subroutine InitHistory(this, bounds, carbon_type) avgflag='A', long_name='dead coarse root to CWD due to landcover change', & ptr_col=this%dwt_deadcrootc_to_cwdc_col, default='inactive') + this%gru_conv_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_CONV_CFLUX', units='gC/m^2/s', & + avgflag='A', long_name='gross unrepresented conversion C flux (immediate loss to atm) (0 at all times except first timestep of year)', & + ptr_patch=this%gru_conv_cflux_patch) + + this%gru_conv_cflux_dribbled_grc(begg:endg) = spval + call hist_addfld1d (fname='GRU_CONV_CFLUX_DRIBBLED', units='gC/m^2/s', & + avgflag='A', & + long_name='gross unrepresented conversion C flux (immediate loss to atm), dribbled throughout the year', & + ptr_gcell=this%gru_conv_cflux_dribbled_grc) + + this%gru_wood_productc_gain_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_WOODPRODC_GAIN', units='gC/m^2/s', & + avgflag='A', long_name='gross unrepresented landcover change driven addition to wood product carbon pools (0 at all times except first timestep of year)', & + ptr_patch=this%gru_wood_productc_gain_patch) + + this%gru_slash_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_SLASH_CFLUX', units='gC/m^2/s', & + avgflag='A', long_name='slash gross unrepresented landcover change carbon (to litter)', & + ptr_patch=this%gru_slash_cflux_patch) + this%crop_seedc_to_leaf_patch(begp:endp) = spval call hist_addfld1d (fname='CROP_SEEDC_TO_LEAF', units='gC/m^2/s', & avgflag='A', long_name='crop seed source to leaf', & @@ -3052,10 +3160,19 @@ subroutine InitHistory(this, bounds, carbon_type) long_name='C13 conversion C flux (immediate loss to atm), dribbled throughout the year', & ptr_gcell=this%dwt_conv_cflux_dribbled_grc) - this%dwt_slash_cflux_col(begc:endc) = spval - call hist_addfld1d (fname='C13_DWT_SLASH_CFLUX', units='gC/m^2/s', & - avgflag='A', long_name='C13 slash C flux to litter and CWD due to land use', & - ptr_col=this%dwt_slash_cflux_col) + this%dwt_slash_cflux_grc(begg:endg) = spval + call hist_addfld1d (fname='C13_DWT_SLASH_CFLUX', units='gC13/m^2/s', & + avgflag='A', long_name='C13 slash C flux (to litter diagnostic only)' // & + '(0 at all times except first timestep of year)', & + ptr_gcell=this%dwt_slash_cflux_grc) + + this%dwt_slash_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='C13_DWT_SLASH_CFLUX_PATCH', units='gC13/m^2/s', & + avgflag='A', & + long_name='patch-level C13 slash C flux (to litter diagnostic only) ' // & + '(0 at all times except first timestep of year) ' // & + '(per-area-gridcell; only makes sense with dov2xy=.false.)', & + ptr_patch=this%dwt_slash_cflux_patch, default='inactive') this%dwt_frootc_to_litr_met_c_col(begc:endc,:) = spval call hist_addfld_decomp (fname='C13_DWT_FROOTC_TO_LITR_MET_C', units='gC13/m^2/s', type2d='levdcmp', & @@ -3201,10 +3318,19 @@ subroutine InitHistory(this, bounds, carbon_type) long_name='C14 conversion C flux (immediate loss to atm), dribbled throughout the year', & ptr_gcell=this%dwt_conv_cflux_dribbled_grc) - this%dwt_slash_cflux_col(begc:endc) = spval - call hist_addfld1d (fname='C14_DWT_SLASH_CFLUX', units='gC/m^2/s', & - avgflag='A', long_name='C14 slash C flux to litter and CWD due to land use', & - ptr_col=this%dwt_slash_cflux_col) + this%dwt_slash_cflux_grc(begg:endg) = spval + call hist_addfld1d (fname='C14_DWT_SLASH_CFLUX', units='gC14/m^2/s', & + avgflag='A', long_name='C14 slash C flux (to litter diagnostic only)' // & + '(0 at all times except first timestep of year)', & + ptr_gcell=this%dwt_slash_cflux_grc) + + this%dwt_slash_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='C14_DWT_SLASH_CFLUX_PATCH', units='gC14/m^2/s', & + avgflag='A', & + long_name='patch-level C14 slash C flux (to litter diagnostic only)' // & + '(0 at all times except first timestep of year) ' // & + '(per-area-gridcell; only makes sense with dov2xy=.false.)', & + ptr_patch=this%dwt_slash_cflux_patch, default='inactive') this%dwt_frootc_to_litr_met_c_col(begc:endc,:) = spval call hist_addfld_decomp (fname='C14_DWT_FROOTC_TO_LITR_MET_C', units='gC14/m^2/s', type2d='levdcmp', & @@ -3352,7 +3478,6 @@ subroutine InitCold(this, bounds) ! also initialize dynamic landcover fluxes so that they have ! real values on first timestep, prior to calling pftdyn_cnbal if (lun%itype(l) == istsoil .or. lun%itype(l) == istcrop) then - this%dwt_slash_cflux_col(c) = 0._r8 do j = 1, nlevdecomp_full this%dwt_frootc_to_litr_met_c_col(c,j) = 0._r8 this%dwt_frootc_to_litr_cel_c_col(c,j) = 0._r8 @@ -3363,6 +3488,21 @@ subroutine InitCold(this, bounds) end if end do + do c = bounds%begc, bounds%endc + l = col%landunit(c) + + ! also initialize gross unrepresented landcover fluxes so that they have + ! real values on first timestep, prior to calling pftdyn_cnbal + if (lun%itype(l) == istsoil .or. lun%itype(l) == istcrop) then + do j = 1, nlevdecomp_full + this%gru_c_to_litr_met_c_col(c,j) = 0._r8 + this%gru_c_to_litr_cel_c_col(c,j) = 0._r8 + this%gru_c_to_litr_lig_c_col(c,j) = 0._r8 + this%gru_c_to_cwdc_col(c,j) = 0._r8 + end do + end if + end do + do p = bounds%begp,bounds%endp l = patch%landunit(p) @@ -3596,6 +3736,7 @@ subroutine RestartAllIsotopes ( this, bounds, ncid, flag ) !----------------------------------------------------------------------- call this%dwt_conv_cflux_dribbler%Restart(bounds, ncid, flag) + call this%gru_conv_cflux_dribbler%Restart(bounds, ncid, flag) call this%hrv_xsmrpool_to_atm_dribbler%Restart(bounds, ncid, flag) end subroutine RestartAllIsotopes @@ -3666,6 +3807,32 @@ subroutine SetValues ( this, & this%hrv_gresp_xfer_to_litter_patch(i) = value_patch this%hrv_xsmrpool_to_atm_patch(i) = value_patch + this%gru_leafc_to_litter_patch(i) = value_patch + this%gru_leafc_storage_to_atm_patch(i) = value_patch + this%gru_leafc_xfer_to_atm_patch(i) = value_patch + this%gru_frootc_to_litter_patch(i) = value_patch + this%gru_frootc_storage_to_atm_patch(i) = value_patch + this%gru_frootc_xfer_to_atm_patch(i) = value_patch + this%gru_livestemc_to_atm_patch(i) = value_patch + this%gru_livestemc_storage_to_atm_patch(i) = value_patch + this%gru_livestemc_xfer_to_atm_patch(i) = value_patch + this%gru_deadstemc_to_atm_patch(i) = value_patch + this%gru_deadstemc_storage_to_atm_patch(i) = value_patch + this%gru_deadstemc_xfer_to_atm_patch(i) = value_patch + this%gru_livecrootc_to_litter_patch(i) = value_patch + this%gru_livecrootc_storage_to_atm_patch(i) = value_patch + this%gru_livecrootc_xfer_to_atm_patch(i) = value_patch + this%gru_deadcrootc_to_litter_patch(i) = value_patch + this%gru_deadcrootc_storage_to_atm_patch(i) = value_patch + this%gru_deadcrootc_xfer_to_atm_patch(i) = value_patch + this%gru_gresp_storage_to_atm_patch(i) = value_patch + this%gru_gresp_xfer_to_atm_patch(i) = value_patch + this%gru_xsmrpool_to_atm_patch(i) = value_patch + + this%gru_conv_cflux_patch(i) = value_patch + this%gru_wood_productc_gain_patch(i) = value_patch + this%gru_slash_cflux_patch(i) = value_patch + this%m_leafc_to_fire_patch(i) = value_patch this%m_leafc_storage_to_fire_patch(i) = value_patch this%m_leafc_xfer_to_fire_patch(i) = value_patch @@ -3840,6 +4007,11 @@ subroutine SetValues ( this, & this%harvest_c_to_litr_lig_c_col(i,j) = value_column this%harvest_c_to_cwdc_col(i,j) = value_column + this%gru_c_to_litr_met_c_col(i,j) = value_column + this%gru_c_to_litr_cel_c_col(i,j) = value_column + this%gru_c_to_litr_lig_c_col(i,j) = value_column + this%gru_c_to_cwdc_col(i,j) = value_column + end do end do @@ -3924,6 +4096,8 @@ subroutine SetValues ( this, & this%fire_closs_col(i) = value_column this%wood_harvestc_col(i) = value_column this%hrv_xsmrpool_to_atm_col(i) = value_column + this%gru_conv_cflux_col(i) = value_column + this%gru_wood_productc_gain_col(i) = value_column this%nep_col(i) = value_column @@ -3951,10 +4125,7 @@ subroutine ZeroDwt( this, bounds ) this%dwt_seedc_to_leaf_grc(g) = 0._r8 this%dwt_seedc_to_deadstem_grc(g) = 0._r8 this%dwt_conv_cflux_grc(g) = 0._r8 - end do - - do c = bounds%begc,bounds%endc - this%dwt_slash_cflux_col(c) = 0._r8 + this%dwt_slash_cflux_grc(g) = 0._r8 end do do j = 1, nlevdecomp_full @@ -3969,6 +4140,28 @@ subroutine ZeroDwt( this, bounds ) end subroutine ZeroDwt + !----------------------------------------------------------------------- + subroutine ZeroGru( this, bounds ) + ! + ! !DESCRIPTION + ! Initialize flux variables needed for dynamic land use. + ! + ! !ARGUMENTS: + class(cnveg_carbonflux_type) :: this + type(bounds_type), intent(in) :: bounds + ! + ! !LOCAL VARIABLES: + integer :: c, g, j ! indices + !----------------------------------------------------------------------- + + ! set conversion and product pool fluxes to 0 at the beginning of every timestep + + do g = bounds%begg, bounds%endg + this%gru_conv_cflux_grc(g) = 0._r8 + end do + + end subroutine ZeroGru + !----------------------------------------------------------------------- subroutine Summary_carbonflux(this, & bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & @@ -4011,6 +4204,7 @@ subroutine Summary_carbonflux(this, & real(r8) :: hrv_xsmrpool_to_atm_delta_grc(bounds%begg:bounds%endg) ! hrv_xsmrpool_to_atm_col averaged to gridcell, expressed as a delta (not a flux) (gC/m2) real(r8) :: hrv_xsmrpool_to_atm_dribbled_grc(bounds%begg:bounds%endg) ! hrv_xsmrpool_to_atm, dribbled over the year (gC/m2/s) real(r8) :: dwt_conv_cflux_delta_grc(bounds%begg:bounds%endg) ! dwt_conv_cflux_grc expressed as a total delta (not a flux) (gC/m2) + real(r8) :: gru_conv_cflux_delta_grc(bounds%begg:bounds%endg) ! gru_conv_cflux_grc expressed as a total delta (not a flux) (gC/m2) !----------------------------------------------------------------------- SHR_ASSERT_ALL((ubound(product_closs_grc) == (/bounds%endg/)), errMsg(sourcefile, __LINE__)) @@ -4243,7 +4437,12 @@ subroutine Summary_carbonflux(this, & this%hrv_deadcrootc_storage_to_litter_patch(p) + & this%hrv_deadcrootc_xfer_to_litter_patch(p) + & this%hrv_gresp_storage_to_litter_patch(p) + & - this%hrv_gresp_xfer_to_litter_patch(p) + this%hrv_gresp_xfer_to_litter_patch(p) + & + + this%gru_leafc_to_litter_patch(p) + & + this%gru_frootc_to_litter_patch(p) + & + this%gru_livecrootc_to_litter_patch(p) + & + this%gru_deadcrootc_to_litter_patch(p) if ( use_crop .and. patch%itype(p) >= npcropmin )then this%litfall_patch(p) = & @@ -4316,6 +4515,7 @@ subroutine Summary_carbonflux(this, & this%m_leafc_to_fire_patch(p) + & this%m_leafc_to_litter_fire_patch(p) + & this%hrv_leafc_to_litter_patch(p) + & + this%gru_leafc_to_litter_patch(p) + & this%leafc_to_litter_patch(p) ! (WOODC_ALLOC) - wood C allocation @@ -4350,7 +4550,20 @@ subroutine Summary_carbonflux(this, & this%hrv_livecrootc_xfer_to_litter_patch(p) + & this%hrv_deadcrootc_to_litter_patch(p) + & this%hrv_deadcrootc_storage_to_litter_patch(p) + & - this%hrv_deadcrootc_xfer_to_litter_patch(p) + this%hrv_deadcrootc_xfer_to_litter_patch(p) + & + this%gru_livestemc_to_atm_patch(p) + & + this%gru_livestemc_storage_to_atm_patch(p) + & + this%gru_livestemc_xfer_to_atm_patch(p) + & + this%gru_deadstemc_to_atm_patch(p) + & + this%gru_wood_productc_gain_patch(p) + & + this%gru_deadstemc_storage_to_atm_patch(p) + & + this%gru_deadstemc_xfer_to_atm_patch(p) + & + this%gru_livecrootc_to_litter_patch(p) + & + this%gru_livecrootc_storage_to_atm_patch(p) + & + this%gru_livecrootc_xfer_to_atm_patch(p) + & + this%gru_deadcrootc_to_litter_patch(p) + & + this%gru_deadcrootc_storage_to_atm_patch(p) + & + this%gru_deadcrootc_xfer_to_atm_patch(p) ! (Slash Harvest Flux) - Additional Wood Harvest Veg C Losses this%slash_harvestc_patch(p) = & @@ -4371,10 +4584,36 @@ subroutine Summary_carbonflux(this, & this%hrv_deadcrootc_to_litter_patch(p) + & this%hrv_deadcrootc_storage_to_litter_patch(p) + & this%hrv_deadcrootc_xfer_to_litter_patch(p) + & - this%hrv_xsmrpool_to_atm_patch(p) + & this%hrv_gresp_storage_to_litter_patch(p) + & this%hrv_gresp_xfer_to_litter_patch(p) + ! (Gross Unrepresented Landcover Change Conversion Flux) - Direct Veg C Loss to Atmosphere + this%gru_conv_cflux_patch(p) = & + this%gru_livestemc_to_atm_patch(p) + & + this%gru_deadstemc_to_atm_patch(p) + & + this%gru_xsmrpool_to_atm_patch(p) + & + this%gru_leafc_storage_to_atm_patch(p) + & + this%gru_frootc_storage_to_atm_patch(p) + & + this%gru_livestemc_storage_to_atm_patch(p) + & + this%gru_deadstemc_storage_to_atm_patch(p) + & + this%gru_livecrootc_storage_to_atm_patch(p) + & + this%gru_deadcrootc_storage_to_atm_patch(p) + & + this%gru_gresp_storage_to_atm_patch(p) + & + this%gru_leafc_xfer_to_atm_patch(p) + & + this%gru_frootc_xfer_to_atm_patch(p) + & + this%gru_livestemc_xfer_to_atm_patch(p) + & + this%gru_deadstemc_xfer_to_atm_patch(p) + & + this%gru_livecrootc_xfer_to_atm_patch(p) + & + this%gru_deadcrootc_xfer_to_atm_patch(p) + & + this%gru_gresp_xfer_to_atm_patch(p) + + ! (Gross Unrepresented Landcover Change Slash Flux) - Direct Veg C Loss to Atmosphere + this%gru_slash_cflux_patch(p) = & + this%gru_leafc_to_litter_patch(p) + & + this%gru_frootc_to_litter_patch(p) + & + this%gru_livecrootc_to_litter_patch(p) + & + this%gru_deadcrootc_to_litter_patch(p) + end do ! end of patches loop !------------------------------------------------ @@ -4387,6 +4626,10 @@ subroutine Summary_carbonflux(this, & this%hrv_xsmrpool_to_atm_patch(bounds%begp:bounds%endp), & this%hrv_xsmrpool_to_atm_col(bounds%begc:bounds%endc)) + call p2c(bounds, num_soilc, filter_soilc, & + this%gru_conv_cflux_patch(bounds%begp:bounds%endp), & + this%gru_conv_cflux_col(bounds%begc:bounds%endc)) + call p2c(bounds, num_soilc, filter_soilc, & this%fire_closs_patch(bounds%begp:bounds%endp), & this%fire_closs_p2c_col(bounds%begc:bounds%endc)) @@ -4515,6 +4758,18 @@ subroutine Summary_carbonflux(this, & call this%dwt_conv_cflux_dribbler%get_curr_flux(bounds, & this%dwt_conv_cflux_dribbled_grc(bounds%begg:bounds%endg)) + call c2g( bounds = bounds, & + carr = this%gru_conv_cflux_col(bounds%begc:bounds%endc), & + garr = this%gru_conv_cflux_grc(bounds%begg:bounds%endg), & + c2l_scale_type = 'unity', & + l2g_scale_type = 'unity') + gru_conv_cflux_delta_grc(bounds%begg:bounds%endg) = & + this%gru_conv_cflux_grc(bounds%begg:bounds%endg) * dtime + call this%gru_conv_cflux_dribbler%set_curr_delta(bounds, & + gru_conv_cflux_delta_grc(bounds%begg:bounds%endg)) + call this%gru_conv_cflux_dribbler%get_curr_flux(bounds, & + this%gru_conv_cflux_dribbled_grc(bounds%begg:bounds%endg)) + do g = bounds%begg, bounds%endg ! net ecosystem exchange of carbon, includes fire flux and hrv_xsmrpool flux, ! positive for source (NEE) @@ -4525,6 +4780,7 @@ subroutine Summary_carbonflux(this, & this%landuseflux_grc(g) = & this%dwt_conv_cflux_dribbled_grc(g) + & + this%gru_conv_cflux_dribbled_grc(g) + & product_closs_grc(g) ! net biome production of carbon, positive for sink diff --git a/src/biogeochem/CNVegNitrogenFluxType.F90 b/src/biogeochem/CNVegNitrogenFluxType.F90 index 1bb630c687..1bb3e223f9 100644 --- a/src/biogeochem/CNVegNitrogenFluxType.F90 +++ b/src/biogeochem/CNVegNitrogenFluxType.F90 @@ -205,6 +205,37 @@ module CNVegNitrogenFluxType real(r8), pointer :: dwt_livecrootn_to_cwdn_col (:,:) ! col (gN/m3/s) live coarse root to CWD due to landcover change real(r8), pointer :: dwt_deadcrootn_to_cwdn_col (:,:) ! col (gN/m3/s) dead coarse root to CWD due to landcover change + ! gross unrepresented landcover fluxes + real(r8), pointer :: gru_leafn_to_litter_patch (:) ! patch leaf N gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_leafn_storage_to_atm_patch (:) ! patch leaf N storage gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_leafn_xfer_to_atm_patch (:) ! patch leaf N transfer gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_frootn_to_litter_patch (:) ! patch fine root N gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_frootn_storage_to_atm_patch (:) ! patch fine root N storage gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_frootn_xfer_to_atm_patch (:) ! patch fine root N transfer gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_livestemn_to_atm_patch (:) ! patch live stem N gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_livestemn_storage_to_atm_patch (:) ! patch live stem N storage gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_livestemn_xfer_to_atm_patch (:) ! patch live stem N transfer gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_deadstemn_to_atm_patch (:) ! patch dead stem N gross unrepresented landcover change mortality to the atmosphere (gC/m2/s) + real(r8), pointer :: gru_deadstemn_storage_to_atm_patch (:) ! patch dead stem N storage gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_deadstemn_xfer_to_atm_patch (:) ! patch dead stem N transfer gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_livecrootn_to_litter_patch (:) ! patch live coarse root N gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_livecrootn_storage_to_atm_patch (:) ! patch live coarse root N storage gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_livecrootn_xfer_to_atm_patch (:) ! patch live coarse root N transfer gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_deadcrootn_to_litter_patch (:) ! patch dead coarse root N gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_deadcrootn_storage_to_atm_patch (:) ! patch dead coarse root N storage gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_deadcrootn_xfer_to_atm_patch (:) ! patch dead coarse root N transfer gross unrepresented landcover change mortality (gN/m2/s) + real(r8), pointer :: gru_retransn_to_litter_patch (:) ! patch retranslocated N pool gross unrepresented landcover change mortality (gN/m2/s) + + real(r8), pointer :: gru_conv_nflux_patch (:) ! (gN/m2/s) conversion N flux (immediate loss to atm) + real(r8), pointer :: gru_conv_nflux_col (:) ! (gN/m2/s) conversion N flux (immediate loss to atm) + real(r8), pointer :: gru_conv_nflux_grc (:) ! (gN/m2/s) gru_conv_nflux_patch summed to the gridcell-level + real(r8), pointer :: gru_wood_productn_gain_patch (:) ! patch (gN/m2/s) addition to wood product pools from gross unrepresented landcover change + real(r8), pointer :: gru_wood_productn_gain_col (:) ! patch (gN/m2/s) addition to wood product pools from gross unrepresented landcover change + real(r8), pointer :: gru_n_to_litr_met_n_col (:,:) ! col (gN/m3/s) N to litter due to gross unrepresented landcover change + real(r8), pointer :: gru_n_to_litr_cel_n_col (:,:) ! col (gN/m3/s) N to litter due to gross unrepresented landcover change + real(r8), pointer :: gru_n_to_litr_lig_n_col (:,:) ! col (gN/m3/s) N to litter due to gross unrepresented landcover change + real(r8), pointer :: gru_n_to_cwdn_col (:,:) ! col (gN/m3/s) N to CWD due to gross unrepresented landcover change + ! crop fluxes real(r8), pointer :: crop_seedn_to_leaf_patch (:) ! patch (gN/m2/s) seed source to leaf, for crops @@ -249,6 +280,7 @@ module CNVegNitrogenFluxType procedure , public :: Restart procedure , public :: SetValues procedure , public :: ZeroDWT + procedure , public :: ZeroGRU procedure , public :: Summary => Summary_nitrogenflux procedure , private :: InitAllocate procedure , private :: InitHistory @@ -447,6 +479,36 @@ subroutine InitAllocate(this, bounds) allocate(this%dwt_livecrootn_to_cwdn_col (begc:endc,1:nlevdecomp_full)) ; this%dwt_livecrootn_to_cwdn_col (:,:) = nan allocate(this%dwt_deadcrootn_to_cwdn_col (begc:endc,1:nlevdecomp_full)) ; this%dwt_deadcrootn_to_cwdn_col (:,:) = nan + allocate(this%gru_leafn_to_litter_patch (begp:endp)) ; this%gru_leafn_to_litter_patch (:) = nan + allocate(this%gru_leafn_storage_to_atm_patch (begp:endp)) ; this%gru_leafn_storage_to_atm_patch (:) = nan + allocate(this%gru_leafn_xfer_to_atm_patch (begp:endp)) ; this%gru_leafn_xfer_to_atm_patch (:) = nan + allocate(this%gru_frootn_to_litter_patch (begp:endp)) ; this%gru_frootn_to_litter_patch (:) = nan + allocate(this%gru_frootn_storage_to_atm_patch (begp:endp)) ; this%gru_frootn_storage_to_atm_patch (:) = nan + allocate(this%gru_frootn_xfer_to_atm_patch (begp:endp)) ; this%gru_frootn_xfer_to_atm_patch (:) = nan + allocate(this%gru_livestemn_to_atm_patch (begp:endp)) ; this%gru_livestemn_to_atm_patch (:) = nan + allocate(this%gru_livestemn_storage_to_atm_patch (begp:endp)) ; this%gru_livestemn_storage_to_atm_patch (:) = nan + allocate(this%gru_livestemn_xfer_to_atm_patch (begp:endp)) ; this%gru_livestemn_xfer_to_atm_patch (:) = nan + allocate(this%gru_deadstemn_to_atm_patch (begp:endp)) ; this%gru_deadstemn_storage_to_atm_patch (:) = nan + allocate(this%gru_deadstemn_storage_to_atm_patch (begp:endp)) ; this%gru_deadstemn_storage_to_atm_patch (:) = nan + allocate(this%gru_deadstemn_xfer_to_atm_patch (begp:endp)) ; this%gru_deadstemn_xfer_to_atm_patch (:) = nan + allocate(this%gru_livecrootn_storage_to_atm_patch (begp:endp)) ; this%gru_livecrootn_storage_to_atm_patch (:) = nan + allocate(this%gru_livecrootn_xfer_to_atm_patch (begp:endp)) ; this%gru_livecrootn_xfer_to_atm_patch (:) = nan + allocate(this%gru_livecrootn_to_litter_patch (begp:endp)) ; this%gru_livecrootn_to_litter_patch (:) = nan + allocate(this%gru_deadcrootn_storage_to_atm_patch (begp:endp)) ; this%gru_deadcrootn_storage_to_atm_patch (:) = nan + allocate(this%gru_deadcrootn_xfer_to_atm_patch (begp:endp)) ; this%gru_deadcrootn_xfer_to_atm_patch (:) = nan + allocate(this%gru_deadcrootn_to_litter_patch (begp:endp)) ; this%gru_deadcrootn_to_litter_patch (:) = nan + allocate(this%gru_retransn_to_litter_patch (begp:endp)) ; this%gru_retransn_to_litter_patch (:) = nan + + allocate(this%gru_conv_nflux_patch (begp:endp)) ; this%gru_conv_nflux_patch (:) =nan + allocate(this%gru_conv_nflux_col (begc:endc)) ; this%gru_conv_nflux_col (:) =nan + allocate(this%gru_conv_nflux_grc (begg:endg)) ; this%gru_conv_nflux_grc (:) =nan + allocate(this%gru_wood_productn_gain_patch (begp:endp)) ; this%gru_wood_productn_gain_patch (:) =nan + allocate(this%gru_wood_productn_gain_col (begc:endc)) ; this%gru_wood_productn_gain_col (:) =nan + allocate(this%gru_n_to_litr_met_n_col (begc:endc,1:nlevdecomp_full)); this%gru_n_to_litr_met_n_col (:,:)=nan + allocate(this%gru_n_to_litr_cel_n_col (begc:endc,1:nlevdecomp_full)); this%gru_n_to_litr_cel_n_col (:,:)=nan + allocate(this%gru_n_to_litr_lig_n_col (begc:endc,1:nlevdecomp_full)); this%gru_n_to_litr_lig_n_col (:,:)=nan + allocate(this%gru_n_to_cwdn_col (begc:endc,1:nlevdecomp_full)); this%gru_n_to_cwdn_col (:,:)=nan + allocate(this%crop_seedn_to_leaf_patch (begp:endp)) ; this%crop_seedn_to_leaf_patch (:) = nan allocate(this%m_decomp_npools_to_fire_vr_col (begc:endc,1:nlevdecomp_full,1:ndecomp_pools)) @@ -1059,6 +1121,16 @@ subroutine InitHistory(this, bounds) avgflag='A', long_name='dead coarse root to CWD due to landcover change', & ptr_col=this%dwt_deadcrootn_to_cwdn_col, default='inactive') + this%gru_conv_nflux_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_CONV_NFLUX', units='gN/m^2/s', & + avgflag='A', long_name='gross unrepresented conversion N flux (immediate loss to atm) (0 at all times except first timestep of year)', & + ptr_patch=this%gru_conv_nflux_patch) + + this%gru_wood_productn_gain_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_WOODPRODN_GAIN', units='gN/m^2/s', & + avgflag='A', long_name='gross unrepresented landcover change driven addition to wood product nitrogen pools (0 at all times except first timestep of year)', & + ptr_patch=this%gru_wood_productn_gain_patch) + this%crop_seedn_to_leaf_patch(begp:endp) = spval call hist_addfld1d (fname='CROP_SEEDN_TO_LEAF', units='gN/m^2/s', & avgflag='A', long_name='crop seed source to leaf', & @@ -1577,6 +1649,26 @@ subroutine SetValues ( this, & this%hrv_deadcrootn_to_litter_patch(i) = value_patch this%hrv_retransn_to_litter_patch(i) = value_patch + this%gru_leafn_to_litter_patch(i) = value_patch + this%gru_leafn_storage_to_atm_patch(i) = value_patch + this%gru_leafn_xfer_to_atm_patch(i) = value_patch + this%gru_frootn_to_litter_patch(i) = value_patch + this%gru_frootn_storage_to_atm_patch(i) = value_patch + this%gru_frootn_xfer_to_atm_patch(i) = value_patch + this%gru_livestemn_to_atm_patch(i) = value_patch + this%gru_livestemn_storage_to_atm_patch(i) = value_patch + this%gru_livestemn_xfer_to_atm_patch(i) = value_patch + this%gru_deadstemn_to_atm_patch(i) = value_patch + this%gru_deadstemn_storage_to_atm_patch(i) = value_patch + this%gru_deadstemn_xfer_to_atm_patch(i) = value_patch + this%gru_livecrootn_to_litter_patch(i) = value_patch + this%gru_livecrootn_storage_to_atm_patch(i) = value_patch + this%gru_livecrootn_xfer_to_atm_patch(i) = value_patch + this%gru_deadcrootn_storage_to_atm_patch(i) = value_patch + this%gru_deadcrootn_xfer_to_atm_patch(i) = value_patch + this%gru_deadcrootn_to_litter_patch(i) = value_patch + this%gru_retransn_to_litter_patch(i) = value_patch + this%m_leafn_to_fire_patch(i) = value_patch this%m_leafn_storage_to_fire_patch(i) = value_patch this%m_leafn_xfer_to_fire_patch(i) = value_patch @@ -1597,6 +1689,8 @@ subroutine SetValues ( this, & this%m_deadcrootn_xfer_to_fire_patch(i) = value_patch this%m_retransn_to_fire_patch(i) = value_patch + this%gru_conv_nflux_patch(i) = value_patch + this%gru_wood_productn_gain_patch(i) = value_patch this%m_leafn_to_litter_fire_patch(i) = value_patch this%m_leafn_storage_to_litter_fire_patch(i) = value_patch @@ -1703,6 +1797,12 @@ subroutine SetValues ( this, & this%harvest_n_to_litr_cel_n_col(i,j) = value_column this%harvest_n_to_litr_lig_n_col(i,j) = value_column this%harvest_n_to_cwdn_col(i,j) = value_column + + ! gross unrepresented landcover change + this%gru_n_to_litr_met_n_col(i,j) = value_column + this%gru_n_to_litr_cel_n_col(i,j) = value_column + this%gru_n_to_litr_lig_n_col(i,j) = value_column + this%gru_n_to_cwdn_col(i,j) = value_column end do end do @@ -1715,6 +1815,8 @@ subroutine SetValues ( this, & ! Zero p2c column fluxes this%fire_nloss_col(i) = value_column this%wood_harvestn_col(i) = value_column + this%gru_conv_nflux_col(i) = value_column + this%gru_wood_productn_gain_col(i) = value_column end do do k = 1, ndecomp_pools @@ -1767,13 +1869,33 @@ subroutine ZeroDwt( this, bounds ) end subroutine ZeroDwt + !----------------------------------------------------------------------- + subroutine ZeroGru( this, bounds ) + ! + ! !DESCRIPTION + ! Initialize flux variables needed for dynamic land use. + ! + ! !ARGUMENTS: + class(cnveg_nitrogenflux_type) :: this + type(bounds_type), intent(in) :: bounds + ! + ! !LOCAL VARIABLES: + integer :: c, g, j ! indices + !----------------------------------------------------------------------- + + do g = bounds%begg, bounds%endg + this%gru_conv_nflux_grc(g) = 0._r8 + end do + + end subroutine ZeroGru + !----------------------------------------------------------------------- subroutine Summary_nitrogenflux(this, bounds, num_soilc, filter_soilc, num_soilp, filter_soilp) ! ! !USES: use clm_varpar , only: nlevdecomp,ndecomp_cascade_transitions,ndecomp_pools use clm_varctl , only: use_nitrif_denitrif - use subgridAveMod , only: p2c + use subgridAveMod , only: p2c, c2g ! ! !ARGUMENTS: class (cnveg_nitrogenflux_type) :: this @@ -1820,12 +1942,39 @@ subroutine Summary_nitrogenflux(this, bounds, num_soilc, filter_soilc, num_soilp this%m_deadcrootn_xfer_to_fire_patch(p) + & this%m_retransn_to_fire_patch(p) + ! (Gross Unrepresented Landcover Change Conversion Flux) - Direct Veg N Loss to Atmosphere + this%gru_conv_nflux_patch(p) = & + this%gru_livestemn_to_atm_patch(p) + & + this%gru_deadstemn_to_atm_patch(p) + & + this%gru_leafn_storage_to_atm_patch(p) + & + this%gru_frootn_storage_to_atm_patch(p) + & + this%gru_livestemn_storage_to_atm_patch(p) + & + this%gru_deadstemn_storage_to_atm_patch(p) + & + this%gru_livecrootn_storage_to_atm_patch(p) + & + this%gru_deadcrootn_storage_to_atm_patch(p) + & + this%gru_leafn_xfer_to_atm_patch(p) + & + this%gru_frootn_xfer_to_atm_patch(p) + & + this%gru_livestemn_xfer_to_atm_patch(p) + & + this%gru_deadstemn_xfer_to_atm_patch(p) + & + this%gru_livecrootn_xfer_to_atm_patch(p) + & + this%gru_deadcrootn_xfer_to_atm_patch(p) + end do call p2c(bounds, num_soilc, filter_soilc, & this%fire_nloss_patch(bounds%begp:bounds%endp), & this%fire_nloss_p2c_col(bounds%begc:bounds%endc)) + call p2c(bounds, num_soilc, filter_soilc, & + this%gru_conv_nflux_patch(bounds%begp:bounds%endp), & + this%gru_conv_nflux_col(bounds%begc:bounds%endc)) + + call c2g( bounds = bounds, & + carr = this%gru_conv_nflux_col(bounds%begc:bounds%endc), & + garr = this%gru_conv_nflux_grc(bounds%begg:bounds%endg), & + c2l_scale_type = 'unity', & + l2g_scale_type = 'unity') + ! vertically integrate column-level fire N losses do k = 1, ndecomp_pools do j = 1, nlevdecomp diff --git a/src/biogeochem/CNVegetationFacade.F90 b/src/biogeochem/CNVegetationFacade.F90 index 10c8164baa..c1ff66f675 100644 --- a/src/biogeochem/CNVegetationFacade.F90 +++ b/src/biogeochem/CNVegetationFacade.F90 @@ -558,6 +558,15 @@ subroutine InitEachTimeStep(this, bounds, num_soilc, filter_soilc) call this%cnveg_carbonstate_inst%ZeroDWT(bounds) call this%cnveg_nitrogenstate_inst%ZeroDWT(bounds) + call this%cnveg_carbonflux_inst%ZeroGRU(bounds) + if (use_c13) then + call this%c13_cnveg_carbonflux_inst%ZeroGRU(bounds) + end if + if (use_c14) then + call this%c14_cnveg_carbonflux_inst%ZeroGRU(bounds) + end if + call this%cnveg_nitrogenflux_inst%ZeroGRU(bounds) + end subroutine InitEachTimeStep !----------------------------------------------------------------------- diff --git a/src/biogeochem/dynConsBiogeochemMod.F90 b/src/biogeochem/dynConsBiogeochemMod.F90 index e233653ca7..096ab102fb 100644 --- a/src/biogeochem/dynConsBiogeochemMod.F90 +++ b/src/biogeochem/dynConsBiogeochemMod.F90 @@ -587,36 +587,6 @@ subroutine dyn_cnbal_patch(bounds, & end do - ! calculate patch-to-column slash fluxes into litter and CWD pools - do p = bounds%begp, bounds%endp - c = patch%column(p) - - ! fine and coarse root to litter and CWD slash carbon fluxes - cnveg_carbonflux_inst%dwt_slash_cflux_col(c) = & - cnveg_carbonflux_inst%dwt_slash_cflux_col(c) + & - dwt_frootc_to_litter(p)/dt + & - dwt_livecrootc_to_litter(p)/dt + & - dwt_deadcrootc_to_litter(p)/dt - - if ( use_c13 ) then - c13_cnveg_carbonflux_inst%dwt_slash_cflux_col(c) = & - c13_cnveg_carbonflux_inst%dwt_slash_cflux_col(c) + & - dwt_frootc13_to_litter(p)/dt + & - dwt_livecrootc13_to_litter(p)/dt + & - dwt_deadcrootc13_to_litter(p)/dt - endif - - if ( use_c14 ) then - c14_cnveg_carbonflux_inst%dwt_slash_cflux_col(c) = & - c14_cnveg_carbonflux_inst%dwt_slash_cflux_col(c) + & - dwt_frootc14_to_litter(p)/dt + & - dwt_livecrootc14_to_litter(p)/dt + & - dwt_deadcrootc14_to_litter(p)/dt - endif - - end do - - ! calculate patch-to-column for fluxes into litter and CWD pools do j = 1, nlevdecomp do p = bounds%begp, bounds%endp @@ -787,6 +757,46 @@ subroutine dyn_cnbal_patch(bounds, & end do + ! calculate patch-to-gridcell slash fluxes into litter and CWD pools + ! Note that patch-level fluxes are stored per unit GRIDCELL area - thus, we don't + ! need to multiply by the patch's gridcell weight when translating patch-level + ! fluxes into gridcell-level fluxes. + + do p = bounds%begp, bounds%endp + g = patch%gridcell(p) + + ! fine and coarse root to litter and CWD slash carbon fluxes + cnveg_carbonflux_inst%dwt_slash_cflux_patch(p) = & + dwt_frootc_to_litter(p)/dt + & + dwt_livecrootc_to_litter(p)/dt + & + dwt_deadcrootc_to_litter(p)/dt + cnveg_carbonflux_inst%dwt_slash_cflux_grc(g) = & + cnveg_carbonflux_inst%dwt_slash_cflux_grc(g) + & + cnveg_carbonflux_inst%dwt_slash_cflux_patch(p) + + if ( use_c13 ) then + c13_cnveg_carbonflux_inst%dwt_slash_cflux_patch(p) = & + dwt_frootc13_to_litter(p)/dt + & + dwt_livecrootc13_to_litter(p)/dt + & + dwt_deadcrootc13_to_litter(p)/dt + c13_cnveg_carbonflux_inst%dwt_slash_cflux_grc(g) = & + c13_cnveg_carbonflux_inst%dwt_slash_cflux_grc(g) + & + c13_cnveg_carbonflux_inst%dwt_slash_cflux_patch(p) + endif + + if ( use_c14 ) then + c14_cnveg_carbonflux_inst%dwt_slash_cflux_patch(p) = & + dwt_frootc14_to_litter(p)/dt + & + dwt_livecrootc14_to_litter(p)/dt + & + dwt_deadcrootc14_to_litter(p)/dt + c14_cnveg_carbonflux_inst%dwt_slash_cflux_grc(g) = & + c14_cnveg_carbonflux_inst%dwt_slash_cflux_grc(g) + & + c14_cnveg_carbonflux_inst%dwt_slash_cflux_patch(p) + endif + + end do + + ! Deallocate patch-level flux arrays deallocate(dwt) deallocate(dwt_leafc_seed) diff --git a/src/biogeochem/dynGrossUnrepMod.F90 b/src/biogeochem/dynGrossUnrepMod.F90 new file mode 100644 index 0000000000..15d5db95a6 --- /dev/null +++ b/src/biogeochem/dynGrossUnrepMod.F90 @@ -0,0 +1,528 @@ +module dynGrossUnrepMod + +#include "shr_assert.h" + + !--------------------------------------------------------------------------- + ! !DESCRIPTION: + ! Handle reading of the gross unrepresented landcover change data, as well as the + ! state updates that happen as a result. + ! + ! !USES: + use shr_kind_mod , only : r8 => shr_kind_r8 + use shr_log_mod , only : errMsg => shr_log_errMsg + use decompMod , only : bounds_type, BOUNDS_LEVEL_PROC + use abortutils , only : endrun + use dynFileMod , only : dyn_file_type + use dynVarTimeUninterpMod , only : dyn_var_time_uninterp_type + use CNVegCarbonStateType , only : cnveg_carbonstate_type + use CNVegCarbonFluxType , only : cnveg_carbonflux_type + use CNVegNitrogenStateType , only : cnveg_nitrogenstate_type + use CNVegNitrogenFluxType , only : cnveg_nitrogenflux_type + use SoilBiogeochemStateType , only : soilbiogeochem_state_type + use pftconMod , only : pftcon + use clm_varcon , only : grlnd + use clm_varpar , only : natpft_size + use ColumnType , only : col + use PatchType , only : patch + ! + ! !PUBLIC MEMBER FUNCTIONS: + implicit none + private + ! + public :: dynGrossUnrep_init ! initialize data structures for grossunrep information + public :: dynGrossUnrep_interp ! get grossunrep data for current time step, if needed + public :: CNGrossUnrep ! grossunrep mortality routine for CN code + ! + ! !PRIVATE MEMBER FUNCTIONS: + private :: CNGrossUnrepPftToColumn ! gather patch-level grossunrep fluxes to the column level + ! + ! !PRIVATE TYPES: + + ! Note that, since we have our own dyngrossunrep_file object (distinct from dynpft_file), + ! we could theoretically have a separate file providing grossunrep data from that providing + ! the pftdyn data + type(dyn_file_type), target :: dyngrossunrep_file ! information for the file containing grossunrep data + + ! Define the underlying grossunrep variables + character(len=64), parameter :: grossunrep_varname = 'UNREPRESENTED_PFT_LULCC' + + type(dyn_var_time_uninterp_type) :: gru_inst ! value of gross unrepresented variable + + real(r8) , allocatable :: grossunrepfrac(:,:) ! gross unrepresented landcover change fraction + logical :: do_grossunrep ! whether we're in a period when we should do gross unrepresented landcover change + character(len=*), parameter, private :: sourcefile = & + __FILE__ + !--------------------------------------------------------------------------- + +contains + + !----------------------------------------------------------------------- + subroutine dynGrossUnrep_init(bounds, grossunrep_filename) + ! + ! !DESCRIPTION: + ! Initialize data structures for grossunrep information. + ! This should be called once, during model initialization. + ! + ! !USES: + use dynVarTimeUninterpMod , only : dyn_var_time_uninterp_type + use dynTimeInfoMod , only : YEAR_POSITION_START_OF_TIMESTEP + ! + ! !ARGUMENTS: + type(bounds_type) , intent(in) :: bounds ! proc-level bounds + character(len=*) , intent(in) :: grossunrep_filename ! name of file containing grossunrep information + ! + ! !LOCAL VARIABLES: + integer :: num_points ! number of spatial points + integer :: ier ! error code + + character(len=*), parameter :: subname = 'dynGrossUnrep_init' + !----------------------------------------------------------------------- + + SHR_ASSERT_ALL(bounds%level == BOUNDS_LEVEL_PROC, subname // ': argument must be PROC-level bounds') + + allocate(grossunrepfrac(bounds%begg:bounds%endg,0:natpft_size-1),stat=ier) + if (ier /= 0) then + call endrun(msg=' allocation error for grossunrep'//errMsg(sourcefile, __LINE__)) + end if + + ! Get the year from the START of the timestep for consistency with other dyn file + ! stuff (though it shouldn't actually matter for grossunrep, given the way the + ! once-per-year grossunrep is applied). + dyngrossunrep_file = dyn_file_type(grossunrep_filename, YEAR_POSITION_START_OF_TIMESTEP) + + ! Get initial grossunrep data + num_points = (bounds%endg - bounds%begg + 1) + gru_inst = dyn_var_time_uninterp_type( & + dyn_file=dyngrossunrep_file, varname=grossunrep_varname, & + dim1name=grlnd, conversion_factor=1.0_r8, & + do_check_sums_equal_1=.false., data_shape=[num_points,natpft_size]) + + end subroutine dynGrossUnrep_init + + + !----------------------------------------------------------------------- + subroutine dynGrossUnrep_interp(bounds) + ! + ! !DESCRIPTION: + ! Get grossunrep data for model time, when needed. + ! + ! Note that grossunrep data are stored as rates (not weights) and so time interpolation + ! is not necessary - the grossunrep rate is held constant through the year. This is + ! consistent with the treatment of changing PFT weights, where interpolation of the + ! annual endpoint weights leads to a constant rate of change in PFT weight through the + ! year, with abrupt changes in the rate at annual boundaries. + ! + ! !USES: + use dynTimeInfoMod , only : time_info_type + ! + ! !ARGUMENTS: + type(bounds_type), intent(in) :: bounds ! proc-level bounds + ! + ! !LOCAL VARIABLES: + + character(len=*), parameter :: subname = 'dyngru_interp' + !----------------------------------------------------------------------- + + SHR_ASSERT_ALL(bounds%level == BOUNDS_LEVEL_PROC, subname // ': argument must be PROC-level bounds') + + call dyngrossunrep_file%time_info%set_current_year() + + ! Get total grossunrep for this time step + grossunrepfrac(bounds%begg:bounds%endg,0:natpft_size-1) = 0._r8 + + if (dyngrossunrep_file%time_info%is_before_time_series()) then + ! Turn off grossunrep before the start of the grossunrep time series + do_grossunrep = .false. + else + ! Note that do_grossunrep stays true even past the end of the time series. This + ! means that grossunrep rates will be maintained at the rate given in the last + ! year of the file for all years past the end of this specified time series. + do_grossunrep = .true. + call gru_inst%get_current_data(grossunrepfrac) + end if + + end subroutine dynGrossUnrep_interp + + + !----------------------------------------------------------------------- + subroutine CNGrossUnrep (num_soilc, filter_soilc, num_soilp, filter_soilp, & + soilbiogeochem_state_inst, cnveg_carbonstate_inst, cnveg_nitrogenstate_inst, & + cnveg_carbonflux_inst, cnveg_nitrogenflux_inst) + ! + ! !DESCRIPTION: + ! GrossUnrep mortality routine for coupled carbon-nitrogen code (CN) + ! + ! !USES: + use pftconMod , only : noveg, nbrdlf_evr_shrub, nc4_grass + use clm_varcon , only : secspday + use clm_time_manager, only : get_step_size_real, is_beg_curr_year + ! + ! !ARGUMENTS: + integer , intent(in) :: num_soilc ! number of soil columns in filter + integer , intent(in) :: filter_soilc(:) ! column filter for soil points + integer , intent(in) :: num_soilp ! number of soil patches in filter + integer , intent(in) :: filter_soilp(:) ! patch filter for soil points + type(soilbiogeochem_state_type) , intent(in) :: soilbiogeochem_state_inst + type(cnveg_carbonstate_type) , intent(in) :: cnveg_carbonstate_inst + type(cnveg_nitrogenstate_type) , intent(in) :: cnveg_nitrogenstate_inst + type(cnveg_carbonflux_type) , intent(inout) :: cnveg_carbonflux_inst + type(cnveg_nitrogenflux_type) , intent(inout) :: cnveg_nitrogenflux_inst + ! + ! !LOCAL VARIABLES: + integer :: p ! patch index + integer :: g ! gridcell index + integer :: fp ! patch filter index + real(r8):: thistreec ! carbon in this tree for calculating grossunrep fraction (gC/m2) + real(r8):: cm ! rate for carbon grossunrep mortality (gC/m2/yr) + real(r8):: am ! rate for fractional grossunrep mortality (1/yr) + real(r8):: m ! rate for fractional grossunrep mortality (1/s) + real(r8):: dtime ! model time step (s) + !----------------------------------------------------------------------- + + associate(& + ivt => patch%itype , & ! Input: [integer (:)] pft vegetation type + convfrac => pftcon%pconv , & ! Input: [real (:)] pft conversion to atmosphere fraction + + leafc => cnveg_carbonstate_inst%leafc_patch , & ! Input: [real(r8) (:)] (gC/m2) leaf C + frootc => cnveg_carbonstate_inst%frootc_patch , & ! Input: [real(r8) (:)] (gC/m2) fine root C + livestemc => cnveg_carbonstate_inst%livestemc_patch , & ! Input: [real(r8) (:)] (gC/m2) live stem C + deadstemc => cnveg_carbonstate_inst%deadstemc_patch , & ! Input: [real(r8) (:)] (gC/m2) dead stem C + livecrootc => cnveg_carbonstate_inst%livecrootc_patch , & ! Input: [real(r8) (:)] (gC/m2) live coarse root C + deadcrootc => cnveg_carbonstate_inst%deadcrootc_patch , & ! Input: [real(r8) (:)] (gC/m2) dead coarse root C + xsmrpool => cnveg_carbonstate_inst%xsmrpool_patch , & ! Input: [real(r8) (:)] (gC/m2) abstract C pool to meet excess MR demand + leafc_storage => cnveg_carbonstate_inst%leafc_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) leaf C storage + frootc_storage => cnveg_carbonstate_inst%frootc_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) fine root C storage + livestemc_storage => cnveg_carbonstate_inst%livestemc_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) live stem C storage + deadstemc_storage => cnveg_carbonstate_inst%deadstemc_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) dead stem C storage + livecrootc_storage => cnveg_carbonstate_inst%livecrootc_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) live coarse root C storage + deadcrootc_storage => cnveg_carbonstate_inst%deadcrootc_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) dead coarse root C storage + gresp_storage => cnveg_carbonstate_inst%gresp_storage_patch , & ! Input: [real(r8) (:)] (gC/m2) growth respiration storage + leafc_xfer => cnveg_carbonstate_inst%leafc_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) leaf C transfer + frootc_xfer => cnveg_carbonstate_inst%frootc_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) fine root C transfer + livestemc_xfer => cnveg_carbonstate_inst%livestemc_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) live stem C transfer + deadstemc_xfer => cnveg_carbonstate_inst%deadstemc_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) dead stem C transfer + livecrootc_xfer => cnveg_carbonstate_inst%livecrootc_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) live coarse root C transfer + deadcrootc_xfer => cnveg_carbonstate_inst%deadcrootc_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) dead coarse root C transfer + gresp_xfer => cnveg_carbonstate_inst%gresp_xfer_patch , & ! Input: [real(r8) (:)] (gC/m2) growth respiration transfer + + leafn => cnveg_nitrogenstate_inst%leafn_patch , & ! Input: [real(r8) (:)] (gN/m2) leaf N + frootn => cnveg_nitrogenstate_inst%frootn_patch , & ! Input: [real(r8) (:)] (gN/m2) fine root N + livestemn => cnveg_nitrogenstate_inst%livestemn_patch , & ! Input: [real(r8) (:)] (gN/m2) live stem N + deadstemn => cnveg_nitrogenstate_inst%deadstemn_patch , & ! Input: [real(r8) (:)] (gN/m2) dead stem N + livecrootn => cnveg_nitrogenstate_inst%livecrootn_patch , & ! Input: [real(r8) (:)] (gN/m2) live coarse root N + deadcrootn => cnveg_nitrogenstate_inst%deadcrootn_patch , & ! Input: [real(r8) (:)] (gN/m2) dead coarse root N + retransn => cnveg_nitrogenstate_inst%retransn_patch , & ! Input: [real(r8) (:)] (gN/m2) plant pool of retranslocated N + leafn_storage => cnveg_nitrogenstate_inst%leafn_storage_patch , & ! Input: [real(r8) (:)] (gN/m2) leaf N storage + frootn_storage => cnveg_nitrogenstate_inst%frootn_storage_patch , & ! Input: [real(r8) (:)] (gN/m2) fine root N storage + livestemn_storage => cnveg_nitrogenstate_inst%livestemn_storage_patch , & ! Input: [real(r8) (:)] (gN/m2) live stem N storage + deadstemn_storage => cnveg_nitrogenstate_inst%deadstemn_storage_patch , & ! Input: [real(r8) (:)] (gN/m2) dead stem N storage + livecrootn_storage => cnveg_nitrogenstate_inst%livecrootn_storage_patch , & ! Input: [real(r8) (:)] (gN/m2) live coarse root N storage + deadcrootn_storage => cnveg_nitrogenstate_inst%deadcrootn_storage_patch , & ! Input: [real(r8) (:)] (gN/m2) dead coarse root N storage + leafn_xfer => cnveg_nitrogenstate_inst%leafn_xfer_patch , & ! Input: [real(r8) (:)] (gN/m2) leaf N transfer + frootn_xfer => cnveg_nitrogenstate_inst%frootn_xfer_patch , & ! Input: [real(r8) (:)] (gN/m2) fine root N transfer + livestemn_xfer => cnveg_nitrogenstate_inst%livestemn_xfer_patch , & ! Input: [real(r8) (:)] (gN/m2) live stem N transfer + deadstemn_xfer => cnveg_nitrogenstate_inst%deadstemn_xfer_patch , & ! Input: [real(r8) (:)] (gN/m2) dead stem N transfer + livecrootn_xfer => cnveg_nitrogenstate_inst%livecrootn_xfer_patch , & ! Input: [real(r8) (:)] (gN/m2) live coarse root N transfer + deadcrootn_xfer => cnveg_nitrogenstate_inst%deadcrootn_xfer_patch , & ! Input: [real(r8) (:)] (gN/m2) dead coarse root N transfer + + gru_leafc_to_litter => cnveg_carbonflux_inst%gru_leafc_to_litter_patch , & ! Output: [real(r8) (:)] + gru_frootc_to_litter => cnveg_carbonflux_inst%gru_frootc_to_litter_patch , & ! Output: [real(r8) (:)] + gru_livestemc_to_atm => cnveg_carbonflux_inst%gru_livestemc_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadstemc_to_atm => cnveg_carbonflux_inst%gru_deadstemc_to_atm_patch , & ! Output: [real(r8) (:)] + gru_wood_productc_gain => cnveg_carbonflux_inst%gru_wood_productc_gain_patch , & ! Output: [real(r8) (:)] + gru_livecrootc_to_litter => cnveg_carbonflux_inst%gru_livecrootc_to_litter_patch , & ! Output: [real(r8) (:)] + gru_deadcrootc_to_litter => cnveg_carbonflux_inst%gru_deadcrootc_to_litter_patch , & ! Output: [real(r8) (:)] + gru_xsmrpool_to_atm => cnveg_carbonflux_inst%gru_xsmrpool_to_atm_patch , & ! Output: [real(r8) (:)] + gru_leafc_storage_to_atm => cnveg_carbonflux_inst%gru_leafc_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_frootc_storage_to_atm => cnveg_carbonflux_inst%gru_frootc_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livestemc_storage_to_atm => cnveg_carbonflux_inst%gru_livestemc_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadstemc_storage_to_atm => cnveg_carbonflux_inst%gru_deadstemc_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livecrootc_storage_to_atm => cnveg_carbonflux_inst%gru_livecrootc_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadcrootc_storage_to_atm => cnveg_carbonflux_inst%gru_deadcrootc_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_gresp_storage_to_atm => cnveg_carbonflux_inst%gru_gresp_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_leafc_xfer_to_atm => cnveg_carbonflux_inst%gru_leafc_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_frootc_xfer_to_atm => cnveg_carbonflux_inst%gru_frootc_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livestemc_xfer_to_atm => cnveg_carbonflux_inst%gru_livestemc_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadstemc_xfer_to_atm => cnveg_carbonflux_inst%gru_deadstemc_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livecrootc_xfer_to_atm => cnveg_carbonflux_inst%gru_livecrootc_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadcrootc_xfer_to_atm => cnveg_carbonflux_inst%gru_deadcrootc_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_gresp_xfer_to_atm => cnveg_carbonflux_inst%gru_gresp_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + + gru_leafn_to_litter => cnveg_nitrogenflux_inst%gru_leafn_to_litter_patch , & ! Output: [real(r8) (:)] + gru_frootn_to_litter => cnveg_nitrogenflux_inst%gru_frootn_to_litter_patch , & ! Output: [real(r8) (:)] + gru_livestemn_to_atm => cnveg_nitrogenflux_inst%gru_livestemn_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadstemn_to_atm => cnveg_nitrogenflux_inst%gru_deadstemn_to_atm_patch , & ! Output: [real(r8) (:)] + gru_wood_productn_gain => cnveg_nitrogenflux_inst%gru_wood_productn_gain_patch , & ! Output: [real(r8) (:)] + gru_livecrootn_to_litter => cnveg_nitrogenflux_inst%gru_livecrootn_to_litter_patch , & ! Output: [real(r8) (:)] + gru_deadcrootn_to_litter => cnveg_nitrogenflux_inst%gru_deadcrootn_to_litter_patch , & ! Output: [real(r8) (:)] + gru_retransn_to_litter => cnveg_nitrogenflux_inst%gru_retransn_to_litter_patch , & ! Output: [real(r8) (:)] + gru_leafn_storage_to_atm => cnveg_nitrogenflux_inst%gru_leafn_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_frootn_storage_to_atm => cnveg_nitrogenflux_inst%gru_frootn_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livestemn_storage_to_atm => cnveg_nitrogenflux_inst%gru_livestemn_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadstemn_storage_to_atm => cnveg_nitrogenflux_inst%gru_deadstemn_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livecrootn_storage_to_atm => cnveg_nitrogenflux_inst%gru_livecrootn_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadcrootn_storage_to_atm => cnveg_nitrogenflux_inst%gru_deadcrootn_storage_to_atm_patch , & ! Output: [real(r8) (:)] + gru_leafn_xfer_to_atm => cnveg_nitrogenflux_inst%gru_leafn_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_frootn_xfer_to_atm => cnveg_nitrogenflux_inst%gru_frootn_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livestemn_xfer_to_atm => cnveg_nitrogenflux_inst%gru_livestemn_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadstemn_xfer_to_atm => cnveg_nitrogenflux_inst%gru_deadstemn_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_livecrootn_xfer_to_atm => cnveg_nitrogenflux_inst%gru_livecrootn_xfer_to_atm_patch , & ! Output: [real(r8) (:)] + gru_deadcrootn_xfer_to_atm => cnveg_nitrogenflux_inst%gru_deadcrootn_xfer_to_atm_patch & ! Output: [real(r8) (:)] + ) + + dtime = get_step_size_real() + + ! patch loop + do fp = 1,num_soilp + p = filter_soilp(fp) + g = patch%gridcell(p) + + ! If this is a natural pft, then + ! get the annual grossunrep "mortality" rate (am) from grossunrep array + ! and convert to rate per second + if (ivt(p) > noveg .and. ivt(p) <= nc4_grass) then + + if (do_grossunrep) then + am = grossunrepfrac(g,ivt(p)) + + ! Apply all grossunrep at the start of the year + if (is_beg_curr_year()) then + m = am/dtime + else + m = 0._r8 + end if + else + m = 0._r8 + end if + + ! patch-level gross unrepresented landcover change carbon fluxes + ! displayed pools + gru_leafc_to_litter(p) = leafc(p) * m + gru_frootc_to_litter(p) = frootc(p) * m + gru_livestemc_to_atm(p) = livestemc(p) * m + gru_deadstemc_to_atm(p) = deadstemc(p) * m * convfrac(ivt(p)) + gru_wood_productc_gain(p) = deadstemc(p) * m * (1._r8 - convfrac(ivt(p))) + gru_livecrootc_to_litter(p) = livecrootc(p) * m + gru_deadcrootc_to_litter(p) = deadcrootc(p) * m + gru_xsmrpool_to_atm(p) = xsmrpool(p) * m + + ! storage pools + gru_leafc_storage_to_atm(p) = leafc_storage(p) * m + gru_frootc_storage_to_atm(p) = frootc_storage(p) * m + gru_livestemc_storage_to_atm(p) = livestemc_storage(p) * m + gru_deadstemc_storage_to_atm(p) = deadstemc_storage(p) * m + gru_livecrootc_storage_to_atm(p) = livecrootc_storage(p) * m + gru_deadcrootc_storage_to_atm(p) = deadcrootc_storage(p) * m + gru_gresp_storage_to_atm(p) = gresp_storage(p) * m + + ! transfer pools + gru_leafc_xfer_to_atm(p) = leafc_xfer(p) * m + gru_frootc_xfer_to_atm(p) = frootc_xfer(p) * m + gru_livestemc_xfer_to_atm(p) = livestemc_xfer(p) * m + gru_deadstemc_xfer_to_atm(p) = deadstemc_xfer(p) * m + gru_livecrootc_xfer_to_atm(p) = livecrootc_xfer(p) * m + gru_deadcrootc_xfer_to_atm(p) = deadcrootc_xfer(p) * m + gru_gresp_xfer_to_atm(p) = gresp_xfer(p) * m + + ! patch-level gross unrepresented landcover change mortality nitrogen fluxes + ! displayed pools + gru_leafn_to_litter(p) = leafn(p) * m + gru_frootn_to_litter(p) = frootn(p) * m + gru_livestemn_to_atm(p) = livestemn(p) * m + gru_deadstemn_to_atm(p) = deadstemn(p) * m * convfrac(ivt(p)) + gru_wood_productn_gain(p) = deadstemn(p) * m * (1._r8 - convfrac(ivt(p))) + gru_livecrootn_to_litter(p) = livecrootn(p) * m + gru_deadcrootn_to_litter(p) = deadcrootn(p) * m + gru_retransn_to_litter(p) = retransn(p) * m + + ! storage pools + gru_leafn_storage_to_atm(p) = leafn_storage(p) * m + gru_frootn_storage_to_atm(p) = frootn_storage(p) * m + gru_livestemn_storage_to_atm(p) = livestemn_storage(p) * m + gru_deadstemn_storage_to_atm(p) = deadstemn_storage(p) * m + gru_livecrootn_storage_to_atm(p) = livecrootn_storage(p) * m + gru_deadcrootn_storage_to_atm(p) = deadcrootn_storage(p) * m + + ! transfer pools + gru_leafn_xfer_to_atm(p) = leafn_xfer(p) * m + gru_frootn_xfer_to_atm(p) = frootn_xfer(p) * m + gru_livestemn_xfer_to_atm(p) = livestemn_xfer(p) * m + gru_deadstemn_xfer_to_atm(p) = deadstemn_xfer(p) * m + gru_livecrootn_xfer_to_atm(p) = livecrootn_xfer(p) * m + gru_deadcrootn_xfer_to_atm(p) = deadcrootn_xfer(p) * m + + end if ! end tree block + + end do ! end of pft loop + + ! gather all patch-level litterfall fluxes from grossunrep to the column + ! for litter C and N inputs + + call CNGrossUnrepPftToColumn(num_soilc, filter_soilc, & + soilbiogeochem_state_inst, cnveg_carbonflux_inst, cnveg_nitrogenflux_inst) + + end associate + + end subroutine CNGrossUnrep + + !----------------------------------------------------------------------- + subroutine CNGrossUnrepPftToColumn (num_soilc, filter_soilc, & + soilbiogeochem_state_inst, CNVeg_carbonflux_inst, cnveg_nitrogenflux_inst) + ! + ! !DESCRIPTION: + ! called at the end of CNGrossUnrep to gather all patch-level grossunrep litterfall fluxes + ! to the column level and assign them to the three litter pools + ! + ! !USES: + use clm_varpar , only : maxpatch_pft, nlevdecomp, numpft + ! + ! !ARGUMENTS: + integer , intent(in) :: num_soilc ! number of soil columns in filter + integer , intent(in) :: filter_soilc(:) ! soil column filter + type(soilbiogeochem_state_type) , intent(in) :: soilbiogeochem_state_inst + type(cnveg_carbonflux_type) , intent(inout) :: cnveg_carbonflux_inst + type(cnveg_nitrogenflux_type) , intent(inout) :: cnveg_nitrogenflux_inst + ! + ! !LOCAL VARIABLES: + integer :: fc,c,pi,p,j ! indices + !----------------------------------------------------------------------- + + associate( & + ivt => patch%itype , & ! Input: [integer (:) ] pft vegetation type + wtcol => patch%wtcol , & ! Input: [real(r8) (:) ] pft weight relative to column (0-1) + + lf_flab => pftcon%lf_flab , & ! Input: leaf litter labile fraction + lf_fcel => pftcon%lf_fcel , & ! Input: leaf litter cellulose fraction + lf_flig => pftcon%lf_flig , & ! Input: leaf litter lignin fraction + fr_flab => pftcon%fr_flab , & ! Input: fine root litter labile fraction + fr_fcel => pftcon%fr_fcel , & ! Input: fine root litter cellulose fraction + fr_flig => pftcon%fr_flig , & ! Input: fine root litter lignin fraction + + leaf_prof => soilbiogeochem_state_inst%leaf_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of leaves + froot_prof => soilbiogeochem_state_inst%froot_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of fine roots + croot_prof => soilbiogeochem_state_inst%croot_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of coarse roots + stem_prof => soilbiogeochem_state_inst%stem_prof_patch , & ! Input: [real(r8) (:,:) ] (1/m) profile of stems + + gru_leafc_to_litter => cnveg_carbonflux_inst%gru_leafc_to_litter_patch , & ! Input: [real(r8) (:)] + gru_frootc_to_litter => cnveg_carbonflux_inst%gru_frootc_to_litter_patch , & ! Input: [real(r8) (:)] + gru_wood_productc_gain => cnveg_carbonflux_inst%gru_wood_productc_gain_patch , & ! Input: [real(r8) (:)] + gru_livecrootc_to_litter => cnveg_carbonflux_inst%gru_livecrootc_to_litter_patch , & ! Input: [real(r8) (:)] + gru_deadcrootc_to_litter => cnveg_carbonflux_inst%gru_deadcrootc_to_litter_patch , & ! Input: [real(r8) (:)] + gru_conv_cflux => cnveg_carbonflux_inst%gru_conv_cflux_patch , & ! Input: [real(r8) (:) ] C fluxes associated with conversion (immediate loss to atm) + + gru_c_to_litr_met_c => cnveg_carbonflux_inst%gru_c_to_litr_met_c_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with grossunrep to litter metabolic pool (gC/m3/s) + gru_c_to_litr_cel_c => cnveg_carbonflux_inst%gru_c_to_litr_cel_c_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with grossunrep to litter cellulose pool (gC/m3/s) + gru_c_to_litr_lig_c => cnveg_carbonflux_inst%gru_c_to_litr_lig_c_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with grossunrep to litter lignin pool (gC/m3/s) + gru_c_to_cwdc_c => cnveg_carbonflux_inst%gru_c_to_cwdc_col , & ! Output: [real(r8) (:,:) ] C fluxes associated with grossunrep to CWD pool (gC/m3/s) + + gru_wood_productc_gain_c => cnveg_carbonflux_inst%gru_wood_productc_gain_col , & ! Input: [real(r8) (:)] + + gru_leafn_to_litter => cnveg_nitrogenflux_inst%gru_leafn_to_litter_patch , & ! Input: [real(r8) (:)] + gru_frootn_to_litter => cnveg_nitrogenflux_inst%gru_frootn_to_litter_patch , & ! Input: [real(r8) (:)] + gru_wood_productn_gain => cnveg_nitrogenflux_inst%gru_wood_productn_gain_patch , & ! Input: [real(r8) (:)] + gru_livecrootn_to_litter => cnveg_nitrogenflux_inst%gru_livecrootn_to_litter_patch , & ! Input: [real(r8) (:)] + gru_deadcrootn_to_litter => cnveg_nitrogenflux_inst%gru_deadcrootn_to_litter_patch , & ! Input: [real(r8) (:)] + gru_retransn_to_litter => cnveg_nitrogenflux_inst%gru_retransn_to_litter_patch , & ! Input: [real(r8) (:)] + gru_conv_nflux => cnveg_nitrogenflux_inst%gru_conv_nflux_patch , & ! Input: [real(r8) (:) ] N fluxes associated with conversion (immediate loss to atm) + + gru_n_to_litr_met_c => cnveg_nitrogenflux_inst%gru_n_to_litr_met_n_col , & ! Output: [real(r8) (:,:) ] N fluxes associated with grossunrep to litter metabolic pool (gN/m3/s) + gru_n_to_litr_cel_c => cnveg_nitrogenflux_inst%gru_n_to_litr_cel_n_col , & ! Output: [real(r8) (:,:) ] N fluxes associated with grossunrep to litter cellulose pool (gN/m3/s) + gru_n_to_litr_lig_c => cnveg_nitrogenflux_inst%gru_n_to_litr_lig_n_col , & ! Output: [real(r8) (:,:) ] N fluxes associated with grossunrep to litter lignin pool (gN/m3/s) + gru_n_to_cwdn_c => cnveg_nitrogenflux_inst%gru_n_to_cwdn_col , & ! Output: [real(r8) (:,:) ] N fluxes associated with grossunrep to CWD pool (gN/m3/s) + + gru_wood_productn_gain_c => cnveg_nitrogenflux_inst%gru_wood_productn_gain_col & ! Input: [real(r8) (:)] + ) + + do j = 1, nlevdecomp + do pi = 1,maxpatch_pft + do fc = 1,num_soilc + c = filter_soilc(fc) + + if (pi <= col%npatches(c)) then + p = col%patchi(c) + pi - 1 + + if (patch%active(p)) then + + ! leaf gross unrepresented landcover change mortality carbon fluxes + gru_c_to_litr_met_c(c,j) = gru_c_to_litr_met_c(c,j) + & + gru_leafc_to_litter(p) * lf_flab(ivt(p)) * wtcol(p) * leaf_prof(p,j) + gru_c_to_litr_cel_c(c,j) = gru_c_to_litr_cel_c(c,j) + & + gru_leafc_to_litter(p) * lf_fcel(ivt(p)) * wtcol(p) * leaf_prof(p,j) + gru_c_to_litr_lig_c(c,j) = gru_c_to_litr_lig_c(c,j) + & + gru_leafc_to_litter(p) * lf_flig(ivt(p)) * wtcol(p) * leaf_prof(p,j) + + ! fine root gross unrepresented landcover change mortality carbon fluxes + gru_c_to_litr_met_c(c,j) = gru_c_to_litr_met_c(c,j) + & + gru_frootc_to_litter(p) * fr_flab(ivt(p)) * wtcol(p) * froot_prof(p,j) + gru_c_to_litr_cel_c(c,j) = gru_c_to_litr_cel_c(c,j) + & + gru_frootc_to_litter(p) * fr_fcel(ivt(p)) * wtcol(p) * froot_prof(p,j) + gru_c_to_litr_lig_c(c,j) = gru_c_to_litr_lig_c(c,j) + & + gru_frootc_to_litter(p) * fr_flig(ivt(p)) * wtcol(p) * froot_prof(p,j) + + ! coarse root gross unrepresented landcover change mortality carbon fluxes + gru_c_to_cwdc_c(c,j) = gru_c_to_cwdc_c(c,j) + & + gru_livecrootc_to_litter(p) * wtcol(p) * croot_prof(p,j) + gru_c_to_cwdc_c(c,j) = gru_c_to_cwdc_c(c,j) + & + gru_deadcrootc_to_litter(p) * wtcol(p) * croot_prof(p,j) + + ! leaf gross unrepresented landcover change mortality nitrogen fluxes + gru_n_to_litr_met_c(c,j) = gru_n_to_litr_met_c(c,j) + & + gru_leafn_to_litter(p) * lf_flab(ivt(p)) * wtcol(p) * leaf_prof(p,j) + gru_n_to_litr_cel_c(c,j) = gru_n_to_litr_cel_c(c,j) + & + gru_leafn_to_litter(p) * lf_fcel(ivt(p)) * wtcol(p) * leaf_prof(p,j) + gru_n_to_litr_lig_c(c,j) = gru_n_to_litr_lig_c(c,j) + & + gru_leafn_to_litter(p) * lf_flig(ivt(p)) * wtcol(p) * leaf_prof(p,j) + + ! fine root unrepresented landcover change mortality nitrogen fluxes + gru_n_to_litr_met_c(c,j) = gru_n_to_litr_met_c(c,j) + & + gru_frootn_to_litter(p) * fr_flab(ivt(p)) * wtcol(p) * froot_prof(p,j) + gru_n_to_litr_cel_c(c,j) = gru_n_to_litr_cel_c(c,j) + & + gru_frootn_to_litter(p) * fr_fcel(ivt(p)) * wtcol(p) * froot_prof(p,j) + gru_n_to_litr_lig_c(c,j) = gru_n_to_litr_lig_c(c,j) + & + gru_frootn_to_litter(p) * fr_flig(ivt(p)) * wtcol(p) * froot_prof(p,j) + + ! coarse root gross unrepresented landcover change mortality nitrogen fluxes + gru_n_to_cwdn_c(c,j) = gru_n_to_cwdn_c(c,j) + & + gru_livecrootn_to_litter(p) * wtcol(p) * croot_prof(p,j) + gru_n_to_cwdn_c(c,j) = gru_n_to_cwdn_c(c,j) + & + gru_deadcrootn_to_litter(p) * wtcol(p) * croot_prof(p,j) + + ! retranslocated N pool gross unrepresented landcover change mortality fluxes + gru_n_to_litr_met_c(c,j) = gru_n_to_litr_met_c(c,j) + & + gru_retransn_to_litter(p) * wtcol(p) * leaf_prof(p,j) + + end if + end if + + end do + + end do + end do + + do pi = 1,maxpatch_pft + do fc = 1,num_soilc + c = filter_soilc(fc) + + if (pi <= col%npatches(c)) then + p = col%patchi(c) + pi - 1 + + if (patch%active(p)) then + ! wood gross unrepresented landcover change mortality carbon fluxes to product pools + gru_wood_productc_gain_c(c) = gru_wood_productc_gain_c(c) + & + gru_wood_productc_gain(p) * wtcol(p) + + ! wood gross unrepresented landcover change mortality nitrogen fluxes to product pools + gru_wood_productn_gain_c(c) = gru_wood_productn_gain_c(c) + & + gru_wood_productn_gain(p) * wtcol(p) + end if + end if + + end do + + end do + + end associate + + end subroutine CNGrossUnrepPftToColumn + +end module dynGrossUnrepMod diff --git a/src/dyn_subgrid/dynSubgridControlMod.F90 b/src/dyn_subgrid/dynSubgridControlMod.F90 index 58e3a40359..3491057dc0 100644 --- a/src/dyn_subgrid/dynSubgridControlMod.F90 +++ b/src/dyn_subgrid/dynSubgridControlMod.F90 @@ -26,6 +26,7 @@ module dynSubgridControlMod public :: get_do_transient_crops ! return the value of the do_transient_crops control flag public :: run_has_transient_landcover ! returns true if any aspects of prescribed transient landcover are enabled public :: get_do_harvest ! return the value of the do_harvest control flag + public :: get_do_grossunrep ! return the value of the do_grossunrep control flag public :: get_for_testing_allow_non_annual_changes ! return true if user has requested to allow area changes at times other than the year boundary, for testing purposes public :: get_for_testing_zero_dynbal_fluxes ! return true if user has requested to set the dynbal water and energy fluxes to zero, for testing purposes ! @@ -40,6 +41,7 @@ module dynSubgridControlMod logical :: do_transient_pfts = .false. ! whether to apply transient natural PFTs from dataset logical :: do_transient_crops = .false. ! whether to apply transient crops from dataset logical :: do_harvest = .false. ! whether to apply harvest from dataset + logical :: do_grossunrep = .false. ! whether to apply gross unrepresented landcover change from dataset ! The following is only meant for testing: Whether area changes are allowed at times ! other than the year boundary. This should only arise in some test configurations @@ -114,6 +116,7 @@ subroutine read_namelist( NLFilename ) logical :: do_transient_pfts logical :: do_transient_crops logical :: do_harvest + logical :: do_grossunrep logical :: for_testing_allow_non_annual_changes logical :: for_testing_zero_dynbal_fluxes ! other local variables: @@ -136,6 +139,7 @@ subroutine read_namelist( NLFilename ) do_transient_pfts = .false. do_transient_crops = .false. do_harvest = .false. + do_grossunrep = .false. for_testing_allow_non_annual_changes = .false. for_testing_zero_dynbal_fluxes = .false. @@ -153,12 +157,14 @@ subroutine read_namelist( NLFilename ) end if close(nu_nml) call relavu( nu_nml ) + do_grossunrep = do_harvest endif call shr_mpi_bcast (flanduse_timeseries, mpicom) call shr_mpi_bcast (do_transient_pfts, mpicom) call shr_mpi_bcast (do_transient_crops, mpicom) call shr_mpi_bcast (do_harvest, mpicom) + call shr_mpi_bcast (do_grossunrep, mpicom) call shr_mpi_bcast (for_testing_allow_non_annual_changes, mpicom) call shr_mpi_bcast (for_testing_zero_dynbal_fluxes, mpicom) @@ -167,6 +173,7 @@ subroutine read_namelist( NLFilename ) do_transient_pfts = do_transient_pfts, & do_transient_crops = do_transient_crops, & do_harvest = do_harvest, & + do_grossunrep = do_grossunrep, & for_testing_allow_non_annual_changes = for_testing_allow_non_annual_changes, & for_testing_zero_dynbal_fluxes = for_testing_zero_dynbal_fluxes) @@ -211,6 +218,11 @@ subroutine check_namelist_consistency write(iulog,*) 'a flanduse_timeseries file (currently flanduse_timeseries is blank)' call endrun(msg=errMsg(sourcefile, __LINE__)) end if + if (dyn_subgrid_control_inst%do_grossunrep) then + write(iulog,*) 'ERROR: do_grossunrep can only be true if you are running with' + write(iulog,*) 'a flanduse_timeseries file (currently flanduse_timeseries is blank)' + call endrun(msg=errMsg(sourcefile, __LINE__)) + end if end if if (dyn_subgrid_control_inst%do_transient_pfts) then @@ -249,6 +261,17 @@ subroutine check_namelist_consistency end if end if + if (dyn_subgrid_control_inst%do_grossunrep) then + if (.not. use_cn) then + write(iulog,*) 'ERROR: do_grossunrep can only be true if use_cn is true' + call endrun(msg=errMsg(sourcefile, __LINE__)) + end if + if (use_fates) then + write(iulog,*) 'ERROR: do_grossunrep currently does not work with use_fates' + call endrun(msg=errMsg(sourcefile, __LINE__)) + end if + end if + end subroutine check_namelist_consistency !----------------------------------------------------------------------- @@ -312,6 +335,18 @@ logical function get_do_harvest() end function get_do_harvest + !----------------------------------------------------------------------- + logical function get_do_grossunrep() + ! !DESCRIPTION: + ! Return the value of the do_grossunrep control flag + !----------------------------------------------------------------------- + + SHR_ASSERT(dyn_subgrid_control_inst%initialized, errMsg(sourcefile, __LINE__)) + + get_do_grossunrep = dyn_subgrid_control_inst%do_grossunrep + + end function get_do_grossunrep + !----------------------------------------------------------------------- logical function get_for_testing_allow_non_annual_changes() ! diff --git a/src/dyn_subgrid/dynSubgridDriverMod.F90 b/src/dyn_subgrid/dynSubgridDriverMod.F90 index 8703fcb2d9..90bbd05abe 100644 --- a/src/dyn_subgrid/dynSubgridDriverMod.F90 +++ b/src/dyn_subgrid/dynSubgridDriverMod.F90 @@ -14,12 +14,14 @@ module dynSubgridDriverMod use dynSubgridControlMod , only : get_flanduse_timeseries use dynSubgridControlMod , only : get_do_transient_pfts, get_do_transient_crops use dynSubgridControlMod , only : get_do_harvest + use dynSubgridControlMod , only : get_do_grossunrep use dynPriorWeightsMod , only : prior_weights_type use dynPatchStateUpdaterMod , only : patch_state_updater_type use dynColumnStateUpdaterMod , only : column_state_updater_type use dynpftFileMod , only : dynpft_init, dynpft_interp use dyncropFileMod , only : dyncrop_init, dyncrop_interp use dynHarvestMod , only : dynHarvest_init, dynHarvest_interp + use dynGrossUnrepMod , only : dynGrossUnrep_init, dynGrossUnrep_interp use dynLandunitAreaMod , only : update_landunit_weights use subgridWeightsMod , only : compute_higher_order_weights, set_subgrid_diagnostic_fields use reweightMod , only : reweight_wrapup @@ -121,6 +123,11 @@ subroutine dynSubgrid_init(bounds_proc, glc_behavior, crop_inst) call dynHarvest_init(bounds_proc, harvest_filename=get_flanduse_timeseries()) end if + ! Initialize stuff for gross unrepresented landuse data. + if (get_do_grossunrep()) then + call dynGrossUnrep_init(bounds_proc, grossunrep_filename=get_flanduse_timeseries()) + end if + ! ------------------------------------------------------------------------ ! Set initial subgrid weights for aspects that are read from file. This is relevant ! for cold start and use_init_interp-based initialization. @@ -248,6 +255,10 @@ subroutine dynSubgrid_driver(bounds_proc, call dynHarvest_interp(bounds_proc) end if + if (get_do_grossunrep()) then + call dynGrossUnrep_interp(bounds_proc) + end if + ! ========================================================================== ! Do land cover change that does not require I/O ! ========================================================================== diff --git a/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 b/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 index 78be5efd44..62683d4256 100644 --- a/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 +++ b/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 @@ -38,7 +38,7 @@ module SoilBiogeochemCarbonStateType real(r8), pointer :: decomp_cpools_col (:,:) ! (gC/m2) decomposing (litter, cwd, soil) c pools real(r8), pointer :: dyn_cbal_adjustments_col (:) ! (gC/m2) adjustments to each column made in this timestep via dynamic column area adjustments (note: this variable only makes sense at the column-level: it is meaningless if averaged to the gridcell-level) integer :: restart_file_spinup_state ! spinup state as read from restart file, for determining whether to enter or exit spinup mode. - real(r8) :: totvegcthresh ! threshold for total vegetation carbon to zero out decomposition pools + real(r8) :: ccrit ! critical carbon value contains @@ -46,7 +46,7 @@ module SoilBiogeochemCarbonStateType procedure , public :: SetValues procedure , public :: Restart procedure , public :: Summary - procedure , public :: SetTotVgCThresh + procedure , public :: SetCCrit procedure , public :: DynamicColumnAdjustments ! adjust state variables when column areas change procedure , private :: InitAllocate procedure , private :: InitHistory @@ -70,7 +70,7 @@ subroutine Init(this, bounds, carbon_type, ratio, c12_soilbiogeochem_carbonstate real(r8) , intent(in) :: ratio type(soilbiogeochem_carbonstate_type) , intent(in), optional :: c12_soilbiogeochem_carbonstate_inst - this%totvegcthresh = nan + this%ccrit = nan call this%InitAllocate ( bounds) call this%InitHistory ( bounds, carbon_type ) if (present(c12_soilbiogeochem_carbonstate_inst)) then @@ -674,10 +674,6 @@ subroutine Restart ( this, bounds, ncid, flag, carbon_type, totvegc_col, c12_so call endrun(msg=' CNRest: error in entering/exiting spinup - should occur only when nstep = 1'//& errMsg(sourcefile, __LINE__)) endif - if ( exit_spinup .and. isnan(this%totvegcthresh) )then - call endrun(msg=' CNRest: error in exit spinup - totvegcthresh was not set with SetTotVgCThresh'//& - errMsg(sourcefile, __LINE__)) - end if do k = 1, ndecomp_pools if ( exit_spinup ) then m = decomp_cascade_con%spinup_factor(k) @@ -694,7 +690,10 @@ subroutine Restart ( this, bounds, ncid, flag, carbon_type, totvegc_col, c12_so ! reset decomp pools to near zero during exit_spinup to avoid very ! large and inert soil carbon stocks; note that only pools with spinup factor > 1 ! will be affected, which means that total SOMC and LITC pools will not be set to 0. - if (totvegc_col(c) <= this%totvegcthresh .and. lun%itype(l) /= istcrop) then +!KO if (totvegc_col(c) <= this%ccrit .and. lun%itype(l) /= istcrop) then +!KO + if (totvegc_col(c) <= 0.1_r8 .and. lun%itype(l) /= istcrop) then +!KO this%decomp_cpools_vr_col(c,j,k) = 0.0_r8 endif elseif ( abs(m - 1._r8) .gt. 0.000001_r8 .and. enter_spinup) then @@ -927,18 +926,14 @@ subroutine Summary(this, bounds, num_allc, filter_allc) end subroutine Summary !------------------------------------------------------------------------ - subroutine SetTotVgCThresh(this, totvegcthresh) + subroutine SetCCrit(this, ccrit) class(soilbiogeochem_carbonstate_type) :: this - real(r8) , intent(in) :: totvegcthresh + real(r8) , intent(in) :: ccrit - if ( totvegcthresh <= 0.0_r8 )then - call endrun(msg=' ERROR totvegcthresh is zero or negative and should be > 0'//& - errMsg(sourcefile, __LINE__)) - end if - this%totvegcthresh = totvegcthresh + this%ccrit = ccrit - end subroutine SetTotVgCThresh + end subroutine SetCCrit !----------------------------------------------------------------------- diff --git a/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 b/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 index ca09e63624..b776f77731 100644 --- a/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 +++ b/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 @@ -52,7 +52,7 @@ module SoilBiogeochemNitrogenStateType ! the N balance check real(r8), pointer :: dyn_no3bal_adjustments_col (:) ! (gN/m2) NO3 adjustments to each column made in this timestep via dynamic column area adjustments (only makes sense at the column-level: meaningless if averaged to the gridcell-level) real(r8), pointer :: dyn_nh4bal_adjustments_col (:) ! (gN/m2) NH4 adjustments to each column made in this timestep via dynamic column adjustments (only makes sense at the column-level: meaningless if averaged to the gridcell-level) - real(r8) :: totvegcthresh ! threshold for total vegetation carbon to zero out decomposition pools + real(r8) :: ncrit ! (gN/m2) critical value of nitrogen before truncation contains @@ -61,7 +61,7 @@ module SoilBiogeochemNitrogenStateType procedure , public :: SetValues procedure , public :: Summary procedure , public :: DynamicColumnAdjustments ! adjust state variables when column areas change - procedure , public :: SetTotVgCThresh ! Set value for totvegcthresh needed in Restart + procedure , public :: SetNCrit procedure , private :: InitAllocate procedure , private :: InitHistory procedure , private :: InitCold @@ -84,7 +84,7 @@ subroutine Init(this, bounds, & real(r8) , intent(in) :: decomp_cpools_col (bounds%begc:, 1:) real(r8) , intent(in) :: decomp_cpools_1m_col (bounds%begc:, 1:) - this%totvegcthresh = nan + this%ncrit = nan call this%InitAllocate (bounds ) call this%InitHistory (bounds) @@ -610,10 +610,6 @@ subroutine Restart ( this, bounds, ncid, flag, totvegc_col ) call endrun(msg=' Error in entering/exiting spinup - should occur only when nstep = 1'//& errMsg(sourcefile, __LINE__)) endif - if ( exit_spinup .and. isnan(this%totvegcthresh) )then - call endrun(msg=' Error in exit spinup - totvegcthresh was not set with SetTotVgCThresh'//& - errMsg(sourcefile, __LINE__)) - end if do k = 1, ndecomp_pools if ( exit_spinup ) then m = decomp_cascade_con%spinup_factor(k) @@ -635,7 +631,10 @@ subroutine Restart ( this, bounds, ncid, flag, totvegc_col ) ! pools with spinup factor > 1 ! will be affected, which means that total SOMN and LITN ! pools will not be set to 0. - if (totvegc_col(c) <= this%totvegcthresh .and. lun%itype(l) /= istcrop) then +!KO if (totvegc_col(c) <= this%ncrit .and. lun%itype(l) /= istcrop) then +!KO + if (totvegc_col(c) <= 0.1_r8 .and. lun%itype(l) /= istcrop) then +!KO this%decomp_npools_vr_col(c,j,k) = 0._r8 endif elseif ( abs(m - 1._r8) .gt. 0.000001_r8 .and. enter_spinup) then @@ -1007,17 +1006,13 @@ subroutine DynamicColumnAdjustments(this, bounds, clump_index, column_state_upda end subroutine DynamicColumnAdjustments !------------------------------------------------------------------------ - subroutine SetTotVgCThresh(this, totvegcthresh) + subroutine SetNCrit(this, ncrit) class(soilbiogeochem_nitrogenstate_type) :: this - real(r8) , intent(in) :: totvegcthresh + real(r8) , intent(in) :: ncrit - if ( totvegcthresh <= 0.0_r8 )then - call endrun(msg=' Error totvegcthresh is zero or negative and should be > 0'//& - errMsg(sourcefile, __LINE__)) - end if - this%totvegcthresh = totvegcthresh + this%ncrit = ncrit - end subroutine SetTotVgCThresh + end subroutine SetNCrit end module SoilBiogeochemNitrogenStateType From 30aac3ebd04e4c0e571bcadc1e2ffd6b5aa344ba Mon Sep 17 00:00:00 2001 From: Peter Lawrence Date: Mon, 5 Mar 2018 14:54:46 -0700 Subject: [PATCH 002/332] Removed soil biogeochem files that were not part of the shifting cultivation. These files were actually changes in r267 for Keith's simulations. By including the updated files this resulted in a build error. --- .../SoilBiogeochemCarbonStateType.F90 | 27 +++++++++++-------- .../SoilBiogeochemNitrogenStateType.F90 | 27 +++++++++++-------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 b/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 index 62683d4256..78be5efd44 100644 --- a/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 +++ b/src/soilbiogeochem/SoilBiogeochemCarbonStateType.F90 @@ -38,7 +38,7 @@ module SoilBiogeochemCarbonStateType real(r8), pointer :: decomp_cpools_col (:,:) ! (gC/m2) decomposing (litter, cwd, soil) c pools real(r8), pointer :: dyn_cbal_adjustments_col (:) ! (gC/m2) adjustments to each column made in this timestep via dynamic column area adjustments (note: this variable only makes sense at the column-level: it is meaningless if averaged to the gridcell-level) integer :: restart_file_spinup_state ! spinup state as read from restart file, for determining whether to enter or exit spinup mode. - real(r8) :: ccrit ! critical carbon value + real(r8) :: totvegcthresh ! threshold for total vegetation carbon to zero out decomposition pools contains @@ -46,7 +46,7 @@ module SoilBiogeochemCarbonStateType procedure , public :: SetValues procedure , public :: Restart procedure , public :: Summary - procedure , public :: SetCCrit + procedure , public :: SetTotVgCThresh procedure , public :: DynamicColumnAdjustments ! adjust state variables when column areas change procedure , private :: InitAllocate procedure , private :: InitHistory @@ -70,7 +70,7 @@ subroutine Init(this, bounds, carbon_type, ratio, c12_soilbiogeochem_carbonstate real(r8) , intent(in) :: ratio type(soilbiogeochem_carbonstate_type) , intent(in), optional :: c12_soilbiogeochem_carbonstate_inst - this%ccrit = nan + this%totvegcthresh = nan call this%InitAllocate ( bounds) call this%InitHistory ( bounds, carbon_type ) if (present(c12_soilbiogeochem_carbonstate_inst)) then @@ -674,6 +674,10 @@ subroutine Restart ( this, bounds, ncid, flag, carbon_type, totvegc_col, c12_so call endrun(msg=' CNRest: error in entering/exiting spinup - should occur only when nstep = 1'//& errMsg(sourcefile, __LINE__)) endif + if ( exit_spinup .and. isnan(this%totvegcthresh) )then + call endrun(msg=' CNRest: error in exit spinup - totvegcthresh was not set with SetTotVgCThresh'//& + errMsg(sourcefile, __LINE__)) + end if do k = 1, ndecomp_pools if ( exit_spinup ) then m = decomp_cascade_con%spinup_factor(k) @@ -690,10 +694,7 @@ subroutine Restart ( this, bounds, ncid, flag, carbon_type, totvegc_col, c12_so ! reset decomp pools to near zero during exit_spinup to avoid very ! large and inert soil carbon stocks; note that only pools with spinup factor > 1 ! will be affected, which means that total SOMC and LITC pools will not be set to 0. -!KO if (totvegc_col(c) <= this%ccrit .and. lun%itype(l) /= istcrop) then -!KO - if (totvegc_col(c) <= 0.1_r8 .and. lun%itype(l) /= istcrop) then -!KO + if (totvegc_col(c) <= this%totvegcthresh .and. lun%itype(l) /= istcrop) then this%decomp_cpools_vr_col(c,j,k) = 0.0_r8 endif elseif ( abs(m - 1._r8) .gt. 0.000001_r8 .and. enter_spinup) then @@ -926,14 +927,18 @@ subroutine Summary(this, bounds, num_allc, filter_allc) end subroutine Summary !------------------------------------------------------------------------ - subroutine SetCCrit(this, ccrit) + subroutine SetTotVgCThresh(this, totvegcthresh) class(soilbiogeochem_carbonstate_type) :: this - real(r8) , intent(in) :: ccrit + real(r8) , intent(in) :: totvegcthresh - this%ccrit = ccrit + if ( totvegcthresh <= 0.0_r8 )then + call endrun(msg=' ERROR totvegcthresh is zero or negative and should be > 0'//& + errMsg(sourcefile, __LINE__)) + end if + this%totvegcthresh = totvegcthresh - end subroutine SetCCrit + end subroutine SetTotVgCThresh !----------------------------------------------------------------------- diff --git a/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 b/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 index b776f77731..ca09e63624 100644 --- a/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 +++ b/src/soilbiogeochem/SoilBiogeochemNitrogenStateType.F90 @@ -52,7 +52,7 @@ module SoilBiogeochemNitrogenStateType ! the N balance check real(r8), pointer :: dyn_no3bal_adjustments_col (:) ! (gN/m2) NO3 adjustments to each column made in this timestep via dynamic column area adjustments (only makes sense at the column-level: meaningless if averaged to the gridcell-level) real(r8), pointer :: dyn_nh4bal_adjustments_col (:) ! (gN/m2) NH4 adjustments to each column made in this timestep via dynamic column adjustments (only makes sense at the column-level: meaningless if averaged to the gridcell-level) - real(r8) :: ncrit ! (gN/m2) critical value of nitrogen before truncation + real(r8) :: totvegcthresh ! threshold for total vegetation carbon to zero out decomposition pools contains @@ -61,7 +61,7 @@ module SoilBiogeochemNitrogenStateType procedure , public :: SetValues procedure , public :: Summary procedure , public :: DynamicColumnAdjustments ! adjust state variables when column areas change - procedure , public :: SetNCrit + procedure , public :: SetTotVgCThresh ! Set value for totvegcthresh needed in Restart procedure , private :: InitAllocate procedure , private :: InitHistory procedure , private :: InitCold @@ -84,7 +84,7 @@ subroutine Init(this, bounds, & real(r8) , intent(in) :: decomp_cpools_col (bounds%begc:, 1:) real(r8) , intent(in) :: decomp_cpools_1m_col (bounds%begc:, 1:) - this%ncrit = nan + this%totvegcthresh = nan call this%InitAllocate (bounds ) call this%InitHistory (bounds) @@ -610,6 +610,10 @@ subroutine Restart ( this, bounds, ncid, flag, totvegc_col ) call endrun(msg=' Error in entering/exiting spinup - should occur only when nstep = 1'//& errMsg(sourcefile, __LINE__)) endif + if ( exit_spinup .and. isnan(this%totvegcthresh) )then + call endrun(msg=' Error in exit spinup - totvegcthresh was not set with SetTotVgCThresh'//& + errMsg(sourcefile, __LINE__)) + end if do k = 1, ndecomp_pools if ( exit_spinup ) then m = decomp_cascade_con%spinup_factor(k) @@ -631,10 +635,7 @@ subroutine Restart ( this, bounds, ncid, flag, totvegc_col ) ! pools with spinup factor > 1 ! will be affected, which means that total SOMN and LITN ! pools will not be set to 0. -!KO if (totvegc_col(c) <= this%ncrit .and. lun%itype(l) /= istcrop) then -!KO - if (totvegc_col(c) <= 0.1_r8 .and. lun%itype(l) /= istcrop) then -!KO + if (totvegc_col(c) <= this%totvegcthresh .and. lun%itype(l) /= istcrop) then this%decomp_npools_vr_col(c,j,k) = 0._r8 endif elseif ( abs(m - 1._r8) .gt. 0.000001_r8 .and. enter_spinup) then @@ -1006,13 +1007,17 @@ subroutine DynamicColumnAdjustments(this, bounds, clump_index, column_state_upda end subroutine DynamicColumnAdjustments !------------------------------------------------------------------------ - subroutine SetNCrit(this, ncrit) + subroutine SetTotVgCThresh(this, totvegcthresh) class(soilbiogeochem_nitrogenstate_type) :: this - real(r8) , intent(in) :: ncrit + real(r8) , intent(in) :: totvegcthresh - this%ncrit = ncrit + if ( totvegcthresh <= 0.0_r8 )then + call endrun(msg=' Error totvegcthresh is zero or negative and should be > 0'//& + errMsg(sourcefile, __LINE__)) + end if + this%totvegcthresh = totvegcthresh - end subroutine SetNCrit + end subroutine SetTotVgCThresh end module SoilBiogeochemNitrogenStateType From ac4d0d4cfcde04003511f2ea8901e2158ccf5454 Mon Sep 17 00:00:00 2001 From: Peter Lawrence Date: Mon, 2 Apr 2018 15:57:40 -0600 Subject: [PATCH 003/332] Update these two files to address Fail status in CLM5 testing. CLM tests all pass now with the note that the new code changes the DWT_SLASH_CFLUX history variable from the column to the grid cell to be consistent with the other DWT fluxes. An additional variable DWT_SLASH_CFLUX_PATCH has been added to allow subgrid calculation at the patch and column levels as done with other DWT fluxes. This change results in a failure in the ERS_Ly3.f10_f10.I1850Clm50BgcCrop test that requires the user_nl_clm file to specify the hist_fincl3 to use DWT_SLASH_CFLUX_PATCH rather than DWT_SLASH_CFLUX. After this change the test passes. . The result from the test suite are: ================================================================================ These tests failed ================================================================================ ERP_D_Lm9.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_monthly EXPECTED ERS_Ly5_P72x1.f10_f10_musgs.IHistClm45BgcCrop.cheyenne_intel.clm-cropMonthOutput EXPECTED NCK_Ld1.f10_f10_musgs.I2000Clm50Sp.cheyenne_intel.clm-default EXPECTED SMS_D_Ld5.f45_f45_mg37.I2000Clm50Fates.cheyenne_intel.clm-Fates EXPECTED SMS_D_Lm13.f10_f10_musgs.IHistClm50BgcCrop.cheyenne_intel.clm-ciso_monthly EXPECTED SMS_D_Lm6.f45_f45_mg37.I2000Clm50Fates.cheyenne_intel.clm-Fates EXPECTED SMS_D_Lm6_P144x1.f45_f45_mg37.I2000Clm50Fates.cheyenne_intel.clm-Fates EXPECTED SMS_Ld1.f09_g16.I1850Clm40SpCruGs.cheyenne_intel.clm-40default EXPECTED --- src/biogeochem/CNVegNitrogenFluxType.F90 | 2 +- src/biogeochem/dynGrossUnrepMod.F90 | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/biogeochem/CNVegNitrogenFluxType.F90 b/src/biogeochem/CNVegNitrogenFluxType.F90 index 1bb3e223f9..64930e6c7c 100644 --- a/src/biogeochem/CNVegNitrogenFluxType.F90 +++ b/src/biogeochem/CNVegNitrogenFluxType.F90 @@ -488,7 +488,7 @@ subroutine InitAllocate(this, bounds) allocate(this%gru_livestemn_to_atm_patch (begp:endp)) ; this%gru_livestemn_to_atm_patch (:) = nan allocate(this%gru_livestemn_storage_to_atm_patch (begp:endp)) ; this%gru_livestemn_storage_to_atm_patch (:) = nan allocate(this%gru_livestemn_xfer_to_atm_patch (begp:endp)) ; this%gru_livestemn_xfer_to_atm_patch (:) = nan - allocate(this%gru_deadstemn_to_atm_patch (begp:endp)) ; this%gru_deadstemn_storage_to_atm_patch (:) = nan + allocate(this%gru_deadstemn_to_atm_patch (begp:endp)) ; this%gru_deadstemn_to_atm_patch (:) = nan allocate(this%gru_deadstemn_storage_to_atm_patch (begp:endp)) ; this%gru_deadstemn_storage_to_atm_patch (:) = nan allocate(this%gru_deadstemn_xfer_to_atm_patch (begp:endp)) ; this%gru_deadstemn_xfer_to_atm_patch (:) = nan allocate(this%gru_livecrootn_storage_to_atm_patch (begp:endp)) ; this%gru_livecrootn_storage_to_atm_patch (:) = nan diff --git a/src/biogeochem/dynGrossUnrepMod.F90 b/src/biogeochem/dynGrossUnrepMod.F90 index 15d5db95a6..0774a3a1a5 100644 --- a/src/biogeochem/dynGrossUnrepMod.F90 +++ b/src/biogeochem/dynGrossUnrepMod.F90 @@ -95,7 +95,8 @@ subroutine dynGrossUnrep_init(bounds, grossunrep_filename) gru_inst = dyn_var_time_uninterp_type( & dyn_file=dyngrossunrep_file, varname=grossunrep_varname, & dim1name=grlnd, conversion_factor=1.0_r8, & - do_check_sums_equal_1=.false., data_shape=[num_points,natpft_size]) + do_check_sums_equal_1=.false., data_shape=[num_points,natpft_size], & + allow_nodata=.true.) end subroutine dynGrossUnrep_init From ffc742eefaf10e516217117ebe90ecbcde4468af Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Apr 2018 12:27:45 -0600 Subject: [PATCH 004/332] Switch DWT_SLASH_CFLUX for DWT_SLASH_CFLUX_PATCH as required in this branch for shifting cultivation --- cime_config/usermods_dirs/cmip6_output/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/cmip6_output/user_nl_clm b/cime_config/usermods_dirs/cmip6_output/user_nl_clm index bd333a213b..88a9399a31 100644 --- a/cime_config/usermods_dirs/cmip6_output/user_nl_clm +++ b/cime_config/usermods_dirs/cmip6_output/user_nl_clm @@ -31,7 +31,7 @@ hist_fincl2 = 'GPP', 'NPP', 'AGNPP', 'TLAI', 'TOTVEGC', 'TSA','TREFMNAV','TREFMX hist_fincl3 = 'FSR', 'H2OSNO', 'Q2M', 'SNOWDP', 'TSA', 'TREFMNAV', 'TREFMXAV', 'TG', 'QRUNOFF', 'FSH', 'FIRE', 'FIRA', 'FGR', 'EFLX_LH_TOT', 'RH2M', 'TLAI', 'GPP', 'NPP', 'SOILWATER_10CM', 'TOTSOILLIQ', 'TOTSOILICE', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'U10', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'GRAINC_TO_FOOD', 'SLASH_HARVESTC', 'TSOI_10CM', 'COL_FIRE_CLOSS', -'DWT_SLASH_CFLUX','TOTSOMC:I', 'TOTSOMC_1m:I', 'TOTECOSYSC:I', 'TOTVEGC:I', 'WOODC:I', 'TOTLITC:I', 'LIVECROOTC:I', 'DEADCROOTC:I', +'DWT_SLASH_CFLUX_PATCH','TOTSOMC:I', 'TOTSOMC_1m:I', 'TOTECOSYSC:I', 'TOTVEGC:I', 'WOODC:I', 'TOTLITC:I', 'LIVECROOTC:I', 'DEADCROOTC:I', 'FROOTC:I' hist_fincl4 = 'PCT_CFT', 'PCT_GLC_MEC', 'SOIL1C_vr', 'SOIL1N_vr', 'SOIL2C_vr', 'SOIL2N_vr', 'SOIL3C_vr', 'SOIL3N_vr', 'CWDC_vr', From ee439c181d74130aefb9d08b432d3db61cb370dd Mon Sep 17 00:00:00 2001 From: Peter Lawrence Date: Sat, 22 Sep 2018 08:36:58 -0600 Subject: [PATCH 005/332] New changes to set up the namelist variable do_grossunrep. By default do_grossunrep is set to .false. When set to .true. do_grossunrep turns on shifting cultivation which is read off the landuse_timeseries file. --- bld/CLMBuildNamelist.pm | 60 +++++++++++++++++++ .../namelist_definition_clm4_5.xml | 7 +++ src/dyn_subgrid/dynSubgridControlMod.F90 | 2 +- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 3595ec7461..58389e83e4 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -2473,6 +2473,7 @@ sub setup_logic_dynamic_subgrid { setup_logic_do_transient_pfts($opts, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_do_transient_crops($opts, $nl_flags, $definition, $defaults, $nl, $physv); setup_logic_do_harvest($opts, $nl_flags, $definition, $defaults, $nl, $physv); + setup_logic_do_grossunrep($opts, $nl_flags, $definition, $defaults, $nl, $physv); } @@ -2657,6 +2658,65 @@ sub setup_logic_do_harvest { #------------------------------------------------------------------------------- +sub setup_logic_do_grossunrep { + # + # Set do_grossunrep default value, and perform error checking on do_grossunrep + # + # Assumes the following are already set in the namelist (although it's okay + # for them to be unset if that will be their final state): + # - flanduse_timeseries + # - use_cn + # - use_fates + # + my ($opts, $nl_flags, $definition, $defaults, $nl, $physv) = @_; + + my $var = 'do_grossunrep'; + + if ($physv->as_long() >= $physv->as_long("clm4_5")) { + # Start by assuming a default value of '.true.'. Then check a number of + # conditions under which do_grossunrep cannot be true. Under these + # conditions: (1) set default value to '.false.'; (2) make sure that the + # value is indeed false (e.g., that the user didn't try to set it to true). + + my $default_val = ".false."; + + # cannot_be_true will be set to a non-empty string in any case where + # do_grossunrep should not be true; if it turns out that do_grossunrep IS true + # in any of these cases, a fatal error will be generated + my $cannot_be_true = ""; + + if (string_is_undef_or_empty($nl->get_value('flanduse_timeseries'))) { + $cannot_be_true = "$var can only be set to true when running a transient case (flanduse_timeseries non-blank)"; + } + elsif (!&value_is_true($nl->get_value('use_cn'))) { + $cannot_be_true = "$var can only be set to true when running with CN (use_cn = true)"; + } + elsif (&value_is_true($nl->get_value('use_fates'))) { + $cannot_be_true = "$var currently doesn't work with ED"; + } + + if ($cannot_be_true) { + $default_val = ".false."; + } + + if (!$cannot_be_true) { + # Note that, if the variable cannot be true, we don't call add_default + # - so that we don't clutter up the namelist with variables that don't + # matter for this case + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, val=>$default_val); + } + + # Make sure the value is false when it needs to be false - i.e., that the + # user hasn't tried to set a true value at an inappropriate time. + + if (&value_is_true($nl->get_value($var)) && $cannot_be_true) { + $log->fatal_error($cannot_be_true); + } + + } +} + +#------------------------------------------------------------------------------- sub setup_logic_spinup { my ($opts, $nl_flags, $definition, $defaults, $nl, $physv) = @_; diff --git a/bld/namelist_files/namelist_definition_clm4_5.xml b/bld/namelist_files/namelist_definition_clm4_5.xml index 0a4bbd9a28..db030cd2a8 100644 --- a/bld/namelist_files/namelist_definition_clm4_5.xml +++ b/bld/namelist_files/namelist_definition_clm4_5.xml @@ -2225,6 +2225,13 @@ If TRUE, apply harvest from flanduse_timeseries file. (Also, only valid for use_cn = true.) + +If TRUE, apply grossunrep from flanduse_timeseries file. +(Only valid for transient runs, where there is a flanduse_timeseries file.) +(Also, only valid for use_cn = true.) + + If TRUE, allow area changes at times other than the year boundary. This should diff --git a/src/dyn_subgrid/dynSubgridControlMod.F90 b/src/dyn_subgrid/dynSubgridControlMod.F90 index 3491057dc0..ca057e9202 100644 --- a/src/dyn_subgrid/dynSubgridControlMod.F90 +++ b/src/dyn_subgrid/dynSubgridControlMod.F90 @@ -131,6 +131,7 @@ subroutine read_namelist( NLFilename ) do_transient_pfts, & do_transient_crops, & do_harvest, & + do_grossunrep, & for_testing_allow_non_annual_changes, & for_testing_zero_dynbal_fluxes @@ -157,7 +158,6 @@ subroutine read_namelist( NLFilename ) end if close(nu_nml) call relavu( nu_nml ) - do_grossunrep = do_harvest endif call shr_mpi_bcast (flanduse_timeseries, mpicom) From 9532135c3ddc51c5462f6cda4be4dab16b4f3205 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 6 Aug 2019 15:54:13 -0600 Subject: [PATCH 006/332] Change notes about ED to FATES, and add a test for do_grossunrep=T with transient, but SP instead of CN --- bld/CLMBuildNamelist.pm | 12 ++++++------ bld/unit_testers/build-namelist_test.pl | 7 ++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 74ca2c8910..f34aff7993 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -734,7 +734,7 @@ sub setup_cmdl_fates_mode { $var = "use_fates"; if ( &value_is_true($nl_flags->{$var}) ) { - # This section is a place-holder to test for modules that are not allowed with ED + # This section is a place-holder to test for modules that are not allowed with FATES # the defaults which are set in the logic section of the namelist builder will # automatically set these correctly (well that is the assumption), but here we # want to set a catch to fail and warn users if they explicitly set incompatible user namelist @@ -1138,7 +1138,7 @@ sub setup_cmdl_spinup { $log->fatal_error("$var has an invalid value ($val). Valid values are: @valid_values"); } if ( $nl_flags->{'bgc_spinup'} eq "on" && (not &value_is_true( $nl_flags->{'use_cn'} )) && (not &value_is_true($nl_flags->{'use_fates'})) ) { - $log->fatal_error("$var can not be '$nl_flags->{'bgc_spinup'}' if neither CN nor ED is turned on (use_cn=$nl_flags->{'use_cn'}, use_fates=$nl_flags->{'use_fates'})."); + $log->fatal_error("$var can not be '$nl_flags->{'bgc_spinup'}' if neither CN nor FATES is turned on (use_cn=$nl_flags->{'use_cn'}, use_fates=$nl_flags->{'use_fates'})."); } if ( $nl->get_value("spinup_state") eq 0 && $nl_flags->{'bgc_spinup'} eq "on" ) { $log->fatal_error("Namelist spinup_state contradicts the command line option bgc_spinup" ); @@ -2488,7 +2488,7 @@ sub setup_logic_do_transient_crops { # In principle, use_fates should be compatible with # do_transient_crops. However, this hasn't been tested, so to be safe, # we are not allowing this combination for now. - $cannot_be_true = "$var has not been tested with ED, so for now these two options cannot be combined"; + $cannot_be_true = "$var has not been tested with FATES, so for now these two options cannot be combined"; } if ($cannot_be_true) { @@ -2559,7 +2559,7 @@ sub setup_logic_do_harvest { } elsif (!&value_is_true($nl->get_value('use_cn'))) { $cannot_be_true = "$var can only be set to true when running with CN (use_cn = true)"; } elsif (&value_is_true($nl->get_value('use_fates'))) { - $cannot_be_true = "$var currently doesn't work with ED"; + $cannot_be_true = "$var currently doesn't work with FATES"; } if ($cannot_be_true) { @@ -2616,7 +2616,7 @@ sub setup_logic_do_grossunrep { $cannot_be_true = "$var can only be set to true when running with CN (use_cn = true)"; } elsif (&value_is_true($nl->get_value('use_fates'))) { - $cannot_be_true = "$var currently doesn't work with ED"; + $cannot_be_true = "$var currently doesn't work with FATES"; } if ($cannot_be_true) { @@ -3419,7 +3419,7 @@ sub setup_logic_megan { if ($nl_flags->{'megan'} ) { if ( &value_is_true( $nl_flags->{'use_fates'} ) ) { - $log->fatal_error("MEGAN can NOT be on when ED is also on.\n" . + $log->fatal_error("MEGAN can NOT be on when FATES is also on.\n" . " Use the '-no-megan' option when '-bgc fates' is activated"); } add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'megan_specifier'); diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 39756d8055..79253617e9 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -138,7 +138,7 @@ sub make_config_cache { # # Figure out number of tests that will run # -my $ntests = 927; +my $ntests = 928; if ( defined($opts{'compare'}) ) { $ntests += 588; } @@ -849,6 +849,11 @@ sub make_config_cache { GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm5_0", }, + "dogrossandsp" =>{ options=>"-envxml_dir . -bgc sp -use_case 20thC_transient", + namelst=>"do_grossunrep=.true.", + GLC_TWO_WAY_COUPLING=>"FALSE", + phys=>"clm5_0", + }, "nocropwfert" =>{ options=>"-envxml_dir . -bgc sp -no-crop", namelst=>"use_fertilizer=T", GLC_TWO_WAY_COUPLING=>"FALSE", From 9704782c55eb657542abeb1411a66f5cb5234f7a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 6 Aug 2019 16:24:10 -0600 Subject: [PATCH 007/332] Check that turning on do_grossunrep=T works for 20th century transient cases, so also turn on BGC and crop --- .../namelist_definition_ctsm.xml | 2 +- bld/unit_testers/build-namelist_test.pl | 2 +- .../usermods_dirs/cmip6_output/user_nl_clm | 77 ------------------- 3 files changed, 2 insertions(+), 79 deletions(-) delete mode 100644 cime_config/usermods_dirs/cmip6_output/user_nl_clm diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 9d94b10be8..cd3eb0d5c6 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -2344,7 +2344,7 @@ If TRUE, apply harvest from flanduse_timeseries file. -If TRUE, apply grossunrep from flanduse_timeseries file. +If TRUE, apply gross unrepresented landuse/land-cover change from flanduse_timeseries file. (Only valid for transient runs, where there is a flanduse_timeseries file.) (Also, only valid for use_cn = true.) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 79253617e9..5535712803 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -1224,7 +1224,7 @@ sub make_config_cache { my $usecase = "20thC_transient"; my $GLC_NEC = 10; foreach my $res ( @tran_res ) { - $options = "-res $res -use_case $usecase -envxml_dir . "; + $options = "-res $res -use_case $usecase -envxml_dir . -bgc bgc -crop -namelist '&a do_grossunrep=T/'"; &make_env_run(); eval{ system( "$bldnml $options > $tempfile 2>&1 " ); }; is( $@, '', "$options" ); diff --git a/cime_config/usermods_dirs/cmip6_output/user_nl_clm b/cime_config/usermods_dirs/cmip6_output/user_nl_clm deleted file mode 100644 index 0df7c685eb..0000000000 --- a/cime_config/usermods_dirs/cmip6_output/user_nl_clm +++ /dev/null @@ -1,77 +0,0 @@ -!---------------------------------------------------------------------------------- -! Users should add all user specific namelist changes below in the form of -! namelist_var = new_namelist_value -! -! EXCEPTIONS: -! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting -! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting -! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting -! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting -! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting -! Set irrigate by the CLM_BLDNML_OPTS -irrig setting -! Set co2_ppmv with CCSM_CO2_PPMV option -! Set dtime with L_NCPL option -! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options -! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases -! (includes $inst_string for multi-ensemble cases) -! Set maxpatch_glcmec with GLC_NEC option -! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable -!---------------------------------------------------------------------------------- - -! h0 stream (monthly average, gridcell-level) -! Emon, Lmon, LIMon -hist_fexcl1 = 'PCT_CFT', 'PCT_GLC_MEC', 'SOIL1C_vr', 'SOIL1N_vr', 'SOIL2C_vr', 'SOIL2N_vr', 'SOIL3C_vr', 'SOIL3N_vr', 'CWDC_vr', -'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', -'PCT_NAT_PFT','SMIN_NO3_vr','CONC_O2_UNSAT', 'CONC_O2_SAT','SMIN_NH4_vr','SMINN_vr' - -! h0 stream ICE variables and other variables inactive by default (monthly average, gridcell-level) -! Emon, Lmon, LIMon -hist_fincl1 = 'EFLX_LH_TOT_ICE', 'FIRE_ICE', 'FLDS_ICE', 'FSH_ICE', 'FSNO_ICE', 'FSR_ICE', -'QFLX_SUB_SNOW_ICE', 'QRUNOFF_ICE', 'QSNOFRZ_ICE', 'QSNOMELT_ICE', 'RAIN_ICE', 'SNOW_ICE', -'SNOWICE_ICE', 'SNOWLIQ_ICE', 'SNOTXMASS_ICE', 'TG_ICE', 'TOPO_COL_ICE', 'TSA_ICE', 'TSOI_ICE', -'LEAFC_TO_LITTER', 'FROOTC_TO_LITTER','LITR1C_TO_SOIL1C','LITR1N_TO_SOIL1N','LITR2C_TO_SOIL1C', -'LITR2N_TO_SOIL1N','LITR3C_TO_SOIL2C','LITR3N_TO_SOIL2N','DWT_WOOD_PRODUCTC_GAIN_PATCH' - -! h1 stream (monthly average, finest sub-grid) -! Emon, Lmon -hist_fincl2 = 'GPP', 'NPP', 'AGNPP', 'TLAI', 'TOTVEGC', 'TSA','TREFMNAV','TREFMXAV', 'BTRANMN', 'NPP_NUPTAKE', 'GRAINC_TO_FOOD', -'Vcmx25Z', 'FSH', 'NFERTILIZATION', 'AR', 'VEGWP', 'FCTR', 'FCEV', 'FGEV', 'FIRE', 'FSR', 'HTOP', -'FIRA', 'FSA', 'HR', 'GSSUNLN', 'GSSHALN', 'TSKIN' - -! h2 stream (monthly average, landunit-level) -! Emon -! TOT_WOODPRODC:I, CROPPROD1C:I, and NEE are not available at the landunit level -hist_fincl3 = 'FSR', 'H2OSNO', 'Q2M', 'SNOWDP', 'TSA', 'TREFMNAV', 'TREFMXAV', 'TG', 'QRUNOFF', 'FSH', 'FIRE', 'FIRA', 'FGR', -'EFLX_LH_TOT', 'RH2M', 'TLAI', 'GPP', 'NPP', 'SOILWATER_10CM', 'TOTSOILLIQ', 'TOTSOILICE', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', -'WOOD_HARVESTC', 'U10', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'GRAINC_TO_FOOD', 'SLASH_HARVESTC', 'TSOI_10CM', 'COL_FIRE_CLOSS', -'DWT_SLASH_CFLUX_PATCH', 'QIRRIG', 'URBAN_HEAT', 'WASTEHEAT', 'HTOP', 'TSKIN', 'FROOTC:I' - -! h3 stream (yearly average, gridcell-level) -! Eyr -hist_fincl4 = 'PCT_CFT', 'PCT_GLC_MEC', 'SOIL1C_vr', 'SOIL1N_vr', 'SOIL2C_vr', 'SOIL2N_vr', 'SOIL3C_vr', 'SOIL3N_vr', 'CWDC_vr', -'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', 'QICE_FORC', 'TSRF_FORC', 'TOPO_FORC' -'PCT_NAT_PFT','PCT_LANDUNIT','TOTLITC:I','TOT_WOODPRODC:I','CROPPROD1C:I','TOTSOMC:I','TOTVEGC:I','FSNO_ICE' - -! h4 stream (yearly average, landunit-level) -! Eyr landuse tiles -hist_fincl5 = 'TOTSOMC:I', 'TOTSOMC_1m:I', 'TOTECOSYSC:I', 'TOTVEGC:I', 'WOODC:I', 'TOTLITC:I', 'LIVECROOTC:I', 'DEADCROOTC:I', 'FROOTC:I' - -! h5 stream (daily average, gridcell-level) -! day, Eday -hist_fincl6 = 'SOILWATER_10CM', 'TOTSOILLIQ', 'TOTSOILICE', 'GPP', 'NPP', 'AR', 'HR', 'EFLX_LH_TOT', 'FSH', 'FGR12', 'FSM', 'QSNOEVAP', 'TLAI', 'QDRAI', 'QDRAI_PERCH', 'QOVER', 'QFLX_SUB_SNOW', 'FSA', 'FIRA', 'H2OSNO', 'SNOCAN', 'QSNOFRZ', 'QFLX_SNOW_DRAIN', 'SNOWDP', 'H2OSFC', 'TV', 'TG', 'TAUX', 'TAUY', 'QVEGT', 'TWS', 'H2OCAN', 'ALT', 'QVEGE', 'QSOIL', 'TSKIN', 'FSDS','FSNO','SNOFSRVD','SNOFSRVI','SNOFSRND','SNOFSRNI','FSDSVD','FSDSVI','FSDSND','FSDSNI','SNOWLIQ','SOILICE','SOILLIQ','QINTR','SNOBCMSL','TSOI','SNOTXMASS','SNOWICE','SNOWLIQ','QRUNOFF','RAIN','SNOW' - -! h6 stream (daily average, landunit-level) -! Eday -hist_fincl7 = 'TREFMXAV','TREFMNAV' - -! h7 stream (3-hourly average, gridcell-level) -! 3hr, E3hr, CF3hr -! 3hr requires QRUNOFF for time mean, and SOILWATER_10CM, TSKIN for time point (I) -! E3hr requires GPP, AR, HR for time mean -! CF3hr requires QFLX_SUB_SNOW for time point (I) -hist_fincl8 = 'GPP','AR','HR','TSA','RH2M','SOILWATER_10CM:I','FSH','EFLX_LH_TOT','FSDS','QRUNOFF','QFLX_SUB_SNOW:I','TSKIN:I' - -hist_mfilt = 1,1,1,1,1,365,365,2920 -hist_dov2xy = .true.,.false.,.false.,.true.,.false.,.true.,.false.,.true. -hist_nhtfrq = 0,0,0,-8760,-8760,-24,-24,-3 -hist_type1d_pertape = ' ',' ','LAND',' ','LAND',' ','LAND',' ' From 9f73f6e9f0c6d6d8753c5b8f94c19764643b2d20 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 6 Aug 2019 22:41:50 -0600 Subject: [PATCH 008/332] Change maxpatch_pft to maxsoil_patches as it was changed, so now works compilers and runs --- src/biogeochem/CNCIsoFluxMod.F90 | 4 ++-- src/biogeochem/dynGrossUnrepMod.F90 | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/biogeochem/CNCIsoFluxMod.F90 b/src/biogeochem/CNCIsoFluxMod.F90 index 7c9697bbaf..f07e6b7a62 100644 --- a/src/biogeochem/CNCIsoFluxMod.F90 +++ b/src/biogeochem/CNCIsoFluxMod.F90 @@ -1767,7 +1767,7 @@ subroutine CNCIsoGrossUnrepPftToColumn (num_soilc, filter_soilc, & ) do j = 1, nlevdecomp - do pi = 1,maxpatch_pft + do pi = 1,maxsoil_patches do fc = 1,num_soilc c = filter_soilc(fc) @@ -1806,7 +1806,7 @@ subroutine CNCIsoGrossUnrepPftToColumn (num_soilc, filter_soilc, & end do end do - do pi = 1,maxpatch_pft + do pi = 1,maxsoil_patches do fc = 1,num_soilc c = filter_soilc(fc) diff --git a/src/biogeochem/dynGrossUnrepMod.F90 b/src/biogeochem/dynGrossUnrepMod.F90 index 0774a3a1a5..1ad01c9348 100644 --- a/src/biogeochem/dynGrossUnrepMod.F90 +++ b/src/biogeochem/dynGrossUnrepMod.F90 @@ -375,7 +375,7 @@ subroutine CNGrossUnrepPftToColumn (num_soilc, filter_soilc, & ! to the column level and assign them to the three litter pools ! ! !USES: - use clm_varpar , only : maxpatch_pft, nlevdecomp, numpft + use clm_varpar , only : maxsoil_patches, nlevdecomp ! ! !ARGUMENTS: integer , intent(in) :: num_soilc ! number of soil columns in filter @@ -435,7 +435,7 @@ subroutine CNGrossUnrepPftToColumn (num_soilc, filter_soilc, & ) do j = 1, nlevdecomp - do pi = 1,maxpatch_pft + do pi = 1,maxsoil_patches do fc = 1,num_soilc c = filter_soilc(fc) @@ -500,7 +500,7 @@ subroutine CNGrossUnrepPftToColumn (num_soilc, filter_soilc, & end do end do - do pi = 1,maxpatch_pft + do pi = 1,maxsoil_patches do fc = 1,num_soilc c = filter_soilc(fc) From a7c384e38c83c3e7ad8bc2bb5dde799b7ddcfc2b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 6 Aug 2019 22:43:07 -0600 Subject: [PATCH 009/332] Change a test to do a test with transient Gross Unrepresented LULCC --- cime_config/testdefs/testlist_clm.xml | 4 ++-- .../clm/ciso_decStart_GU_LULCC/include_user_mods | 1 + .../testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods create mode 100644 cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 8409089613..8906c6c4bd 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -314,14 +314,14 @@ - + - + diff --git a/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods new file mode 100644 index 0000000000..c0adfb5904 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods @@ -0,0 +1 @@ +../ciso_decStart diff --git a/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm new file mode 100644 index 0000000000..8d3ca17778 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm @@ -0,0 +1 @@ +do_grossunrep = .true. From b91c593028ca1e28b0b7ba4c7168e649e0ca10a0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 14 Aug 2019 15:51:28 -0600 Subject: [PATCH 010/332] Remove a git conflict leftover, and close the toosmall_* entries, so it will pass xmllint --- bld/namelist_files/namelist_definition_ctsm.xml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index cd3eb0d5c6..f7d24e077a 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -1111,37 +1111,37 @@ If true, this directs the model to collapse the urban landunits to the dominant Percentage threshold above which the model keeps the soil landunit. Selecting the value 0 means DO NOTHING. Default: 0 Percentage threshold above which the model keeps the crop landunit. Selecting the value 0 means DO NOTHING. Default: 0 Percentage threshold above which the model keeps the glacier landunit. Selecting the value 0 means DO NOTHING. Default: 0 Percentage threshold above which the model keeps the lake landunit. Selecting the value 0 means DO NOTHING. Default: 0 Percentage threshold above which the model keeps the wetland landunit. Selecting the value 0 means DO NOTHING. Default: 0 Percentage threshold above which the model keeps the urban landunits. Selecting the value 0 means DO NOTHING. If collapse_urban = .false., the same threshold will apply to all three urban landunits if they are present. If collapse_urban = .true., this threshold will apply to the single collapsed urban landunit if presnet. Default: 0 @@ -2395,7 +2395,6 @@ continue run; it is an error for this to be set in a branch run. Furthermore, this setting has no effect in a cold start run. ->>>>>>> ctsm1.0.dev055:bld/namelist_files/namelist_definition_ctsm.xml If TRUE, allow area changes at times other than the year boundary. This should From cc7d56567044959b937d8ce4a2986279aeeb044d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 15 Aug 2019 14:35:56 -0600 Subject: [PATCH 011/332] Move SLASH_HARVESTC to gridcell level list --- cime_config/usermods_dirs/output_bgc/user_nl_clm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/output_bgc/user_nl_clm b/cime_config/usermods_dirs/output_bgc/user_nl_clm index de136ad4b4..607c2ee25c 100644 --- a/cime_config/usermods_dirs/output_bgc/user_nl_clm +++ b/cime_config/usermods_dirs/output_bgc/user_nl_clm @@ -11,10 +11,10 @@ hist_fincl2 += 'GPP', 'NPP', 'AGNPP', 'TOTVEGC', 'NPP_NUPTAKE', 'AR', 'HR', 'HTO ! h2 stream (monthly average, landunit-level) ! TOT_WOODPRODC:I, CROPPROD1C:I, and NEE are not available at the landunit level -hist_fincl3 += 'GPP', 'NPP', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'SLASH_HARVESTC', 'COL_FIRE_CLOSS', 'DWT_SLASH_CFLUX', 'FROOTC:I', 'HTOP' +hist_fincl3 += 'GPP', 'NPP', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'COL_FIRE_CLOSS', 'DWT_SLASH_CFLUX', 'FROOTC:I', 'HTOP' ! h3 stream (yearly average, gridcell-level) -hist_fincl4 += 'SOILC_vr', 'SOILN_vr', 'CWDC_vr', 'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', 'TOTLITC:I', 'TOT_WOODPRODC:I', 'TOTSOMC:I','TOTVEGC:I' +hist_fincl4 += 'SOILC_vr', 'SOILN_vr', 'CWDC_vr', 'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', 'TOTLITC:I', 'TOT_WOODPRODC:I', 'TOTSOMC:I','TOTVEGC:I', 'SLASH_HARVESTC' ! h4 stream (yearly average, landunit-level) hist_fincl5 += 'TOTSOMC:I', 'TOTSOMC_1m:I', 'TOTECOSYSC:I', 'TOTVEGC:I', 'WOODC:I', 'TOTLITC:I', 'LIVECROOTC:I', 'DEADCROOTC:I', 'FROOTC:I' From 570e192f51338c417ee5cba07d56ec4c47ac3394 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 15 Aug 2019 14:53:34 -0600 Subject: [PATCH 012/332] Revert "Move SLASH_HARVESTC to gridcell level list" This reverts commit cc7d56567044959b937d8ce4a2986279aeeb044d. --- cime_config/usermods_dirs/output_bgc/user_nl_clm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/output_bgc/user_nl_clm b/cime_config/usermods_dirs/output_bgc/user_nl_clm index 607c2ee25c..de136ad4b4 100644 --- a/cime_config/usermods_dirs/output_bgc/user_nl_clm +++ b/cime_config/usermods_dirs/output_bgc/user_nl_clm @@ -11,10 +11,10 @@ hist_fincl2 += 'GPP', 'NPP', 'AGNPP', 'TOTVEGC', 'NPP_NUPTAKE', 'AR', 'HR', 'HTO ! h2 stream (monthly average, landunit-level) ! TOT_WOODPRODC:I, CROPPROD1C:I, and NEE are not available at the landunit level -hist_fincl3 += 'GPP', 'NPP', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'COL_FIRE_CLOSS', 'DWT_SLASH_CFLUX', 'FROOTC:I', 'HTOP' +hist_fincl3 += 'GPP', 'NPP', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'SLASH_HARVESTC', 'COL_FIRE_CLOSS', 'DWT_SLASH_CFLUX', 'FROOTC:I', 'HTOP' ! h3 stream (yearly average, gridcell-level) -hist_fincl4 += 'SOILC_vr', 'SOILN_vr', 'CWDC_vr', 'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', 'TOTLITC:I', 'TOT_WOODPRODC:I', 'TOTSOMC:I','TOTVEGC:I', 'SLASH_HARVESTC' +hist_fincl4 += 'SOILC_vr', 'SOILN_vr', 'CWDC_vr', 'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', 'TOTLITC:I', 'TOT_WOODPRODC:I', 'TOTSOMC:I','TOTVEGC:I' ! h4 stream (yearly average, landunit-level) hist_fincl5 += 'TOTSOMC:I', 'TOTSOMC_1m:I', 'TOTECOSYSC:I', 'TOTVEGC:I', 'WOODC:I', 'TOTLITC:I', 'LIVECROOTC:I', 'DEADCROOTC:I', 'FROOTC:I' From 5523c2d47338fec99ae7bbb210138186016734d2 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 15 Aug 2019 15:00:14 -0600 Subject: [PATCH 013/332] Remove DWT_SLASH_CFLUX from landunit average, and have it just output monthly on gridcell which it is by default --- cime_config/usermods_dirs/output_bgc/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/output_bgc/user_nl_clm b/cime_config/usermods_dirs/output_bgc/user_nl_clm index de136ad4b4..0c0bcd52ff 100644 --- a/cime_config/usermods_dirs/output_bgc/user_nl_clm +++ b/cime_config/usermods_dirs/output_bgc/user_nl_clm @@ -11,7 +11,7 @@ hist_fincl2 += 'GPP', 'NPP', 'AGNPP', 'TOTVEGC', 'NPP_NUPTAKE', 'AR', 'HR', 'HTO ! h2 stream (monthly average, landunit-level) ! TOT_WOODPRODC:I, CROPPROD1C:I, and NEE are not available at the landunit level -hist_fincl3 += 'GPP', 'NPP', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'SLASH_HARVESTC', 'COL_FIRE_CLOSS', 'DWT_SLASH_CFLUX', 'FROOTC:I', 'HTOP' +hist_fincl3 += 'GPP', 'NPP', 'AR', 'HR', 'DWT_CONV_CFLUX_PATCH', 'WOOD_HARVESTC', 'DWT_WOOD_PRODUCTC_GAIN_PATCH', 'SLASH_HARVESTC', 'COL_FIRE_CLOSS', 'FROOTC:I', 'HTOP' ! h3 stream (yearly average, gridcell-level) hist_fincl4 += 'SOILC_vr', 'SOILN_vr', 'CWDC_vr', 'LITR1C_vr', 'LITR2C_vr', 'LITR3C_vr', 'LITR1N_vr', 'LITR2N_vr', 'LITR3N_vr','CWDN_vr', 'TOTLITC:I', 'TOT_WOODPRODC:I', 'TOTSOMC:I','TOTVEGC:I' From df7eb92131a074bd1b0a9a73faa243b14a3ee375 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 17 Aug 2019 02:23:14 -0600 Subject: [PATCH 014/332] Only output the GRU fields if do_grossunrep is actually turned on --- src/biogeochem/CNVegCarbonFluxType.F90 | 44 +++++++++++++----------- src/biogeochem/CNVegNitrogenFluxType.F90 | 19 +++++----- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index b9b7c69e4a..396a8088cc 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -22,7 +22,7 @@ module CNVegCarbonFluxType use ColumnType , only : col use PatchType , only : patch use AnnualFluxDribbler , only : annual_flux_dribbler_type, annual_flux_dribbler_gridcell - use dynSubgridControlMod , only : get_for_testing_allow_non_annual_changes + use dynSubgridControlMod , only : get_for_testing_allow_non_annual_changes, get_do_grossunrep use abortutils , only : endrun ! ! !PUBLIC TYPES: @@ -2993,26 +2993,28 @@ subroutine InitHistory(this, bounds, carbon_type) avgflag='A', long_name='dead coarse root to CWD due to landcover change', & ptr_col=this%dwt_deadcrootc_to_cwdc_col, default='inactive') - this%gru_conv_cflux_patch(begp:endp) = spval - call hist_addfld1d (fname='GRU_CONV_CFLUX', units='gC/m^2/s', & - avgflag='A', long_name='gross unrepresented conversion C flux (immediate loss to atm) (0 at all times except first timestep of year)', & - ptr_patch=this%gru_conv_cflux_patch) - - this%gru_conv_cflux_dribbled_grc(begg:endg) = spval - call hist_addfld1d (fname='GRU_CONV_CFLUX_DRIBBLED', units='gC/m^2/s', & - avgflag='A', & - long_name='gross unrepresented conversion C flux (immediate loss to atm), dribbled throughout the year', & - ptr_gcell=this%gru_conv_cflux_dribbled_grc) - - this%gru_wood_productc_gain_patch(begp:endp) = spval - call hist_addfld1d (fname='GRU_WOODPRODC_GAIN', units='gC/m^2/s', & - avgflag='A', long_name='gross unrepresented landcover change driven addition to wood product carbon pools (0 at all times except first timestep of year)', & - ptr_patch=this%gru_wood_productc_gain_patch) - - this%gru_slash_cflux_patch(begp:endp) = spval - call hist_addfld1d (fname='GRU_SLASH_CFLUX', units='gC/m^2/s', & - avgflag='A', long_name='slash gross unrepresented landcover change carbon (to litter)', & - ptr_patch=this%gru_slash_cflux_patch) + if ( get_do_grossunrep() )then + this%gru_conv_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_CONV_CFLUX', units='gC/m^2/s', & + avgflag='A', long_name='gross unrepresented conversion C flux (immediate loss to atm) (0 at all times except first timestep of year)', & + ptr_patch=this%gru_conv_cflux_patch) + + this%gru_conv_cflux_dribbled_grc(begg:endg) = spval + call hist_addfld1d (fname='GRU_CONV_CFLUX_DRIBBLED', units='gC/m^2/s', & + avgflag='A', & + long_name='gross unrepresented conversion C flux (immediate loss to atm), dribbled throughout the year', & + ptr_gcell=this%gru_conv_cflux_dribbled_grc) + + this%gru_wood_productc_gain_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_WOODPRODC_GAIN', units='gC/m^2/s', & + avgflag='A', long_name='gross unrepresented landcover change driven addition to wood product carbon pools (0 at all times except first timestep of year)', & + ptr_patch=this%gru_wood_productc_gain_patch) + + this%gru_slash_cflux_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_SLASH_CFLUX', units='gC/m^2/s', & + avgflag='A', long_name='slash gross unrepresented landcover change carbon (to litter)', & + ptr_patch=this%gru_slash_cflux_patch) + end if this%crop_seedc_to_leaf_patch(begp:endp) = spval call hist_addfld1d (fname='CROP_SEEDC_TO_LEAF', units='gC/m^2/s', & diff --git a/src/biogeochem/CNVegNitrogenFluxType.F90 b/src/biogeochem/CNVegNitrogenFluxType.F90 index d9e121ed78..d3bf84e1eb 100644 --- a/src/biogeochem/CNVegNitrogenFluxType.F90 +++ b/src/biogeochem/CNVegNitrogenFluxType.F90 @@ -11,6 +11,7 @@ module CNVegNitrogenFluxType use decompMod , only : bounds_type use abortutils , only : endrun use SoilBiogeochemDecompCascadeConType , only : decomp_cascade_con + use dynSubgridControlMod , only : get_do_grossunrep use LandunitType , only : lun use ColumnType , only : col use PatchType , only : patch @@ -1120,15 +1121,17 @@ subroutine InitHistory(this, bounds) avgflag='A', long_name='dead coarse root to CWD due to landcover change', & ptr_col=this%dwt_deadcrootn_to_cwdn_col, default='inactive') - this%gru_conv_nflux_patch(begp:endp) = spval - call hist_addfld1d (fname='GRU_CONV_NFLUX', units='gN/m^2/s', & - avgflag='A', long_name='gross unrepresented conversion N flux (immediate loss to atm) (0 at all times except first timestep of year)', & - ptr_patch=this%gru_conv_nflux_patch) + if ( get_do_grossunrep() )then + this%gru_conv_nflux_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_CONV_NFLUX', units='gN/m^2/s', & + avgflag='A', long_name='gross unrepresented conversion N flux (immediate loss to atm) (0 at all times except first timestep of year)', & + ptr_patch=this%gru_conv_nflux_patch) - this%gru_wood_productn_gain_patch(begp:endp) = spval - call hist_addfld1d (fname='GRU_WOODPRODN_GAIN', units='gN/m^2/s', & - avgflag='A', long_name='gross unrepresented landcover change driven addition to wood product nitrogen pools (0 at all times except first timestep of year)', & - ptr_patch=this%gru_wood_productn_gain_patch) + this%gru_wood_productn_gain_patch(begp:endp) = spval + call hist_addfld1d (fname='GRU_WOODPRODN_GAIN', units='gN/m^2/s', & + avgflag='A', long_name='gross unrepresented landcover change driven addition to wood product nitrogen pools (0 at all times except first timestep of year)', & + ptr_patch=this%gru_wood_productn_gain_patch) + end if this%crop_seedn_to_leaf_patch(begp:endp) = spval call hist_addfld1d (fname='CROP_SEEDN_TO_LEAF', units='gN/m^2/s', & From c3a29531bce74523c6f1e76f19aa1f64d243ffdd Mon Sep 17 00:00:00 2001 From: "Miles A. Curry" Date: Wed, 17 Nov 2021 16:01:11 -0700 Subject: [PATCH 015/332] Add maps, fsurdat for MPAS grids 480, 240, 120, 60, 30, 15 --- bld/namelist_files/namelist_defaults_ctsm.xml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 8a10a96ebf..338aebc95d 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -801,6 +801,36 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 + + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 + + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 + + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 + + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 + + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 + @@ -1099,6 +1129,19 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_numaIA_hist_78pfts_CMIP6_si lnd/clm2/surfdata_map/ctsm1.0.dev094-2-g633be0eb/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c200521.nc + +lnd/clm2/surfdata_map/surfdata_mpasa480_hist_78pfts_CMIP6_simyr2000_c211110.nc + +lnd/clm2/surfdata_map/surfdata_mpasa240_hist_78pfts_CMIP6_simyr2000_c211115.nc + +lnd/clm2/surfdata_map/surfdata_mpasa120_hist_78pfts_CMIP6_simyr2000_c211108.nc + +lnd/clm2/surfdata_map/surfdata_mpasa60_hist_78pfts_CMIP6_simyr2000_c211110.nc + +lnd/clm2/surfdata_map/surfdata_mpasa30_hist_78pfts_CMIP6_simyr2000_c211111.nc + +lnd/clm2/surfdata_map/surfdata_mpasa15_hist_78pfts_CMIP6_simyr2000_c211111.nc + lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc @@ -1144,6 +1187,19 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_16pfts_Irrig_CMIP6_sim lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc + +lnd/clm2/surfdata_map/surfdata_mpasa480_hist_78pfts_CMIP6_simyr1850_c211110.nc + +lnd/clm2/surfdata_map/surfdata_mpasa240_hist_78pfts_CMIP6_simyr1850_c211115.nc + +lnd/clm2/surfdata_map/surfdata_mpasa120_hist_78pfts_CMIP6_simyr1850_c211108.nc + +lnd/clm2/surfdata_map/surfdata_mpasa60_hist_78pfts_CMIP6_simyr1850_c211110.nc + +lnd/clm2/surfdata_map/surfdata_mpasa30_hist_78pfts_CMIP6_simyr1850_c211111.nc + +lnd/clm2/surfdata_map/surfdata_mpasa15_hist_78pfts_CMIP6_simyr1850_c211111.nc + lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850_c190303.nc @@ -1245,6 +1301,19 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc +lnd/clm2/surfdata_map/landuse.timeseries_mpasa480_hist_78pfts_CMIP6_simyr1850-2015_c211110.nc +lnd/clm2/surfdata_map/landuse.timeseries_mpasa240_hist_78pfts_CMIP6_simyr1850-2015_c211115.nc +lnd/clm2/surfdata_map/landuse.timeseries_mpasa120_hist_78pfts_CMIP6_simyr1850-2015_c211108.nc +lnd/clm2/surfdata_map/landuse.timeseries_mpasa60_hist_78pfts_CMIP6_simyr1850-2015_c211110.nc +lnd/clm2/surfdata_map/landuse.timeseries_mpasa30_hist_78pfts_CMIP6_simyr1850-2015_c211111.nc +lnd/clm2/surfdata_map/landuse.timeseries_mpasa15_hist_78pfts_CMIP6_simyr1850-2015_c211111.nc + lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc Date: Wed, 17 Nov 2021 16:03:55 -0700 Subject: [PATCH 016/332] Add MPAS resolutions to res namelist default --- bld/namelist_files/namelist_definition_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml index 67929c4bd4..875e4691e5 100644 --- a/bld/namelist_files/namelist_definition_ctsm.xml +++ b/bld/namelist_files/namelist_definition_ctsm.xml @@ -1985,7 +1985,7 @@ CLM run type. + valid_values="512x1024,360x720cru,128x256,64x128,48x96,94x192,0.23x0.31,0.47x0.63,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,0.125nldas2,5x5_amazon,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.25x0.25,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,0.125x0.125,ne4np4,ne16np4,ne30np4.pg2,ne30np4.pg3,ne30np4,ne60np4,ne120np4,ne120np4.pg2,ne120np4.pg3,ne0np4CONUS.ne30x8,ne0np4.ARCTIC.ne30x4,ne0np4.ARCTICGRIS.ne30x8,ne240np4,1km-merge-10min,C24,C48,C96,C192,C384,mpasa480,mpasa240,mpasa120,mpasa60,mpasa30,mpasa15"> Horizontal resolutions Note: 0.25x0.25, 0.5x0.5, 5x5min, 10x10min, 3x3min, 1km-merge-10min and 0.33x0.33 are only used for CLM toolsI From 1f01f3d5112896d71541863a7206354342d8f6ae Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 22 Mar 2022 14:26:35 -0600 Subject: [PATCH 017/332] Add live course root maintence respiration to root respiration fixing #1676 --- src/biogeochem/CNVegCarbonFluxType.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index 718f7b0330..ddd1126008 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -4169,6 +4169,7 @@ subroutine Summary_carbonflux(this, & ! root respiration (RR) this%rr_patch(p) = & this%froot_mr_patch(p) + & + this%livecroot_mr_patch(p) + & this%cpool_froot_gr_patch(p) + & this%cpool_livecroot_gr_patch(p) + & this%cpool_deadcroot_gr_patch(p) + & From 6f97bb5c4a36c4dea90ae459cba76252706ef742 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 28 Mar 2022 17:04:32 -0600 Subject: [PATCH 018/332] Start working on the makefile --- tools/mksurfdata_map/Makefile.data | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index d0c000ba63..15ca410f0f 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -53,7 +53,12 @@ else endif MKSURFDATA = $(BATCHJOBS) $(PWD)/mksurfdata.pl +SUBSETDATA = $(BATCHJOBS) $(PWD)/../site_and_regional/subset_data +# subset_data options +# +SUBSETDATA_1X1_BRAZIL := --lat --lon --site 1x1_brazil --outdir . --user-mods-dir ./1x1_brazil \ + --overwrite --create-user-mods --uniform-snowpack --include-nonveg --cap-saturation # f19 and f09 are standard resolutions, f10 is used for testing, f45 is used for FATES # ne30np4 is standard resolution for SE dycore in CAM, C96 is standard for fv3 dycore # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore @@ -119,14 +124,18 @@ global-present-nldas : FORCE # tropics : $(TROPICS) -crop-tropics-present : FORCE - $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon,1x1_brazil $(BACKGROUND) +crop-tropics-present : FORCE brazil-tropics-present + $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon $(BACKGROUND) + +1x1_brazil-tropics-present : FORCE + $(SUBSETDATA) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop $(BACKGROUND) + crop-tropics-historical : FORCE - $(MKSURFDATA) -glc_nec 10 -y 1850 -res 1x1_brazil $(BACKGROUND) + $(SUBSETDATA) -glc_nec 10 -y 1850 -res 1x1_brazil $(BACKGROUND) crop-tropics-transient : FORCE - $(MKSURFDATA) -glc_nec 10 -no_surfdata -y 1850-2000 -res 1x1_brazil $(BACKGROUND) + $(SUBSETDATA) -glc_nec 10 -no_surfdata -y 1850-2000 -res 1x1_brazil $(BACKGROUND) # # crop @@ -143,13 +152,13 @@ crop-global-present-f05 : FORCE $(MKSURFDATA) -glc_nec 10 -y 1850,2000 -res 0.47x0.63 $(BACKGROUND) crop-numa-present : FORCE - $(MKSURFDATA) -glc_nec 10 -y 2000 -r 1x1_numaIA $(BACKGROUND) + $(SUBSETDATA) -glc_nec 10 -y 2000 -r 1x1_numaIA $(BACKGROUND) crop-numa-historical : FORCE - $(MKSURFDATA) -glc_nec 10 -y 1850 -r 1x1_numaIA $(BACKGROUND) + $(SUBSETDATA) -glc_nec 10 -y 1850 -r 1x1_numaIA $(BACKGROUND) crop-smallville : FORCE - $(MKSURFDATA) -glc_nec 10 -y 2000 -r 1x1_smallvilleIA \ + $(SUBSETDATA) -glc_nec 10 -y 2000 -r 1x1_smallvilleIA \ -pft_idx 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78 \ -pft_frc 6.5,1.5,1.6,1.7,1.8,1.9,1.5,1.6,1.7,1.8,1.9,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5 \ $(BACKGROUND) @@ -165,7 +174,7 @@ crop-global-present-ne120np4 : FORCE # adds crop (to make sure that it works properly to add crop in a grid cell # where there used to be no crop). crop-smallville-historical : FORCE - $(MKSURFDATA) -glc_nec 10 -y 1850 -r 1x1_smallvilleIA -pft_idx 13 -pft_frc 100 $(BACKGROUND) + $(SUBSETDATA) -glc_nec 10 -y 1850 -r 1x1_smallvilleIA -pft_idx 13 -pft_frc 100 $(BACKGROUND) # Setup the historical case for SSP5-8.5 so that historical can be used to go into the future. crop-global-historical : FORCE @@ -230,12 +239,12 @@ crop-global-SSP5-8.5 : FORCE urban : urban-present urban-alpha urban-present : FORCE - $(MKSURFDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_vancouverCAN,1x1_mexicocityMEX $(BACKGROUND) + $(SUBSETDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_vancouverCAN,1x1_mexicocityMEX $(BACKGROUND) # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 urban-alpha : FORCE - $(MKSURFDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_urbanc_alpha -urban_skip_abort_on_invalid_data_check $(BACKGROUND) + $(SUBSETDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_urbanc_alpha -urban_skip_abort_on_invalid_data_check $(BACKGROUND) # From 03c67869b1e699a27b30b1919cab47e29b77f32c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 6 Jul 2022 14:28:10 -0600 Subject: [PATCH 019/332] Working version of fsurdatmodifyctsm system test that uses conda environment rather than ncar_pylib --- cime_config/SystemTests/fsurdatmodifyctsm.py | 73 ++++++++------------ 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 4071316ea9..3cfef3d6fa 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -65,54 +65,37 @@ def _create_config_file(self): def _run_modify_fsurdat(self): tool_path = os.path.join(self._ctsm_root, 'tools/modify_fsurdat/fsurdat_modifier') - # Need to specify a specific python version that has the required - # dependencies - python_path = _get_python_path() - subprocess.check_call([python_path, tool_path, self._cfg_file_path]) + + self._case.load_env(reset=True) + conda_env = ". "+self._get_caseroot()+"/.env_mach_specific.sh; " + # Preprend the commands to get the conda environment for python first + conda_env += self._get_conda_env() + # Source the env + subprocess.run( conda_env+"python3 "+tool_path+" "+self._cfg_file_path, shell=True, check=True) def _modify_user_nl(self): append_to_user_nl_files(caseroot = self._get_caseroot(), component = "clm", contents = "fsurdat = '{}'".format(self._fsurdat_out)) -def _get_python_path(): - """Get path to ncar_pylib's python on cheyenne - - This is needed because we need a python environment that includes xarray - and its dependencies. This is currently hard-coded for cheyenne until we - come up with a robust way in CIME of ensuring that the correc python - environment is loaded. - - """ - out = subprocess.check_output(['/glade/u/apps/opt/ncar_pylib/ncar_pylib', - '-l'], universal_newlines=True) - - # First look for a loaded ('L') python - path = _find_path_from_pylib_output(out, 'L') - # If no loaded python found, look for a default ('D') python - if path is None: - path = _find_path_from_pylib_output(out, 'D') - - if path is None: - raise RuntimeError('No python found') - - return os.path.join(path, 'bin', 'python') - -def _find_path_from_pylib_output(ncar_pylib_output, char): - """Given line-by-line output from ncar_pylib, return the path to python if found - - Args: - - ncar_pylib_output: line-by-line output from ncar_pylib - - char: the character to look for in the leading parenthetical expression (typically 'L' or 'D') - - Returns a path to python, or None if not found - """ - # The line of interest looks like the following (for char = 'L'): - # (L) ... /path/to/python - regex = r'\(' + char + r'\).* (/\S+)' - for line in ncar_pylib_output.splitlines(): - match_line = re.match(regex, line) - if match_line: - return match_line.group(1) - - return None + def _get_conda_env(self): + # + # Add specific commands needed on different machines to get conda available + # Use semicolon here since it's OK to fail + # + # Remove python and add conda to environment for cheyennne + conda_env = "module unload python; module load conda" + # CGD machines should already have conda loaded + conda_env += "; module load lang/python" + # End above machine specific with a semicolon + conda_env += "; " + + # The following uses "&&" as they all need to work + # Run the manage_python_env script + conda_env += self._ctsm_root+"/manage_python_env" + # Activate the python environment + conda_env += " && conda activate ctsm_py" + # End above to get to actual command + conda_env += " && " + + return( conda_env ) From 384d3176979b098640ee001c10c56ecbba93ad18 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Jul 2022 16:43:32 -0600 Subject: [PATCH 020/332] Set zetamaxstable to 2 for ctsm5_1 for all cases --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2562f9ce88..3492816773 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -211,7 +211,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 2.0d00 0.5d00 -0.5d00 +2.0d00 .true. From 08f556b9c16d1b8c053041dcba159274069c6327 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Jul 2022 16:50:15 -0600 Subject: [PATCH 021/332] Apply same zetamaxstable limit for biomass heat storage as for everything else, rather than a hardcoded 100 --- src/biogeophys/CanopyFluxesMod.F90 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/biogeophys/CanopyFluxesMod.F90 b/src/biogeophys/CanopyFluxesMod.F90 index 1d5298bac2..f1cc884d83 100644 --- a/src/biogeophys/CanopyFluxesMod.F90 +++ b/src/biogeophys/CanopyFluxesMod.F90 @@ -1329,12 +1329,7 @@ subroutine CanopyFluxes(bounds, num_exposedvegp, filter_exposedvegp, zeta(p) = zldis(p)*vkc*grav*thvstar/(ustar(p)**2*thv(c)) if (zeta(p) >= 0._r8) then !stable - ! remove stability cap when biomass heat storage is active - if(use_biomass_heat_storage) then - zeta(p) = min(100._r8,max(zeta(p),0.01_r8)) - else - zeta(p) = min(zetamax,max(zeta(p),0.01_r8)) - endif + zeta(p) = min(zetamax,max(zeta(p),0.01_r8)) um(p) = max(ur(p),0.1_r8) else !unstable zeta(p) = max(-100._r8,min(zeta(p),-0.01_r8)) From 0f39e0e056f7432f46d2c79e6c9fd4dbfcf093ae Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Jul 2022 17:54:16 -0600 Subject: [PATCH 022/332] Set zetmaxstable to 2.0 when BHS is on --- bld/CLMBuildNamelist.pm | 20 ++++++++++++------- bld/namelist_files/namelist_defaults_ctsm.xml | 7 ++++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 7c5859e0ad..73a857f718 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -1663,11 +1663,6 @@ sub process_namelist_inline_logic { ############################################# setup_logic_rooting_profile($opts, $nl_flags, $definition, $defaults, $nl); - ############################################# - # namelist group: friction_velocity # - ############################################# - setup_logic_friction_vel($opts, $nl_flags, $definition, $defaults, $nl); - ############################# # namelist group: cngeneral # ############################# @@ -1687,6 +1682,11 @@ sub process_namelist_inline_logic { ############################################# setup_logic_canopyfluxes($opts, $nl_flags, $definition, $defaults, $nl); + ########################################################## + # namelist group: friction_velocity (after canopyfluxes) # + ########################################################## + setup_logic_friction_vel($opts, $nl_flags, $definition, $defaults, $nl); + ############################################# # namelist group: canopyhydrology_inparm # ############################################# @@ -3867,10 +3867,11 @@ sub setup_logic_rooting_profile { #------------------------------------------------------------------------------- sub setup_logic_friction_vel { - # + # Must be after canopyfluxes so that use_biomass_heat_storagge will be set my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; - add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'zetamaxstable' ); + add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'zetamaxstable', + 'use_biomass_heat_storage'=>$nl_flags->{'use_biomass_heat_storage'}, 'phys'=>$nl_flags->{'phys'} ); } #------------------------------------------------------------------------------- @@ -3895,6 +3896,11 @@ sub setup_logic_canopyfluxes { if ( &value_is_true($nl->get_value('use_biomass_heat_storage') ) && &value_is_true( $nl_flags->{'use_fates'}) ) { $log->fatal_error('use_biomass_heat_storage can NOT be set to true when fates is on'); } + if ( &value_is_true($nl->get_value('use_biomass_heat_storage')) ) { + $nl_flags->{'use_biomass_heat_storage'} = ".true."; + } else { + $nl_flags->{'use_biomass_heat_storage'} = ".false."; + } } #------------------------------------------------------------------------------- diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 3492816773..1651e99f26 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -209,9 +209,10 @@ attributes from the config_cache.xml file (with keys converted to upper-case). 1.d-2 -2.0d00 -0.5d00 -2.0d00 +2.0d00 +2.0d00 +0.5d00 +0.5d00 .true. From 8ed188ae02fb1e5456934d1f95521920a35db366 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 14 Jul 2022 14:32:25 -0600 Subject: [PATCH 023/332] Fix typo @olyson pointed out --- bld/CLMBuildNamelist.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index 6b8f90eef9..9126ca2302 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -3899,7 +3899,7 @@ sub setup_logic_rooting_profile { #------------------------------------------------------------------------------- sub setup_logic_friction_vel { - # Must be after canopyfluxes so that use_biomass_heat_storagge will be set + # Must be after canopyfluxes so that use_biomass_heat_storage will be set my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'zetamaxstable', From c3bb13d1a0ca744d808cac08caefecc141238b22 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 14 Jul 2022 16:59:38 -0600 Subject: [PATCH 024/332] Get the 2000 1x1_brazil subset_data case working --- tools/mksurfdata_map/Makefile.data | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 15ca410f0f..c7ff1f85a8 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -53,12 +53,15 @@ else endif MKSURFDATA = $(BATCHJOBS) $(PWD)/mksurfdata.pl -SUBSETDATA = $(BATCHJOBS) $(PWD)/../site_and_regional/subset_data +SUBSETDATA = $(PWD)/../site_and_regional/subset_data # subset_data options # -SUBSETDATA_1X1_BRAZIL := --lat --lon --site 1x1_brazil --outdir . --user-mods-dir ./1x1_brazil \ - --overwrite --create-user-mods --uniform-snowpack --include-nonveg --cap-saturation +SUBSETDATA_POINT = $(SUBSETDATA) point --overwrite --create-user-mods --uniform-snowpack --include-nonveg --cap-saturation + +# Subset data sites... +SUBSETDATA_1X1_BRAZIL := --lat -7 --lon -55 --site 1x1_brazil --user-mods-dir ./1x1_brazil --outdir ./1x1_brazil +SUBSETDATA_1X1_NUMAIA := --lat 40.6878 --lon 267.0228 --site 1x1_numaIA --user-mods-dir ./1x1_numaIA --outdir ./1x1_numaIA # f19 and f09 are standard resolutions, f10 is used for testing, f45 is used for FATES # ne30np4 is standard resolution for SE dycore in CAM, C96 is standard for fv3 dycore # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore @@ -96,7 +99,6 @@ CROP = \ crop-global-historical \ crop-global-transient \ crop-global-future - all : standard tropics crop urban landuse-timeseries DEBUG: @@ -124,18 +126,18 @@ global-present-nldas : FORCE # tropics : $(TROPICS) -crop-tropics-present : FORCE brazil-tropics-present +crop-tropics-present : brazil-tropics-present $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon $(BACKGROUND) 1x1_brazil-tropics-present : FORCE - $(SUBSETDATA) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop $(BACKGROUND) + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop crop-tropics-historical : FORCE - $(SUBSETDATA) -glc_nec 10 -y 1850 -res 1x1_brazil $(BACKGROUND) + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop crop-tropics-transient : FORCE - $(SUBSETDATA) -glc_nec 10 -no_surfdata -y 1850-2000 -res 1x1_brazil $(BACKGROUND) + $(SUBSETDATA_POINT) -glc_nec 10 -no_surfdata -y 1850-2000 -res 1x1_brazil # # crop @@ -152,16 +154,15 @@ crop-global-present-f05 : FORCE $(MKSURFDATA) -glc_nec 10 -y 1850,2000 -res 0.47x0.63 $(BACKGROUND) crop-numa-present : FORCE - $(SUBSETDATA) -glc_nec 10 -y 2000 -r 1x1_numaIA $(BACKGROUND) + $(SUBSETDATA_POINT) -glc_nec 10 -y 2000 -r 1x1_numaIA crop-numa-historical : FORCE - $(SUBSETDATA) -glc_nec 10 -y 1850 -r 1x1_numaIA $(BACKGROUND) + $(SUBSETDATA_POINT) -glc_nec 10 -y 1850 -r 1x1_numaIA crop-smallville : FORCE - $(SUBSETDATA) -glc_nec 10 -y 2000 -r 1x1_smallvilleIA \ + $(SUBSETDATA_POINT) -glc_nec 10 -y 2000 -r 1x1_smallvilleIA \ -pft_idx 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78 \ - -pft_frc 6.5,1.5,1.6,1.7,1.8,1.9,1.5,1.6,1.7,1.8,1.9,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5 \ - $(BACKGROUND) + -pft_frc 6.5,1.5,1.6,1.7,1.8,1.9,1.5,1.6,1.7,1.8,1.9,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5 crop-global-present-ne16np4 : FORCE $(MKSURFDATA) -glc_nec 10 -y 2000 -res ne16np4 $(BACKGROUND) @@ -174,7 +175,7 @@ crop-global-present-ne120np4 : FORCE # adds crop (to make sure that it works properly to add crop in a grid cell # where there used to be no crop). crop-smallville-historical : FORCE - $(SUBSETDATA) -glc_nec 10 -y 1850 -r 1x1_smallvilleIA -pft_idx 13 -pft_frc 100 $(BACKGROUND) + $(SUBSETDATA_POINT) -glc_nec 10 -y 1850 -r 1x1_smallvilleIA -pft_idx 13 -pft_frc 100 # Setup the historical case for SSP5-8.5 so that historical can be used to go into the future. crop-global-historical : FORCE @@ -239,12 +240,12 @@ crop-global-SSP5-8.5 : FORCE urban : urban-present urban-alpha urban-present : FORCE - $(SUBSETDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_vancouverCAN,1x1_mexicocityMEX $(BACKGROUND) + $(SUBSETDATA_POINT) -y 2000 -no-crop -glc_nec 10 -r 1x1_vancouverCAN,1x1_mexicocityMEX # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 urban-alpha : FORCE - $(SUBSETDATA) -y 2000 -no-crop -glc_nec 10 -r 1x1_urbanc_alpha -urban_skip_abort_on_invalid_data_check $(BACKGROUND) + $(SUBSETDATA_POINT) -y 2000 -no-crop -glc_nec 10 -r 1x1_urbanc_alpha -urban_skip_abort_on_invalid_data_check # From 50c941d3e32749c5bfa8e506d6129fc7003ebd61 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 14 Jul 2022 17:00:29 -0600 Subject: [PATCH 025/332] Add ability to change the default config file for the default filenames fixing #1809 --- python/ctsm/subset_data.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 7a33d9c2fa..4fc92615f4 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -75,7 +75,7 @@ process_logging_args, ) -DEFAULTS_FILE = "default_data.cfg" +DEFAULTS_CONFIG = "tools/site_and_regional/default_data.cfg" logger = logging.getLogger(__name__) @@ -308,6 +308,16 @@ def get_parser(): type=str, default="", ) + cesmroot = path_to_ctsm_root() + DEFAULTS_FILE = os.path.join(cesmroot, DEFAULTS_CONFIG) + subparser.add_argument( + "--file", + help="Default configure file to use for default filenames.", + action="store", + dest="config_file", + type=str, + default=DEFAULTS_FILE, + ) subparser.add_argument( "--overwrite", help="Flag to overwrite if the files already exists.", @@ -588,7 +598,7 @@ def main(): # parse defaults file cesmroot = path_to_ctsm_root() defaults = configparser.ConfigParser() - defaults.read(os.path.join(cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) + defaults.read(args.config_file) # --------------------------------- # myname = getuser() From d93b414673b1aea23f72e3dc0f94620b416cccd1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Jul 2022 16:40:26 -0600 Subject: [PATCH 026/332] Get subset_data single point sites working --- tools/mksurfdata_map/Makefile.data | 51 ++++++++++++++++------ tools/mksurfdata_map/default_data_1850.cfg | 29 ++++++++++++ 2 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 tools/mksurfdata_map/default_data_1850.cfg diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index c7ff1f85a8..caaef030e3 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -57,11 +57,23 @@ SUBSETDATA = $(PWD)/../site_and_regional/subset_data # subset_data options # -SUBSETDATA_POINT = $(SUBSETDATA) point --overwrite --create-user-mods --uniform-snowpack --include-nonveg --cap-saturation +SUBSETDATA_POINT = $(SUBSETDATA) point --overwrite --uniform-snowpack --cap-saturation --outdir . +SUBSETDATA_POINT_ALLLU = $(SUBSETDATA_POINT) --include-nonveg +# NOTE: TODO: We need settings in subset_data to handle urban datasets see issue #1665 +SUBSETDATA_POINT_URBAN = $(SUBSETDATA_POINT) --include-nonveg # Subset data sites... -SUBSETDATA_1X1_BRAZIL := --lat -7 --lon -55 --site 1x1_brazil --user-mods-dir ./1x1_brazil --outdir ./1x1_brazil -SUBSETDATA_1X1_NUMAIA := --lat 40.6878 --lon 267.0228 --site 1x1_numaIA --user-mods-dir ./1x1_numaIA --outdir ./1x1_numaIA +SUBSETDATA_1X1_BRAZIL := --lat -7 --lon -55 --site 1x1_brazil +SUBSETDATA_1X1_NUMAIA := --lat 40.6878 --lon 267.0228 --site 1x1_numaIA +SUBSETDATA_1X1_SMALL := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA \ + --dompft 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 \ + --pctpft 6.5 1.5 1.6 1.7 1.8 1.9 1.5 1.6 1.7 1.8 1.9 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 1.5 +# NOTE: The 1850 smallvilleIA site is constructed to start with 100% natural vegetation, so we can test transition to crops +SUBSETDATA_1X1_SMALL1850 := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA --dompft 13 --pctpft 100 + +SUBSETDATA_1X1_MEXICOCITY := --lat 19.5 --lon 260.5 --site 1x1_mexicocityMEX +SUBSETDATA_1X1_VANCOUVER := --lat 49.5 --lon 236.5 --site 1x1_vancouverCAN +SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha # f19 and f09 are standard resolutions, f10 is used for testing, f45 is used for FATES # ne30np4 is standard resolution for SE dycore in CAM, C96 is standard for fv3 dycore # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore @@ -101,11 +113,22 @@ CROP = \ crop-global-future all : standard tropics crop urban landuse-timeseries +all-subset : \ + 1x1_brazil-tropics-present \ + crop-tropics-historical \ + crop-tropics-transient \ + crop-numa-present \ + crop-numa-historical \ + crop-smallville \ + crop-smallville-historical \ + urban-present urban-alpha + DEBUG: @echo "HOST := $(HOST)" @echo "PROJECT := $(PROJECT)" @echo "BATCHJOBS := $(BATCHJOBS)" @echo "BACKGROUND := $(BACKGROUND)" + # # standard # @@ -130,14 +153,14 @@ crop-tropics-present : brazil-tropics-present $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon $(BACKGROUND) 1x1_brazil-tropics-present : FORCE - $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop crop-tropics-historical : FORCE - $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --file default_data_1850.cfg --crop crop-tropics-transient : FORCE - $(SUBSETDATA_POINT) -glc_nec 10 -no_surfdata -y 1850-2000 -res 1x1_brazil + $(SUBSETDATA_POINT_ALLLU) --create-landuse $(SUBSETDATA_1X1_BRAZIL) --crop # # crop @@ -154,15 +177,13 @@ crop-global-present-f05 : FORCE $(MKSURFDATA) -glc_nec 10 -y 1850,2000 -res 0.47x0.63 $(BACKGROUND) crop-numa-present : FORCE - $(SUBSETDATA_POINT) -glc_nec 10 -y 2000 -r 1x1_numaIA + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --crop crop-numa-historical : FORCE - $(SUBSETDATA_POINT) -glc_nec 10 -y 1850 -r 1x1_numaIA + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --file default_data_1850.cfg --crop crop-smallville : FORCE - $(SUBSETDATA_POINT) -glc_nec 10 -y 2000 -r 1x1_smallvilleIA \ - -pft_idx 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78 \ - -pft_frc 6.5,1.5,1.6,1.7,1.8,1.9,1.5,1.6,1.7,1.8,1.9,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5,1.5 + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL) --crop crop-global-present-ne16np4 : FORCE $(MKSURFDATA) -glc_nec 10 -y 2000 -res ne16np4 $(BACKGROUND) @@ -175,7 +196,7 @@ crop-global-present-ne120np4 : FORCE # adds crop (to make sure that it works properly to add crop in a grid cell # where there used to be no crop). crop-smallville-historical : FORCE - $(SUBSETDATA_POINT) -glc_nec 10 -y 1850 -r 1x1_smallvilleIA -pft_idx 13 -pft_frc 100 + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL1850) --file default_data_1850.cfg --crop # Setup the historical case for SSP5-8.5 so that historical can be used to go into the future. crop-global-historical : FORCE @@ -240,12 +261,13 @@ crop-global-SSP5-8.5 : FORCE urban : urban-present urban-alpha urban-present : FORCE - $(SUBSETDATA_POINT) -y 2000 -no-crop -glc_nec 10 -r 1x1_vancouverCAN,1x1_mexicocityMEX + $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) + $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 urban-alpha : FORCE - $(SUBSETDATA_POINT) -y 2000 -no-crop -glc_nec 10 -r 1x1_urbanc_alpha -urban_skip_abort_on_invalid_data_check + $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) # @@ -253,6 +275,7 @@ urban-alpha : FORCE # landuse-timeseries : landuse-timeseries-smallville +# NOTE: TODO: This needs to be chagned to use subset_data when transient configurations are resolved (see Issue #1673 landuse-timeseries-smallville : FORCE $(MKSURFDATA) -no_surfdata -glc_nec 10 -y 1850-1855 -r 1x1_smallvilleIA \ -pft_idx 17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78 \ diff --git a/tools/mksurfdata_map/default_data_1850.cfg b/tools/mksurfdata_map/default_data_1850.cfg new file mode 100644 index 0000000000..311aeef13d --- /dev/null +++ b/tools/mksurfdata_map/default_data_1850.cfg @@ -0,0 +1,29 @@ +[main] +clmforcingindir = /glade/p/cesmdata/inputdata + +[datm_gswp3] +dir = /glade/p/cgd/tss/CTSM_datm_forcing_data/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 +domain = domain.lnd.360x720_gswp3.0v1.c170606.nc +solardir = Solar +precdir = Precip +tpqwdir = TPHWL +solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. +prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. +tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. +solarname = CLMGSWP3v1.Solar +precname = CLMGSWP3v1.Precip +tpqwname = CLMGSWP3v1.TPQW + +[surfdat] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +surfdat_16pft = surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc +surfdat_78pft = surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr1850_c190214.nc + +[landuse] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +landuse_16pft = landuse.timeseries_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c190214.nc +landuse_78pft = landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_c190214.nc + +[domain] +file = share/domains/domain.lnd.fv0.9x1.25_gx1v7.151020.nc + From 5eabd919534f7897acc65f66f4703b8a2aa4f014 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Jul 2022 16:59:52 -0600 Subject: [PATCH 027/332] Update to version that has the fix for coszen --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index ade436e8db..871aafdbfd 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,7 +55,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.41 +tag = cdeps0.12.42 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps From 44a44a5d8a2a27fbaaa27db8aa774c2b194af678 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Jul 2022 17:02:19 -0600 Subject: [PATCH 028/332] Update to version of cdeps that fixes the ZBOT read issue --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 871aafdbfd..0b0124c9f3 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,7 +55,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.42 +tag = cdeps0.12.52 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps From dca339ec1777f70a0f43181bbf03947c87f70cd5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 28 Jul 2022 15:58:55 -0600 Subject: [PATCH 029/332] Change the name of the conda environment for CTSM from ctsm_py to ctsm_pylib to match ncar_pylib and to use a different name from the Hammon script that uses a ctsm_py environment name --- .github/workflows/black.yml | 2 +- cime_config/SystemTests/fsurdatmodifyctsm.py | 6 +++--- py_env_create | 2 +- python/README.md | 2 +- python/conda_env_ctsm_py.txt | 10 +++++----- python/conda_env_ctsm_py_cgd.txt | 10 +++++----- test/tools/test_driver.sh | 8 ++++---- tools/contrib/README | 2 +- tools/contrib/ssp_anomaly_forcing_smooth | 2 +- tools/modify_fsurdat/README | 2 +- tools/site_and_regional/README | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index a2115d2833..438b48e918 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -17,5 +17,5 @@ jobs: # Using pyproject.toml makes sure this testing is consistent with our python directory testing options: "--check --config python/pyproject.toml" src: "./python" - # Version should be coordinated with the ctsm_py conda environment under the python directory + # Version should be coordinated with the ctsm_pylib conda environment under the python directory version: "22.3.0" diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 3cfef3d6fa..74b7eff7f2 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -91,10 +91,10 @@ def _get_conda_env(self): conda_env += "; " # The following uses "&&" as they all need to work - # Run the manage_python_env script - conda_env += self._ctsm_root+"/manage_python_env" + # Run the py_env_create script + conda_env += self._ctsm_root+"/py_env_create" # Activate the python environment - conda_env += " && conda activate ctsm_py" + conda_env += " && conda activate ctsm_pylib" # End above to get to actual command conda_env += " && " diff --git a/py_env_create b/py_env_create index 59bf13c222..c323a374df 100755 --- a/py_env_create +++ b/py_env_create @@ -23,7 +23,7 @@ if [ $error != 0 ]; then exit -1 fi rm condahelp.txt -ctsm_python=ctsm_py +ctsm_python=ctsm_pylib condadir="$dir/python" diff --git a/python/README.md b/python/README.md index 63868bee63..2cb488be22 100644 --- a/python/README.md +++ b/python/README.md @@ -14,7 +14,7 @@ You can also use the script in the top level directory to do all the conda commands and do this for you. ../py_env_create - conda activate ctsm_py + conda activate ctsm_pylib Conda requirements files: diff --git a/python/conda_env_ctsm_py.txt b/python/conda_env_ctsm_py.txt index 8c110e8f3f..445300c754 100644 --- a/python/conda_env_ctsm_py.txt +++ b/python/conda_env_ctsm_py.txt @@ -3,12 +3,12 @@ # # To install this on cheyenne with conda loaded in modules # use the top level bash script: -# ../py_env_create # Do this each time you update your CTSM Version -# conda activate ctsm_py # Do this anytime you want to run a CTSM python script +# ../py_env_create # Do this each time you update your CTSM Version +# conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script # Or the individual conda commands: -# conda create -n ctsm_py # Do this one time for each machine -# conda install -n ctsm_py --file conda_env_ctsm_py.txt # Do this each time you update your CTSM Version -# conda activate ctsm_py # Do this anytime you want to run a CTSM python script +# conda create -n ctsm_pylib # Do this one time for each machine +# conda install -n ctsm_pylib --file conda_env_ctsm_py.txt # Do this each time you update your CTSM Version +# conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script # python=3.7.9 scipy diff --git a/python/conda_env_ctsm_py_cgd.txt b/python/conda_env_ctsm_py_cgd.txt index 45025506a8..1dc52a873b 100644 --- a/python/conda_env_ctsm_py_cgd.txt +++ b/python/conda_env_ctsm_py_cgd.txt @@ -5,12 +5,12 @@ # # To install this on cheyenne with conda loaded in modules # use the top level bash script: -# ../py_env_create # Do this each time you update your CTSM Version -# conda activate ctsm_py # Do this anytime you want to run a CTSM python script +# ../py_env_create # Do this each time you update your CTSM Version +# conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script # Or the individual conda commands: -# conda create -n ctsm_py # Do this one time for each machine -# conda install -n ctsm_py --file conda_env_ctsm_py.txt # Do this each time you update your CTSM Version -# conda activate ctsm_py # Do this anytime you want to run a CTSM python script +# conda create -n ctsm_pylib # Do this one time for each machine +# conda install -n ctsm_pylib --file conda_env_ctsm_py.txt # Do this each time you update your CTSM Version +# conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script # python=3.7.0 # The python version MUST match the python version available on CGD systems through modules exactly scipy diff --git a/test/tools/test_driver.sh b/test/tools/test_driver.sh index 1b3c141d79..db3466e5a8 100755 --- a/test/tools/test_driver.sh +++ b/test/tools/test_driver.sh @@ -55,7 +55,7 @@ module load ncl module load conda $CESMDATAROOT/py_env_create -conda activate ctsm_py +conda activate ctsm_pylib ##omp threads @@ -120,7 +120,7 @@ module load nco module load conda module load ncl $CESMDATAROOT/py_env_create -conda activate ctsm_py +conda activate ctsm_pylib ##omp threads @@ -221,7 +221,7 @@ module load tool/nco module load tool/netcdf module load lang/python $CESMDATAROOT/py_env_create -conda activate ctsm_py +conda activate ctsm_pylib export NETCDF_DIR=\$NETCDF_PATH export INC_NETCDF=\${NETCDF_PATH}/include @@ -304,7 +304,7 @@ module load tool/nco module load tool/netcdf module load lang/python $CESMDATAROOT/py_env_create -conda activate ctsm_py +conda activate ctsm_pylib export NETCDF_DIR=\$NETCDF_PATH export INC_NETCDF=\${NETCDF_PATH}/include diff --git a/tools/contrib/README b/tools/contrib/README index 43884abe69..4cf9c68fc4 100644 --- a/tools/contrib/README +++ b/tools/contrib/README @@ -10,7 +10,7 @@ The python scripts require the following settings before running on cheyenne: module load conda ../../py_env_create -conda activate ctsm_py +conda activate ctsm_pylib Brief description of scripts: diff --git a/tools/contrib/ssp_anomaly_forcing_smooth b/tools/contrib/ssp_anomaly_forcing_smooth index 94658f3d54..ae3d189a8b 100755 --- a/tools/contrib/ssp_anomaly_forcing_smooth +++ b/tools/contrib/ssp_anomaly_forcing_smooth @@ -8,7 +8,7 @@ Create anomoly forcing datasets for SSP scenarios that can be used by CESM datm load proper modules first, i.e. ../../py_env_create -conda activate ctsm_py +conda activate ctsm_pylib """ import sys diff --git a/tools/modify_fsurdat/README b/tools/modify_fsurdat/README index b8337c93f4..2295eabf32 100644 --- a/tools/modify_fsurdat/README +++ b/tools/modify_fsurdat/README @@ -32,7 +32,7 @@ To run on Cheyenne/Casper/Izumi module unload python module load conda ./py_env_create -conda activate ctsm_py +conda activate ctsm_pylib (Use "deactivate" to reverse the latter.) 2) Copy, then modify the configure file named modify_template.cfg, which contains all the arguments needed by the script. diff --git a/tools/site_and_regional/README b/tools/site_and_regional/README index d6cc55eac8..723ddb4a97 100644 --- a/tools/site_and_regional/README +++ b/tools/site_and_regional/README @@ -7,7 +7,7 @@ The python scripts require the following settings before running on cheyenne: module load conda ../../py_env_create -conda activate ctsm_py +conda activate ctsm_pylib Brief description of scripts: From d943730ee25661aeeec140dff13443f7130b3668 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 28 Jul 2022 16:14:25 -0600 Subject: [PATCH 030/332] Add a workflow for python directory testing --- .github/workflows/python_testing.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/python_testing.yml diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml new file mode 100644 index 0000000000..e2bd3567ca --- /dev/null +++ b/.github/workflows/python_testing.yml @@ -0,0 +1,26 @@ +name: python testing on push and PR +# +# Run the python directory testing on push and pull request +# +on: [push, pull_request] + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + # Checkout the code + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: ctsm_pylib + environment-file: python/conda_env_ctsm_py.txt + auto-activate-base: false + - shell: bash -l {0} + run: | + conda info + conda list + - name: Run python tests + shell: bash -l {0} + run: | + cd python + make all From f48503405edde55c4fe3697568f940c4e935e9ae Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 28 Jul 2022 16:23:21 -0600 Subject: [PATCH 031/332] Add conda-forge channel to action, and only do test and lint test so black check isn't duplicated --- .github/workflows/python_testing.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml index e2bd3567ca..c7cbff63a5 100644 --- a/.github/workflows/python_testing.yml +++ b/.github/workflows/python_testing.yml @@ -15,6 +15,7 @@ jobs: activate-environment: ctsm_pylib environment-file: python/conda_env_ctsm_py.txt auto-activate-base: false + channels: conda-forge - shell: bash -l {0} run: | conda info @@ -23,4 +24,5 @@ jobs: shell: bash -l {0} run: | cd python - make all + make test + make lint From 82adf407b1623c176b33bc6efb65dd5d6f7b4367 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 28 Jul 2022 16:30:33 -0600 Subject: [PATCH 032/332] Have to manage externals so that cime exists --- .github/workflows/python_testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml index c7cbff63a5..5098a9495d 100644 --- a/.github/workflows/python_testing.yml +++ b/.github/workflows/python_testing.yml @@ -18,6 +18,7 @@ jobs: channels: conda-forge - shell: bash -l {0} run: | + manage_externals/checkout_externals conda info conda list - name: Run python tests From e7c69a6ac87eae95e432a15c304a8182328d7e59 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 28 Jul 2022 16:41:36 -0600 Subject: [PATCH 033/332] Seperate tests out, and add git lfs --- .github/workflows/python_testing.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml index 5098a9495d..e2137d91f7 100644 --- a/.github/workflows/python_testing.yml +++ b/.github/workflows/python_testing.yml @@ -16,14 +16,18 @@ jobs: environment-file: python/conda_env_ctsm_py.txt auto-activate-base: false channels: conda-forge - - shell: bash -l {0} + - name: manage_exernals and LFS + shell: bash -l {0} run: | + git lfs install manage_externals/checkout_externals - conda info - conda list - name: Run python tests shell: bash -l {0} run: | cd python make test + - name: Run pylint + shell: bash -l {0} + run: | + cd python make lint From ae3acd0cce5af3df8023ade93d662022e252c342 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 1 Aug 2022 17:02:31 -0600 Subject: [PATCH 034/332] Add an explicit check for the make returning an exit code --- .github/workflows/python_testing.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml index e2137d91f7..790528c2be 100644 --- a/.github/workflows/python_testing.yml +++ b/.github/workflows/python_testing.yml @@ -26,8 +26,14 @@ jobs: run: | cd python make test + if [ $? != 0 ]; then + exit 1 + fi - name: Run pylint shell: bash -l {0} run: | cd python make lint + if [ $? != 0 ]; then + exit 1 + fi From c9d4861a5fe88351d7893a2ec84f3381ebc8ce6f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 1 Aug 2022 17:49:57 -0600 Subject: [PATCH 035/332] Remove set of bash for login, and add explicit rule for cd to python directory --- .github/workflows/python_testing.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml index 790528c2be..f9234acc2f 100644 --- a/.github/workflows/python_testing.yml +++ b/.github/workflows/python_testing.yml @@ -17,23 +17,15 @@ jobs: auto-activate-base: false channels: conda-forge - name: manage_exernals and LFS - shell: bash -l {0} run: | git lfs install manage_externals/checkout_externals - - name: Run python tests - shell: bash -l {0} + - name: chdirectory python run: | cd python + - name: Run python tests + run: | make test - if [ $? != 0 ]; then - exit 1 - fi - name: Run pylint - shell: bash -l {0} run: | - cd python make lint - if [ $? != 0 ]; then - exit 1 - fi From b1b5960f28cf20be635418b11cad0a53415a6b20 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Wed, 19 Oct 2022 14:11:08 -0400 Subject: [PATCH 036/332] Updating fates parameter file to a place-holder api25 --- Externals_CLM.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg index 2284518c58..1761ceffcc 100644 --- a/Externals_CLM.cfg +++ b/Externals_CLM.cfg @@ -1,8 +1,8 @@ [fates] local_path = src/fates protocol = git -repo_url = https://github.com/NGEET/fates -tag = sci.1.58.1_api.24.1.0 +repo_url = https://github.com/rgknox/fates +branch = cnp-dynamic-root-allom-dfdd-api21-symfix required = True [externals_description] From 21d826b67bba1f3e6ac7e93fc46da88c37001b35 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Wed, 19 Oct 2022 14:11:23 -0400 Subject: [PATCH 037/332] Updating fates parameter file to a place-holder api25 p2 --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2686d62b9a..6f693e2af0 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -491,7 +491,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.24.0.0_12pft_c220608.nc +lnd/clm2/paramdata/fates_params_api.25.0.0_12pft_c221019.nc From 26affffe1e3629e6caa5dff8b8a95d2728e26841 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Wed, 19 Oct 2022 14:11:52 -0400 Subject: [PATCH 038/332] Adding the dimension levclscpf --- src/main/histFileMod.F90 | 3 +++ src/utils/clmfates_interfaceMod.F90 | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/histFileMod.F90 b/src/main/histFileMod.F90 index a8c1c639b6..d5175342f0 100644 --- a/src/main/histFileMod.F90 +++ b/src/main/histFileMod.F90 @@ -2468,6 +2468,7 @@ subroutine htape_create (t, histrest) call ncd_defdim(lnfid, 'fates_levelcwd', num_elements_fates * ncwd, dimid) call ncd_defdim(lnfid, 'fates_levelage', num_elements_fates * nlevage, dimid) call ncd_defdim(lnfid, 'fates_levagefuel', nlevage * nfsc, dimid) + call ncd_defdim(lnfid, 'fates_levclscpf', nclmax*nlevsclass*numpft_fates, dimid) end if if ( .not. lhistrest )then @@ -5512,6 +5513,8 @@ subroutine hist_addfld2d (fname, type2d, units, avgflag, long_name, type1d_out, num2d = num_elements_fates*nlevage case ('fates_levagefuel') num2d = nlevage*nfsc + case('fates_levclscpf') + num2d = nclmax * nclmax * numpft_fates case('cft') if (cft_size > 0) then num2d = cft_size diff --git a/src/utils/clmfates_interfaceMod.F90 b/src/utils/clmfates_interfaceMod.F90 index 9a04a9d66f..c6d4d16624 100644 --- a/src/utils/clmfates_interfaceMod.F90 +++ b/src/utils/clmfates_interfaceMod.F90 @@ -2745,7 +2745,7 @@ subroutine init_history_io(this,bounds_proc) use FatesIOVariableKindMod, only : site_can_r8, site_cnlf_r8, site_cnlfpft_r8 use FatesIOVariableKindMod, only : site_height_r8, site_elem_r8, site_elpft_r8 use FatesIOVariableKindMod, only : site_elcwd_r8, site_elage_r8, site_agefuel_r8 - use FatesIOVariableKindMod, only : site_cdpf_r8, site_cdsc_r8 + use FatesIOVariableKindMod, only : site_cdpf_r8, site_cdsc_r8, site_clscpf_r8 use FatesIODimensionsMod, only : fates_bounds_type @@ -2844,7 +2844,7 @@ subroutine init_history_io(this,bounds_proc) case(site_soil_r8, site_size_pft_r8, site_size_r8, site_pft_r8, & site_age_r8, site_height_r8, site_coage_r8,site_coage_pft_r8, & - site_fuel_r8, site_cwdsc_r8, & + site_fuel_r8, site_cwdsc_r8, site_clscpf_r8, & site_can_r8,site_cnlf_r8, site_cnlfpft_r8, site_scag_r8, & site_scagpft_r8, site_agepft_r8, site_elem_r8, site_elpft_r8, & site_elcwd_r8, site_elage_r8, site_agefuel_r8, & @@ -3227,6 +3227,10 @@ subroutine hlm_bounds_to_fates_bounds(hlm, fates) fates%cdam_begin = 1 fates%cdam_end = nlevdamage + + fates%clscpf_begin = 1 + fates%clscpf_end = numpft_fates * nlevsclass * nclmax + call t_stopf('fates_hlm2fatesbnds') From 3ad2e7790225f361c986151e6173e2958f4d3b42 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Wed, 19 Oct 2022 14:27:20 -0400 Subject: [PATCH 039/332] Refactor of history interface with fates to accomodate api25 and new dimensions --- src/utils/clmfates_interfaceMod.F90 | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/utils/clmfates_interfaceMod.F90 b/src/utils/clmfates_interfaceMod.F90 index c6d4d16624..16a5f0fd7f 100644 --- a/src/utils/clmfates_interfaceMod.F90 +++ b/src/utils/clmfates_interfaceMod.F90 @@ -1014,6 +1014,8 @@ subroutine dynamics_driv(this, nc, bounds_clump, & ! --------------------------------------------------------------------------------- call fates_hist%flush_hvars(nc,upfreq_in=1) + call fates_hist%flush_hvars(nc,upfreq_in=5) + ! --------------------------------------------------------------------------------- ! Part II: Call the FATES model now that input boundary conditions have been @@ -1088,7 +1090,8 @@ subroutine dynamics_driv(this, nc, bounds_clump, & ! --------------------------------------------------------------------------------- call fates_hist%update_history_dyn( nc, & this%fates(nc)%nsites, & - this%fates(nc)%sites) + this%fates(nc)%sites, & + this%fates(nc)%bc_in) if (masterproc) then write(iulog, *) 'clm: leaving fates model', bounds_clump%begg, & @@ -1659,12 +1662,13 @@ subroutine restart( this, bounds_proc, ncid, flag, waterdiagnosticbulk_inst, & ! ------------------------------------------------------------------------ call fates_hist%flush_hvars(nc,upfreq_in=1) do s = 1,this%fates(nc)%nsites - call fates_hist%zero_site_hvars(this%fates(nc)%sites(s), & + call fates_hist%zero_site_hvars(this%fates(nc)%sites(s), & upfreq_in=1) end do - call fates_hist%update_history_dyn( nc, & - this%fates(nc)%nsites, & - this%fates(nc)%sites) + call fates_hist%update_history_dyn( nc, & + this%fates(nc)%nsites, & + this%fates(nc)%sites, & + this%fates(nc)%bc_in) end if @@ -1831,12 +1835,13 @@ subroutine init_coldstart(this, waterstatebulk_inst, waterdiagnosticbulk_inst, & ! ------------------------------------------------------------------------ call fates_hist%flush_hvars(nc,upfreq_in=1) do s = 1,this%fates(nc)%nsites - call fates_hist%zero_site_hvars(this%fates(nc)%sites(s), & + call fates_hist%zero_site_hvars(this%fates(nc)%sites(s), & upfreq_in=1) end do - call fates_hist%update_history_dyn( nc, & - this%fates(nc)%nsites, & - this%fates(nc)%sites) + call fates_hist%update_history_dyn( nc, & + this%fates(nc)%nsites, & + this%fates(nc)%sites, & + this%fates(nc)%bc_in(s)) From d01a98d623dd3e837e147c6f64d8fb9e50f55e4c Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Mon, 14 Nov 2022 07:44:46 -0700 Subject: [PATCH 040/332] updates for memory scaling --- .../share_esmf/lnd_set_decomp_and_domain.F90 | 162 +++++++++++------- 1 file changed, 96 insertions(+), 66 deletions(-) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index d1d29672b3..602b83d897 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -1,9 +1,6 @@ module lnd_set_decomp_and_domain - use ESMF , only : ESMF_VM, ESMF_Mesh, ESMF_DistGrid, ESMF_LogFoundError, ESMF_LOGERR_PASSTHRU - use ESMF , only : ESMF_Array, ESMF_ArrayCreate, ESMF_SUCCESS, ESMF_MeshCreate, ESMF_FILEFORMAT_ESMFMESH - use ESMF , only : ESMF_MeshGet, ESMF_DistGridGet, ESMF_Grid, ESMF_Field, ESMF_FieldGet, ESMF_FieldCreate, ESMF_FieldDestroy - use ESMF , only : ESMF_TYPEKIND_R8, ESMF_MESHLOC_ELEMENT, ESMF_VMAllReduce, ESMF_REDUCE_SUM + use ESMF use shr_kind_mod , only : r8 => shr_kind_r8, cl=>shr_kind_cl use shr_sys_mod , only : shr_sys_abort use shr_log_mod , only : errMsg => shr_log_errMsg @@ -45,7 +42,6 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes use decompMod , only : gindex_global, bounds_type, get_proc_bounds use clm_varpar , only : nlevsoi use clm_varctl , only : use_soil_moisture_streams - use ESMF , only : ESMF_DistGridCreate ! input/output variables character(len=*) , intent(in) :: driver ! cmeps or lilac @@ -57,20 +53,25 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes integer , intent(out) :: rc ! local variables - type(ESMF_Mesh) :: mesh_maskinput - type(ESMF_Mesh) :: mesh_lndinput - type(ESMF_DistGrid) :: distgrid_ctsm - integer :: g,n ! indices - integer :: nlnd, nocn ! local size of arrays - integer :: gsize ! global size of grid - logical :: isgrid2d ! true => grid is 2d - type(bounds_type) :: bounds ! bounds - integer :: begg,endg ! local bounds - integer , pointer :: gindex_lnd(:) ! global index space for just land points - integer , pointer :: gindex_ocn(:) ! global index space for just ocean points - integer , pointer :: gindex_ctsm(:) ! global index space for land and ocean points - integer , pointer :: lndmask_glob(:) - real(r8) , pointer :: lndfrac_glob(:) + type(ESMF_Mesh) :: mesh_maskinput + type(ESMF_Mesh) :: mesh_lndinput + type(ESMF_DistGrid) :: distgrid_ctsm + type(ESMF_Field) :: field_lnd + type(ESMF_Field) :: field_ctsm + type(ESMF_RouteHandle) :: rhandle_lnd2ctsm + integer :: g,n ! indices + integer :: nlnd, nocn ! local size of arrays + integer :: gsize ! global size of grid + logical :: isgrid2d ! true => grid is 2d + type(bounds_type) :: bounds ! bounds + integer :: begg,endg ! local bounds + integer , pointer :: gindex_lnd(:) ! global index space for just land points + integer , pointer :: gindex_ocn(:) ! global index space for just ocean points + integer , pointer :: gindex_ctsm(:) ! global index space for land and ocean points + integer , pointer :: lndmask_glob(:) + real(r8) , pointer :: lndfrac_glob(:) + real(r8) , pointer :: lndfrac_loc_input(:) + real(r8) , pointer :: dataptr1d(:) !------------------------------------------------------------------------------- rc = ESMF_SUCCESS @@ -90,8 +91,6 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes ! Determine global 2d sizes from read of dimensions of surface dataset and allocate global memory call lnd_get_global_dims(ni, nj, gsize, isgrid2d) - allocate(lndmask_glob(gsize)); lndmask_glob(:) = 0 - allocate(lndfrac_glob(gsize)); lndfrac_glob(:) = 0._r8 ! Read in the land mesh from the file mesh_lndinput = ESMF_MeshCreate(filename=trim(meshfile_lnd), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc) @@ -100,21 +99,27 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes if (trim(driver) == 'cmeps') then ! Read in mask meshfile if needed if (trim(meshfile_mask) /= trim(meshfile_lnd)) then + call ESMF_VMLogMemInfo("clm: Before lnd mesh create in ") mesh_maskinput = ESMF_MeshCreate(filename=trim(meshfile_mask), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - end if + call ESMF_VMLogMemInfo("clm: After lnd mesh create in ") + print *,__FILE__,__LINE__ - ! Determine lndmask_glob and lndfrac_glob - if (trim(meshfile_mask) /= trim(meshfile_lnd)) then + ! Determine lndmask_glob and lndfrac_glob ! obain land mask and land fraction by mapping ocean mesh conservatively to land mesh - call lnd_set_lndmask_from_maskmesh(mesh_lndinput, mesh_maskinput, vm, gsize, lndmask_glob, lndfrac_glob, rc) + ! Note that lndmask_glob and lndfrac_loc_input are allocated in lnd_set_lndmask_from_maskmesh + call lnd_set_lndmask_from_maskmesh(mesh_lndinput, mesh_maskinput, vm, gsize, lndmask_glob, & + lndfrac_loc_input, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + call ESMF_VMLogMemInfo("clm: After lnd_set_lndmask_from_maskmesh ") + print *,__FILE__,__LINE__ else ! obtain land mask from land mesh file - assume that land frac is identical to land mask - call lnd_set_lndmask_from_lndmesh(mesh_lndinput, vm, gsize, lndmask_glob, lndfrac_glob, rc) + call lnd_set_lndmask_from_lndmesh(mesh_lndinput, vm, gsize, lndmask_glob, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return end if else if (trim(driver) == 'lilac') then + allocate(lndfrac_glob(ni*nj)) call lnd_set_lndmask_from_fatmlndfrc(lndmask_glob, lndfrac_glob, ni,nj) else call shr_sys_abort('driver '//trim(driver)//' is not supported, must be lilac or cmeps') @@ -146,16 +151,14 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes ! Initialize domain data structure call domain_init(domain=ldomain, isgrid2d=isgrid2d, ni=ni, nj=nj, nbeg=begg, nend=endg) - ! Determine ldomain%mask and ldomain%frac using ctsm decomposition + ! Determine ldomain%mask do g = begg, endg n = 1 + (g - begg) ldomain%mask(g) = lndmask_glob(gindex_lnd(n)) - ldomain%frac(g) = lndfrac_glob(gindex_lnd(n)) end do ! Deallocate global pointer memory deallocate(lndmask_glob) - deallocate(lndfrac_glob) ! Generate a ctsm global index that includes both land and ocean points nocn = size(gindex_ocn) @@ -178,6 +181,55 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes call lnd_set_ldomain_gridinfo_from_mesh(mesh_ctsm, vm, gindex_ctsm, begg, endg, isgrid2d, ni, nj, ldomain, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return + ! Set ldomain%lfrac + ! Create fields on the input decomp and ctsm decomp + ! Determine route handle to do a redist from input mesh read decomp to ctsm decomp + ! Fill in the field on the input mesh read decomp with lndfrac_loc_input values + ! Redistribute field_lnd to field_ctsm + + ! Determine ldomain%frac using ctsm decomposition + if (trim(driver) == 'cmeps') then + + if (trim(meshfile_mask) /= trim(meshfile_lnd)) then + field_lnd = ESMF_FieldCreate(mesh_lndinput, ESMF_TYPEKIND_R8, meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + print *,__FILE__,__LINE__ + field_ctsm = ESMF_FieldCreate(mesh_ctsm, ESMF_TYPEKIND_R8, meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + print *,__FILE__,__LINE__ + call ESMF_FieldRedistStore(field_lnd, field_ctsm, routehandle=rhandle_lnd2ctsm, & + ignoreUnmatchedIndices=.true., rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldGet(field_lnd, farrayptr=dataptr1d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + do n = 1,size(dataptr1d) + dataptr1d(n) = lndfrac_loc_input(n) + end do + call ESMF_FieldRedist(field_lnd, field_ctsm, routehandle=rhandle_lnd2ctsm, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + call ESMF_FieldGet(field_ctsm, farrayptr=dataptr1d, rc=rc) + if (chkerr(rc,__LINE__,u_FILE_u)) return + do g = begg, endg + n = 1 + (g - begg) + ldomain%frac(g) = dataptr1d(n) + end do + else + ! ASSUME that land fraction is identical to land mask in this case + do g = begg, endg + ldomain%frac(g) = ldomain%mask(g) + end do + end if + + else + + do g = begg, endg + n = 1 + (g - begg) + ldomain%frac(g) = lndfrac_glob(gindex_lnd(n)) + end do + deallocate(lndfrac_glob) + + end if + ! Deallocate local pointer memory deallocate(gindex_lnd) deallocate(gindex_ocn) @@ -189,8 +241,8 @@ end subroutine lnd_set_decomp_and_domain_from_readmesh subroutine lnd_set_mesh_for_single_column(scol_lon, scol_lat, mesh, rc) ! Generate a mesh for single column + use netcdf use clm_varcon, only : spval - use ESMF , only : ESMF_Grid, ESMF_GridCreateNoPeriDimUfrm, ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER ! input/output variables real(r8) , intent(in) :: scol_lon @@ -343,7 +395,7 @@ subroutine lnd_get_global_dims(ni, nj, gsize, isgrid2d) end subroutine lnd_get_global_dims !=============================================================================== - subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask_glob, lndfrac_glob, rc) + subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask_glob, lndfrac_loc, rc) ! If the landfrac/landmask file does not exists then determine the ! land fraction and land mask on the land grid by mapping the mask @@ -352,19 +404,13 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask ! exist then simply read in the global land fraction and land mask ! from the file - ! Uses: - use ESMF , only : ESMF_RouteHandle, ESMF_FieldRegridStore, ESMF_FieldRegrid - use ESMF , only : ESMF_REGRIDMETHOD_CONSERVE, ESMF_NORMTYPE_DSTAREA, ESMF_UNMAPPEDACTION_IGNORE - use ESMF , only : ESMF_TERMORDER_SRCSEQ, ESMF_REGION_TOTAL - - ! input/out variables type(ESMF_Mesh) , intent(in) :: mesh_lnd type(ESMF_Mesh) , intent(in) :: mesh_mask type(ESMF_VM) , intent(in) :: vm integer , intent(in) :: gsize integer , pointer :: lndmask_glob(:) - real(r8) , pointer :: lndfrac_glob(:) + real(r8) , pointer :: lndfrac_loc(:) integer , intent(out) :: rc ! local variables: @@ -376,8 +422,6 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask integer , pointer :: gindex_input(:) ! global index space for land and ocean points integer , pointer :: lndmask_loc(:) integer , pointer :: itemp_glob(:) - real(r8) , pointer :: rtemp_glob(:) - real(r8) , pointer :: lndfrac_loc(:) real(r8) , pointer :: maskmask_loc(:) ! on ocean mesh real(r8) , pointer :: maskfrac_loc(:) ! on land mesh real(r8) , pointer :: dataptr1d(:) @@ -403,9 +447,13 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask klen = len_trim(flandfrac) - 3 ! remove the .nc flandfrac_status = flandfrac(1:klen)//'.status' + allocate(lndmask_glob(gsize)); lndmask_glob(:) = 0 + ! Determine if lndfrac/lndmask file exists inquire(file=trim(flandfrac), exist=lexist) + lexist = .false. ! TODO: fix this + if (lexist) then ! If file exists - read in lndmask and lndfrac @@ -413,8 +461,8 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask write(iulog,*) write(iulog,'(a)')' Reading in land fraction and land mask from '//trim(flandfrac) end if - call lnd_set_read_write_landmask(trim(flandfrac), trim(flandfrac_status), .false., .true., & - lndmask_glob, lndfrac_glob, size(lndmask_glob)) + ! call lnd_set_read_write_landmask(trim(flandfrac), trim(flandfrac_status), .false., .true., & + ! lndmask_glob, lndfrac_glob, size(lndmask_glob)) else @@ -493,39 +541,25 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask lndmask_glob(:) = int(itemp_glob(:)) deallocate(itemp_glob) - ! Determine ldomain%frac using both input and ctsm decompositions - ! lndfrac_glob is filled using the input decomposition and - ! ldomin%frac is set using the ctsm decomposition - allocate(rtemp_glob(gsize)) - do n = 1,lsize_lnd - lndfrac_glob(gindex_input(n)) = lndfrac_loc(n) - end do - call ESMF_VMAllReduce(vm, sendData=lndfrac_glob, recvData=rtemp_glob, count=gsize, & - reduceflag=ESMF_REDUCE_SUM, rc=rc) - lndfrac_glob(:) = rtemp_glob(:) - deallocate(rtemp_glob) - ! deallocate memory deallocate(maskmask_loc) deallocate(lndmask_loc) - deallocate(lndfrac_loc) - call lnd_set_read_write_landmask(trim(flandfrac), trim(flandfrac_status), .true., .false., & - lndmask_glob, lndfrac_glob, size(lndmask_glob)) + ! call lnd_set_read_write_landmask(trim(flandfrac), trim(flandfrac_status), .true., .false., & + ! lndmask_glob, lndfrac_glob, size(lndmask_glob)) end if end subroutine lnd_set_lndmask_from_maskmesh !=============================================================================== - subroutine lnd_set_lndmask_from_lndmesh(mesh_lnd, vm, gsize, lndmask_glob, lndfrac_glob, rc) + subroutine lnd_set_lndmask_from_lndmesh(mesh_lnd, vm, gsize, lndmask_glob, rc) ! input/out variables type(ESMF_Mesh) , intent(in) :: mesh_lnd type(ESMF_VM) , intent(in) :: vm integer , intent(in) :: gsize integer , pointer :: lndmask_glob(:) - real(r8) , pointer :: lndfrac_glob(:) integer , intent(out) :: rc ! local variables: @@ -570,9 +604,6 @@ subroutine lnd_set_lndmask_from_lndmesh(mesh_lnd, vm, gsize, lndmask_glob, lndfr deallocate(gindex) deallocate(lndmask_loc) - ! ASSUME that land fraction is identical to land mask in this case - lndfrac_glob(:) = lndmask_glob(:) - end subroutine lnd_set_lndmask_from_lndmesh !=============================================================================== @@ -602,7 +633,7 @@ subroutine lnd_set_lndmask_from_fatmlndfrc(mask, frac, ni, nj) logical :: readvar ! read variable in or not integer , allocatable :: idata2d(:,:) real(r8), allocatable :: rdata2d(:,:) - integer :: unitn + integer :: unitn character(len=32) :: subname = 'lnd_set_mask_from_fatmlndfrc' ! subroutine name !----------------------------------------------------------------------- @@ -672,7 +703,6 @@ subroutine lnd_set_ldomain_gridinfo_from_mesh(mesh, vm, gindex, begg, endg, isgr use clm_varcon , only : grlnd use fileutils , only : getfil use ncdio_pio , only : ncd_io, file_desc_t, ncd_pio_openfile, ncd_pio_closefile - use ESMF , only : ESMF_FieldRegridGetArea ! input/output variables type(ESMF_Mesh) , intent(in) :: mesh @@ -815,7 +845,7 @@ subroutine lnd_set_read_write_landmask(flandfrac, flandfrac_status, write_file, integer :: dimid integer :: iun integer :: ioe - integer :: ier + integer :: ier logical :: lexists !------------------------------------------------------------------------------- @@ -856,7 +886,7 @@ subroutine lnd_set_read_write_landmask(flandfrac, flandfrac_status, write_file, close(iun) write(iulog,'(a)')' Successfully wrote land fraction/mask status file '//trim(flandfrac_status) end if - + else if (read_file) then if (masterproc) then From b91dc65404331fc61d41039ad60b20479d176e0e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 14 Nov 2022 08:28:34 -0700 Subject: [PATCH 041/332] add only to use esmf, remove commented code --- .../share_esmf/lnd_set_decomp_and_domain.F90 | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index 602b83d897..ff144ed49a 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -1,6 +1,19 @@ module lnd_set_decomp_and_domain - use ESMF + use ESMF , only : ESMF_VM, ESMF_MESH, ESMF_DistGrid, ESMF_Field + use ESMF , only : ESMF_RouteHandle, ESMF_SUCCESS, ESMF_MeshCreate + use ESMF , only : ESMF_FileFormat_ESMFMESH, ESMF_VMLogMemInfo + use ESMF , only : ESMF_FieldCreate, ESMF_FieldRedistStore, ESMF_FieldRedist + use ESMF , only : ESMF_FieldGet, ESMF_GridCreateNoPeriDimUfrm + use ESMF , only : ESMF_FieldRegrid, ESMF_FieldRegridStore + use ESMF , only : ESMF_Grid, ESMF_ARRAY, ESMF_DistGridCreate, ESMF_TYPEKIND_R8 + use ESMF , only : ESMF_MESHLOC_ELEMENT, ESMF_FieldCreate, ESMF_STAGGERLOC_CORNER, ESMF_STAGGERLOC_CENTER + use ESMF , only : ESMF_REGRIDMETHOD_CONSERVE, ESMF_NORMTYPE_DSTAREA + use ESMF , only : ESMF_UNMAPPEDACTION_IGNORE, ESMF_TERMORDER_SRCSEQ + use ESMF , only : ESMF_REGION_TOTAL, ESMF_REDUCE_SUM, ESMF_ARRAYCREATE + use ESMF , only : ESMF_MeshGet, ESMF_DistGridGet, ESMF_LOGFOUNDERROR + use ESMF , only : ESMF_LOGERR_PASSTHRU, ESMF_VMAllReduce, ESMF_FieldRegridGetArea + use ESMF , only : ESMF_FieldDestroy use shr_kind_mod , only : r8 => shr_kind_r8, cl=>shr_kind_cl use shr_sys_mod , only : shr_sys_abort use shr_log_mod , only : errMsg => shr_log_errMsg @@ -99,20 +112,23 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes if (trim(driver) == 'cmeps') then ! Read in mask meshfile if needed if (trim(meshfile_mask) /= trim(meshfile_lnd)) then +#ifdef DEBUG call ESMF_VMLogMemInfo("clm: Before lnd mesh create in ") +#endif mesh_maskinput = ESMF_MeshCreate(filename=trim(meshfile_mask), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return +#ifdef DEBUG call ESMF_VMLogMemInfo("clm: After lnd mesh create in ") - print *,__FILE__,__LINE__ - +#endif ! Determine lndmask_glob and lndfrac_glob ! obain land mask and land fraction by mapping ocean mesh conservatively to land mesh ! Note that lndmask_glob and lndfrac_loc_input are allocated in lnd_set_lndmask_from_maskmesh call lnd_set_lndmask_from_maskmesh(mesh_lndinput, mesh_maskinput, vm, gsize, lndmask_glob, & lndfrac_loc_input, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return +#ifdef DEBUG call ESMF_VMLogMemInfo("clm: After lnd_set_lndmask_from_maskmesh ") - print *,__FILE__,__LINE__ +#endif else ! obtain land mask from land mesh file - assume that land frac is identical to land mask call lnd_set_lndmask_from_lndmesh(mesh_lndinput, vm, gsize, lndmask_glob, rc) @@ -461,9 +477,6 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask write(iulog,*) write(iulog,'(a)')' Reading in land fraction and land mask from '//trim(flandfrac) end if - ! call lnd_set_read_write_landmask(trim(flandfrac), trim(flandfrac_status), .false., .true., & - ! lndmask_glob, lndfrac_glob, size(lndmask_glob)) - else ! If file does not exist - compute lndmask and lndfrac and write to output file @@ -545,9 +558,6 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask deallocate(maskmask_loc) deallocate(lndmask_loc) - ! call lnd_set_read_write_landmask(trim(flandfrac), trim(flandfrac_status), .true., .false., & - ! lndmask_glob, lndfrac_glob, size(lndmask_glob)) - end if end subroutine lnd_set_lndmask_from_maskmesh From d016cd6605a7114ce95d665fbbb9a8a6d61b1810 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 14 Nov 2022 08:35:03 -0700 Subject: [PATCH 042/332] remove unused use statement --- src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index ff144ed49a..1db72296b4 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -257,7 +257,6 @@ end subroutine lnd_set_decomp_and_domain_from_readmesh subroutine lnd_set_mesh_for_single_column(scol_lon, scol_lat, mesh, rc) ! Generate a mesh for single column - use netcdf use clm_varcon, only : spval ! input/output variables From bffff16b40abc2e85bfc9d90f9895e9232bfaa3d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 14 Nov 2022 13:41:28 -0700 Subject: [PATCH 043/332] allocate mask and frac in all cases --- src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index 1db72296b4..f7b356e0aa 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -135,7 +135,6 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes if (ChkErr(rc,__LINE__,u_FILE_u)) return end if else if (trim(driver) == 'lilac') then - allocate(lndfrac_glob(ni*nj)) call lnd_set_lndmask_from_fatmlndfrc(lndmask_glob, lndfrac_glob, ni,nj) else call shr_sys_abort('driver '//trim(driver)//' is not supported, must be lilac or cmeps') @@ -603,6 +602,9 @@ subroutine lnd_set_lndmask_from_lndmesh(mesh_lnd, vm, gsize, lndmask_glob, rc) allocate(itemp_glob(gsize)) call ESMF_DistGridGet(distgrid, 0, seqIndexList=gindex, rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return + + allocate(lndmask_glob(gsize)); lndmask_glob(:) = 0 + do n = 1,lsize lndmask_glob(gindex(n)) = lndmask_loc(n) end do @@ -655,6 +657,7 @@ subroutine lnd_set_lndmask_from_fatmlndfrc(mask, frac, ni, nj) if (masterproc) then write(iulog,*)'lat/lon grid flag (isgrid2d) is ',isgrid2d end if + allocate(frac(ni*nj), mask(ni*nj)) if (isgrid2d) then ! Grid is 2d From 6d2515f212a91f19e2ee7f287a4a946667707a0f Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Mon, 28 Nov 2022 20:03:26 -0700 Subject: [PATCH 044/332] Updated fates parameter file location --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 6f693e2af0..50f33ffcbb 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -491,7 +491,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.25.0.0_12pft_c221019.nc +lnd/clm2/paramdata/fates_params_api.25.0.0_12pft_c221128.nc From 19e39b8ba41c1e2c7337100eaf6f340e1f692f98 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Tue, 29 Nov 2022 16:50:32 -0700 Subject: [PATCH 045/332] Fixed test variable list and fixed interface to fates history for dynamics --- cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm | 2 +- .../testdefs/testmods_dirs/clm/FatesColdDefAllVars/user_nl_clm | 2 +- src/utils/clmfates_interfaceMod.F90 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm index 59b639fa84..2a915f2fd7 100644 --- a/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm @@ -15,7 +15,7 @@ hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_FUEL_AMOUNT', 'FATES_LITTER_IN', 'FATES_LITTER_OUT', 'FATES_SEED_BANK', 'FATES_SEEDS_IN', 'FATES_STOREC', 'FATES_VEGC', 'FATES_SAPWOODC', 'FATES_LEAFC', 'FATES_FROOTC', 'FATES_REPROC', -'FATES_CEFFLUX', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', +'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', 'FATES_CANOPY_VEGC', 'FATES_USTORY_VEGC', 'FATES_PRIMARY_PATCHFUSION_ERR', 'FATES_DISTURBANCE_RATE_P2P', 'FATES_DISTURBANCE_RATE_P2S', 'FATES_DISTURBANCE_RATE_S2S', 'FATES_DISTURBANCE_RATE_FIRE', diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdDefAllVars/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdDefAllVars/user_nl_clm index b7d33c87d2..0a0726dcce 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdDefAllVars/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdDefAllVars/user_nl_clm @@ -70,4 +70,4 @@ hist_fincl1 = 'FATES_CROWNAREA_PF', 'FATES_CANOPYCROWNAREA_PF', 'FATES_CROOTMAINTAR_USTORY_SZ', 'FATES_FROOTMAINTAR_USTORY_SZ', 'FATES_GROWAR_USTORY_SZ', 'FATES_MAINTAR_USTORY_SZ', 'FATES_VEGC_SZPF', 'FATES_LEAFC_SZPF', 'FATES_FROOTC_SZPF', 'FATES_SAPWOODC_SZPF', -'FATES_STOREC_SZPF', 'FATES_REPROC_SZPF', 'FATES_CEFFLUX_SZPF' +'FATES_STOREC_SZPF', 'FATES_REPROC_SZPF' diff --git a/src/utils/clmfates_interfaceMod.F90 b/src/utils/clmfates_interfaceMod.F90 index 16a5f0fd7f..3f55959f16 100644 --- a/src/utils/clmfates_interfaceMod.F90 +++ b/src/utils/clmfates_interfaceMod.F90 @@ -1841,7 +1841,7 @@ subroutine init_coldstart(this, waterstatebulk_inst, waterdiagnosticbulk_inst, & call fates_hist%update_history_dyn( nc, & this%fates(nc)%nsites, & this%fates(nc)%sites, & - this%fates(nc)%bc_in(s)) + this%fates(nc)%bc_in) From ed9762d44cbd0ebc095741a6b54b2aa565180777 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Wed, 30 Nov 2022 08:25:31 -0700 Subject: [PATCH 046/332] removed cefflux from fates hist list --- cime_config/usermods_dirs/fates_sp/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/fates_sp/user_nl_clm b/cime_config/usermods_dirs/fates_sp/user_nl_clm index cc3d9339e8..f2f68eb74c 100644 --- a/cime_config/usermods_dirs/fates_sp/user_nl_clm +++ b/cime_config/usermods_dirs/fates_sp/user_nl_clm @@ -18,7 +18,7 @@ hist_fexcl1 = 'FATES_TRIMMING', 'FATES_COLD_STATUS', 'FATES_DROUGHT_STATUS', 'FA 'FATES_FUEL_AMOUNT_AP', 'FATES_FUEL_BURNT_BURNFRAC_FC', 'FATES_LITTER_IN', 'FATES_LITTER_OUT', 'FATES_SEED_BANK', 'FATES_SEEDS_IN', 'FATES_LITTER_IN_EL', 'FATES_LITTER_OUT_EL', 'FATES_SEED_BANK_EL', 'FATES_SEEDS_IN_LOCAL_EL', 'FATES_SEEDS_IN_EXTERN_EL', 'FATES_SEED_GERM_EL', 'FATES_SEED_DECAY_EL', 'FATES_STOREC', 'FATES_VEGC', - 'FATES_SAPWOODC', 'FATES_FROOTC', 'FATES_REPROC', 'FATES_CEFFLUX', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', + 'FATES_SAPWOODC', 'FATES_FROOTC', 'FATES_REPROC', 'FATES_STRUCTC', 'FATES_NONSTRUCTC', 'FATES_VEGC_ABOVEGROUND', 'FATES_CANOPY_VEGC', 'FATES_USTORY_VEGC', 'FATES_PRIMARY_PATCHFUSION_ERR', 'FATES_DISTURBANCE_RATE_P2P', 'FATES_DISTURBANCE_RATE_P2S', 'FATES_DISTURBANCE_RATE_S2S', 'FATES_DISTURBANCE_RATE_FIRE', 'FATES_DISTURBANCE_RATE_LOGGING', 'FATES_DISTURBANCE_RATE_TREEFALL', From 48bcbfc1b041fa6c95cd0a793ea64cc9975bf06e Mon Sep 17 00:00:00 2001 From: olyson Date: Wed, 30 Nov 2022 15:55:09 -0700 Subject: [PATCH 047/332] Fix issue #1864 --- .../CLM50_Tech_Note_Plant_Hydraulics.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/source/tech_note/Plant_Hydraulics/CLM50_Tech_Note_Plant_Hydraulics.rst b/doc/source/tech_note/Plant_Hydraulics/CLM50_Tech_Note_Plant_Hydraulics.rst index f33a6d96b5..2d8843510a 100644 --- a/doc/source/tech_note/Plant_Hydraulics/CLM50_Tech_Note_Plant_Hydraulics.rst +++ b/doc/source/tech_note/Plant_Hydraulics/CLM50_Tech_Note_Plant_Hydraulics.rst @@ -376,22 +376,23 @@ from the characteristic root spacing (section :numref:`Root Spacing`). .. math:: :label: 11.109 - q_{3,i}=k_{3,i} \cdot RAI \cdot \left(\psi_{soil,i}-\psi_{root} + \Delta\psi_{z,i} \right) + q_{3,i}=k_{3,i} \cdot \left(\psi_{soil,i}-\psi_{root} + \Delta\psi_{z,i} \right) .. math:: :label: 11.110 - RAI=\left(LAI+SAI \right) \cdot r_i \cdot f_{root-leaf} + k_{3,i}=\dfrac{k_{r,i} \cdot k_{s,i}}{k_{r,i}+k_{s,i}} .. math:: :label: 11.111 - k_{3,i}=\dfrac{k_{r,i} \cdot k_{s,i}}{k_{r,i}+k_{s,i}} + k_{r,i}=\dfrac{k_{3,max}}{z_{3,i}} \cdot RAI \cdot 2^{-\left(\dfrac{\psi_{soil,i}}{p50_3}\right)^{c_k}} .. math:: :label: 11.112 - k_{r,i}=\dfrac{k_{3,max}}{z_{3,i}} \cdot 2^{-\left(\dfrac{\psi_{soil,i}}{p50_3}\right)^{c_k}} + RAI=\left(LAI+SAI \right) \cdot r_i \cdot f_{root-leaf} + .. math:: :label: 11.113 From 757d22a823e347b97d463c2f9d62d93c6d993b1a Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Thu, 1 Dec 2022 07:33:08 -0700 Subject: [PATCH 048/332] Updated fates prt2 testdefs --- .../testmods_dirs/clm/FatesColdPRT2/user_nl_clm | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm index bde61cee2e..37ba2f814d 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm @@ -1 +1,12 @@ -fates_parteh_mode = 2 \ No newline at end of file +fates_parteh_mode = 2 +hist_fincl1 = 'FATES_L2FR','FATES_L2FR_CANOPY_REC_PF','FATES_L2FR_USTORY_REC_PF', +'FATES_NH4UPTAKE_SZPF','FATES_NO3UPTAKE_SZPF','FATES_NEFFLUX_SZPF', +'FATES_NDEMAND_SZPF','FATES_NFIX_SYM_SZPF','FATES_NH4UPTAKE','FATES_NO3UPTAKE', +'FATES_NEFFLUX','FATES_NDEMAND','FATES_NFIX_SYM','FATES_STOREN','FATES_STOREN_TF', +'FATES_VEGN','FATES_SAPWOODN','FATES_LEAFN','FATES_FROOTN','FATES_REPRON','FATES_VEGN_SZPF', +'FATES_LEAFN_SZPF','FATES_FROOTN_SZPF','FATES_SAPWOODN_SZPF','FATES_STOREN_SZPF','FATES_STOREN_TF_CANOPY_SZPF', +'FATES_STOREN_TF_USTORY_SZPF','FATES_REPRON_SZPF','FATES_STOREP','FATES_STOREP_TF','FATES_VEGP','FATES_SAPWOODP', +'FATES_LEAFP','FATES_FROOTP','FATES_REPROP','FATES_PUPTAKE','FATES_PEFFLUX','FATES_PDEMAND', +'FATES_VEGP_SZPF','FATES_LEAFP_SZPF','FATES_FROOTP_SZPF','FATES_SAPWOODP_SZPF','FATES_STOREP_SZPF', +'FATES_STOREP_TF_CANOPY_SZPF','FATES_STOREP_TF_USTORY_SZPF','FATES_REPROP_SZPF','FATES_PUPTAKE_SZPF', +'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF' \ No newline at end of file From 9aadaab9bd0b9b5e9f69fa44b7961c21043a4fca Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Fri, 2 Dec 2022 15:40:15 -0700 Subject: [PATCH 049/332] Added new line character to end of user_nl_clm in prt2 testdef --- .../testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm index 37ba2f814d..679f025b60 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdPRT2/user_nl_clm @@ -9,4 +9,4 @@ hist_fincl1 = 'FATES_L2FR','FATES_L2FR_CANOPY_REC_PF','FATES_L2FR_USTORY_R 'FATES_LEAFP','FATES_FROOTP','FATES_REPROP','FATES_PUPTAKE','FATES_PEFFLUX','FATES_PDEMAND', 'FATES_VEGP_SZPF','FATES_LEAFP_SZPF','FATES_FROOTP_SZPF','FATES_SAPWOODP_SZPF','FATES_STOREP_SZPF', 'FATES_STOREP_TF_CANOPY_SZPF','FATES_STOREP_TF_USTORY_SZPF','FATES_REPROP_SZPF','FATES_PUPTAKE_SZPF', -'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF' \ No newline at end of file +'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF' From 43974b24b360e495da8678b0a47d52f25c5d8e96 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Fri, 2 Dec 2022 16:06:28 -0700 Subject: [PATCH 050/332] Updated change-log --- doc/ChangeLog | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 78 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 5f16bf5634..49f49c235f 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,81 @@ =============================================================== +Tag name: ctsm5.1.dev115 +Originator(s): rgknox (Ryan Knox) +Date: Fri Dec 2 15:45:32 MST 2022 +One-line Summary: API compatability with FATES V2 nutrient dynamics + +Purpose and description of changes +---------------------------------- + +This set of changes allows CTSM to continue API compatability with changes to the FATES API. FATES has updated its nutrient dynamics routine, and required a modification to the test environment, some minor updates to variable dimensions in the history, and a call to a new FATES history routine. Implicitly, the updating of the FATES tag introduces new content in the FATES model since the last API update (mostly bug fixes). + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + + +The changes here will only affect FATES simulations. Any FATES simulation will be affected. Carbon-only FATES simulations will not have qualitatively different results since the last API update (but will have bit-for-bit differences). Nutrient enabled FATES simulations (not fully coupled to CLM, only via prognosed plant N,P boundaries), and FATES-Hydro simulations (bug fix) will be different. + +Bugs fixed or introduced +------------------------ + +See the descriptions in FATES tags between sci.1.57.4_api.24.0.0 to sci.1.60.0_api.25.0.0 for details on FATES changes in this tag. + +CTSM issues fixed (include CTSM Issue #): None + +Externals issues fixed (include issue #): None + +Known bugs introduced in this tag (include issue #): None + +Known bugs found since the previous tag (include issue #): None + + +Notes of particular relevance for users +--------------------------------------- + +This set of changes comes with an updated FATES parameter file. This includes format changes only. No changes to variable values were introduced. Format changes are relegated to new parameters and/or name changes only. These changes are encapsulated in: fates/parameter_files/archive/apichange_24.2to25.xml + + +Notes of particular relevance for developers: +--------------------------------------------- + +Nothing of note regarding changes for developers. + +Changes to tests or testing: New history variables were added to the FATES PRT2 user_nl_clm. + + +Testing summary: +---------------- + +regular + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- nominal: /glade/scratch/rgknox/tests_1201-121507ch + izumi ------- nominal: /scratch/cluster/rgknox/tests_1201-122133iz + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- nominal against ctsm5.1_dev112 /glade/scratch/rgknox/tests_1130-082657ch + izumi ------- NA + +Answer changes +-------------- + +Changes answers relative to baseline: FATES answers changed relative to base. Explanation already provided (nutrient and hydro changes are qualitative). + + +Other details +------------- + +FATES external was updated. + +Pull Requests that document the changes (include PR ids): + +https://github.com/ESCOMP/CTSM/pull/1874 +https://github.com/NGEET/fates/pull/880 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev114 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326)/@wwieder/@olyson/@ka7eh Date: Sat Nov 19 18:11:15 MST 2022 diff --git a/doc/ChangeSum b/doc/ChangeSum index 772545d00c..1d42a3522d 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics ctsm5.1.dev114 multiple 11/19/2022 Some NEON updates fixing AG sites, update MOSART, small fixes ctsm5.1.dev113 multiple 10/28/2022 Fix some compsets; add only clauses for ESMF use statements ctsm5.1.dev112 multiple 10/15/2022 Rework fates test definitions and add new fates tests From 128eb553419f94c56da03a449e26eb0f130e915e Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Fri, 2 Dec 2022 16:13:04 -0700 Subject: [PATCH 051/332] Updated fates external pointer to sci.1.61.0_api.25.0.0 --- Externals_CLM.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg index 1761ceffcc..219d1a9704 100644 --- a/Externals_CLM.cfg +++ b/Externals_CLM.cfg @@ -1,8 +1,8 @@ [fates] local_path = src/fates protocol = git -repo_url = https://github.com/rgknox/fates -branch = cnp-dynamic-root-allom-dfdd-api21-symfix +repo_url = https://github.com/NGEET/fates +tag = sci.1.61.0_api.25.0.0 required = True [externals_description] From 8dd3a2c7c855542ec4bd0517bba874ba65b6d9a9 Mon Sep 17 00:00:00 2001 From: Ryan Knox Date: Fri, 2 Dec 2022 16:15:35 -0700 Subject: [PATCH 052/332] Whitespace fix to clmfates_interfaceMod.F90 --- src/utils/clmfates_interfaceMod.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/clmfates_interfaceMod.F90 b/src/utils/clmfates_interfaceMod.F90 index 3f55959f16..c169d710ef 100644 --- a/src/utils/clmfates_interfaceMod.F90 +++ b/src/utils/clmfates_interfaceMod.F90 @@ -1015,7 +1015,6 @@ subroutine dynamics_driv(this, nc, bounds_clump, & call fates_hist%flush_hvars(nc,upfreq_in=1) call fates_hist%flush_hvars(nc,upfreq_in=5) - ! --------------------------------------------------------------------------------- ! Part II: Call the FATES model now that input boundary conditions have been From cb224da9cd17e1532181621b2db4f2455133ca05 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Dec 2022 16:34:48 -0700 Subject: [PATCH 053/332] Convert variable name to snake_case so that pylint passes --- python/ctsm/subset_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 6ef1acca85..080c7f3003 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -310,14 +310,14 @@ def get_parser(): default="", ) cesmroot = path_to_ctsm_root() - DEFAULTS_FILE = os.path.join(cesmroot, DEFAULTS_CONFIG) + defaults_file = os.path.join(cesmroot, DEFAULTS_CONFIG) subparser.add_argument( "--file", help="Default configure file to use for default filenames.", action="store", dest="config_file", type=str, - default=DEFAULTS_FILE, + default=defaults_file, ) subparser.add_argument( "--overwrite", From 51318489675196c3b0e904e5f9b3a2d735186b12 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Dec 2022 17:55:13 -0700 Subject: [PATCH 054/332] Add a test to fsurdat_modifier sys tester that if you don't give the fsurdat_in/fsurdat_out files it fails --- python/ctsm/test/test_sys_fsurdat_modifier.py | 11 +++++++++++ python/ctsm/test/testinputs/modify_fsurdat_short.cfg | 12 ++++++++++++ .../test/testinputs/modify_fsurdat_short_nofiles.cfg | 9 +++++++++ 3 files changed, 32 insertions(+) create mode 100644 python/ctsm/test/testinputs/modify_fsurdat_short.cfg create mode 100644 python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 6ee00604be..647a37e3db 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -39,6 +39,7 @@ def setUp(self): path_to_ctsm_root(), "tools/modify_input_files/modify_fsurdat_template.cfg" ) testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") + self._testinputs_path = testinputs_path self._fsurdat_in = os.path.join( testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", @@ -53,6 +54,16 @@ def tearDown(self): """ shutil.rmtree(self._tempdir, ignore_errors=True) + def test_no_files_given_fail(self): + """ + Test that if no input or output files are given that it will gracefully fail + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short_nofiles.cfg") + with self.assertRaisesRegex( + SystemExit, "must contain item 'fsurdat_in'" + ): + fsurdat_modifier(self._cfg_file_path) + def test_minimalInfo(self): """ This test specifies a minimal amount of information diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg new file mode 100644 index 0000000000..b0c94a11f2 --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg @@ -0,0 +1,12 @@ +[modify_input] + +fsurdat_in = FILL_THIS_IN +fsurdat_out = FILL_THIS_IN + +idealized = False +include_nonveg = False + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg new file mode 100644 index 0000000000..e4ba31912a --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg @@ -0,0 +1,9 @@ +[modify_input] + +idealized = False +include_nonveg = False + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 From 4c3339f81ddd4ea4e3dd241efdcde1bfc3775a24 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Dec 2022 18:03:15 -0700 Subject: [PATCH 055/332] Add a test for a config file that doesn't exist that fails --- python/ctsm/test/test_sys_fsurdat_modifier.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 647a37e3db..8dad324230 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -64,6 +64,17 @@ def test_no_files_given_fail(self): ): fsurdat_modifier(self._cfg_file_path) + def test_cfg_file_DNE_fail(self): + """ + Test that if the config file does not exist that it gracefully fails + """ + self._cfg_file_path = os.path.join(self._tempdir, "FILE_DOES_NOT_EXIST.cfg" ) + with self.assertRaisesRegex( + SystemExit, "Config file does NOT exist" + ): + fsurdat_modifier(self._cfg_file_path) + + def test_minimalInfo(self): """ This test specifies a minimal amount of information From df5ad3eeb8e42aeb4fdfe45067b638725b7fd761 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 12 Dec 2022 22:11:08 -0700 Subject: [PATCH 056/332] Add a check for the config file not existing --- python/ctsm/modify_input_files/fsurdat_modifier.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 7b7abbc646..7750d9eb20 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -45,6 +45,8 @@ def fsurdat_modifier(cfg_path): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file config = ConfigParser() + if not os.path.exists(cfg_path): + abort( "Config file does NOT exist: "+str(cfg_path) ) config.read(cfg_path) section = config.sections()[0] # name of the first section From c08ce1c0cc7e5cedcb992a6aaddf21369bc9bdf8 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 13 Dec 2022 00:45:36 -0700 Subject: [PATCH 057/332] Add a check for an empty file failing gracefully --- python/ctsm/test/test_sys_fsurdat_modifier.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 8dad324230..5ed0e328fb 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -74,6 +74,17 @@ def test_cfg_file_DNE_fail(self): ): fsurdat_modifier(self._cfg_file_path) + def test_cfg_file_empty_fail(self): + """ + Test that if the config file is empty it gracefully fails + """ + self._cfg_file_path = os.path.join(self._tempdir, "EMPTY_FILE.cfg" ) + fil = open( self._cfg_file_path, "w" ) + fil.close() + with self.assertRaisesRegex( + SystemExit, "Config file does not have the expected section" + ): + fsurdat_modifier(self._cfg_file_path) def test_minimalInfo(self): """ From 0c3a179fc4573cdf8a54de1d004aa831704452aa Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 13 Dec 2022 00:51:31 -0700 Subject: [PATCH 058/332] Add check for a config file that does not have the expected main section name --- python/ctsm/modify_input_files/fsurdat_modifier.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 7750d9eb20..24bddb9e65 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -48,7 +48,9 @@ def fsurdat_modifier(cfg_path): if not os.path.exists(cfg_path): abort( "Config file does NOT exist: "+str(cfg_path) ) config.read(cfg_path) - section = config.sections()[0] # name of the first section + section = "modify_input" + if not config.has_section(section): + abort( "Config file does not have the expected section: "+section ) # required: user must set these in the .cfg file fsurdat_in = get_config_value( From a4a35dec4f2036bd748a0c647efc4a4c70e29560 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 13 Dec 2022 01:09:57 -0700 Subject: [PATCH 059/332] Add test for short config file and add all needed settings in the config file so that it will work --- python/ctsm/test/test_sys_fsurdat_modifier.py | 7 +++++++ .../test/testinputs/modify_fsurdat_short.cfg | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 5ed0e328fb..f7dd3fbc42 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -64,6 +64,13 @@ def test_no_files_given_fail(self): ): fsurdat_modifier(self._cfg_file_path) + def test_short_config(self): + """ + Test that a short config file works + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") + fsurdat_modifier(self._cfg_file_path) + def test_cfg_file_DNE_fail(self): """ Test that if the config file does not exist that it gracefully fails diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg index b0c94a11f2..e298b3575e 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg @@ -1,11 +1,26 @@ [modify_input] -fsurdat_in = FILL_THIS_IN -fsurdat_out = FILL_THIS_IN +fsurdat_in = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc +fsurdat_out = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc idealized = False include_nonveg = False +landmask_file = UNSET + +lat_dimname = lsmlat +lon_dimname = lsmlon + +dom_pft = UNSET + +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET +soil_color = UNSET +std_elev = UNSET +max_sat_area = UNSET + lnd_lat_1 = -90 lnd_lat_2 = 90 lnd_lon_1 = 0 From a4b717385ba41e2257fe7751748f05994161d9f3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 13 Dec 2022 16:16:00 -0700 Subject: [PATCH 060/332] Remove the output file created after the test --- python/ctsm/test/test_sys_fsurdat_modifier.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index f7dd3fbc42..9fd349b2ae 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -70,6 +70,8 @@ def test_short_config(self): """ self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") fsurdat_modifier(self._cfg_file_path) + fsurdat_out = "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" + os.remove( fsurdat_out ) def test_cfg_file_DNE_fail(self): """ From d8eedefdbeed69f581056010b528b8e068fa3796 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Thu, 15 Dec 2022 15:19:27 -0700 Subject: [PATCH 061/332] Minor corrections for system test to pass FSURDATMODIFYCTSM_D_Mmpi-serial_Ld1.5x5_amazon.I2000Clm50SpRs.cheyenne_intel --- cime_config/SystemTests/fsurdatmodifyctsm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index fa29f9acd6..a7c7bbf61b 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -64,7 +64,7 @@ def _create_config_file(self): def _run_modify_fsurdat(self): tool_path = os.path.join(self._ctsm_root, - 'tools/modify_fsurdat/fsurdat_modifier') + 'tools/modify_input_files/fsurdat_modifier') self._case.load_env(reset=True) conda_env = ". "+self._get_caseroot()+"/.env_mach_specific.sh; " @@ -85,8 +85,6 @@ def _get_conda_env(self): # # Remove python and add conda to environment for cheyennne conda_env = "module unload python; module load conda" - # CGD machines should already have conda loaded - conda_env += "; module load lang/python" # End above machine specific with a semicolon conda_env += "; " From 7d50ef6b74dd9c42338a34c593cd093cc0516483 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 16 Dec 2022 12:06:50 -0700 Subject: [PATCH 062/332] Stop running py_env_create so test will work in clm_pymods test-suite run_sys_tests (which starts ctsm's test-suites) builds cases on cheyenne's compute nodes. Running py_env_create on cheyenne's compute nodes fails, so we will leave it up to users to run py_env_create when necessary, manually. --- cime_config/SystemTests/fsurdatmodifyctsm.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index a7c7bbf61b..6b89c393c1 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -88,11 +88,8 @@ def _get_conda_env(self): # End above machine specific with a semicolon conda_env += "; " - # The following uses "&&" as they all need to work - # Run the py_env_create script - conda_env += self._ctsm_root+"/py_env_create" # Activate the python environment - conda_env += " && conda activate ctsm_pylib" + conda_env += " conda activate ctsm_pylib" # End above to get to actual command conda_env += " && " From 5fe8cfdf6cff482c72350a9515303aaa44ff7dfe Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 16 Dec 2022 12:53:21 -0700 Subject: [PATCH 063/332] Add new error msg when fsurdat_modifier tool fails New msg explains that user's ctsm_pylib environmt may be out-of_date and what to do. As written, the code lumps getting the conda environmt and running the tool in one subprocess. I will separate these to enable more informative error messages. --- cime_config/SystemTests/fsurdatmodifyctsm.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 6b89c393c1..a6c609187f 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -71,7 +71,18 @@ def _run_modify_fsurdat(self): # Preprend the commands to get the conda environment for python first conda_env += self._get_conda_env() # Source the env - subprocess.run( conda_env+"python3 "+tool_path+" "+self._cfg_file_path, shell=True, check=True) + try: + subprocess.run( conda_env+"python3 "+tool_path+" "+self._cfg_file_path, shell=True, check=True) + except subprocess.CalledProcessError as error: + print("ERROR while running fsurdat_modifier tool. ") + print("Your ctsm_pylib environment may be out of date. If so, ") + print("you may get past this error by running ./py_env_create ") + print("in your ctsm directory and trying this test again. ") + print("Default error message: ") + print(error.output) + except: + print("ERROR trying to run fsurdat_modifier tool.") + raise def _modify_user_nl(self): append_to_user_nl_files(caseroot = self._get_caseroot(), From 09d3639b413f728c32bc0d3f37e1c474d0f22c81 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 16 Dec 2022 14:45:59 -0700 Subject: [PATCH 064/332] Update error message when running the fsurdat_modifier tool I haven't found a way to separate the single subprocess that currently (1) loads conda and activates the ctsm_pylib environment and (2) runs the fsurdat_modifier tool into two or more subprocesses without losing access to the ctsm_pylib environment. So instead I have written a more elaborate error message with the various possible scenarios leading to an error in the subprocess. I have tested it by commenting out the "conda activate" line and confirming that I see my error message in TestStatus.log. --- cime_config/SystemTests/fsurdatmodifyctsm.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index a6c609187f..f8255624b5 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -74,10 +74,16 @@ def _run_modify_fsurdat(self): try: subprocess.run( conda_env+"python3 "+tool_path+" "+self._cfg_file_path, shell=True, check=True) except subprocess.CalledProcessError as error: - print("ERROR while running fsurdat_modifier tool. ") - print("Your ctsm_pylib environment may be out of date. If so, ") - print("you may get past this error by running ./py_env_create ") + print("ERROR while getting the conda environment and/or ") + print("running the fsurdat_modifier tool: ") + print("(1) If your ctsm_pylib environment is out of date or you ") + print("have not created the ctsm_pylib environment, yet, you may ") + print("get past this error by running ./py_env_create ") print("in your ctsm directory and trying this test again. ") + print("(2) If conda is not available, install and load conda, ") + print("run ./py_env_create, and then try this test again. ") + print("(3) If (1) and (2) are not the issue, then you may be ") + print("getting an error within the fsurdat_modifier tool itself. ") print("Default error message: ") print(error.output) except: From f0fdd2bd3e194dfc3dfe4cf2b6b7bcf051a59095 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 16 Dec 2022 18:42:21 -0700 Subject: [PATCH 065/332] Add fsurdat_modifier_arg_process method for argument processing so it can be tested, fix some lint and black issues --- .../modify_input_files/fsurdat_modifier.py | 17 +++++++--- python/ctsm/subset_data.py | 2 +- python/ctsm/test/test_sys_fsurdat_modifier.py | 33 ++++++++++--------- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 24bddb9e65..b9cb44de4c 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -29,6 +29,12 @@ def main(): Calls function that modifies an fsurdat (surface dataset) """ + args = fsurdat_modifier_arg_process() + fsurdat_modifier(args.cfg_path) + + +def fsurdat_modifier_arg_process(): + """Argument processing for fsurdat_modifier script""" # set up logging allowing user control setup_logging_pre_config() @@ -38,19 +44,22 @@ def main(): add_logging_args(parser) args = parser.parse_args() process_logging_args(args) - fsurdat_modifier(args.cfg_path) + # Error checking of arguments + if not os.path.exists(args.cfg_path): + abort("Config file does NOT exist: " + str(args.cfg_path)) + print(args.cfg_path) + + return args def fsurdat_modifier(cfg_path): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file config = ConfigParser() - if not os.path.exists(cfg_path): - abort( "Config file does NOT exist: "+str(cfg_path) ) config.read(cfg_path) section = "modify_input" if not config.has_section(section): - abort( "Config file does not have the expected section: "+section ) + abort("Config file does not have the expected section: " + section) # required: user must set these in the .cfg file fsurdat_in = get_config_value( diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 080c7f3003..8545870ee5 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -582,7 +582,7 @@ def main(): # --------------------------------- # # print help and exit when no option is chosen - if args.run_type != "point" and args.run_type != "region": + if args.run_type not in ("point", "region"): err_msg = textwrap.dedent( """\ \n ------------------------------------ diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 9fd349b2ae..4f8466c6c1 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -10,12 +10,14 @@ import unittest import tempfile import shutil +import sys import xarray as xr from ctsm.path_utils import path_to_ctsm_root from ctsm import unit_testing from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier +from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -58,10 +60,10 @@ def test_no_files_given_fail(self): """ Test that if no input or output files are given that it will gracefully fail """ - self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short_nofiles.cfg") - with self.assertRaisesRegex( - SystemExit, "must contain item 'fsurdat_in'" - ): + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" + ) + with self.assertRaisesRegex(SystemExit, "must contain item 'fsurdat_in'"): fsurdat_modifier(self._cfg_file_path) def test_short_config(self): @@ -70,29 +72,28 @@ def test_short_config(self): """ self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") fsurdat_modifier(self._cfg_file_path) - fsurdat_out = "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" - os.remove( fsurdat_out ) + fsurdat_out = ( + "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" + ) + os.remove(fsurdat_out) def test_cfg_file_DNE_fail(self): """ Test that if the config file does not exist that it gracefully fails """ - self._cfg_file_path = os.path.join(self._tempdir, "FILE_DOES_NOT_EXIST.cfg" ) - with self.assertRaisesRegex( - SystemExit, "Config file does NOT exist" - ): - fsurdat_modifier(self._cfg_file_path) + self._cfg_file_path = os.path.join(self._tempdir, "FILE_DOES_NOT_EXIST.cfg") + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + with self.assertRaisesRegex(SystemExit, "Config file does NOT exist"): + fsurdat_modifier_arg_process() def test_cfg_file_empty_fail(self): """ Test that if the config file is empty it gracefully fails """ - self._cfg_file_path = os.path.join(self._tempdir, "EMPTY_FILE.cfg" ) - fil = open( self._cfg_file_path, "w" ) + self._cfg_file_path = os.path.join(self._tempdir, "EMPTY_FILE.cfg") + fil = open(self._cfg_file_path, "w") fil.close() - with self.assertRaisesRegex( - SystemExit, "Config file does not have the expected section" - ): + with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): fsurdat_modifier(self._cfg_file_path) def test_minimalInfo(self): From 5fe2dbd85b4137b6e8f56adbcfcd5a637a7f4e97 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 16 Dec 2022 23:46:56 -0700 Subject: [PATCH 066/332] Change the name of the main section for fsurdat_modifier --- python/ctsm/modify_input_files/fsurdat_modifier.py | 2 +- python/ctsm/test/testinputs/modify_fsurdat_short.cfg | 2 +- python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg | 2 +- tools/modify_input_files/modify_fsurdat_template.cfg | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index b9cb44de4c..0b686249f7 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -57,7 +57,7 @@ def fsurdat_modifier(cfg_path): # read the .cfg (config) file config = ConfigParser() config.read(cfg_path) - section = "modify_input" + section = "modify_fsurdat_basic_options" if not config.has_section(section): abort("Config file does not have the expected section: " + section) diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg index e298b3575e..b2393b3251 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg @@ -1,4 +1,4 @@ -[modify_input] +[modify_fsurdat_basic_options] fsurdat_in = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc fsurdat_out = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg index e4ba31912a..1706613e54 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg @@ -1,4 +1,4 @@ -[modify_input] +[modify_fsurdat_basic_options] idealized = False include_nonveg = False diff --git a/tools/modify_input_files/modify_fsurdat_template.cfg b/tools/modify_input_files/modify_fsurdat_template.cfg index 0694174666..7919c5f98c 100644 --- a/tools/modify_input_files/modify_fsurdat_template.cfg +++ b/tools/modify_input_files/modify_fsurdat_template.cfg @@ -1,4 +1,4 @@ -[modify_input] +[modify_fsurdat_basic_options] # ------------------------------------------------------------------------ # .cfg file with inputs for fsurdat_modifier. From 41b537e8824559fb6123975f2412d8a104c96b01 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 16 Dec 2022 23:57:53 -0700 Subject: [PATCH 067/332] Add two failing tests one to set an input file that does not exist, and the other to set an existing output file --- python/ctsm/test/test_sys_fsurdat_modifier.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 4f8466c6c1..948ea10691 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -86,6 +86,22 @@ def test_cfg_file_DNE_fail(self): with self.assertRaisesRegex(SystemExit, "Config file does NOT exist"): fsurdat_modifier_arg_process() + def test_input_fsurdat_DNE_fail(self): + """ + Test that if the input fsurdat file does not exist that it gracefully fails + """ + sys.argv = ["fsurdat_modifier", self._cfg_file_path, "-i", "FILE_DOES_NOT_EXIST.nc"] + with self.assertRaisesRegex(SystemExit, "Input surface dataset file does NOT exist"): + fsurdat_modifier_arg_process() + + def test_output_fsurdat_EXISTS_fail(self): + """ + Test that if the output fsurdat file does exist that it gracefully fails + """ + sys.argv = ["fsurdat_modifier", self._cfg_file_path, "-o", self._cfg_file_path] + with self.assertRaisesRegex(SystemExit, "Output file already exists"): + fsurdat_modifier_arg_process() + def test_cfg_file_empty_fail(self): """ Test that if the config file is empty it gracefully fails From 31798f104f2bad893366943403e1c1509ab22f18 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 19 Dec 2022 16:53:59 -0700 Subject: [PATCH 068/332] Update test_driver.sh according to Erik's review --- test/tools/test_driver.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/tools/test_driver.sh b/test/tools/test_driver.sh index b4dac7f3a0..a16799ae87 100755 --- a/test/tools/test_driver.sh +++ b/test/tools/test_driver.sh @@ -54,8 +54,6 @@ module load nco module load ncl module load conda -$CESMDATAROOT/py_env_create -conda activate ctsm_pylib ##omp threads @@ -119,8 +117,6 @@ module load openmpi module load nco module load conda module load ncl -$CESMDATAROOT/py_env_create -conda activate ctsm_pylib ##omp threads @@ -220,8 +216,6 @@ module load compiler/intel module load tool/nco module load tool/netcdf module load lang/python -$CESMDATAROOT/py_env_create -conda activate ctsm_pylib export NETCDF_DIR=\$NETCDF_PATH export INC_NETCDF=\${NETCDF_PATH}/include @@ -303,8 +297,6 @@ module load compiler/intel module load tool/nco module load tool/netcdf module load lang/python -$CESMDATAROOT/py_env_create -conda activate ctsm_pylib export NETCDF_DIR=\$NETCDF_PATH export INC_NETCDF=\${NETCDF_PATH}/include @@ -382,7 +374,7 @@ fi # Setup conda environement \$CLM_ROOT/py_env_create -conda activate ctsm_py +conda activate ctsm_pylib ##output files clm_log=\${initdir}/td.\${JOBID}.log From 4057f9aac8ba6b1ad2dff90520cdc35140d007c8 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 19 Dec 2022 18:07:27 -0700 Subject: [PATCH 069/332] If "which conda" fails, execute module load/unload, eg on cheyenne --- cime_config/SystemTests/fsurdatmodifyctsm.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index f8255624b5..0569683553 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -100,10 +100,14 @@ def _get_conda_env(self): # Add specific commands needed on different machines to get conda available # Use semicolon here since it's OK to fail # - # Remove python and add conda to environment for cheyennne - conda_env = "module unload python; module load conda" - # End above machine specific with a semicolon - conda_env += "; " + # Execute the module unload/load when "which conda" fails + # eg on cheyenne + try: + subprocess.run( "which conda", shell=True, check=True) + conda_env = " " + except subprocess.CalledProcessError: + # Remove python and add conda to environment for cheyennne + conda_env = "module unload python; module load conda;" # Activate the python environment conda_env += " conda activate ctsm_pylib" From 098153d0ee2bdd7f971dd943cfb6e7441a491308 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 20 Dec 2022 16:47:13 -0700 Subject: [PATCH 070/332] Add ability to set fsurdat_in and fsurdat_out on command line, get tests working, fix pylint issues, run in black --- .../modify_input_files/fsurdat_modifier.py | 57 ++++++++++++++----- python/ctsm/subset_data.py | 4 +- python/ctsm/test/test_sys_fsurdat_modifier.py | 47 +++++++++++---- 3 files changed, 81 insertions(+), 27 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 0b686249f7..6bcebc92cf 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -30,7 +30,7 @@ def main(): """ args = fsurdat_modifier_arg_process() - fsurdat_modifier(args.cfg_path) + fsurdat_modifier(args) def fsurdat_modifier_arg_process(): @@ -41,33 +41,65 @@ def fsurdat_modifier_arg_process(): # read the command line argument to obtain the path to the .cfg file parser = argparse.ArgumentParser() parser.add_argument("cfg_path", help="/path/name.cfg of input file, eg ./modify.cfg") + parser.add_argument( + "-i", + "--fsurdat_in", + default="UNSET", + required=False, + type=str, + help="The input surface dataset to modify. ", + ) + parser.add_argument( + "-o", + "--fsurdat_out", + required=False, + default="UNSET", + type=str, + help="The output surface dataset with the modifications. ", + ) add_logging_args(parser) args = parser.parse_args() process_logging_args(args) # Error checking of arguments if not os.path.exists(args.cfg_path): abort("Config file does NOT exist: " + str(args.cfg_path)) - print(args.cfg_path) return args -def fsurdat_modifier(cfg_path): +def fsurdat_modifier(parser): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file + cfg_path = str(parser.cfg_path) config = ConfigParser() config.read(cfg_path) section = "modify_fsurdat_basic_options" if not config.has_section(section): abort("Config file does not have the expected section: " + section) - # required: user must set these in the .cfg file - fsurdat_in = get_config_value( - config=config, section=section, item="fsurdat_in", file_path=cfg_path - ) - fsurdat_out = get_config_value( - config=config, section=section, item="fsurdat_out", file_path=cfg_path - ) + if parser.fsurdat_in == "UNSET": + # required: user must set these in the .cfg file + fsurdat_in = get_config_value( + config=config, section=section, item="fsurdat_in", file_path=cfg_path + ) + else: + fsurdat_in = str(parser.fsurdat_in) + + # Error checking of input file + if not os.path.exists(fsurdat_in): + abort("Input fsurdat_in file does NOT exist: " + str(fsurdat_in)) + + if parser.fsurdat_out == "UNSET": + fsurdat_out = get_config_value( + config=config, section=section, item="fsurdat_out", file_path=cfg_path + ) + else: + fsurdat_out = str(parser.fsurdat_out) + + # If output file exists, abort before starting work + if os.path.exists(fsurdat_out): + errmsg = "Output file already exists: " + fsurdat_out + abort(errmsg) # required but fallback values available for variables omitted # entirely from the .cfg file @@ -142,11 +174,6 @@ def fsurdat_modifier(cfg_path): lon_dimname, ) - # If output file exists, abort before starting work - if os.path.exists(fsurdat_out): - errmsg = "Output file already exists: " + fsurdat_out - abort(errmsg) - # not required: user may set these in the .cfg file max_pft = int(max(modify_fsurdat.file.lsmpft)) dom_pft = get_config_value( diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 8545870ee5..8015fe1858 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -589,7 +589,7 @@ def main(): \n Must supply a positional argument: 'point' or 'region'. """ ) - raise parser.error(err_msg) + raise args.error(err_msg) if not any( [ @@ -606,7 +606,7 @@ def main(): \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n """ ) - raise parser.error(err_msg) + raise args.error(err_msg) # --------------------------------- # # process logging args (i.e. debug and verbose) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 948ea10691..a09d6a7655 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -63,15 +63,19 @@ def test_no_files_given_fail(self): self._cfg_file_path = os.path.join( self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" ) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() with self.assertRaisesRegex(SystemExit, "must contain item 'fsurdat_in'"): - fsurdat_modifier(self._cfg_file_path) + fsurdat_modifier(parser) def test_short_config(self): """ Test that a short config file works """ self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") - fsurdat_modifier(self._cfg_file_path) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) fsurdat_out = ( "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" ) @@ -90,17 +94,32 @@ def test_input_fsurdat_DNE_fail(self): """ Test that if the input fsurdat file does not exist that it gracefully fails """ + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" + ) sys.argv = ["fsurdat_modifier", self._cfg_file_path, "-i", "FILE_DOES_NOT_EXIST.nc"] - with self.assertRaisesRegex(SystemExit, "Input surface dataset file does NOT exist"): - fsurdat_modifier_arg_process() + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex(SystemExit, "Input fsurdat_in file does NOT exist"): + fsurdat_modifier(parser) def test_output_fsurdat_EXISTS_fail(self): """ Test that if the output fsurdat file does exist that it gracefully fails """ - sys.argv = ["fsurdat_modifier", self._cfg_file_path, "-o", self._cfg_file_path] + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" + ) + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + self._cfg_file_path, + "-o", + self._cfg_file_path, + ] + parser = fsurdat_modifier_arg_process() with self.assertRaisesRegex(SystemExit, "Output file already exists"): - fsurdat_modifier_arg_process() + fsurdat_modifier(parser) def test_cfg_file_empty_fail(self): """ @@ -109,8 +128,10 @@ def test_cfg_file_empty_fail(self): self._cfg_file_path = os.path.join(self._tempdir, "EMPTY_FILE.cfg") fil = open(self._cfg_file_path, "w") fil.close() + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): - fsurdat_modifier(self._cfg_file_path) + fsurdat_modifier(parser) def test_minimalInfo(self): """ @@ -121,7 +142,9 @@ def test_minimalInfo(self): self._create_config_file_minimal() # run the fsurdat_modifier tool - fsurdat_modifier(self._cfg_file_path) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) # the critical piece of this test is that the above command # doesn't generate errors; however, we also do some assertions below @@ -139,7 +162,9 @@ def test_crop(self): self._create_config_file_crop() # run the fsurdat_modifier tool - fsurdat_modifier(self._cfg_file_path) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) # the critical piece of this test is that the above command # doesn't generate errors; however, we also do some assertions below @@ -164,7 +189,9 @@ def test_allInfo(self): self._create_config_file_complete() # run the fsurdat_modifier tool - fsurdat_modifier(self._cfg_file_path) + sys.argv = ["fsurdat_modifier", self._cfg_file_path] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) # the critical piece of this test is that the above command # doesn't generate errors; however, we also do some assertions below From 32a88225ae1ac4b45a540844e4e7c3e18c57eb87 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 20 Dec 2022 16:57:03 -0700 Subject: [PATCH 071/332] Add tests when in and out fsurdat files are specified in both the cmd-line and config file, stest fails --- python/ctsm/test/test_sys_fsurdat_modifier.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index a09d6a7655..d5bd80b4b0 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -81,6 +81,46 @@ def test_short_config(self): ) os.remove(fsurdat_out) + def test_short_infile_both_cmdline_and_cfg(self): + """ + Test that a graceful fail happens when the infile + is given both in the command line and the config file + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + "specify_fsurdat_in_on_cmd_line.nc", + ] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex( + SystemExit, + "fsurdat_in is specified in both the command line and the config file, pick one", + ): + fsurdat_modifier(parser) + + def test_short_outfile_both_cmdline_and_cfg(self): + """ + Test that a graceful fail happens when the outfile is given + both in the command line and the config file + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + "specify_fsurdat_in_on_cmd_line.nc", + "-o", + "specify_fsurdat_out_on_cmd_line.nc", + ] + parser = fsurdat_modifier_arg_process() + with self.assertRaisesRegex( + SystemExit, + "fsurdat_out is specified in both the command line and the config file, pick one", + ): + fsurdat_modifier(parser) + def test_cfg_file_DNE_fail(self): """ Test that if the config file does not exist that it gracefully fails From 5368ae7f32137b7026b1980a24639d4319c80184 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 20 Dec 2022 17:04:04 -0700 Subject: [PATCH 072/332] Get tests of specifying both on command line and in config file working --- python/ctsm/modify_input_files/fsurdat_modifier.py | 4 ++++ python/ctsm/test/test_sys_fsurdat_modifier.py | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 6bcebc92cf..a26d711da1 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -83,6 +83,8 @@ def fsurdat_modifier(parser): config=config, section=section, item="fsurdat_in", file_path=cfg_path ) else: + if config.has_option(section=section, option="fsurdat_in"): + abort("fsurdat_in is specified in both the command line and the config file, pick one") fsurdat_in = str(parser.fsurdat_in) # Error checking of input file @@ -94,6 +96,8 @@ def fsurdat_modifier(parser): config=config, section=section, item="fsurdat_out", file_path=cfg_path ) else: + if config.has_option(section=section, option="fsurdat_out"): + abort("fsurdat_out is specified in both the command line and the config file, pick one") fsurdat_out = str(parser.fsurdat_out) # If output file exists, abort before starting work diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index d5bd80b4b0..458f325d27 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -109,8 +109,6 @@ def test_short_outfile_both_cmdline_and_cfg(self): sys.argv = [ "fsurdat_modifier", self._cfg_file_path, - "-i", - "specify_fsurdat_in_on_cmd_line.nc", "-o", "specify_fsurdat_out_on_cmd_line.nc", ] From 40478eccbdadeed8d9e87502407ef3a0cc826001 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Dec 2022 00:02:54 -0700 Subject: [PATCH 073/332] Addd booleans about handling new optional sections --- python/ctsm/modify_input_files/fsurdat_modifier.py | 14 ++++++++++++++ .../ctsm/test/testinputs/modify_fsurdat_short.cfg | 2 ++ .../testinputs/modify_fsurdat_short_nofiles.cfg | 2 ++ .../modify_input_files/modify_fsurdat_template.cfg | 7 +++++++ 4 files changed, 25 insertions(+) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index a26d711da1..8e39d58a89 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -114,6 +114,20 @@ def fsurdat_modifier(parser): file_path=cfg_path, convert_to_type=bool, ) + process_subgrid = get_config_value( + config=config, + section=section, + item="process_subgrid_section", + file_path=cfg_path, + convert_to_type=bool, + ) + process_var_list = get_config_value( + config=config, + section=section, + item="process_var_list_section", + file_path=cfg_path, + convert_to_type=bool, + ) include_nonveg = get_config_value( config=config, section=section, diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg index b2393b3251..74c6639899 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_short.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_short.cfg @@ -4,6 +4,8 @@ fsurdat_in = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr20 fsurdat_out = ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc idealized = False +process_subgrid_section = False +process_var_list_section = False include_nonveg = False landmask_file = UNSET diff --git a/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg index 1706613e54..3e5aada24b 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_short_nofiles.cfg @@ -1,6 +1,8 @@ [modify_fsurdat_basic_options] idealized = False +process_subgrid_section = False +process_var_list_section = False include_nonveg = False lnd_lat_1 = -90 diff --git a/tools/modify_input_files/modify_fsurdat_template.cfg b/tools/modify_input_files/modify_fsurdat_template.cfg index 7919c5f98c..aebe42a4f8 100644 --- a/tools/modify_input_files/modify_fsurdat_template.cfg +++ b/tools/modify_input_files/modify_fsurdat_template.cfg @@ -37,6 +37,13 @@ fsurdat_out = FILL_THIS_IN # ORGANIC = 0 idealized = False + +# Process the optional section that handles modifying subgrid fractions +process_subgrid_section = False + +# Process the optional section that handles modifying an arbitrary list of variables +process_var_list_section = False + # Boundaries of user-defined rectangle (float) # If lat_1 > lat_2, the code creates two rectangles, one in the north and # one in the south. From d05b3ff956c72fde40b965d15127dab479256218 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Dec 2022 12:45:50 -0700 Subject: [PATCH 074/332] Start adding handling of the optional sections, and add a simple test for it --- .../modify_input_files/fsurdat_modifier.py | 23 ++++++++++ python/ctsm/test/test_sys_fsurdat_modifier.py | 21 +++++++++ .../modify_fsurdat_opt_sections.cfg | 44 +++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 8e39d58a89..e0d833a076 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -67,6 +67,20 @@ def fsurdat_modifier_arg_process(): return args +def read_subgrid(config): + """Read the subgrid fraction section from the config file""" + section = "modify_fsurdat_subgrid_fractions" + if not config.has_section(section): + abort("Config file does not have the expected section: " + section) + + +def read_var_list(config): + """Read the variable list section from the config file""" + section = "modify_fsurdat_variable_list" + if not config.has_section(section): + abort("Config file does not have the expected section: " + section) + + def fsurdat_modifier(parser): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file @@ -308,6 +322,15 @@ def fsurdat_modifier(parser): ) logger.info("dom_pft complete") + # + # Handle optional sections + # + if process_subgrid: + read_subgrid(config) + + if process_var_list: + read_var_list(config) + # ---------------------------------------------- # Output the now modified CTSM surface data file # ---------------------------------------------- diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 458f325d27..5645ec152f 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -119,6 +119,27 @@ def test_short_outfile_both_cmdline_and_cfg(self): ): fsurdat_modifier(parser) + def test_opt_sections(self): + """ + Test that a simple file with the optional sections works + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_opt_sections.cfg") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + os.path.join( + self._testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc" + ), + "-o", + os.path.join( + self._tempdir, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_output_urban.nc", + ), + ] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) + def test_cfg_file_DNE_fail(self): """ Test that if the config file does not exist that it gracefully fails diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg new file mode 100644 index 0000000000..80389e1c38 --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -0,0 +1,44 @@ +[modify_fsurdat_basic_options] + +idealized = False +process_subgrid_section = True +process_var_list_section = True +include_nonveg = True + +landmask_file = UNSET + +lat_dimname = lsmlat +lon_dimname = lsmlon + +dom_pft = UNSET + +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET +soil_color = UNSET +std_elev = UNSET +max_sat_area = UNSET + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# Set to 100% urban for the test +PCT_NATVEG = 0.0 +PCT_CROP = 0.0 +PCT_LAKE = 0.0 +PCT_GLACIER = 0.0 +PCT_WETLAND = 0.0 +PCT_URBAN = 100.0 + + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# Set three variables to 200 for the test +CANYON_HWR = 200.00 +HT_ROOF = 200.0 +T_BUILDING_MIN = 200 From 3683a91e269db8062cb44413a59facacdb9d9b1a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Dec 2022 15:46:20 -0700 Subject: [PATCH 075/332] More work on processing subgrid fraction and variable list sections --- .../modify_input_files/fsurdat_modifier.py | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index e0d833a076..660146e627 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -67,19 +67,52 @@ def fsurdat_modifier_arg_process(): return args -def read_subgrid(config): +def read_subgrid(config, cfg_path): """Read the subgrid fraction section from the config file""" section = "modify_fsurdat_subgrid_fractions" if not config.has_section(section): abort("Config file does not have the expected section: " + section) + subgrid_settings = {} + var_list = config.options(section) + valid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] + for var in var_list: + if var_list.count(var) == 0: + abort( + "Variable " + + var + + " in " + + section + + " is not a valid variable name. Valid vars =" + + str(valid_list) + ) + value = get_config_value( + config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float + ) + if value < 0.0 or value > 100.0: + abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + value) + + subgrid_settings[var.upper()] = value + + return subgrid_settings + -def read_var_list(config): +def read_var_list(config, cfg_path): """Read the variable list section from the config file""" section = "modify_fsurdat_variable_list" if not config.has_section(section): abort("Config file does not have the expected section: " + section) + varlist_settings = {} + var_list = config.options(section) + for var in var_list: + value = get_config_value( + config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float + ) + varlist_settings[var] = value + + return varlist_settings + def fsurdat_modifier(parser): """Implementation of fsurdat_modifier command""" @@ -326,10 +359,10 @@ def fsurdat_modifier(parser): # Handle optional sections # if process_subgrid: - read_subgrid(config) + subgrid = read_subgrid(config, cfg_path) if process_var_list: - read_var_list(config) + varlist = read_var_list(config, cfg_path) # ---------------------------------------------- # Output the now modified CTSM surface data file From 8514bd3747b6d7a9f413e720e9bed025773fa5c5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Dec 2022 19:13:30 -0700 Subject: [PATCH 076/332] Add unit tester for fsurdat_modifier subroutines --- .../ctsm/test/test_unit_fsurdat_modifier.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 python/ctsm/test/test_unit_fsurdat_modifier.py diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py new file mode 100755 index 0000000000..a0c273fcf9 --- /dev/null +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +""" +Unit tests for fsurdat_modifier subroutines: read_subgrid, read_varlist +""" + +import unittest +import os +import sys +import shutil + +import tempfile +from configparser import ConfigParser + +from ctsm import unit_testing +from ctsm.path_utils import path_to_ctsm_root +from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process +from ctsm.modify_input_files.fsurdat_modifier import read_subgrid +from ctsm.modify_input_files.fsurdat_modifier import read_var_list + +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable +# pylint: disable=invalid-name + +# pylint: disable=protected-access + + +class TestFSurdatModifier(unittest.TestCase): + """Tests the read_subgrid and read_var_list methods""" + + def setUp(self): + """Setup for trying out the methods""" + testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") + self._cfg_file_path = os.path.join(testinputs_path, "modify_fsurdat_opt_sections.cfg") + self._testinputs_path = testinputs_path + self._fsurdat_in = os.path.join( + testinputs_path, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", + ) + self._tempdir = tempfile.mkdtemp() + self._fsurdat_in = os.path.join( + testinputs_path, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", + ) + self._fsurdat_out = os.path.join(self._tempdir, "fsurdat_out.nc") + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + self._fsurdat_in, + "-o", + self._fsurdat_out, + ] + parser = fsurdat_modifier_arg_process() + self.cfg_path = str(parser.cfg_path) + self.config = ConfigParser() + self.config.read(self.cfg_path) + + def tearDown(self): + """ + Remove temporary directory + """ + shutil.rmtree(self._tempdir, ignore_errors=True) + + def test_read_subgrid(self): + """test a simple read of subgrid""" + read_subgrid(self.config, self.cfg_path) + + def test_read_var_list(self): + """test a simple read of var_list""" + read_var_list(self.config, self.cfg_path) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From cd202a052ebd0655fd55e2a019eac85ea804fde1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 22 Dec 2022 01:45:28 -0700 Subject: [PATCH 077/332] Add a out of range test --- python/ctsm/modify_input_files/fsurdat_modifier.py | 2 +- python/ctsm/test/test_unit_fsurdat_modifier.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 660146e627..b860655035 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -90,7 +90,7 @@ def read_subgrid(config, cfg_path): config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float ) if value < 0.0 or value > 100.0: - abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + value) + abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + str(value)) subgrid_settings[var.upper()] = value diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index a0c273fcf9..e5d37bf848 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -70,6 +70,13 @@ def test_read_var_list(self): """test a simple read of var_list""" read_var_list(self.config, self.cfg_path) + def test_subgrid_outofrange(self): + """test a read of subgrid that's out of range""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "101." ) + with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): + read_subgrid(self.config, self.cfg_path) + if __name__ == "__main__": unit_testing.setup_for_tests() From 5efda44e0035070f4d9aced80861a00bb973e9b8 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 22 Dec 2022 01:52:54 -0700 Subject: [PATCH 078/332] Add a check for valid variables in read_subgrid and get it working --- python/ctsm/modify_input_files/fsurdat_modifier.py | 2 +- python/ctsm/test/test_unit_fsurdat_modifier.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index b860655035..bdfceb9966 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -77,7 +77,7 @@ def read_subgrid(config, cfg_path): var_list = config.options(section) valid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] for var in var_list: - if var_list.count(var) == 0: + if valid_list.count(var) == 0: abort( "Variable " + var diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index e5d37bf848..f87e718e6d 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -77,6 +77,12 @@ def test_subgrid_outofrange(self): with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): read_subgrid(self.config, self.cfg_path) + def test_subgrid_badvar(self): + """test a read of subgrid for a variable thats not in the list""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "badvariable", "100." ) + with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): + read_subgrid(self.config, self.cfg_path) if __name__ == "__main__": unit_testing.setup_for_tests() From 957042576e0755b930f1de5246e9c7172a23872e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 22 Dec 2022 01:59:06 -0700 Subject: [PATCH 079/332] Add fail tests when the optional sections have been removed --- python/ctsm/test/test_unit_fsurdat_modifier.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index f87e718e6d..7e5cadef92 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -84,6 +84,21 @@ def test_subgrid_badvar(self): with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): read_subgrid(self.config, self.cfg_path) + def test_subgrid_remove(self): + """test a read of subgrid when it's section has been removed""" + section = "modify_fsurdat_subgrid_fractions" + self.config.remove_section(section) + with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): + read_subgrid(self.config, self.cfg_path) + + def test_varlist_remove(self): + """test a read of varlist when it's section has been removed""" + section = "modify_fsurdat_variable_list" + self.config.remove_section(section) + with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): + read_var_list(self.config, self.cfg_path) + + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() From c9bb37335a38d08c43a497f71e064286954c258e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 22 Dec 2022 14:43:18 -0700 Subject: [PATCH 080/332] Add test and check that optional sections aren't there if they aren't turned on --- .../modify_input_files/fsurdat_modifier.py | 16 ++++++++++++++ .../ctsm/test/test_unit_fsurdat_modifier.py | 21 +++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index bdfceb9966..ab8cb47587 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -66,6 +66,18 @@ def fsurdat_modifier_arg_process(): return args +def check_no_subgrid_section(config): + """ Check that there isn't a subgrid section when it's processing is turned off""" + section = "modify_fsurdat_subgrid_fractions" + if config.has_section(section): + abort("Config file does have a section: " + section + " that should NOT be there since it is turned off") + +def check_no_varlist_section(config): + """ Check that there isn't a var list section when it's processing is turned off""" + section = "modify_fsurdat_variable_list" + if config.has_section(section): + abort("Config file does have a section: " + section + " that should NOT be there since it is turned off") + def read_subgrid(config, cfg_path): """Read the subgrid fraction section from the config file""" @@ -360,9 +372,13 @@ def fsurdat_modifier(parser): # if process_subgrid: subgrid = read_subgrid(config, cfg_path) + else: + check_no_subgrid_section(config) if process_var_list: varlist = read_var_list(config, cfg_path) + else: + check_no_varlist_section(config) # ---------------------------------------------- # Output the now modified CTSM surface data file diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 7e5cadef92..1070fa429b 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -17,6 +17,8 @@ from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process from ctsm.modify_input_files.fsurdat_modifier import read_subgrid from ctsm.modify_input_files.fsurdat_modifier import read_var_list +from ctsm.modify_input_files.fsurdat_modifier import check_no_subgrid_section +from ctsm.modify_input_files.fsurdat_modifier import check_no_varlist_section # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -91,12 +93,19 @@ def test_subgrid_remove(self): with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): read_subgrid(self.config, self.cfg_path) - def test_varlist_remove(self): - """test a read of varlist when it's section has been removed""" - section = "modify_fsurdat_variable_list" - self.config.remove_section(section) - with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): - read_var_list(self.config, self.cfg_path) + def test_subgrid_not_thereifoff(self): + """test that a graceful error happens if subgrid section is off, but it appears in the file""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "process_subgrid_section", "False" ) + with self.assertRaisesRegex(SystemExit, "Config file does have a section"): + check_no_subgrid_section(self.config) + + def test_varlist_not_thereifoff(self): + """test that a graceful error happens if varlist section is off, but it appears in the file""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "process_var_list_section", "False" ) + with self.assertRaisesRegex(SystemExit, "Config file does have a section"): + check_no_varlist_section(self.config) if __name__ == "__main__": From 891e1305c8b43b717e26fc3d1355cb929264f67b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 00:11:09 -0700 Subject: [PATCH 081/332] Get working in black and pylint, utest and stest working, add set_varlist and check_varlist methods and start adding tests for them, add Setup method for fsurdat modify unit tester and use it for an old and now a new test, simplify the modify test config file so that it will work --- .../modify_input_files/fsurdat_modifier.py | 30 +++++-- .../ctsm/modify_input_files/modify_fsurdat.py | 50 +++++++++++ .../ctsm/test/test_unit_fsurdat_modifier.py | 24 ++--- python/ctsm/test/test_unit_modify_fsurdat.py | 88 +++++++++++-------- .../modify_fsurdat_opt_sections.cfg | 9 +- 5 files changed, 142 insertions(+), 59 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index ab8cb47587..2d5f602166 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -66,17 +66,27 @@ def fsurdat_modifier_arg_process(): return args + def check_no_subgrid_section(config): - """ Check that there isn't a subgrid section when it's processing is turned off""" + """Check that there isn't a subgrid section when it's processing is turned off""" section = "modify_fsurdat_subgrid_fractions" if config.has_section(section): - abort("Config file does have a section: " + section + " that should NOT be there since it is turned off") + abort( + "Config file does have a section: " + + section + + " that should NOT be there since it is turned off" + ) + def check_no_varlist_section(config): - """ Check that there isn't a var list section when it's processing is turned off""" + """Check that there isn't a var list section when it's processing is turned off""" section = "modify_fsurdat_variable_list" if config.has_section(section): - abort("Config file does have a section: " + section + " that should NOT be there since it is turned off") + abort( + "Config file does have a section: " + + section + + " that should NOT be there since it is turned off" + ) def read_subgrid(config, cfg_path): @@ -102,7 +112,14 @@ def read_subgrid(config, cfg_path): config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float ) if value < 0.0 or value > 100.0: - abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + str(value)) + abort( + "Variable " + + var + + " in " + + section + + " is out of range of 0 to 100 = " + + str(value) + ) subgrid_settings[var.upper()] = value @@ -372,11 +389,14 @@ def fsurdat_modifier(parser): # if process_subgrid: subgrid = read_subgrid(config, cfg_path) + modify_fsurdat.set_varlist(subgrid) else: check_no_subgrid_section(config) if process_var_list: varlist = read_var_list(config, cfg_path) + modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) + modify_fsurdat.set_varlist(varlist) else: check_no_varlist_section(config) diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index ba1fd18fdb..4a38ed4d96 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -214,6 +214,56 @@ def set_dom_pft(self, dom_pft, lai, sai, hgt_top, hgt_bot): if val is not None: self.set_lai_sai_hgts(dom_pft=dom_pft, var=var, val=val) + def check_varlist(self, settings, allow_uppercase_vars=False): + """ + Check a list of variables from a dictionary of settings + """ + # exclude_list = ( "MONTHLY_LAI", "MONTHLY_SAI", + # "MONTHLY_HEIGHT_TOP", "MONTHLY_HEIGHT_BOT" ) + for var in settings.keys(): + # if exclude_list.count(var) > 0: + # abort("Error: Variable "+var+" is already handled outside "+ + # "of the optional variable list sections") + if not var in self.file: + if not allow_uppercase_vars: + errmsg = "Error: Variable " + var + " is NOT in the file" + abort(errmsg) + if not var.upper() in self.file: + errmsg = "Error: Variable " + var.upper() + " is NOT in the file" + abort(errmsg) + val = settings[var] + del settings[var] + var = var.upper() + settings[var] = val + + def set_varlist(self, settings): + """ + Set a list of variables from a dictionary of settings + """ + for var in settings.keys(): + # if exclude_list.count(var) > 0: + # abort("Error: Variable "+var+" is already handled outside "+ + # "of the optional variable list sections") + if var in self.file: + if len(self.file[var].dims) == 2: + self.setvar_lev0(var, settings[var]) + elif self.file[var].dims == 3: + self.setvar_lev1(var, settings[var], lev1_dim=self.file[var].dims[0]) + elif self.file[var].dims == 4: + self.setvar_lev2( + var, + settings[var], + lev1_dim=self.file[var].dims[1], + lev2_dim=self.file[var].dims[0], + ) + else: + abort( + "Error: Variable " + var + " is a higher dimension than currently allowed" + ) + else: + errmsg = "Error: Variable " + var + " is NOT in the file" + abort(errmsg) + def set_lai_sai_hgts(self, dom_pft, var, val): """ Description diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 1070fa429b..3508022afa 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -75,37 +75,39 @@ def test_read_var_list(self): def test_subgrid_outofrange(self): """test a read of subgrid that's out of range""" section = "modify_fsurdat_subgrid_fractions" - self.config.set(section, "pct_urban", "101." ) + self.config.set(section, "pct_urban", "101.") with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): - read_subgrid(self.config, self.cfg_path) + read_subgrid(self.config, self.cfg_path) def test_subgrid_badvar(self): """test a read of subgrid for a variable thats not in the list""" section = "modify_fsurdat_subgrid_fractions" - self.config.set(section, "badvariable", "100." ) + self.config.set(section, "badvariable", "100.") with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): - read_subgrid(self.config, self.cfg_path) + read_subgrid(self.config, self.cfg_path) def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" section = "modify_fsurdat_subgrid_fractions" self.config.remove_section(section) with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): - read_subgrid(self.config, self.cfg_path) + read_subgrid(self.config, self.cfg_path) def test_subgrid_not_thereifoff(self): - """test that a graceful error happens if subgrid section is off, but it appears in the file""" + """test that a graceful error happens if subgrid section is off, + but it appears in the file""" section = "modify_fsurdat_basic_options" - self.config.set(section, "process_subgrid_section", "False" ) + self.config.set(section, "process_subgrid_section", "False") with self.assertRaisesRegex(SystemExit, "Config file does have a section"): - check_no_subgrid_section(self.config) + check_no_subgrid_section(self.config) def test_varlist_not_thereifoff(self): - """test that a graceful error happens if varlist section is off, but it appears in the file""" + """test that a graceful error happens if varlist section is off, + but it appears in the file""" section = "modify_fsurdat_basic_options" - self.config.set(section, "process_var_list_section", "False" ) + self.config.set(section, "process_var_list_section", "False") with self.assertRaisesRegex(SystemExit, "Config file does have a section"): - check_no_varlist_section(self.config) + check_no_varlist_section(self.config) if __name__ == "__main__": diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index e53175b9b7..df84bd9c4c 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -19,87 +19,96 @@ # pylint: disable=protected-access +## Too many instant variables as part of the class (too many self. in the SetUp) +# pylint: disable=too-many-instance-attributes + class TestModifyFsurdat(unittest.TestCase): """Tests the setvar_lev functions and the - _get_rectangle function + _get_rectangle function and + set_varlist method """ - def test_setvarLev(self): - """ - Tests that setvar_lev0, setvar_lev1, and setvar_lev2 update values of - variables within a rectangle defined by user-specified - lon_1, lon_2, lat_1, lat_2 - """ + def setUp(self): # get longxy, latixy that would normally come from an fsurdat file # self._get_longxy_latixy will convert -180 to 180 to 0-360 longitudes # get cols, rows also - min_lon = 2 # expects min_lon < max_lon - min_lat = 3 # expects min_lat < max_lat - longxy, latixy, cols, rows = self._get_longxy_latixy( - _min_lon=min_lon, _max_lon=10, _min_lat=min_lat, _max_lat=12 + self.min_lon = 2 # expects min_lon < max_lon + self.min_lat = 3 # expects min_lat < max_lat + longxy, latixy, self.cols, self.rows = self._get_longxy_latixy( + _min_lon=self.min_lon, _max_lon=10, _min_lat=self.min_lat, _max_lat=12 ) # get not_rectangle from user-defined lon_1, lon_2, lat_1, lat_2 - lon_1 = 3 - lon_2 = 5 # lon_1 < lon_2 - lat_1 = 5 - lat_2 = 7 # lat_1 < lat_2 + self.lon_1 = 3 + self.lon_2 = 5 # lon_1 < lon_2 + self.lat_1 = 5 + self.lat_2 = 7 # lat_1 < lat_2 # create xarray dataset containing lev0, lev1, and lev2 variables; # the fsurdat_modify tool reads variables like this from fsurdat file - var_1d = np.arange(cols) - var_lev2 = var_1d * np.ones((rows, cols, rows, cols)) - var_lev1 = var_1d * np.ones((cols, rows, cols)) + var_1d = np.arange(self.cols) + var_lev0 = var_1d * np.ones((self.rows, self.cols)) + var_lev2 = var_1d * np.ones((self.rows, self.cols, self.rows, self.cols)) + var_lev1 = var_1d * np.ones((self.cols, self.rows, self.cols)) my_data = xr.Dataset( data_vars=dict( LONGXY=(["x", "y"], longxy), # use LONGXY as var_lev0 LATIXY=(["x", "y"], latixy), # __init__ expects LONGXY, LATIXY + var_lev0=(["x", "y"], var_lev0), var_lev1=(["w", "x", "y"], var_lev1), var_lev2=(["v", "w", "x", "y"], var_lev2), ) ) # create ModifyFsurdat object - modify_fsurdat = ModifyFsurdat( + self.modify_fsurdat = ModifyFsurdat( my_data=my_data, - lon_1=lon_1, - lon_2=lon_2, - lat_1=lat_1, - lat_2=lat_2, + lon_1=self.lon_1, + lon_2=self.lon_2, + lat_1=self.lat_1, + lat_2=self.lat_2, landmask_file=None, lat_dimname=None, lon_dimname=None, ) + def test_setvarLev(self): + """ + Tests that setvar_lev0, setvar_lev1, and setvar_lev2 update values of + variables within a rectangle defined by user-specified + lon_1, lon_2, lat_1, lat_2 + """ + # initialize and then modify the comparison matrices - comp_lev0 = modify_fsurdat.file.LONGXY - comp_lev1 = modify_fsurdat.file.var_lev1 - comp_lev2 = modify_fsurdat.file.var_lev2 + comp_lev0 = self.modify_fsurdat.file.LONGXY + comp_lev1 = self.modify_fsurdat.file.var_lev1 + comp_lev2 = self.modify_fsurdat.file.var_lev2 val_for_rectangle = 1.5 comp_lev0[ - lat_1 - min_lat : lat_2 - min_lat + 1, lon_1 - min_lon : lon_2 - min_lon + 1 + self.lat_1 - self.min_lat : self.lat_2 - self.min_lat + 1, + self.lon_1 - self.min_lon : self.lon_2 - self.min_lon + 1, ] = val_for_rectangle comp_lev1[ ..., - lat_1 - min_lat : lat_2 - min_lat + 1, - lon_1 - min_lon : lon_2 - min_lon + 1, + self.lat_1 - self.min_lat : self.lat_2 - self.min_lat + 1, + self.lon_1 - self.min_lon : self.lon_2 - self.min_lon + 1, ] = val_for_rectangle comp_lev2[ ..., - lat_1 - min_lat : lat_2 - min_lat + 1, - lon_1 - min_lon : lon_2 - min_lon + 1, + self.lat_1 - self.min_lat : self.lat_2 - self.min_lat + 1, + self.lon_1 - self.min_lon : self.lon_2 - self.min_lon + 1, ] = val_for_rectangle # test setvar - modify_fsurdat.setvar_lev0("LONGXY", val_for_rectangle) - np.testing.assert_array_equal(modify_fsurdat.file.LONGXY, comp_lev0) + self.modify_fsurdat.setvar_lev0("LONGXY", val_for_rectangle) + np.testing.assert_array_equal(self.modify_fsurdat.file.LONGXY, comp_lev0) - modify_fsurdat.setvar_lev1("var_lev1", val_for_rectangle, cols - 1) - np.testing.assert_array_equal(modify_fsurdat.file.var_lev1, comp_lev1) + self.modify_fsurdat.setvar_lev1("var_lev1", val_for_rectangle, self.cols - 1) + np.testing.assert_array_equal(self.modify_fsurdat.file.var_lev1, comp_lev1) - modify_fsurdat.setvar_lev2("var_lev2", val_for_rectangle, cols - 1, rows - 1) - np.testing.assert_array_equal(modify_fsurdat.file.var_lev2, comp_lev2) + self.modify_fsurdat.setvar_lev2("var_lev2", val_for_rectangle, self.cols - 1, self.rows - 1) + np.testing.assert_array_equal(self.modify_fsurdat.file.var_lev2, comp_lev2) def test_getNotRectangle_lon1leLon2Lat1leLat2(self): """ @@ -359,6 +368,11 @@ def test_getNotRectangle_latsOutOfBounds(self): latixy=latixy, ) + def test_set_varlist(self): + """Test the set_varlist method""" + settings = {"var_lev0": 100.0} + self.modify_fsurdat.set_varlist(settings) + def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ Return longxy, latixy, cols, rows diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index 80389e1c38..fb163e9e72 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -30,15 +30,12 @@ lnd_lon_2 = 360 # Set to 100% urban for the test PCT_NATVEG = 0.0 PCT_CROP = 0.0 -PCT_LAKE = 0.0 +PCT_LAKE = 100.0 PCT_GLACIER = 0.0 PCT_WETLAND = 0.0 -PCT_URBAN = 100.0 # Section with a list of variables to prcoess [modify_fsurdat_variable_list] -# Set three variables to 200 for the test -CANYON_HWR = 200.00 -HT_ROOF = 200.0 -T_BUILDING_MIN = 200 +# Set lake-depth to 200 for the test +LAKEDEPTH = 200.00 From a9873c7c25d88263b4c84c6985419100440bd841 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 18:21:33 -0700 Subject: [PATCH 082/332] Add two tests that check that PCT_* variables for subgrid add to a hundred, and one that checks that a variable handled in the idealized list can't be set in the variable_list section, these tests fail --- .../ctsm/test/test_unit_fsurdat_modifier.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 3508022afa..7047977761 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -79,6 +79,20 @@ def test_subgrid_outofrange(self): with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): read_subgrid(self.config, self.cfg_path) + def test_subgrid_notsumtohundred(self): + """test a read of subgrid that's doesn't sum to a hundred""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "0.") + self.config.set(section, "pct_lake", "0.") + self.config.set(section, "pct_wetland", "0.") + self.config.set(section, "pct_glacier", "0.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + with self.assertRaisesRegex( + SystemExit, "PCT fractions in subgrid section do NOT sum to a hundred as they should" + ): + read_subgrid(self.config, self.cfg_path) + def test_subgrid_badvar(self): """test a read of subgrid for a variable thats not in the list""" section = "modify_fsurdat_subgrid_fractions" @@ -86,6 +100,17 @@ def test_subgrid_badvar(self): with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): read_subgrid(self.config, self.cfg_path) + def test_varlist_varinidealized(self): + """test a read of varlist for a variable thats in the idealized list""" + section = "modify_fsurdat_variable_list" + self.config.set(section, "PCT_SAND", "100.") + with self.assertRaisesRegex( + SystemExit, + "is variable handled in the idealized section." + + " This should NOT be handled in the variiable list section. Idealized vars =", + ): + read_var_list(self.config, self.cfg_path) + def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" section = "modify_fsurdat_subgrid_fractions" From 6e5731f35f0138db0b41482b06051c937263cf02 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 18:51:01 -0700 Subject: [PATCH 083/332] Get the two failing tests to work --- .../modify_input_files/fsurdat_modifier.py | 18 ++++++++++++++++++ python/ctsm/test/test_unit_fsurdat_modifier.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 2d5f602166..256a6e1757 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -98,6 +98,7 @@ def read_subgrid(config, cfg_path): subgrid_settings = {} var_list = config.options(section) valid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] + varsum = 0 for var in var_list: if valid_list.count(var) == 0: abort( @@ -122,6 +123,12 @@ def read_subgrid(config, cfg_path): ) subgrid_settings[var.upper()] = value + varsum += value + if varsum != 100.0: + abort( + "PCT fractions in subgrid section do NOT sum to a hundred as they should. Sum = " + + str(varsum) + ) return subgrid_settings @@ -134,7 +141,18 @@ def read_var_list(config, cfg_path): varlist_settings = {} var_list = config.options(section) + ideal_list = ["soil_color", "pct_sand", "pct_clay", "organic", "pct_cft", "pct_nat_pft"] + # subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", + # "pct_wetland", "pct_urban"] for var in var_list: + if ideal_list.count(var) != 0: + abort( + var + + " is a special variable handled in the idealized section." + + " This should NOT be handled in the variiable list section." + + " Special idealized vars =" + + str(ideal_list) + ) value = get_config_value( config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float ) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 7047977761..8399540b7b 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -106,8 +106,8 @@ def test_varlist_varinidealized(self): self.config.set(section, "PCT_SAND", "100.") with self.assertRaisesRegex( SystemExit, - "is variable handled in the idealized section." - + " This should NOT be handled in the variiable list section. Idealized vars =", + "is a special variable handled in the idealized section." + + " This should NOT be handled in the variiable list section. Special idealized vars =", ): read_var_list(self.config, self.cfg_path) From c88e071b9d5ad41ccdce34deb778afa9e484e19a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 18:54:11 -0700 Subject: [PATCH 084/332] Add a test to make sure a variable read in the variable list section is not from the subgrid section, this fails as expected --- python/ctsm/test/test_unit_fsurdat_modifier.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 8399540b7b..531f2821ed 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -111,6 +111,17 @@ def test_varlist_varinidealized(self): ): read_var_list(self.config, self.cfg_path) + def test_varlist_varinsubgrid(self): + """test a read of varlist for a variable thats in the subgrid list""" + section = "modify_fsurdat_variable_list" + self.config.set(section, "PCT_GLACIER", "100.") + with self.assertRaisesRegex( + SystemExit, + "is a variable handled in the subgrid section." + + " This should NOT be handled in the variiable list section. Subgrid vars =", + ): + read_var_list(self.config, self.cfg_path) + def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" section = "modify_fsurdat_subgrid_fractions" From e06dd5294eeee202ca4330367648ff979b26c7a9 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 18:57:09 -0700 Subject: [PATCH 085/332] Get the test to pass now --- python/ctsm/modify_input_files/fsurdat_modifier.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 256a6e1757..4c210c08c2 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -142,8 +142,7 @@ def read_var_list(config, cfg_path): varlist_settings = {} var_list = config.options(section) ideal_list = ["soil_color", "pct_sand", "pct_clay", "organic", "pct_cft", "pct_nat_pft"] - # subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", - # "pct_wetland", "pct_urban"] + subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] for var in var_list: if ideal_list.count(var) != 0: abort( @@ -153,6 +152,14 @@ def read_var_list(config, cfg_path): + " Special idealized vars =" + str(ideal_list) ) + if subgrid_list.count(var) != 0: + abort( + var + + " is a variable handled in the subgrid section." + + " This should NOT be handled in the variiable list section." + + " Subgrid vars =" + + str(subgrid_list) + ) value = get_config_value( config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float ) From 16a771b3ed882b256a092a2cc3405fd6037daa0c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 19:07:12 -0700 Subject: [PATCH 086/332] Add failing test for trying to set one of the monthly variables in the variable list section --- python/ctsm/test/test_unit_fsurdat_modifier.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 531f2821ed..d40dc4a46c 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -122,6 +122,19 @@ def test_varlist_varinsubgrid(self): ): read_var_list(self.config, self.cfg_path) + def test_varlist_monthlyvar(self): + """test a read of varlist for a variable thats one of the monthly + variables handled in the dom_pft section""" + section = "modify_fsurdat_variable_list" + self.config.set(section, "MONTHLY_LAI", "100.") + with self.assertRaisesRegex( + SystemExit, + "is a variable handled as part of the dom_pft handling." + + " This should NOT be handled in the variiable list section." + + " Monthly vars handled this way =", + ): + read_var_list(self.config, self.cfg_path) + def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" section = "modify_fsurdat_subgrid_fractions" From 63e19beafe7b4f53d747646fcf810e590f4e6991 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 19:10:37 -0700 Subject: [PATCH 087/332] Get latest test to work --- python/ctsm/modify_input_files/fsurdat_modifier.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 4c210c08c2..29ba1cfedb 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -143,6 +143,7 @@ def read_var_list(config, cfg_path): var_list = config.options(section) ideal_list = ["soil_color", "pct_sand", "pct_clay", "organic", "pct_cft", "pct_nat_pft"] subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] + monthly_list = ["monthly_lai", "monthly_sai", "monthly_height_top", "monthly_height_bot"] for var in var_list: if ideal_list.count(var) != 0: abort( @@ -160,6 +161,14 @@ def read_var_list(config, cfg_path): + " Subgrid vars =" + str(subgrid_list) ) + if monthly_list.count(var) != 0: + abort( + var + + " is a variable handled as part of the dom_pft handling." + + " This should NOT be handled in the variiable list section." + + " Monthly vars handled this way =" + + str(monthly_list) + ) value = get_config_value( config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float ) From 843662468c28bbaea355ecd7ee3d1259dae4c264 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 19:20:00 -0700 Subject: [PATCH 088/332] Add std_elev and fmax to the list of variables to not set in the variable list section as they are handled in the main section --- python/ctsm/modify_input_files/fsurdat_modifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 29ba1cfedb..50d9fe611b 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -141,7 +141,7 @@ def read_var_list(config, cfg_path): varlist_settings = {} var_list = config.options(section) - ideal_list = ["soil_color", "pct_sand", "pct_clay", "organic", "pct_cft", "pct_nat_pft"] + ideal_list = ["soil_color", "pct_sand", "pct_clay", "organic", "pct_cft", "pct_nat_pft", "fmax", "std_elev"] subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] monthly_list = ["monthly_lai", "monthly_sai", "monthly_height_top", "monthly_height_bot"] for var in var_list: From 8390289f220930d5f5e7ee94303d5a080bbfc008 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 22:55:54 -0700 Subject: [PATCH 089/332] Black and lint update, add tests for lev1 and lev2 variables for set_varlist, test for badvar, and too high dimension, and get working, with needed for loops --- .../modify_input_files/fsurdat_modifier.py | 11 ++++++- .../ctsm/modify_input_files/modify_fsurdat.py | 32 ++++++++----------- python/ctsm/test/test_unit_modify_fsurdat.py | 25 ++++++++++++--- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 50d9fe611b..62a3a40dd5 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -141,7 +141,16 @@ def read_var_list(config, cfg_path): varlist_settings = {} var_list = config.options(section) - ideal_list = ["soil_color", "pct_sand", "pct_clay", "organic", "pct_cft", "pct_nat_pft", "fmax", "std_elev"] + ideal_list = [ + "soil_color", + "pct_sand", + "pct_clay", + "organic", + "pct_cft", + "pct_nat_pft", + "fmax", + "std_elev", + ] subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] monthly_list = ["monthly_lai", "monthly_sai", "monthly_height_top", "monthly_height_bot"] for var in var_list: diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index 4a38ed4d96..cbfb528487 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -218,12 +218,7 @@ def check_varlist(self, settings, allow_uppercase_vars=False): """ Check a list of variables from a dictionary of settings """ - # exclude_list = ( "MONTHLY_LAI", "MONTHLY_SAI", - # "MONTHLY_HEIGHT_TOP", "MONTHLY_HEIGHT_BOT" ) for var in settings.keys(): - # if exclude_list.count(var) > 0: - # abort("Error: Variable "+var+" is already handled outside "+ - # "of the optional variable list sections") if not var in self.file: if not allow_uppercase_vars: errmsg = "Error: Variable " + var + " is NOT in the file" @@ -241,24 +236,25 @@ def set_varlist(self, settings): Set a list of variables from a dictionary of settings """ for var in settings.keys(): - # if exclude_list.count(var) > 0: - # abort("Error: Variable "+var+" is already handled outside "+ - # "of the optional variable list sections") if var in self.file: if len(self.file[var].dims) == 2: self.setvar_lev0(var, settings[var]) - elif self.file[var].dims == 3: - self.setvar_lev1(var, settings[var], lev1_dim=self.file[var].dims[0]) - elif self.file[var].dims == 4: - self.setvar_lev2( - var, - settings[var], - lev1_dim=self.file[var].dims[1], - lev2_dim=self.file[var].dims[0], - ) + elif len(self.file[var].dims) == 3: + dim1 = int(self.file.sizes[self.file[var].dims[0]]) + for lev1 in range(dim1 - 1): + self.setvar_lev1(var, settings[var], lev1_dim=lev1) + elif len(self.file[var].dims) == 4: + dim1 = int(self.file.sizes[self.file[var].dims[0]]) + dim2 = int(self.file.sizes[self.file[var].dims[1]]) + for lev1 in range(dim1 - 1): + for lev2 in range(dim2 - 1): + self.setvar_lev2(var, settings[var], lev1_dim=lev1, lev2_dim=lev2) else: abort( - "Error: Variable " + var + " is a higher dimension than currently allowed" + "Error: Variable " + + var + + " is a higher dimension than currently allowed = " + + str(self.file[var].dims) ) else: errmsg = "Error: Variable " + var + " is NOT in the file" diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index df84bd9c4c..97275b90af 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -25,8 +25,7 @@ class TestModifyFsurdat(unittest.TestCase): """Tests the setvar_lev functions and the - _get_rectangle function and - set_varlist method + _get_rectangle, check_varlist, and set_varlist methods """ def setUp(self): @@ -49,8 +48,9 @@ def setUp(self): # the fsurdat_modify tool reads variables like this from fsurdat file var_1d = np.arange(self.cols) var_lev0 = var_1d * np.ones((self.rows, self.cols)) - var_lev2 = var_1d * np.ones((self.rows, self.cols, self.rows, self.cols)) var_lev1 = var_1d * np.ones((self.cols, self.rows, self.cols)) + var_lev2 = var_1d * np.ones((self.rows, self.cols, self.rows, self.cols)) + var_lev3 = var_1d * np.ones((self.cols, self.rows, self.cols, self.rows, self.cols)) my_data = xr.Dataset( data_vars=dict( LONGXY=(["x", "y"], longxy), # use LONGXY as var_lev0 @@ -58,6 +58,7 @@ def setUp(self): var_lev0=(["x", "y"], var_lev0), var_lev1=(["w", "x", "y"], var_lev1), var_lev2=(["v", "w", "x", "y"], var_lev2), + var_lev3=(["z", "v", "w", "x", "y"], var_lev3), ) ) @@ -369,10 +370,24 @@ def test_getNotRectangle_latsOutOfBounds(self): ) def test_set_varlist(self): - """Test the set_varlist method""" - settings = {"var_lev0": 100.0} + """Test the set_varlist method for all the dimensions that works""" + settings = {"var_lev0": 100.0, "var_lev1": 100.0, "var_lev2": 100.0} self.modify_fsurdat.set_varlist(settings) + def test_set_varlist_badvar(self): + """Test the set_varlist method for a variable not on the file""" + settings = {"badvar": 100.0} + with self.assertRaisesRegex(SystemExit, "Variable badvar is NOT in the file"): + self.modify_fsurdat.set_varlist(settings) + + def test_set_varlist_toohighdim(self): + """Test the set_varlist method for a variable of too high a dimension""" + settings = {"var_lev3": 100.0} + with self.assertRaisesRegex( + SystemExit, "Variable var_lev3 is a higher dimension than currently allowed" + ): + self.modify_fsurdat.set_varlist(settings) + def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ Return longxy, latixy, cols, rows From df2d2d23013232f52d892a26fd42336d991e1c9b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 23:40:08 -0700 Subject: [PATCH 090/332] Add uppercase variables for testing, and add several tests for check_varlist including the allow_uppercase option --- python/ctsm/test/test_unit_modify_fsurdat.py | 54 ++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index 97275b90af..84be4215f2 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -59,6 +59,10 @@ def setUp(self): var_lev1=(["w", "x", "y"], var_lev1), var_lev2=(["v", "w", "x", "y"], var_lev2), var_lev3=(["z", "v", "w", "x", "y"], var_lev3), + VAR_LEV0_UPPERCASE=(["x", "y"], var_lev0), + VAR_LEV1_UPPERCASE=(["w", "x", "y"], var_lev1), + VAR_LEV2_UPPERCASE=(["v", "w", "x", "y"], var_lev2), + VAR_LEV3_UPPERCASE=(["z", "v", "w", "x", "y"], var_lev3), ) ) @@ -374,11 +378,46 @@ def test_set_varlist(self): settings = {"var_lev0": 100.0, "var_lev1": 100.0, "var_lev2": 100.0} self.modify_fsurdat.set_varlist(settings) - def test_set_varlist_badvar(self): - """Test the set_varlist method for a variable not on the file""" + def test_check_varlist(self): + """Test the check_varlist method for all the dimensions that works""" + settings = {"var_lev0": 100.0, "var_lev1": 100.0, "var_lev2": 100.0} + settings_orig = settings + self.modify_fsurdat.check_varlist(settings) + self.assertEqual( + settings_orig, settings, "list of variable settings not identical as expected" + ) + + def test_check_varlist_uppercase(self): + """Test the check_varlist method for all the dimensions that + works with allowuppercase option""" + expected = { + "VAR_LEV0_UPPERCASE": 100.0, + "VAR_LEV1_UPPERCASE": 200.0, + "VAR_LEV2_UPPERCASE": 300.0, + } + settings = { + "var_lev0_uppercase": 100.0, + "var_lev1_uppercase": 200.0, + "var_lev2_uppercase": 300.0, + } + self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + self.assertEqual( + expected, + settings, + "list of variable settings not converted to uppercase as expected", + ) + + def test_check_varlist_badvar(self): + """Test the check_varlist method for a variable not on the file""" settings = {"badvar": 100.0} with self.assertRaisesRegex(SystemExit, "Variable badvar is NOT in the file"): - self.modify_fsurdat.set_varlist(settings) + self.modify_fsurdat.check_varlist(settings) + + def test_check_varlist_badvar_uppercase(self): + """Test the check_varlist method for a variable not on the file with allow uppercase""" + settings = {"badvar": 100.0} + with self.assertRaisesRegex(SystemExit, "Variable BADVAR is NOT in the file"): + self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) def test_set_varlist_toohighdim(self): """Test the set_varlist method for a variable of too high a dimension""" @@ -388,6 +427,15 @@ def test_set_varlist_toohighdim(self): ): self.modify_fsurdat.set_varlist(settings) + def test_set_varlist_toohighdim_uppercase(self): + """Test the set_varlist method for a variable of too high a dimension in uppercase""" + settings = {"var_lev3_uppercase": 100.0} + with self.assertRaisesRegex( + SystemExit, "Variable VAR_LEV3_UPPERCASE is a higher dimension than currently allowed" + ): + self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + self.modify_fsurdat.set_varlist(settings) + def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ Return longxy, latixy, cols, rows From 9b2ed223d93eb597d2cd5625588d34e10fa6f546 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 29 Dec 2022 23:49:06 -0700 Subject: [PATCH 091/332] Add a test for 100 glacier for subgrid and a sum of special landunits to 100 both for read_subgrid --- .../ctsm/test/test_unit_fsurdat_modifier.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index d40dc4a46c..a49a6dea68 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -68,6 +68,29 @@ def test_read_subgrid(self): """test a simple read of subgrid""" read_subgrid(self.config, self.cfg_path) + def test_read_subgrid_allglacier(self): + """test a read of subgrid that's for all glacier""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "0.") + self.config.set(section, "pct_lake", "0.") + self.config.set(section, "pct_wetland", "0.") + self.config.set(section, "pct_glacier", "100.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + read_subgrid(self.config, self.cfg_path) + + def test_read_subgrid_allspecial(self): + """test a read of subgrid that's all special landunits""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "0.") + self.config.set(section, "pct_lake", "25.") + self.config.set(section, "pct_wetland", "35.") + self.config.set(section, "pct_glacier", "40.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + read_subgrid(self.config, self.cfg_path) + + def test_read_var_list(self): """test a simple read of var_list""" read_var_list(self.config, self.cfg_path) From b23e17eeb0c0e153d49ef928afdeba9c76155293 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 31 Dec 2022 00:28:15 -0700 Subject: [PATCH 092/332] Add back in some tests for some multidimensional urban arrarys to the standard opt testing, also rework check_varlist a bit so that settings isn't changed, but it returns the new list --- python/ctsm/modify_input_files/fsurdat_modifier.py | 4 ++-- python/ctsm/modify_input_files/modify_fsurdat.py | 14 +++++++++----- python/ctsm/test/test_unit_fsurdat_modifier.py | 1 - python/ctsm/test/test_unit_modify_fsurdat.py | 13 ++++++------- .../testinputs/modify_fsurdat_opt_sections.cfg | 7 ++++++- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 62a3a40dd5..9554117abd 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -438,8 +438,8 @@ def fsurdat_modifier(parser): if process_var_list: varlist = read_var_list(config, cfg_path) - modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) - modify_fsurdat.set_varlist(varlist) + update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) + modify_fsurdat.set_varlist(update_list) else: check_no_varlist_section(config) diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index cbfb528487..85d5dcdd30 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -218,7 +218,10 @@ def check_varlist(self, settings, allow_uppercase_vars=False): """ Check a list of variables from a dictionary of settings """ - for var in settings.keys(): + settings_return = {} + varlist = settings.keys() + for var in varlist: + val = settings[var] if not var in self.file: if not allow_uppercase_vars: errmsg = "Error: Variable " + var + " is NOT in the file" @@ -226,10 +229,11 @@ def check_varlist(self, settings, allow_uppercase_vars=False): if not var.upper() in self.file: errmsg = "Error: Variable " + var.upper() + " is NOT in the file" abort(errmsg) - val = settings[var] - del settings[var] - var = var.upper() - settings[var] = val + varu = var.upper() + settings_return[varu] = val + else: + settings_return[var] = val + return settings_return def set_varlist(self, settings): """ diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index a49a6dea68..e8d1cf7c8a 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -90,7 +90,6 @@ def test_read_subgrid_allspecial(self): self.config.set(section, "pct_crop", "0.") read_subgrid(self.config, self.cfg_path) - def test_read_var_list(self): """test a simple read of var_list""" read_var_list(self.config, self.cfg_path) diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index 84be4215f2..580845bfea 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -381,10 +381,9 @@ def test_set_varlist(self): def test_check_varlist(self): """Test the check_varlist method for all the dimensions that works""" settings = {"var_lev0": 100.0, "var_lev1": 100.0, "var_lev2": 100.0} - settings_orig = settings - self.modify_fsurdat.check_varlist(settings) + settings_new = self.modify_fsurdat.check_varlist(settings) self.assertEqual( - settings_orig, settings, "list of variable settings not identical as expected" + settings_new, settings, "list of variable settings not identical as expected" ) def test_check_varlist_uppercase(self): @@ -400,10 +399,10 @@ def test_check_varlist_uppercase(self): "var_lev1_uppercase": 200.0, "var_lev2_uppercase": 300.0, } - self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + settings_new = self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) self.assertEqual( expected, - settings, + settings_new, "list of variable settings not converted to uppercase as expected", ) @@ -433,8 +432,8 @@ def test_set_varlist_toohighdim_uppercase(self): with self.assertRaisesRegex( SystemExit, "Variable VAR_LEV3_UPPERCASE is a higher dimension than currently allowed" ): - self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) - self.modify_fsurdat.set_varlist(settings) + settings_new = self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) + self.modify_fsurdat.set_varlist(settings_new) def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index fb163e9e72..704587c4d3 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -27,7 +27,7 @@ lnd_lon_2 = 360 # Section for subgrid_fractions [modify_fsurdat_subgrid_fractions] -# Set to 100% urban for the test +# Set to 100% lake for the test PCT_NATVEG = 0.0 PCT_CROP = 0.0 PCT_LAKE = 100.0 @@ -39,3 +39,8 @@ PCT_WETLAND = 0.0 [modify_fsurdat_variable_list] # Set lake-depth to 200 for the test LAKEDEPTH = 200.00 + +# Set soem urban multidimensional variables to 200 for the test +CANYON_HWR = 200.00 +HT_ROOF = 200.0 +T_BUILDING_MIN = 200 From 7ece08f2c78fd3d4cbf5ba3b3e69829d7ceca725 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 31 Dec 2022 01:43:01 -0700 Subject: [PATCH 093/332] Add some more variables to the list of variables that can't be changed for fsurdat_modifier --- .../modify_input_files/fsurdat_modifier.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 9554117abd..cb27e988af 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -152,7 +152,26 @@ def read_var_list(config, cfg_path): "std_elev", ] subgrid_list = ["pct_natveg", "pct_crop", "pct_lake", "pct_glacier", "pct_wetland", "pct_urban"] - monthly_list = ["monthly_lai", "monthly_sai", "monthly_height_top", "monthly_height_bot"] + # List of variables that should be excluded because they are changed elsewhere, + # or they shouldn't be changed # Ds, Dsmax, and Ws are excluded because they + # are of mixed case and we only search for varaibles in lowercase + # or uppercase and not mixed case. + monthly_list = [ + "monthly_lai", + "monthly_sai", + "monthly_height_top", + "monthly_height_bot", + "ds", + "mxsoil_color", + "natpft", + "cft", + "time", + "longxy", + "latixy", + "dsmax", + "area", + "ws", + ] for var in var_list: if ideal_list.count(var) != 0: abort( From bb7a9822c960f5eb78a1008f6f6e75637ef8f232 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 31 Dec 2022 15:27:26 -0700 Subject: [PATCH 094/332] Start adding handling of numurbl so that PCT_URBAN can be recognized as an array, and add a check that you can setup read_subgrid to be 100% urban --- python/ctsm/modify_input_files/fsurdat_modifier.py | 11 ++++++++--- python/ctsm/modify_input_files/modify_fsurdat.py | 7 +++++++ python/ctsm/test/test_unit_fsurdat_modifier.py | 12 ++++++++++++ python/ctsm/test/test_unit_modify_fsurdat.py | 8 ++++++++ .../test/testinputs/modify_fsurdat_opt_sections.cfg | 1 + 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index cb27e988af..5e07b64089 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -89,7 +89,7 @@ def check_no_varlist_section(config): ) -def read_subgrid(config, cfg_path): +def read_subgrid(config, cfg_path, numurbl=3): """Read the subgrid fraction section from the config file""" section = "modify_fsurdat_subgrid_fractions" if not config.has_section(section): @@ -123,7 +123,12 @@ def read_subgrid(config, cfg_path): ) subgrid_settings[var.upper()] = value - varsum += value + # Urban is multidimensional so if a scalar value, must be multiplied + # by the density dimension + if var == "pct_urban": + varsum += value * numurbl + else: + varsum += value if varsum != 100.0: abort( "PCT fractions in subgrid section do NOT sum to a hundred as they should. Sum = " @@ -450,7 +455,7 @@ def fsurdat_modifier(parser): # Handle optional sections # if process_subgrid: - subgrid = read_subgrid(config, cfg_path) + subgrid = read_subgrid(config, cfg_path, numurbl=modify_fsurdat.get_urb_dens()) modify_fsurdat.set_varlist(subgrid) else: check_no_subgrid_section(config) diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index 85d5dcdd30..4b4b2f404c 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -29,7 +29,10 @@ def __init__( self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file, lat_dimname, lon_dimname ): + self.numurbl = 3 # Number of urban density types self.file = my_data + if "numurbl" in self.file.dims: + self.numurbl = self.file.dims["numurbl"] self.rectangle = self._get_rectangle( lon_1=lon_1, @@ -115,6 +118,10 @@ def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy): return rectangle + def get_urb_dens(self): + """Get the number of urban density classes""" + return self.numurbl + def write_output(self, fsurdat_in, fsurdat_out): """ Description diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index e8d1cf7c8a..75ee8efd74 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -90,6 +90,18 @@ def test_read_subgrid_allspecial(self): self.config.set(section, "pct_crop", "0.") read_subgrid(self.config, self.cfg_path) + def test_read_subgrid_allurban(self): + """test a read of subgrid that's all urban""" + section = "modify_fsurdat_subgrid_fractions" + third = float(100.0 / 3.0) + self.config.set(section, "pct_urban", str(third)) + self.config.set(section, "pct_lake", "0.") + self.config.set(section, "pct_wetland", "0.") + self.config.set(section, "pct_glacier", "0.") + self.config.set(section, "pct_natveg", "0.") + self.config.set(section, "pct_crop", "0.") + read_subgrid(self.config, self.cfg_path) + def test_read_var_list(self): """test a simple read of var_list""" read_var_list(self.config, self.cfg_path) diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index 580845bfea..07554e07ab 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -386,6 +386,14 @@ def test_check_varlist(self): settings_new, settings, "list of variable settings not identical as expected" ) + def test_get_numurb_dens(self): + """Check that get num urban density types is correct""" + self.assertEqual( + self.modify_fsurdat.get_urb_dens(), + 3, + "Default number of urban density types is correct", + ) + def test_check_varlist_uppercase(self): """Test the check_varlist method for all the dimensions that works with allowuppercase option""" diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index 704587c4d3..6abdc3c350 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -33,6 +33,7 @@ PCT_CROP = 0.0 PCT_LAKE = 100.0 PCT_GLACIER = 0.0 PCT_WETLAND = 0.0 +PCT_URBAN = 0.0 # Section with a list of variables to prcoess From e8f0b62056958745ca97e8660c3a897f4bbe9b8d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 31 Dec 2022 18:10:58 -0700 Subject: [PATCH 095/332] List out two subroutines so will pass lint check, now make all works, make abort if numurbl is not on the fsurdat file, set the test for it to 9 for the test case --- .../modify_input_files/fsurdat_modifier.py | 277 +++++++++++------- .../ctsm/modify_input_files/modify_fsurdat.py | 2 + python/ctsm/test/test_unit_modify_fsurdat.py | 3 +- 3 files changed, 169 insertions(+), 113 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 5e07b64089..e0ba41a35b 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -210,6 +210,147 @@ def read_var_list(config, cfg_path): return varlist_settings +def modify_optional( + modify_fsurdat, + idealized, + include_nonveg, + max_sat_area, + std_elev, + soil_color, + dom_pft, + lai, + sai, + hgt_top, + hgt_bot, +): + """Modify the dataset according to the optional settings""" + + # Set fsurdat variables in a rectangle that could be global (default). + # Note that the land/ocean mask gets specified in the domain file for + # MCT or the ocean mesh files for NUOPC. Here the user may specify + # fsurdat variables inside a box but cannot change which points will + # run as land and which as ocean. + if idealized: + modify_fsurdat.set_idealized() # set 2D variables + # set 3D and 4D variables pertaining to natural vegetation + modify_fsurdat.set_dom_pft(dom_pft=0, lai=[], sai=[], hgt_top=[], hgt_bot=[]) + logger.info("idealized complete") + + if max_sat_area is not None: # overwrite "idealized" value + modify_fsurdat.setvar_lev0("FMAX", max_sat_area) + logger.info("max_sat_area complete") + + if std_elev is not None: # overwrite "idealized" value + modify_fsurdat.setvar_lev0("STD_ELEV", std_elev) + logger.info("std_elev complete") + + if soil_color is not None: # overwrite "idealized" value + modify_fsurdat.setvar_lev0("SOIL_COLOR", soil_color) + logger.info("soil_color complete") + + if not include_nonveg: + modify_fsurdat.zero_nonveg() + logger.info("zero_nonveg complete") + + # set_dom_pft follows zero_nonveg because it modifies PCT_NATVEG + # and PCT_CROP in the user-defined rectangle + if dom_pft is not None: + modify_fsurdat.set_dom_pft( + dom_pft=dom_pft, lai=lai, sai=sai, hgt_top=hgt_top, hgt_bot=hgt_bot + ) + logger.info("dom_pft complete") + + +def read_opt_config(modify_fsurdat, config, cfg_path, section): + """read the optional parts of the main section config file""" + # not required: user may set these in the .cfg file + max_pft = int(max(modify_fsurdat.file.lsmpft)) + dom_pft = get_config_value( + config=config, + section=section, + item="dom_pft", + file_path=cfg_path, + allowed_values=range(max_pft + 1), # integers from 0 to max_pft + convert_to_type=int, + can_be_unset=True, + ) + + lai = get_config_value( + config=config, + section=section, + item="lai", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + sai = get_config_value( + config=config, + section=section, + item="sai", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + hgt_top = get_config_value( + config=config, + section=section, + item="hgt_top", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + hgt_bot = get_config_value( + config=config, + section=section, + item="hgt_bot", + file_path=cfg_path, + is_list=True, + convert_to_type=float, + can_be_unset=True, + ) + + max_soil_color = int(modify_fsurdat.file.mxsoil_color) + soil_color = get_config_value( + config=config, + section=section, + item="soil_color", + file_path=cfg_path, + allowed_values=range(1, max_soil_color + 1), # 1 to max_soil_color + convert_to_type=int, + can_be_unset=True, + ) + + std_elev = get_config_value( + config=config, + section=section, + item="std_elev", + file_path=cfg_path, + convert_to_type=float, + can_be_unset=True, + ) + max_sat_area = get_config_value( + config=config, + section=section, + item="max_sat_area", + file_path=cfg_path, + convert_to_type=float, + can_be_unset=True, + ) + return ( + max_sat_area, + std_elev, + soil_color, + dom_pft, + lai, + sai, + hgt_top, + hgt_bot, + ) + + def fsurdat_modifier(parser): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file @@ -335,122 +476,34 @@ def fsurdat_modifier(parser): lon_dimname, ) - # not required: user may set these in the .cfg file - max_pft = int(max(modify_fsurdat.file.lsmpft)) - dom_pft = get_config_value( - config=config, - section=section, - item="dom_pft", - file_path=cfg_path, - allowed_values=range(max_pft + 1), # integers from 0 to max_pft - convert_to_type=int, - can_be_unset=True, - ) - - lai = get_config_value( - config=config, - section=section, - item="lai", - file_path=cfg_path, - is_list=True, - convert_to_type=float, - can_be_unset=True, - ) - sai = get_config_value( - config=config, - section=section, - item="sai", - file_path=cfg_path, - is_list=True, - convert_to_type=float, - can_be_unset=True, - ) - hgt_top = get_config_value( - config=config, - section=section, - item="hgt_top", - file_path=cfg_path, - is_list=True, - convert_to_type=float, - can_be_unset=True, - ) - hgt_bot = get_config_value( - config=config, - section=section, - item="hgt_bot", - file_path=cfg_path, - is_list=True, - convert_to_type=float, - can_be_unset=True, - ) - - max_soil_color = int(modify_fsurdat.file.mxsoil_color) - soil_color = get_config_value( - config=config, - section=section, - item="soil_color", - file_path=cfg_path, - allowed_values=range(1, max_soil_color + 1), # 1 to max_soil_color - convert_to_type=int, - can_be_unset=True, - ) - - std_elev = get_config_value( - config=config, - section=section, - item="std_elev", - file_path=cfg_path, - convert_to_type=float, - can_be_unset=True, - ) - max_sat_area = get_config_value( - config=config, - section=section, - item="max_sat_area", - file_path=cfg_path, - convert_to_type=float, - can_be_unset=True, - ) - + # Read parts that are optional + ( + max_sat_area, + std_elev, + soil_color, + dom_pft, + lai, + sai, + hgt_top, + hgt_bot, + ) = read_opt_config(modify_fsurdat, config, cfg_path, section) # ------------------------------ # modify surface data properties # ------------------------------ - # Set fsurdat variables in a rectangle that could be global (default). - # Note that the land/ocean mask gets specified in the domain file for - # MCT or the ocean mesh files for NUOPC. Here the user may specify - # fsurdat variables inside a box but cannot change which points will - # run as land and which as ocean. - if idealized: - modify_fsurdat.set_idealized() # set 2D variables - # set 3D and 4D variables pertaining to natural vegetation - modify_fsurdat.set_dom_pft(dom_pft=0, lai=[], sai=[], hgt_top=[], hgt_bot=[]) - logger.info("idealized complete") - - if max_sat_area is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0("FMAX", max_sat_area) - logger.info("max_sat_area complete") - - if std_elev is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0("STD_ELEV", std_elev) - logger.info("std_elev complete") - - if soil_color is not None: # overwrite "idealized" value - modify_fsurdat.setvar_lev0("SOIL_COLOR", soil_color) - logger.info("soil_color complete") - - if not include_nonveg: - modify_fsurdat.zero_nonveg() - logger.info("zero_nonveg complete") - - # set_dom_pft follows zero_nonveg because it modifies PCT_NATVEG - # and PCT_CROP in the user-defined rectangle - if dom_pft is not None: - modify_fsurdat.set_dom_pft( - dom_pft=dom_pft, lai=lai, sai=sai, hgt_top=hgt_top, hgt_bot=hgt_bot - ) - logger.info("dom_pft complete") - + modify_optional( + modify_fsurdat, + idealized, + include_nonveg, + max_sat_area, + std_elev, + soil_color, + dom_pft, + lai, + sai, + hgt_top, + hgt_bot, + ) # # Handle optional sections # diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index 4b4b2f404c..dc55cc55a8 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -33,6 +33,8 @@ def __init__( self.file = my_data if "numurbl" in self.file.dims: self.numurbl = self.file.dims["numurbl"] + else: + abort("numurbl is not a dimension on the input surface dataset file and needs to be") self.rectangle = self._get_rectangle( lon_1=lon_1, diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index 07554e07ab..7a72288d53 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -55,6 +55,7 @@ def setUp(self): data_vars=dict( LONGXY=(["x", "y"], longxy), # use LONGXY as var_lev0 LATIXY=(["x", "y"], latixy), # __init__ expects LONGXY, LATIXY + urbdens=(["numurbl"], var_1d), # numurbl needs to be dimension var_lev0=(["x", "y"], var_lev0), var_lev1=(["w", "x", "y"], var_lev1), var_lev2=(["v", "w", "x", "y"], var_lev2), @@ -390,7 +391,7 @@ def test_get_numurb_dens(self): """Check that get num urban density types is correct""" self.assertEqual( self.modify_fsurdat.get_urb_dens(), - 3, + 9, "Default number of urban density types is correct", ) From cc3120408899617a8b2979fb1eb2e5e4312f03f5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 31 Dec 2022 18:20:24 -0700 Subject: [PATCH 096/332] Add something that says all target is successful --- python/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/Makefile b/python/Makefile index bf219f16b6..a1eaba0928 100644 --- a/python/Makefile +++ b/python/Makefile @@ -24,6 +24,10 @@ PYLINT_SRC = \ ctsm all: test lint black + @echo + @echo + @echo "Successfully ran all standard tests" + test: utest stest .PHONY: utest From a268e9734e71ed33305fd12685328f749bbc743b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 1 Jan 2023 19:08:31 -0700 Subject: [PATCH 097/332] Require PCT_URBAN to be a list of numurbl values, start adding a new config method get_config_value_or_array --- python/ctsm/config_utils.py | 28 ++++++++++++ .../modify_input_files/fsurdat_modifier.py | 44 ++++++++++++------- .../ctsm/test/test_unit_fsurdat_modifier.py | 11 +++-- .../modify_fsurdat_opt_sections.cfg | 2 +- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/python/ctsm/config_utils.py b/python/ctsm/config_utils.py index 847a1804f6..b8a8d6cd77 100644 --- a/python/ctsm/config_utils.py +++ b/python/ctsm/config_utils.py @@ -86,6 +86,34 @@ def get_config_value( return val +def get_config_value_or_array( + config, + section, + item, + file_path, + convert_to_type=None, +): + """Get a config value as a single value or as an array if it's expressed as an array""" + print(config) + print(section) + print(item) + print(file_path) + if convert_to_type is not None: + if convert_to_type is float: + print("float") + elif convert_to_type is int: + print("int") + elif convert_to_type is str: + print("str") + else: + abort( + "get_config_value_or_array can only have convert_to_type as float, int or str not " + + str(convert_to_type) + ) + + abort("This method not implemented yet") + + def _handle_config_value( var, default, item, is_list, convert_to_type, can_be_unset, allowed_values ): diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index e0ba41a35b..7353c16ef5 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -89,6 +89,12 @@ def check_no_varlist_section(config): ) +def check_range(var, section, value, minval, maxval): + """Check that the value is within range""" + if value < minval or value > maxval: + abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + str(value)) + + def read_subgrid(config, cfg_path, numurbl=3): """Read the subgrid fraction section from the config file""" section = "modify_fsurdat_subgrid_fractions" @@ -109,26 +115,30 @@ def read_subgrid(config, cfg_path, numurbl=3): + " is not a valid variable name. Valid vars =" + str(valid_list) ) - value = get_config_value( - config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float - ) - if value < 0.0 or value > 100.0: - abort( - "Variable " - + var - + " in " - + section - + " is out of range of 0 to 100 = " - + str(value) - ) - - subgrid_settings[var.upper()] = value - # Urban is multidimensional so if a scalar value, must be multiplied - # by the density dimension + # Urban is multidimensional if var == "pct_urban": - varsum += value * numurbl + vallist = get_config_value( + config=config, + section=section, + item=var, + file_path=cfg_path, + is_list=True, + convert_to_type=float, + ) + if len(vallist) != numurbl: + abort("PCT_URBAN is not a list of the expect size of " + str(numurbl)) + # so if a scalar value, must be multiplied # by the density dimension + for val in vallist: + check_range(var, section, val, 0.0, 100.0) + varsum += val else: + value = get_config_value( + config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float + ) + check_range(var, section, value, 0.0, 100.0) varsum += value + + subgrid_settings[var.upper()] = value if varsum != 100.0: abort( "PCT fractions in subgrid section do NOT sum to a hundred as they should. Sum = " diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 75ee8efd74..5231d30318 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -71,7 +71,7 @@ def test_read_subgrid(self): def test_read_subgrid_allglacier(self): """test a read of subgrid that's for all glacier""" section = "modify_fsurdat_subgrid_fractions" - self.config.set(section, "pct_urban", "0.") + self.config.set(section, "pct_urban", "0. 0. 0.") self.config.set(section, "pct_lake", "0.") self.config.set(section, "pct_wetland", "0.") self.config.set(section, "pct_glacier", "100.") @@ -82,7 +82,7 @@ def test_read_subgrid_allglacier(self): def test_read_subgrid_allspecial(self): """test a read of subgrid that's all special landunits""" section = "modify_fsurdat_subgrid_fractions" - self.config.set(section, "pct_urban", "0.") + self.config.set(section, "pct_urban", "0. 0. 0.") self.config.set(section, "pct_lake", "25.") self.config.set(section, "pct_wetland", "35.") self.config.set(section, "pct_glacier", "40.") @@ -93,8 +93,7 @@ def test_read_subgrid_allspecial(self): def test_read_subgrid_allurban(self): """test a read of subgrid that's all urban""" section = "modify_fsurdat_subgrid_fractions" - third = float(100.0 / 3.0) - self.config.set(section, "pct_urban", str(third)) + self.config.set(section, "pct_urban", "100.0 0.0 0.0") self.config.set(section, "pct_lake", "0.") self.config.set(section, "pct_wetland", "0.") self.config.set(section, "pct_glacier", "0.") @@ -109,14 +108,14 @@ def test_read_var_list(self): def test_subgrid_outofrange(self): """test a read of subgrid that's out of range""" section = "modify_fsurdat_subgrid_fractions" - self.config.set(section, "pct_urban", "101.") + self.config.set(section, "pct_urban", "101. 0. 0.") with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): read_subgrid(self.config, self.cfg_path) def test_subgrid_notsumtohundred(self): """test a read of subgrid that's doesn't sum to a hundred""" section = "modify_fsurdat_subgrid_fractions" - self.config.set(section, "pct_urban", "0.") + self.config.set(section, "pct_urban", "0. 0. 0.") self.config.set(section, "pct_lake", "0.") self.config.set(section, "pct_wetland", "0.") self.config.set(section, "pct_glacier", "0.") diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index 6abdc3c350..c73dd43a68 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -33,7 +33,7 @@ PCT_CROP = 0.0 PCT_LAKE = 100.0 PCT_GLACIER = 0.0 PCT_WETLAND = 0.0 -PCT_URBAN = 0.0 +PCT_URBAN = 0.0 0.0 0.0 # Section with a list of variables to prcoess From e317a6ce56554d60e5de9908ba20d577e05fa222 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 1 Jan 2023 19:22:44 -0700 Subject: [PATCH 098/332] Add some tests that the list size of PCT_URBAN is correct and will fail gracefully if not --- .../modify_input_files/fsurdat_modifier.py | 2 +- .../ctsm/test/test_unit_fsurdat_modifier.py | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 7353c16ef5..f9f36a664e 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -126,7 +126,7 @@ def read_subgrid(config, cfg_path, numurbl=3): convert_to_type=float, ) if len(vallist) != numurbl: - abort("PCT_URBAN is not a list of the expect size of " + str(numurbl)) + abort("PCT_URBAN is not a list of the expected size of " + str(numurbl)) # so if a scalar value, must be multiplied # by the density dimension for val in vallist: check_range(var, section, val, 0.0, 100.0) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 5231d30318..6a754065bc 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -112,6 +112,33 @@ def test_subgrid_outofrange(self): with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): read_subgrid(self.config, self.cfg_path) + def test_subgrid_pct_urban_toosmall(self): + """test a read of subgrid for PCT_URBAN that's an array too small""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100. 0.") + with self.assertRaisesRegex( + SystemExit, "PCT_URBAN is not a list of the expected size of 3" + ): + read_subgrid(self.config, self.cfg_path) + + def test_subgrid_pct_urban_toobig(self): + """test a read of subgrid for PCT_URBAN that's an array too big""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100. 0. 0. 0.") + with self.assertRaisesRegex( + SystemExit, "PCT_URBAN is not a list of the expected size of 3" + ): + read_subgrid(self.config, self.cfg_path) + + def test_subgrid_pct_urban_singlevalue(self): + """test a read of subgrid for PCT_URBAN that's a single value""" + section = "modify_fsurdat_subgrid_fractions" + self.config.set(section, "pct_urban", "100.") + with self.assertRaisesRegex( + SystemExit, "PCT_URBAN is not a list of the expected size of 3" + ): + read_subgrid(self.config, self.cfg_path) + def test_subgrid_notsumtohundred(self): """test a read of subgrid that's doesn't sum to a hundred""" section = "modify_fsurdat_subgrid_fractions" From 50878c85553c0a0d79faefd65bd01286089b3bf6 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 1 Jan 2023 23:14:36 -0700 Subject: [PATCH 099/332] Add a new unit tester for the config_utils module --- python/ctsm/test/test_unit_config_utils.py | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 python/ctsm/test/test_unit_config_utils.py diff --git a/python/ctsm/test/test_unit_config_utils.py b/python/ctsm/test/test_unit_config_utils.py new file mode 100644 index 0000000000..45b976f528 --- /dev/null +++ b/python/ctsm/test/test_unit_config_utils.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +"""Unit tests for config_utils +""" + +import unittest + +from ctsm import unit_testing +from ctsm.config_utils import lon_range_0_to_360 + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +class TestConfigUtils(unittest.TestCase): + """Tests of config_utils""" + + def test_negative_lon(self): + """ Test lon_range_0_to_360 for a negative longitude""" + lon = -180. + lon_new = lon_range_0_to_360(lon) + self.assertEqual( lon_new, 180., "lon not as expected" ) + + def test_negative_lon(self): + """ Test lon_range_0_to_360 for a negative longitude""" + lon = -5. + lon_new = lon_range_0_to_360(lon) + self.assertEqual( lon_new, 355., "lon not as expected" ) + + + def test_regular_lon(self): + """ Test lon_range_0_to_360 for a regular longitude""" + lon = 22.567 + lon_new = lon_range_0_to_360(lon) + self.assertEqual( lon_new, lon, "lon not as expected" ) + + def test_lon_out_of_range(self): + """ Test lon_range_0_to_360 for longitude out of range""" + lon = 361. + with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): + lon_range_0_to_360(lon) + + def test_lon_out_of_range_negative(self): + """ Test lon_range_0_to_360 for longitude out of range""" + lon = -181. + with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): + lon_range_0_to_360(lon) From 1f7d82a9e9a2dcb39e0ada0aabb10985d0f0b410 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 2 Jan 2023 18:59:10 -0700 Subject: [PATCH 100/332] Start adding tests for get_config_value_or_array to unit testing that fail as expected at this point --- python/ctsm/config_utils.py | 5 +- python/ctsm/test/test_unit_config_utils.py | 85 +++++++++++++++++----- 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/python/ctsm/config_utils.py b/python/ctsm/config_utils.py index b8a8d6cd77..2dc5428a2f 100644 --- a/python/ctsm/config_utils.py +++ b/python/ctsm/config_utils.py @@ -93,7 +93,9 @@ def get_config_value_or_array( file_path, convert_to_type=None, ): - """Get a config value as a single value or as an array if it's expressed as an array""" + """Get a config value as a single value or as an array if it's expressed as an array + for cases when you don't know how it's going to be expressed""" + val = "thing" print(config) print(section) print(item) @@ -112,6 +114,7 @@ def get_config_value_or_array( ) abort("This method not implemented yet") + return val def _handle_config_value( diff --git a/python/ctsm/test/test_unit_config_utils.py b/python/ctsm/test/test_unit_config_utils.py index 45b976f528..cb650e7d04 100644 --- a/python/ctsm/test/test_unit_config_utils.py +++ b/python/ctsm/test/test_unit_config_utils.py @@ -5,44 +5,91 @@ import unittest +from configparser import ConfigParser + from ctsm import unit_testing -from ctsm.config_utils import lon_range_0_to_360 +from ctsm.config_utils import lon_range_0_to_360, get_config_value_or_array -# Allow names that pylint doesn't like, because otherwise I find it hard -# to make readable unit test names +# Allow test names that pylint doesn't like; otherwise hard to make them +# readable # pylint: disable=invalid-name +# pylint: disable=protected-access + class TestConfigUtils(unittest.TestCase): """Tests of config_utils""" - def test_negative_lon(self): - """ Test lon_range_0_to_360 for a negative longitude""" - lon = -180. - lon_new = lon_range_0_to_360(lon) - self.assertEqual( lon_new, 180., "lon not as expected" ) + # Allow these to be set outside of the __init__ method + # pylint: disable=attribute-defined-outside-init + def setUp(self): + """Setup for testing""" + self.config = ConfigParser() + self.section = "main" + self.file_path = "path_to_file" + self.config[self.section] = {} def test_negative_lon(self): - """ Test lon_range_0_to_360 for a negative longitude""" - lon = -5. + """Test lon_range_0_to_360 for a negative longitude""" + lon = -180.0 lon_new = lon_range_0_to_360(lon) - self.assertEqual( lon_new, 355., "lon not as expected" ) + self.assertEqual(lon_new, 180.0, "lon not as expected") + def test_negative2_lon(self): + """Test lon_range_0_to_360 for a negative longitude""" + lon = -5.0 + lon_new = lon_range_0_to_360(lon) + self.assertEqual(lon_new, 355.0, "lon not as expected") def test_regular_lon(self): - """ Test lon_range_0_to_360 for a regular longitude""" + """Test lon_range_0_to_360 for a regular longitude""" lon = 22.567 lon_new = lon_range_0_to_360(lon) - self.assertEqual( lon_new, lon, "lon not as expected" ) + self.assertEqual(lon_new, lon, "lon not as expected") def test_lon_out_of_range(self): - """ Test lon_range_0_to_360 for longitude out of range""" - lon = 361. + """Test lon_range_0_to_360 for longitude out of range""" + lon = 361.0 with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): - lon_range_0_to_360(lon) + lon_range_0_to_360(lon) def test_lon_out_of_range_negative(self): - """ Test lon_range_0_to_360 for longitude out of range""" - lon = -181. + """Test lon_range_0_to_360 for longitude out of range""" + lon = -181.0 with self.assertRaisesRegex(SystemExit, "lon_in needs to be in the range 0 to 360"): - lon_range_0_to_360(lon) + lon_range_0_to_360(lon) + + def test_config_value_or_array_single_value(self): + """Simple test of get_config_value_or_array""" + item = "single_value_thing" + # Test on a string, float and integer + self.config.set(self.section, item, "one-thing") + value = get_config_value_or_array(self.config, self.section, item, self.file_path) + self.assertEqual(value, "one-thing", "Value as expected") + self.config.set(self.section, item, "100.") + value = get_config_value_or_array(self.config, self.section, item, self.file_path) + self.assertEqual(float(value), 100.0, "Value as expected") + self.config.set(self.section, item, "100") + value = get_config_value_or_array(self.config, self.section, item, self.file_path) + self.assertEqual(int(value), 100, "Value as expected") + # Run over again, with an explicit conversion + self.config.set(self.section, item, "one-thing") + value = get_config_value_or_array( + self.config, self.section, item, self.file_path, convert_to_type=str + ) + self.assertEqual(value, "one-thing", "Value as expected") + self.config.set(self.section, item, "100.") + value = get_config_value_or_array( + self.config, self.section, item, self.file_path, convert_to_type=float + ) + self.assertEqual(value, 100.0, "Value as expected") + self.config.set(self.section, item, "100") + value = get_config_value_or_array( + self.config, self.section, item, self.file_path, convert_to_type=int + ) + self.assertEqual(value, 100, "Value as expected") + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From a82e49fff591b2a6fc14af78f711a3679a281d51 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 3 Jan 2023 11:27:49 -0700 Subject: [PATCH 101/332] Implement get_config_value_or_array and get current tests passing --- python/ctsm/config_utils.py | 31 ++++++++++++---------- python/ctsm/test/test_unit_config_utils.py | 22 ++++++--------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/python/ctsm/config_utils.py b/python/ctsm/config_utils.py index 2dc5428a2f..bd53825f14 100644 --- a/python/ctsm/config_utils.py +++ b/python/ctsm/config_utils.py @@ -90,30 +90,33 @@ def get_config_value_or_array( config, section, item, - file_path, convert_to_type=None, ): """Get a config value as a single value or as an array if it's expressed as an array for cases when you don't know how it's going to be expressed""" - val = "thing" - print(config) - print(section) - print(item) - print(file_path) + val = config.get(section, item) + vallist = val.split() if convert_to_type is not None: - if convert_to_type is float: - print("float") - elif convert_to_type is int: - print("int") - elif convert_to_type is str: - print("str") - else: + if ( + convert_to_type is not float + and convert_to_type is not int + and convert_to_type is not str + ): abort( "get_config_value_or_array can only have convert_to_type as float, int or str not " + str(convert_to_type) ) + is_list = bool(len(vallist) > 1) - abort("This method not implemented yet") + val = _handle_config_value( + var=val, + default=None, + item=item, + is_list=is_list, + convert_to_type=convert_to_type, + can_be_unset=False, + allowed_values=None, + ) return val diff --git a/python/ctsm/test/test_unit_config_utils.py b/python/ctsm/test/test_unit_config_utils.py index cb650e7d04..5af2e67df5 100644 --- a/python/ctsm/test/test_unit_config_utils.py +++ b/python/ctsm/test/test_unit_config_utils.py @@ -64,29 +64,23 @@ def test_config_value_or_array_single_value(self): item = "single_value_thing" # Test on a string, float and integer self.config.set(self.section, item, "one-thing") - value = get_config_value_or_array(self.config, self.section, item, self.file_path) + value = get_config_value_or_array(self.config, self.section, item) self.assertEqual(value, "one-thing", "Value as expected") self.config.set(self.section, item, "100.") - value = get_config_value_or_array(self.config, self.section, item, self.file_path) - self.assertEqual(float(value), 100.0, "Value as expected") + value = get_config_value_or_array(self.config, self.section, item) + self.assertEqual(value, "100.", "Value as expected") self.config.set(self.section, item, "100") - value = get_config_value_or_array(self.config, self.section, item, self.file_path) - self.assertEqual(int(value), 100, "Value as expected") + value = get_config_value_or_array(self.config, self.section, item) + self.assertEqual(value, "100", "Value as expected") # Run over again, with an explicit conversion self.config.set(self.section, item, "one-thing") - value = get_config_value_or_array( - self.config, self.section, item, self.file_path, convert_to_type=str - ) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=str) self.assertEqual(value, "one-thing", "Value as expected") self.config.set(self.section, item, "100.") - value = get_config_value_or_array( - self.config, self.section, item, self.file_path, convert_to_type=float - ) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=float) self.assertEqual(value, 100.0, "Value as expected") self.config.set(self.section, item, "100") - value = get_config_value_or_array( - self.config, self.section, item, self.file_path, convert_to_type=int - ) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=int) self.assertEqual(value, 100, "Value as expected") From d393341f9bb92117101587cb931be0d9c3821612 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 3 Jan 2023 13:17:03 -0700 Subject: [PATCH 102/332] Add a test for get_config_value_or_array for lists of size three --- python/ctsm/test/test_unit_config_utils.py | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/ctsm/test/test_unit_config_utils.py b/python/ctsm/test/test_unit_config_utils.py index 5af2e67df5..7c62173572 100644 --- a/python/ctsm/test/test_unit_config_utils.py +++ b/python/ctsm/test/test_unit_config_utils.py @@ -83,6 +83,30 @@ def test_config_value_or_array_single_value(self): value = get_config_value_or_array(self.config, self.section, item, convert_to_type=int) self.assertEqual(value, 100, "Value as expected") + def test_config_value_or_array_for_list(self): + """Simple test of get_config_value_or_array for a list""" + item = "three_things" + # Test on a string, float and integer + mystr = "one two three" + mystrlist = ["one", "two", "three"] + myfloat = "1. 2. 3." + myfloatlist = [1., 2., 3.] + myint = "1 2 3" + myintlist = [1, 2, 3] + self.config.set(self.section, item, mystr) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=str) + self.assertEqual(value, mystrlist, "List as expected") + self.assertEqual(len(value), 3, "List size as expected") + self.config.set(self.section, item, myfloat) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=float) + self.assertEqual(value, myfloatlist, "Value as expected") + self.assertEqual(len(value), 3, "List size as expected") + self.config.set(self.section, item, myint) + value = get_config_value_or_array(self.config, self.section, item, convert_to_type=int) + self.assertEqual(value, myintlist, "Value as expected") + self.assertEqual(len(value), 3, "List size as expected") + + if __name__ == "__main__": unit_testing.setup_for_tests() From 1bcbec8155cb38e604af18c40baa25ab2fa454c9 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 4 Jan 2023 16:32:35 -0700 Subject: [PATCH 103/332] Get testing for get_config_value_or_array for lists working, and change read_var_list to use get_config_value_or_array --- python/ctsm/modify_input_files/fsurdat_modifier.py | 10 +++++----- python/ctsm/test/test_unit_config_utils.py | 13 ++++++------- python/ctsm/test/test_unit_fsurdat_modifier.py | 8 ++++---- .../test/testinputs/modify_fsurdat_opt_sections.cfg | 2 ++ 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index f9f36a664e..c9f7f1cd84 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -11,7 +11,7 @@ from configparser import ConfigParser from ctsm.utils import abort, write_output -from ctsm.config_utils import get_config_value +from ctsm.config_utils import get_config_value, get_config_value_or_array from ctsm.ctsm_logging import ( setup_logging_pre_config, add_logging_args, @@ -148,7 +148,7 @@ def read_subgrid(config, cfg_path, numurbl=3): return subgrid_settings -def read_var_list(config, cfg_path): +def read_var_list(config): """Read the variable list section from the config file""" section = "modify_fsurdat_variable_list" if not config.has_section(section): @@ -212,8 +212,8 @@ def read_var_list(config, cfg_path): + " Monthly vars handled this way =" + str(monthly_list) ) - value = get_config_value( - config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float + value = get_config_value_or_array( + config=config, section=section, item=var, convert_to_type=float ) varlist_settings[var] = value @@ -524,7 +524,7 @@ def fsurdat_modifier(parser): check_no_subgrid_section(config) if process_var_list: - varlist = read_var_list(config, cfg_path) + varlist = read_var_list(config) update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) modify_fsurdat.set_varlist(update_list) else: diff --git a/python/ctsm/test/test_unit_config_utils.py b/python/ctsm/test/test_unit_config_utils.py index 7c62173572..c9ee23bac3 100644 --- a/python/ctsm/test/test_unit_config_utils.py +++ b/python/ctsm/test/test_unit_config_utils.py @@ -87,12 +87,12 @@ def test_config_value_or_array_for_list(self): """Simple test of get_config_value_or_array for a list""" item = "three_things" # Test on a string, float and integer - mystr = "one two three" - mystrlist = ["one", "two", "three"] - myfloat = "1. 2. 3." - myfloatlist = [1., 2., 3.] - myint = "1 2 3" - myintlist = [1, 2, 3] + mystr = "one two three" + mystrlist = ["one", "two", "three"] + myfloat = "1. 2. 3." + myfloatlist = [1.0, 2.0, 3.0] + myint = "1 2 3" + myintlist = [1, 2, 3] self.config.set(self.section, item, mystr) value = get_config_value_or_array(self.config, self.section, item, convert_to_type=str) self.assertEqual(value, mystrlist, "List as expected") @@ -107,7 +107,6 @@ def test_config_value_or_array_for_list(self): self.assertEqual(len(value), 3, "List size as expected") - if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 6a754065bc..149ba5acac 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -103,7 +103,7 @@ def test_read_subgrid_allurban(self): def test_read_var_list(self): """test a simple read of var_list""" - read_var_list(self.config, self.cfg_path) + read_var_list(self.config) def test_subgrid_outofrange(self): """test a read of subgrid that's out of range""" @@ -169,7 +169,7 @@ def test_varlist_varinidealized(self): "is a special variable handled in the idealized section." + " This should NOT be handled in the variiable list section. Special idealized vars =", ): - read_var_list(self.config, self.cfg_path) + read_var_list(self.config) def test_varlist_varinsubgrid(self): """test a read of varlist for a variable thats in the subgrid list""" @@ -180,7 +180,7 @@ def test_varlist_varinsubgrid(self): "is a variable handled in the subgrid section." + " This should NOT be handled in the variiable list section. Subgrid vars =", ): - read_var_list(self.config, self.cfg_path) + read_var_list(self.config) def test_varlist_monthlyvar(self): """test a read of varlist for a variable thats one of the monthly @@ -193,7 +193,7 @@ def test_varlist_monthlyvar(self): + " This should NOT be handled in the variiable list section." + " Monthly vars handled this way =", ): - read_var_list(self.config, self.cfg_path) + read_var_list(self.config) def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index c73dd43a68..fd07d697cf 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -33,6 +33,7 @@ PCT_CROP = 0.0 PCT_LAKE = 100.0 PCT_GLACIER = 0.0 PCT_WETLAND = 0.0 +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area PCT_URBAN = 0.0 0.0 0.0 @@ -42,6 +43,7 @@ PCT_URBAN = 0.0 0.0 0.0 LAKEDEPTH = 200.00 # Set soem urban multidimensional variables to 200 for the test +# If given as a single value set all to the value, if as list set by dimension and if array as an array CANYON_HWR = 200.00 HT_ROOF = 200.0 T_BUILDING_MIN = 200 From 32e303ea8de556d2193315f0e19f50ec597ea229 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 6 Jan 2023 10:29:33 -0700 Subject: [PATCH 104/332] Assume that 3D and higher arrays come in as a list, add more checks that this is correct and correct size, and more tests for it, also get all tests to work --- .../modify_input_files/fsurdat_modifier.py | 6 +- .../ctsm/modify_input_files/modify_fsurdat.py | 62 +++++++++++++++---- python/ctsm/test/test_unit_modify_fsurdat.py | 37 +++++++---- .../modify_fsurdat_opt_sections.cfg | 6 +- 4 files changed, 84 insertions(+), 27 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index c9f7f1cd84..6653a4937d 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -131,6 +131,7 @@ def read_subgrid(config, cfg_path, numurbl=3): for val in vallist: check_range(var, section, val, 0.0, 100.0) varsum += val + value = vallist else: value = get_config_value( config=config, section=section, item=var, file_path=cfg_path, convert_to_type=float @@ -139,6 +140,7 @@ def read_subgrid(config, cfg_path, numurbl=3): varsum += value subgrid_settings[var.upper()] = value + if varsum != 100.0: abort( "PCT fractions in subgrid section do NOT sum to a hundred as they should. Sum = " @@ -519,14 +521,14 @@ def fsurdat_modifier(parser): # if process_subgrid: subgrid = read_subgrid(config, cfg_path, numurbl=modify_fsurdat.get_urb_dens()) - modify_fsurdat.set_varlist(subgrid) + modify_fsurdat.set_varlist(subgrid, cfg_path) else: check_no_subgrid_section(config) if process_var_list: varlist = read_var_list(config) update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) - modify_fsurdat.set_varlist(update_list) + modify_fsurdat.set_varlist(update_list, cfg_path) else: check_no_varlist_section(config) diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index dc55cc55a8..aac888ae39 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -230,38 +230,78 @@ def check_varlist(self, settings, allow_uppercase_vars=False): settings_return = {} varlist = settings.keys() for var in varlist: - val = settings[var] + varname = var + val = settings[varname] if not var in self.file: if not allow_uppercase_vars: - errmsg = "Error: Variable " + var + " is NOT in the file" + errmsg = "Error: Variable " + varname + " is NOT in the file" abort(errmsg) - if not var.upper() in self.file: - errmsg = "Error: Variable " + var.upper() + " is NOT in the file" + if not varname.upper() in self.file: + errmsg = "Error: Variable " + varname.upper() + " is NOT in the file" abort(errmsg) - varu = var.upper() - settings_return[varu] = val - else: - settings_return[var] = val + varname = varname.upper() + + settings_return[varname] = val + # + # Check that dimensions are as expected + # + if len(self.file[varname].dims) == 2: + if not isinstance(val, float): + abort( + "For 2D vars, there should only be a single value for variable = " + varname + ) + elif len(self.file[varname].dims) >= 3: + dim1 = int(self.file.sizes[self.file[varname].dims[0]]) + if not isinstance(val, list): + abort( + "For higher dimensional vars, the variable needs to be expressed " + + "as a list of values of the dimension size = " + + str(dim1) + + " for variable=" + + varname + ) + if len(val) != dim1: + abort( + "Variable " + varname + " is of the wrong size. It should be = " + str(dim1) + ) return settings_return - def set_varlist(self, settings): + def set_varlist(self, settings, cfg_path="unknown-config-file"): """ Set a list of variables from a dictionary of settings """ for var in settings.keys(): if var in self.file: if len(self.file[var].dims) == 2: + if not isinstance(settings[var], float): + abort( + "For 2D vars, there should only be a single value for variable = " + var + ) self.setvar_lev0(var, settings[var]) elif len(self.file[var].dims) == 3: dim1 = int(self.file.sizes[self.file[var].dims[0]]) + vallist = settings[var] + if not isinstance(vallist, list): + abort( + "For higher dimensional vars, there must be a list of values " + + "for variable= " + + var + + " from the config file = " + + cfg_path + ) + if len(vallist) != dim1: + abort( + "Variable " + var + " is of the wrong size. It should be = " + str(dim1) + ) for lev1 in range(dim1 - 1): - self.setvar_lev1(var, settings[var], lev1_dim=lev1) + self.setvar_lev1(var, vallist[lev1], lev1_dim=lev1) elif len(self.file[var].dims) == 4: dim1 = int(self.file.sizes[self.file[var].dims[0]]) dim2 = int(self.file.sizes[self.file[var].dims[1]]) + vallist = settings[var] for lev1 in range(dim1 - 1): for lev2 in range(dim2 - 1): - self.setvar_lev2(var, settings[var], lev1_dim=lev1, lev2_dim=lev2) + self.setvar_lev2(var, vallist[lev1], lev1_dim=lev1, lev2_dim=lev2) else: abort( "Error: Variable " diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index 7a72288d53..5cbdefcc48 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -376,17 +376,29 @@ def test_getNotRectangle_latsOutOfBounds(self): def test_set_varlist(self): """Test the set_varlist method for all the dimensions that works""" - settings = {"var_lev0": 100.0, "var_lev1": 100.0, "var_lev2": 100.0} + vallist = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + settings = {"var_lev0": 100.0, "var_lev1": vallist, "var_lev2": vallist} self.modify_fsurdat.set_varlist(settings) - def test_check_varlist(self): - """Test the check_varlist method for all the dimensions that works""" - settings = {"var_lev0": 100.0, "var_lev1": 100.0, "var_lev2": 100.0} + def test_check_varlist_lists(self): + """Test the check_varlist method for lists""" + lev1list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + lev2list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] + settings = {"var_lev1": lev1list, "var_lev2": lev2list} settings_new = self.modify_fsurdat.check_varlist(settings) self.assertEqual( settings_new, settings, "list of variable settings not identical as expected" ) + def test_check_varlist_lists_wrongsizes(self): + """Test the check_varlist method for lists to gracefully fail when the sizes are wrong""" + lev1list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + settings = {"var_lev1": lev1list} + with self.assertRaisesRegex( + SystemExit, "Variable var_lev1 is of the wrong size. It should be" + ): + self.modify_fsurdat.check_varlist(settings) + def test_get_numurb_dens(self): """Check that get num urban density types is correct""" self.assertEqual( @@ -398,15 +410,17 @@ def test_get_numurb_dens(self): def test_check_varlist_uppercase(self): """Test the check_varlist method for all the dimensions that works with allowuppercase option""" + vallist = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + vallist2 = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] expected = { "VAR_LEV0_UPPERCASE": 100.0, - "VAR_LEV1_UPPERCASE": 200.0, - "VAR_LEV2_UPPERCASE": 300.0, + "VAR_LEV1_UPPERCASE": vallist, + "VAR_LEV2_UPPERCASE": vallist2, } settings = { "var_lev0_uppercase": 100.0, - "var_lev1_uppercase": 200.0, - "var_lev2_uppercase": 300.0, + "var_lev1_uppercase": vallist, + "var_lev2_uppercase": vallist2, } settings_new = self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) self.assertEqual( @@ -439,10 +453,11 @@ def test_set_varlist_toohighdim_uppercase(self): """Test the set_varlist method for a variable of too high a dimension in uppercase""" settings = {"var_lev3_uppercase": 100.0} with self.assertRaisesRegex( - SystemExit, "Variable VAR_LEV3_UPPERCASE is a higher dimension than currently allowed" + SystemExit, + "For higher dimensional vars, the variable needs to be expressed as a " + + "list of values of the dimension size = 9 for variable=VAR_LEV3_UPPERCASE", ): - settings_new = self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) - self.modify_fsurdat.set_varlist(settings_new) + self.modify_fsurdat.check_varlist(settings, allow_uppercase_vars=True) def _get_longxy_latixy(self, _min_lon, _max_lon, _min_lat, _max_lat): """ diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index fd07d697cf..414f240ebd 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -44,6 +44,6 @@ LAKEDEPTH = 200.00 # Set soem urban multidimensional variables to 200 for the test # If given as a single value set all to the value, if as list set by dimension and if array as an array -CANYON_HWR = 200.00 -HT_ROOF = 200.0 -T_BUILDING_MIN = 200 +CANYON_HWR = 200.00 150.0 100. +HT_ROOF = 200.0 150.0 100. +T_BUILDING_MIN = 200 150.0 100. From 4836ce3f47646eee57bd8992973ddb068ba38c58 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 7 Jan 2023 11:11:10 -0700 Subject: [PATCH 105/332] Make the opt_section tests to 100% urban and add two higher level dimension variables to the test, also in the system test test that the surface dataset gets changed as expected for some variables --- python/ctsm/test/test_sys_fsurdat_modifier.py | 21 +++++++++++++++---- .../modify_fsurdat_opt_sections.cfg | 8 ++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 5645ec152f..7671e10692 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -13,6 +13,7 @@ import sys import xarray as xr +import numpy as np from ctsm.path_utils import path_to_ctsm_root from ctsm import unit_testing @@ -124,6 +125,10 @@ def test_opt_sections(self): Test that a simple file with the optional sections works """ self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_opt_sections.cfg") + outfile = os.path.join( + self._tempdir, + "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_output_urban.nc", + ) sys.argv = [ "fsurdat_modifier", self._cfg_file_path, @@ -132,13 +137,21 @@ def test_opt_sections(self): self._testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc" ), "-o", - os.path.join( - self._tempdir, - "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_output_urban.nc", - ), + outfile, ] parser = fsurdat_modifier_arg_process() fsurdat_modifier(parser) + # Read the resultant output file and make sure the fields are changed as expected + fsurdat_out_data = xr.open_dataset(outfile) + zero0d = np.zeros( (5,5) ) + one0d = np.ones( (5,5) ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_NATVEG, zero0d ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_CROP, zero0d ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_WETLAND, zero0d ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_GLACIER, zero0d ) + np.testing.assert_array_equal(fsurdat_out_data.LAKEDEPTH, one0d*200. ) def test_cfg_file_DNE_fail(self): """ diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index 414f240ebd..b9c006ce03 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -27,14 +27,14 @@ lnd_lon_2 = 360 # Section for subgrid_fractions [modify_fsurdat_subgrid_fractions] -# Set to 100% lake for the test +# Set to 100% urban for the test PCT_NATVEG = 0.0 PCT_CROP = 0.0 -PCT_LAKE = 100.0 +PCT_LAKE = 0.0 PCT_GLACIER = 0.0 PCT_WETLAND = 0.0 # NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area -PCT_URBAN = 0.0 0.0 0.0 +PCT_URBAN = 100.0 0.0 0.0 # Section with a list of variables to prcoess @@ -47,3 +47,5 @@ LAKEDEPTH = 200.00 CANYON_HWR = 200.00 150.0 100. HT_ROOF = 200.0 150.0 100. T_BUILDING_MIN = 200 150.0 100. +#ALB_ROOF_DIR = 200. 100. +#TK_ROOF = 1. 2. 3. 4. 5. From 88638a87f929f2e2708a6a54a6eb312e96c06517 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 9 Jan 2023 14:51:39 -0700 Subject: [PATCH 106/332] add fates-specific helper script --- .../neon_surf_wrapper_fates.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 tools/site_and_regional/neon_surf_wrapper_fates.py diff --git a/tools/site_and_regional/neon_surf_wrapper_fates.py b/tools/site_and_regional/neon_surf_wrapper_fates.py new file mode 100755 index 0000000000..fcda0e17b4 --- /dev/null +++ b/tools/site_and_regional/neon_surf_wrapper_fates.py @@ -0,0 +1,110 @@ +#! /usr/bin/env python3 +""" +|------------------------------------------------------------------| +|--------------------- Instructions -----------------------------| +|------------------------------------------------------------------| +This script is a simple wrapper for neon sites that performs the +following: + 1) For neon sites, subset surface dataset from global dataset + (i.e. ./subset_data.py ) + 2) Download neon and update the created surface dataset + based on the downloaded neon data. + (i.e. modify_singlept_site_neon.py) + +Instructions for running using conda python environments: + +../../py_env_create +conda activate ctsm_py + +""" +# TODO +# Automatic downloading of missing files if they are missing +#-[ ] Download neon sites and dom pft file +#-[ ] Make sure verbose works for printing out commands running + +# Import libraries +from __future__ import print_function + +import os +import sys +import tqdm +import logging +import argparse +import subprocess + +import pandas as pd +#import tqdm as tqdm + + + +def get_parser(): + """ + Get parser object for this script. + """ + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.print_usage = parser.print_help + + parser.add_argument('-v','--verbose', + help='Verbose mode will print more information. ', + action="store_true", + dest="verbose", + default=False) + + + return parser + + +def execute(command): + """ + Function for running a command on shell. + Args: + command (str): + command that we want to run. + Raises: + Error with the return code from shell. + """ + print ('\n',' >> ',*command,'\n') + + try: + subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT) + + except subprocess.CalledProcessError as e: + #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) + #print (e.ouput) + print (e) + + + + + + +def main(): + + args = get_parser().parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + + neon_sites = pd.read_csv('neon_sites_dompft.csv') + + + for i, row in tqdm.tqdm(neon_sites.iterrows()): + lat = row['Lat'] + lon = row['Lon'] + site = row['Site'] + pft = row['pft'] + clmsite = "1x1_NEON_16PFT_"+site + print ("Now processing site :", site) + command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite, + '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] + execute(command) + + #command = ['./modify_singlept_site_neon.py','--neon_site',site, '--surf_dir', + # 'subset_data_single_point'] + #execute(command) + +if __name__ == "__main__": + main() From f56dabfde76fada39d895d621a3c75967dee0cd1 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Mon, 9 Jan 2023 14:56:28 -0700 Subject: [PATCH 107/332] updating so we can create FATES surface datasets --- .../modify_singlept_site_neon.py | 18 +++++++++--------- .../neon_surf_wrapper_fates.py | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 2798f463ec..96f0b7d1ef 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -6,12 +6,12 @@ This script is for modifying surface dataset at neon sites using data available from the neon server. -After creating a single point surface data file from a global -surface data file using subset_data.py, use this script to +After creating a single point surface data file from a global +surface data file using subset_data.py, use this script to overwrite some fields with site-specific data for neon sites. This script will do the following: -- Download neon data for the specified site if it does not exist +- Download neon data for the specified site if it does not exist in the specified directory : (i.e. ../../../neon_surffiles). - Modify surface dataset with downloaded data. @@ -129,7 +129,7 @@ def get_parser(): parser.add_argument( "--surf_dir", help=""" - Directory of single point surface dataset. + Directory of single point surface dataset. [default: %(default)s] """, action="store", @@ -142,7 +142,7 @@ def get_parser(): "--out_dir", help=""" Directory to write updated single point surface dataset. - [default: %(default)s] + [default: %(default)s] """, action="store", dest="out_dir", @@ -154,7 +154,7 @@ def get_parser(): "--inputdata-dir", help=""" Directory to write updated single point surface dataset. - [default: %(default)s] + [default: %(default)s] """, action="store", dest="inputdatadir", @@ -166,8 +166,8 @@ def get_parser(): "-d", "--debug", help=""" - Debug mode will print more information. - [default: %(default)s] + Debug mode will print more information. + [default: %(default)s] """, action="store_true", dest="debug", @@ -260,7 +260,7 @@ def find_surffile(surf_dir, site_name): surf_file (str): name of the surface dataset file """ - sf_name = "surfdata_1x1_NEON_"+site_name+"*hist_78pfts_CMIP6_simyr2000_*.nc" + sf_name = "surfdata_1x1_NEON_16PFT_"+site_name+"*hist_78pfts_CMIP6_simyr2000_*.nc" print (os.path.join(surf_dir , sf_name)) surf_file = sorted(glob.glob(os.path.join(surf_dir , sf_name))) diff --git a/tools/site_and_regional/neon_surf_wrapper_fates.py b/tools/site_and_regional/neon_surf_wrapper_fates.py index fcda0e17b4..60003b857e 100755 --- a/tools/site_and_regional/neon_surf_wrapper_fates.py +++ b/tools/site_and_regional/neon_surf_wrapper_fates.py @@ -102,9 +102,9 @@ def main(): '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] execute(command) - #command = ['./modify_singlept_site_neon.py','--neon_site',site, '--surf_dir', - # 'subset_data_single_point'] - #execute(command) + command = ['./modify_singlept_site_neon.py','--neon_site',site, '--surf_dir', + 'subset_data_single_point'] + execute(command) if __name__ == "__main__": main() From a175a1b4d7b45fafb334c27ead217a8263e56dab Mon Sep 17 00:00:00 2001 From: adrifoster Date: Tue, 10 Jan 2023 07:47:00 -0700 Subject: [PATCH 108/332] fixing some file names --- .../modify_singlept_site_neon.py | 15 ++++++++------- tools/site_and_regional/subset_data | 6 +----- 2 files changed, 9 insertions(+), 12 deletions(-) mode change 100755 => 100644 tools/site_and_regional/subset_data diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 96f0b7d1ef..b9a33e73e1 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -12,7 +12,7 @@ This script will do the following: - Download neon data for the specified site if it does not exist - in the specified directory : (i.e. ../../../neon_surffiles). + in the specified directory : (i.e. ../../../neon_surf_files). - Modify surface dataset with downloaded data. ------------------------------------------------------------------- @@ -181,7 +181,7 @@ def get_neon(neon_dir, site_name): """ Function for finding neon data files and download from neon server if the - file does not exits. + file does not exist. Args: neon_dir (str): local directory for downloading neon data. @@ -202,7 +202,7 @@ def get_neon(neon_dir, site_name): neon_file = os.path.join(neon_dir, site_name + "_surfaceData.csv") - # -- Download the file if it does not exits + # -- Download the file if it does not exist if os.path.isfile(neon_file): print("neon file for", site_name, "already exists! ") print("Skipping download from neon for", site_name, "...") @@ -260,7 +260,7 @@ def find_surffile(surf_dir, site_name): surf_file (str): name of the surface dataset file """ - sf_name = "surfdata_1x1_NEON_16PFT_"+site_name+"*hist_78pfts_CMIP6_simyr2000_*.nc" + sf_name = "surfdata_1x1_NEON_16PFT_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" print (os.path.join(surf_dir , sf_name)) surf_file = sorted(glob.glob(os.path.join(surf_dir , sf_name))) @@ -293,7 +293,7 @@ def find_soil_structure(args, surf_file): This function finds this file for the surface dataset, read it, and find soil layers. - Args: + args: surf_file (str): single point surface data filename Raises: @@ -349,6 +349,7 @@ def update_metadata(nc, surf_file, neon_file, zb_flag): nc (xr Dataset): netcdf file including updated neon surface data surf_file (str): single point surface data filename neon_file (str): filename of neon downloaded surface dataset + zb_flag (bool): update bedrock Returns: nc (xr Dataset): netcdf file including updated neon surface data @@ -380,7 +381,7 @@ def update_time_tag(fname_in): fname_in (str) : file name with the old time tag Raises: - error if the file does not end with with + error if the file does not end with [._]cYYMMDD.nc or [._]YYMMDD.nc Returns: @@ -636,7 +637,7 @@ def main(): # -- inorganic = caco3/100.0869*12.0107 # -- organic = carbon_tot - inorganic # -- else: - # -- oranigc = estimated_oc * bulk_den /0.58 + # -- organic = estimated_oc * bulk_den /0.58 caco3 = df["caco3Conc"][bin_index[soil_lev]] inorganic = caco3 / 100.0869 * 12.0107 diff --git a/tools/site_and_regional/subset_data b/tools/site_and_regional/subset_data old mode 100755 new mode 100644 index 162dcf1d4e..ad8e60e053 --- a/tools/site_and_regional/subset_data +++ b/tools/site_and_regional/subset_data @@ -4,22 +4,18 @@ This is a just top-level skeleton script that calls subset_data.py. The original code (subset_data.py) is located under python/ctsm folder. - For full instructions on how to run the code and different options, please check python/ctsm/subset_data.py file. This script extracts domain files, surface dataset, and DATM files at either a single point or a region using the global dataset. - To run this script the following packages are required: - numpy - xarray ---------------------------------------------------------------- To see all available options for single-point/regional subsetting: ./subset_data --help - ---------------------------------------------------------------- Instructions for running using conda python environments: - ../../py_env_create conda activate ctsm_py """ @@ -36,4 +32,4 @@ sys.path.insert(1, _CTSM_PYTHON) from ctsm.subset_data import main if __name__ == "__main__": - main() + main() \ No newline at end of file From c3921176a13b3605b024bc32f06b267c4f1a0ba1 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Tue, 10 Jan 2023 07:52:01 -0700 Subject: [PATCH 109/332] make executable --- tools/site_and_regional/subset_data | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/site_and_regional/subset_data diff --git a/tools/site_and_regional/subset_data b/tools/site_and_regional/subset_data old mode 100644 new mode 100755 From 1585d0a8c83c3ff4cff69c95b69f2718cf4d53dc Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Tue, 10 Jan 2023 07:59:07 -0700 Subject: [PATCH 110/332] remove packages check --- tools/site_and_regional/modify_singlept_site_neon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index b9a33e73e1..eabf5930f5 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -47,7 +47,7 @@ import numpy as np import pandas as pd import xarray as xr -from packaging import version +#from packaging import version from datetime import date from getpass import getuser @@ -521,9 +521,9 @@ def main(): logging.basicConfig(level=logging.DEBUG) # Check if pandas is a recent enough version - pdvers = pd.__version__ - if version.parse(pdvers) < version.parse("1.1.0"): - sys.exit("The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", pdvers ) + #pdvers = pd.__version__ + #if version.parse(pdvers) < version.parse("1.1.0"): + # sys.exit("The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", pdvers ) file_time = check_neon_time() From 1e930bb071131d39577e2551e049e38a53ba0774 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Tue, 10 Jan 2023 08:14:21 -0700 Subject: [PATCH 111/332] adding reference to new FATES surface files --- .../NEON/FATES/ABBY/include_user_mods | 1 + .../NEON/FATES/ABBY/shell_commands | 3 ++ .../NEON/FATES/BARR/include_user_mods | 1 + .../NEON/FATES/BARR/shell_commands | 11 ++++++ .../NEON/FATES/BART/include_user_mods | 1 + .../NEON/FATES/BART/shell_commands | 3 ++ .../NEON/FATES/BLAN/include_user_mods | 1 + .../NEON/FATES/BLAN/shell_commands | 3 ++ .../NEON/FATES/BONA/include_user_mods | 1 + .../NEON/FATES/BONA/shell_commands | 3 ++ .../NEON/FATES/CLBJ/include_user_mods | 1 + .../NEON/FATES/CLBJ/shell_commands | 3 ++ .../NEON/FATES/CPER/include_user_mods | 1 + .../NEON/FATES/CPER/shell_commands | 6 +++ .../NEON/FATES/DCFS/include_user_mods | 1 + .../NEON/FATES/DCFS/shell_commands | 3 ++ .../NEON/FATES/DEJU/include_user_mods | 1 + .../NEON/FATES/DEJU/shell_commands | 3 ++ .../NEON/FATES/DELA/include_user_mods | 1 + .../NEON/FATES/DELA/shell_commands | 3 ++ .../NEON/FATES/DSNY/include_user_mods | 1 + .../NEON/FATES/DSNY/shell_commands | 3 ++ .../NEON/FATES/GRSM/include_user_mods | 1 + .../NEON/FATES/GRSM/shell_commands | 3 ++ .../NEON/FATES/GUAN/include_user_mods | 1 + .../NEON/FATES/GUAN/shell_commands | 11 ++++++ .../NEON/FATES/HARV/include_user_mods | 1 + .../NEON/FATES/HARV/shell_commands | 3 ++ .../NEON/FATES/HEAL/include_user_mods | 1 + .../NEON/FATES/HEAL/shell_commands | 3 ++ .../NEON/FATES/JERC/include_user_mods | 1 + .../NEON/FATES/JERC/shell_commands | 3 ++ .../NEON/FATES/JORN/include_user_mods | 1 + .../NEON/FATES/JORN/shell_commands | 3 ++ .../NEON/FATES/KONA/include_user_mods | 1 + .../NEON/FATES/KONA/shell_commands | 7 ++++ .../NEON/FATES/KONZ/include_user_mods | 1 + .../NEON/FATES/KONZ/shell_commands | 3 ++ .../NEON/FATES/LAJA/include_user_mods | 1 + .../NEON/FATES/LAJA/shell_commands | 12 ++++++ .../NEON/FATES/LENO/include_user_mods | 1 + .../NEON/FATES/LENO/shell_commands | 12 ++++++ .../NEON/FATES/MLBS/include_user_mods | 1 + .../NEON/FATES/MLBS/shell_commands | 10 +++++ .../NEON/FATES/MOAB/include_user_mods | 1 + .../NEON/FATES/MOAB/shell_commands | 10 +++++ .../NEON/FATES/NIWO/include_user_mods | 1 + .../NEON/FATES/NIWO/shell_commands | 4 ++ .../NEON/FATES/NOGP/include_user_mods | 1 + .../NEON/FATES/NOGP/shell_commands | 3 ++ .../NEON/FATES/OAES/include_user_mods | 1 + .../NEON/FATES/OAES/shell_commands | 3 ++ .../NEON/FATES/ONAQ/include_user_mods | 1 + .../NEON/FATES/ONAQ/shell_commands | 10 +++++ .../NEON/FATES/ORNL/include_user_mods | 1 + .../NEON/FATES/ORNL/shell_commands | 3 ++ .../NEON/FATES/OSBS/include_user_mods | 1 + .../NEON/FATES/OSBS/shell_commands | 3 ++ .../NEON/FATES/PUUM/include_user_mods | 1 + .../NEON/FATES/PUUM/shell_commands | 3 ++ .../NEON/FATES/RMNP/include_user_mods | 1 + .../NEON/FATES/RMNP/shell_commands | 3 ++ .../NEON/FATES/SCBI/include_user_mods | 1 + .../NEON/FATES/SCBI/shell_commands | 3 ++ .../NEON/FATES/SERC/include_user_mods | 1 + .../NEON/FATES/SERC/shell_commands | 3 ++ .../NEON/FATES/SJER/include_user_mods | 1 + .../NEON/FATES/SJER/shell_commands | 11 ++++++ .../NEON/FATES/SOAP/include_user_mods | 1 + .../NEON/FATES/SOAP/shell_commands | 3 ++ .../NEON/FATES/SRER/include_user_mods | 1 + .../NEON/FATES/SRER/shell_commands | 3 ++ .../NEON/FATES/STEI/include_user_mods | 1 + .../NEON/FATES/STEI/shell_commands | 3 ++ .../NEON/FATES/STER/include_user_mods | 1 + .../NEON/FATES/STER/shell_commands | 7 ++++ .../NEON/FATES/TALL/include_user_mods | 1 + .../NEON/FATES/TALL/shell_commands | 3 ++ .../NEON/FATES/TEAK/include_user_mods | 1 + .../NEON/FATES/TEAK/shell_commands | 10 +++++ .../NEON/FATES/TOOL/include_user_mods | 1 + .../NEON/FATES/TOOL/shell_commands | 10 +++++ .../NEON/FATES/TREE/include_user_mods | 1 + .../NEON/FATES/TREE/shell_commands | 3 ++ .../NEON/FATES/UKFS/include_user_mods | 1 + .../NEON/FATES/UKFS/shell_commands | 3 ++ .../NEON/FATES/UNDE/include_user_mods | 1 + .../NEON/FATES/UNDE/shell_commands | 3 ++ .../NEON/FATES/WOOD/include_user_mods | 1 + .../NEON/FATES/WOOD/shell_commands | 3 ++ .../NEON/FATES/WREF/include_user_mods | 1 + .../NEON/FATES/WREF/shell_commands | 10 +++++ .../NEON/FATES/YELL/include_user_mods | 1 + .../NEON/FATES/YELL/shell_commands | 10 +++++ .../NEON/FATES/defaults/shell_commands | 26 +++++++++++++ .../NEON/FATES/defaults/user_nl_clm | 31 +++++++++++++++ .../NEON/FATES/defaults/user_nl_cpl | 20 ++++++++++ .../NEON/FATES/defaults/user_nl_datm_streams | 39 +++++++++++++++++++ 98 files changed, 407 insertions(+) create mode 100644 cime_config/usermods_dirs/NEON/FATES/ABBY/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/ABBY/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/BARR/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/BARR/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/BART/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/BART/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/BLAN/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/BONA/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/BONA/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/CLBJ/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/CLBJ/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/CPER/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/CPER/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/DCFS/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/DCFS/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/DEJU/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/DEJU/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/DELA/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/DELA/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/DSNY/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/DSNY/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/GRSM/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/GRSM/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/GUAN/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/GUAN/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/HARV/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/HARV/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/HEAL/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/HEAL/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/JERC/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/JERC/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/JORN/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/JORN/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/KONZ/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/KONZ/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/LAJA/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/LAJA/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/LENO/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/LENO/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/MLBS/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/MLBS/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/MOAB/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/MOAB/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/NIWO/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/NIWO/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/NOGP/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/NOGP/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/OAES/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/OAES/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/ONAQ/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/ORNL/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/OSBS/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/OSBS/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/PUUM/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/PUUM/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/RMNP/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/RMNP/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/SCBI/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/SCBI/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/SERC/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/SERC/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/SJER/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/SJER/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/SOAP/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/SOAP/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/SRER/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/SRER/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/STEI/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/STEI/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/STER/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/TALL/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/TALL/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/TEAK/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/TEAK/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/TOOL/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/TOOL/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/TREE/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/TREE/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/UKFS/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/UKFS/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/UNDE/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/WOOD/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/WOOD/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/WREF/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/WREF/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/YELL/include_user_mods create mode 100644 cime_config/usermods_dirs/NEON/FATES/YELL/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm create mode 100644 cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_cpl create mode 100644 cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_datm_streams diff --git a/cime_config/usermods_dirs/NEON/FATES/ABBY/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/ABBY/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/ABBY/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/ABBY/shell_commands b/cime_config/usermods_dirs/NEON/FATES/ABBY/shell_commands new file mode 100644 index 0000000000..08f6e7cdef --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/ABBY/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=ABBY +./xmlchange PTS_LON=237.67032799999998 +./xmlchange PTS_LAT=45.762378 diff --git a/cime_config/usermods_dirs/NEON/FATES/BARR/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/BARR/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BARR/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/BARR/shell_commands b/cime_config/usermods_dirs/NEON/FATES/BARR/shell_commands new file mode 100644 index 0000000000..713331c0ee --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BARR/shell_commands @@ -0,0 +1,11 @@ +#!/bin/bash + +./xmlchange NEONSITE=BARR +./xmlchange PTS_LON=203.349781 +./xmlchange PTS_LAT=71.281711 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/BART/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/BART/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BART/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/BART/shell_commands b/cime_config/usermods_dirs/NEON/FATES/BART/shell_commands new file mode 100644 index 0000000000..a4e86a1b8c --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BART/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=BART +./xmlchange PTS_LON=288.71166 +./xmlchange PTS_LAT=44.06516 diff --git a/cime_config/usermods_dirs/NEON/FATES/BLAN/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/BLAN/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BLAN/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands b/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands new file mode 100644 index 0000000000..cb093d806a --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=BLAN +./xmlchange PTS_LON=281.92885 +./xmlchange PTS_LAT=39.06044 diff --git a/cime_config/usermods_dirs/NEON/FATES/BONA/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/BONA/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BONA/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/BONA/shell_commands b/cime_config/usermods_dirs/NEON/FATES/BONA/shell_commands new file mode 100644 index 0000000000..2a66d148b4 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/BONA/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=BONA +./xmlchange PTS_LON=212.49806 +./xmlchange PTS_LAT=65.15333 diff --git a/cime_config/usermods_dirs/NEON/FATES/CLBJ/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/CLBJ/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/CLBJ/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/CLBJ/shell_commands b/cime_config/usermods_dirs/NEON/FATES/CLBJ/shell_commands new file mode 100644 index 0000000000..c1b9154027 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/CLBJ/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=CLBJ +./xmlchange PTS_LON=262.43275 +./xmlchange PTS_LAT=33.40143 diff --git a/cime_config/usermods_dirs/NEON/FATES/CPER/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/CPER/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/CPER/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/CPER/shell_commands b/cime_config/usermods_dirs/NEON/FATES/CPER/shell_commands new file mode 100644 index 0000000000..b6ccbcdf57 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/CPER/shell_commands @@ -0,0 +1,6 @@ +./xmlchange NEONSITE=CPER +./xmlchange PTS_LON=255.25545 +./xmlchange PTS_LAT=40.81297 +if [[ $compset =~ ^HIST ]]; then + ./xmlchange STOP_N=50 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/DCFS/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/DCFS/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DCFS/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/DCFS/shell_commands b/cime_config/usermods_dirs/NEON/FATES/DCFS/shell_commands new file mode 100644 index 0000000000..a6cbed64e1 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DCFS/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=DCFS +./xmlchange PTS_LON=260.88749 +./xmlchange PTS_LAT=47.15919 diff --git a/cime_config/usermods_dirs/NEON/FATES/DEJU/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/DEJU/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DEJU/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/DEJU/shell_commands b/cime_config/usermods_dirs/NEON/FATES/DEJU/shell_commands new file mode 100644 index 0000000000..fce519d559 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DEJU/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=DEJU +./xmlchange PTS_LON=214.25235 +./xmlchange PTS_LAT=63.87983 diff --git a/cime_config/usermods_dirs/NEON/FATES/DELA/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/DELA/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DELA/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/DELA/shell_commands b/cime_config/usermods_dirs/NEON/FATES/DELA/shell_commands new file mode 100644 index 0000000000..f3acbb8fd3 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DELA/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=DELA +./xmlchange PTS_LON=272.19659 +./xmlchange PTS_LAT=32.54092 diff --git a/cime_config/usermods_dirs/NEON/FATES/DSNY/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/DSNY/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DSNY/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/DSNY/shell_commands b/cime_config/usermods_dirs/NEON/FATES/DSNY/shell_commands new file mode 100644 index 0000000000..8304c91d48 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/DSNY/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=DSNY +./xmlchange PTS_LON=278.56606 +./xmlchange PTS_LAT=28.12919 diff --git a/cime_config/usermods_dirs/NEON/FATES/GRSM/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/GRSM/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/GRSM/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/GRSM/shell_commands b/cime_config/usermods_dirs/NEON/FATES/GRSM/shell_commands new file mode 100644 index 0000000000..e52a633408 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/GRSM/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=GRSM +./xmlchange PTS_LON=276.49815 +./xmlchange PTS_LAT=35.68839 diff --git a/cime_config/usermods_dirs/NEON/FATES/GUAN/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/GUAN/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/GUAN/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/GUAN/shell_commands b/cime_config/usermods_dirs/NEON/FATES/GUAN/shell_commands new file mode 100644 index 0000000000..4214a35e3c --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/GUAN/shell_commands @@ -0,0 +1,11 @@ +#!/bin/bash + +./xmlchange NEONSITE=GUAN +./xmlchange PTS_LON=293.13112 +./xmlchange PTS_LAT=17.96882 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/HARV/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/HARV/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/HARV/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/HARV/shell_commands b/cime_config/usermods_dirs/NEON/FATES/HARV/shell_commands new file mode 100644 index 0000000000..839ccf5d8f --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/HARV/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=HARV +./xmlchange PTS_LON=287.82438 +./xmlchange PTS_LAT=42.53562 diff --git a/cime_config/usermods_dirs/NEON/FATES/HEAL/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/HEAL/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/HEAL/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/HEAL/shell_commands b/cime_config/usermods_dirs/NEON/FATES/HEAL/shell_commands new file mode 100644 index 0000000000..21892219e0 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/HEAL/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=HEAL +./xmlchange PTS_LON=210.78461 +./xmlchange PTS_LAT=63.8798 diff --git a/cime_config/usermods_dirs/NEON/FATES/JERC/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/JERC/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/JERC/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/JERC/shell_commands b/cime_config/usermods_dirs/NEON/FATES/JERC/shell_commands new file mode 100644 index 0000000000..80f66d23a2 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/JERC/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=JERC +./xmlchange PTS_LON=275.53353 +./xmlchange PTS_LAT=31.19608 diff --git a/cime_config/usermods_dirs/NEON/FATES/JORN/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/JORN/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/JORN/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/JORN/shell_commands b/cime_config/usermods_dirs/NEON/FATES/JORN/shell_commands new file mode 100644 index 0000000000..87fc3b8c1e --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/JORN/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=JORN +./xmlchange PTS_LON=253.15623 +./xmlchange PTS_LAT=32.59052 diff --git a/cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands b/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands new file mode 100644 index 0000000000..66f274dd36 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands @@ -0,0 +1,7 @@ +./xmlchange NEONSITE=KONA +./xmlchange PTS_LON=263.38956 +./xmlchange PTS_LAT=39.10828 +# Setup to run with prognostic crops for this site +# If you want to explicitly run in SP mode or add other +# options you'll need to add that after this... +./xmlchange CLM_BLDNML_OPTS="--bgc bgc --crop" diff --git a/cime_config/usermods_dirs/NEON/FATES/KONZ/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/KONZ/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/KONZ/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/KONZ/shell_commands b/cime_config/usermods_dirs/NEON/FATES/KONZ/shell_commands new file mode 100644 index 0000000000..bda370c170 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/KONZ/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=KONZ +./xmlchange PTS_LON=263.43773 +./xmlchange PTS_LAT=39.1007 diff --git a/cime_config/usermods_dirs/NEON/FATES/LAJA/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/LAJA/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/LAJA/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/LAJA/shell_commands b/cime_config/usermods_dirs/NEON/FATES/LAJA/shell_commands new file mode 100644 index 0000000000..a7bda447e6 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/LAJA/shell_commands @@ -0,0 +1,12 @@ +#!/bin/bash +./xmlchange NEONSITE=LAJA +./xmlchange PTS_LON=292.92392 +./xmlchange PTS_LAT=18.02184 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi + + diff --git a/cime_config/usermods_dirs/NEON/FATES/LENO/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/LENO/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/LENO/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/LENO/shell_commands b/cime_config/usermods_dirs/NEON/FATES/LENO/shell_commands new file mode 100644 index 0000000000..c5ae590186 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/LENO/shell_commands @@ -0,0 +1,12 @@ +#!/bin/bash + +./xmlchange NEONSITE=LENO +./xmlchange PTS_LON=271.83897 +./xmlchange PTS_LAT=31.8531 +./xmlchange DATM_YR_ALIGN=2021,DATM_YR_START=2021 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2021-01-01 + ./xmlchange STOP_N=15 +fi + diff --git a/cime_config/usermods_dirs/NEON/FATES/MLBS/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/MLBS/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/MLBS/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/FATES/MLBS/shell_commands new file mode 100644 index 0000000000..d5de0f64eb --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/MLBS/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=MLBS +./xmlchange PTS_LON=279.47575 +./xmlchange PTS_LAT=37.37783 +./xmlchange DATM_YR_END=2019 +# Different default number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2020 + ./xmlchange STOP_N=24 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/MOAB/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/MOAB/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/MOAB/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/FATES/MOAB/shell_commands new file mode 100644 index 0000000000..96d0bcbe68 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/MOAB/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=MOAB +./xmlchange PTS_LON=250.61118 +./xmlchange PTS_LAT=38.25136 +./xmlchange DATM_YR_END=2020 +# Different default number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2021 + ./xmlchange STOP_N=36 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/NIWO/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/NIWO/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/NIWO/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/FATES/NIWO/shell_commands new file mode 100644 index 0000000000..a3e73ca343 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/NIWO/shell_commands @@ -0,0 +1,4 @@ +./xmlchange NEONSITE=NIWO +./xmlchange PTS_LON=254.41676 +./xmlchange PTS_LAT=40.05236 +./xmlchange DATM_YR_END=2018 diff --git a/cime_config/usermods_dirs/NEON/FATES/NOGP/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/NOGP/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/NOGP/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/NOGP/shell_commands b/cime_config/usermods_dirs/NEON/FATES/NOGP/shell_commands new file mode 100644 index 0000000000..ad3ef69cd2 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/NOGP/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=NOGP +./xmlchange PTS_LON=259.08168 +./xmlchange PTS_LAT=46.76846 diff --git a/cime_config/usermods_dirs/NEON/FATES/OAES/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/OAES/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/OAES/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/OAES/shell_commands b/cime_config/usermods_dirs/NEON/FATES/OAES/shell_commands new file mode 100644 index 0000000000..2a5cfb87e4 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/OAES/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=OAES +./xmlchange PTS_LON=260.93956000000003 +./xmlchange PTS_LAT=35.41062 diff --git a/cime_config/usermods_dirs/NEON/FATES/ONAQ/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/ONAQ/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/ONAQ/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands new file mode 100644 index 0000000000..43dab69998 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=ONAQ +./xmlchange PTS_LON=276.49815 +./xmlchange PTS_LAT=35.68839 +./xmlchange DATM_YR_END=2019 +# Different default number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2020 + ./xmlchange STOP_N=24 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/ORNL/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/ORNL/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/ORNL/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands b/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands new file mode 100644 index 0000000000..264d451753 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=ORNL +./xmlchange PTS_LON=275.83419000000004 +./xmlchange PTS_LAT=35.57525 diff --git a/cime_config/usermods_dirs/NEON/FATES/OSBS/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/OSBS/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/OSBS/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/OSBS/shell_commands b/cime_config/usermods_dirs/NEON/FATES/OSBS/shell_commands new file mode 100644 index 0000000000..385021f98a --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/OSBS/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=OSBS +./xmlchange PTS_LON=278.00655 +./xmlchange PTS_LAT=29.68819 diff --git a/cime_config/usermods_dirs/NEON/FATES/PUUM/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/PUUM/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/PUUM/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/PUUM/shell_commands b/cime_config/usermods_dirs/NEON/FATES/PUUM/shell_commands new file mode 100644 index 0000000000..07c4331769 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/PUUM/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=PUUM +./xmlchange PTS_LON=204.68269 +./xmlchange PTS_LAT=19.55309 diff --git a/cime_config/usermods_dirs/NEON/FATES/RMNP/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/RMNP/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/RMNP/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/RMNP/shell_commands b/cime_config/usermods_dirs/NEON/FATES/RMNP/shell_commands new file mode 100644 index 0000000000..8dfbf0fa0d --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/RMNP/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=RMNP +./xmlchange PTS_LON=254.45476 +./xmlchange PTS_LAT=40.27707 diff --git a/cime_config/usermods_dirs/NEON/FATES/SCBI/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/SCBI/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SCBI/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/SCBI/shell_commands b/cime_config/usermods_dirs/NEON/FATES/SCBI/shell_commands new file mode 100644 index 0000000000..aa42b8022c --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SCBI/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=SCBI +./xmlchange PTS_LON=281.86235999999997 +./xmlchange PTS_LAT=38.89209 diff --git a/cime_config/usermods_dirs/NEON/FATES/SERC/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/SERC/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SERC/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/SERC/shell_commands b/cime_config/usermods_dirs/NEON/FATES/SERC/shell_commands new file mode 100644 index 0000000000..1053e2dc17 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SERC/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=SERC +./xmlchange PTS_LON=283.44115999999997 +./xmlchange PTS_LAT=38.89124 diff --git a/cime_config/usermods_dirs/NEON/FATES/SJER/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/SJER/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SJER/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/SJER/shell_commands b/cime_config/usermods_dirs/NEON/FATES/SJER/shell_commands new file mode 100644 index 0000000000..6e05d23792 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SJER/shell_commands @@ -0,0 +1,11 @@ +#!/bin/bash +./xmlchange NEONSITE=SJER +./xmlchange PTS_LON=240.267 +./xmlchange PTS_LAT=37.107117 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi + diff --git a/cime_config/usermods_dirs/NEON/FATES/SOAP/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/SOAP/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SOAP/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/SOAP/shell_commands b/cime_config/usermods_dirs/NEON/FATES/SOAP/shell_commands new file mode 100644 index 0000000000..c10274c047 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SOAP/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=SOAP +./xmlchange PTS_LON=240.7379 +./xmlchange PTS_LAT=37.03269 diff --git a/cime_config/usermods_dirs/NEON/FATES/SRER/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/SRER/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SRER/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/SRER/shell_commands b/cime_config/usermods_dirs/NEON/FATES/SRER/shell_commands new file mode 100644 index 0000000000..be1bec52d3 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/SRER/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=SRER +./xmlchange PTS_LON=249.16451 +./xmlchange PTS_LAT=31.91068 diff --git a/cime_config/usermods_dirs/NEON/FATES/STEI/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/STEI/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/STEI/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/STEI/shell_commands b/cime_config/usermods_dirs/NEON/FATES/STEI/shell_commands new file mode 100644 index 0000000000..c2aced2c2e --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/STEI/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=STEI +./xmlchange PTS_LON=270.4112 +./xmlchange PTS_LAT=45.5076 diff --git a/cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands b/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands new file mode 100644 index 0000000000..38b173c309 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands @@ -0,0 +1,7 @@ +./xmlchange NEONSITE=STER +./xmlchange PTS_LON=256.96992 +./xmlchange PTS_LAT=40.45984 +# Setup to run with prognostic crops for this site +# If you want to explicitly run in SP mode or add other +# # options you'll need to add that after this... +./xmlchange CLM_BLDNML_OPTS="--bgc bgc --crop" diff --git a/cime_config/usermods_dirs/NEON/FATES/TALL/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/TALL/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TALL/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/TALL/shell_commands b/cime_config/usermods_dirs/NEON/FATES/TALL/shell_commands new file mode 100644 index 0000000000..1a176ae23f --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TALL/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=TALL +./xmlchange PTS_LON=272.6059 +./xmlchange PTS_LAT=32.95106 diff --git a/cime_config/usermods_dirs/NEON/FATES/TEAK/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/TEAK/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TEAK/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/FATES/TEAK/shell_commands new file mode 100644 index 0000000000..9058eda022 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TEAK/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=TEAK +./xmlchange PTS_LON=240.99424199999999 +./xmlchange PTS_LAT=37.006472 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/TOOL/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/TOOL/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TOOL/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/TOOL/shell_commands b/cime_config/usermods_dirs/NEON/FATES/TOOL/shell_commands new file mode 100644 index 0000000000..f1f2e1771a --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TOOL/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=TOOL +./xmlchange PTS_LON=210.629872 +./xmlchange PTS_LAT=68.66045 +./xmlchange DATM_YR_ALIGN=2020,DATM_YR_START=2020 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2020-01-01 + ./xmlchange STOP_N=27 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/TREE/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/TREE/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TREE/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/TREE/shell_commands b/cime_config/usermods_dirs/NEON/FATES/TREE/shell_commands new file mode 100644 index 0000000000..6d0a4aa1fa --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/TREE/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=TREE +./xmlchange PTS_LON=270.41252 +./xmlchange PTS_LAT=45.49266 diff --git a/cime_config/usermods_dirs/NEON/FATES/UKFS/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/UKFS/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/UKFS/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/UKFS/shell_commands b/cime_config/usermods_dirs/NEON/FATES/UKFS/shell_commands new file mode 100644 index 0000000000..7c8d4f8829 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/UKFS/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=UKFS +./xmlchange PTS_LON=264.79505 +./xmlchange PTS_LAT=39.04168 diff --git a/cime_config/usermods_dirs/NEON/FATES/UNDE/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/UNDE/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/UNDE/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands b/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands new file mode 100644 index 0000000000..79688e0a8f --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=UNDE +./xmlchange PTS_LON=270.6779 +./xmlchange PTS_LAT=46.14103 diff --git a/cime_config/usermods_dirs/NEON/FATES/WOOD/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/WOOD/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/WOOD/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/WOOD/shell_commands b/cime_config/usermods_dirs/NEON/FATES/WOOD/shell_commands new file mode 100644 index 0000000000..48ff0ef999 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/WOOD/shell_commands @@ -0,0 +1,3 @@ +./xmlchange NEONSITE=WOOD +./xmlchange PTS_LON=260.76093000000003 +./xmlchange PTS_LAT=47.12833 diff --git a/cime_config/usermods_dirs/NEON/FATES/WREF/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/WREF/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/WREF/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/WREF/shell_commands b/cime_config/usermods_dirs/NEON/FATES/WREF/shell_commands new file mode 100644 index 0000000000..807055ae6e --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/WREF/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=WREF +./xmlchange PTS_LON=238.04162 +./xmlchange PTS_LAT=45.81637 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/YELL/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/YELL/include_user_mods new file mode 100644 index 0000000000..b152996d95 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/YELL/include_user_mods @@ -0,0 +1 @@ +../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/YELL/shell_commands b/cime_config/usermods_dirs/NEON/FATES/YELL/shell_commands new file mode 100644 index 0000000000..800d7d01ad --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/YELL/shell_commands @@ -0,0 +1,10 @@ +#!/bin/bash +./xmlchange NEONSITE=YELL +./xmlchange PTS_LON=249.45803999999998 +./xmlchange PTS_LAT=44.95597 +./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +# Different default start date and number of months to run for transient case +if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 +fi diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands new file mode 100644 index 0000000000..7095e1def7 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands @@ -0,0 +1,26 @@ +#!/bin/bash +./xmlchange CLM_USRDAT_NAME=NEON +./xmlchange CCSM_CO2_PPMV=408.83 +# Set data forcing data to future scenario so will have data from 2018 to present-day +./xmlchange DATM_PRESAERO=SSP3-7.0 +./xmlchange DATM_PRESNDEP=SSP3-7.0 +./xmlchange DATM_PRESO3=SSP3-7.0 +# Explicitly set the MPI library to mpi-serial so won't have the build/run complexity of a full MPI library +./xmlchange MPILIB=mpi-serial +# Set years to run forcing data over +./xmlchange DATM_YR_ALIGN=2018,DATM_YR_END=2021,DATM_YR_START=2018 +compset=`./xmlquery COMPSET --value` +# For a transient case run the whole length and don't cycle +if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2022 + ./xmlchange RUN_STARTDATE=2018-01-01 + # Number of months that can be run for the full transient case + ./xmlchange STOP_OPTION="nmonths" + ./xmlchange STOP_N=51 + ./xmlchange CLM_NML_USE_CASE="2018-PD_transient" +else + ./xmlchange CLM_NML_USE_CASE="2018_control" +fi + +# Explicitly set PIO Type to NETCDF since this is a single processor case (should already be set this way) +./xmlchange PIO_TYPENAME=netcdf diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm new file mode 100644 index 0000000000..688e651382 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm @@ -0,0 +1,31 @@ +!---------------------------------------------------------------------------------- +! Users should add all user specific namelist changes below in the form of +! namelist_var = new_namelist_value +! +! EXCEPTIONS: +! Set use_cndv by the compset you use and the CLM_BLDNML_OPTS -dynamic_vegetation setting +! Set use_vichydro by the compset you use and the CLM_BLDNML_OPTS -vichydro setting +! Set use_cn by the compset you use and CLM_BLDNML_OPTS -bgc setting +! Set use_crop by the compset you use and CLM_BLDNML_OPTS -crop setting +! Set spinup_state by the CLM_BLDNML_OPTS -bgc_spinup setting +! Set co2_ppmv with CCSM_CO2_PPMV option +! Set fatmlndfrc with LND_DOMAIN_PATH/LND_DOMAIN_FILE options +! Set finidat with RUN_REFCASE/RUN_REFDATE/RUN_REFTOD options for hybrid or branch cases +! (includes $inst_string for multi-ensemble cases) +! or with CLM_FORCE_COLDSTART to do a cold start +! or set it with an explicit filename here. +! Set maxpatch_glcmec with GLC_NEC option +! Set glc_do_dynglacier with GLC_TWO_WAY_COUPLING env variable +!---------------------------------------------------------------------------------- + +flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/FATES/surfdata_1x1_NEON_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230110.nc" + +stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' +stream_meshfile_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/ESMF_MESH.Li_2016.360x720.NEONarea_cdf5_c221104.nc' + +! h1 output stream +hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI', + 'HR','SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FV','NET_NMIN_vr' +hist_mfilt(2) = 48 +hist_nhtfrq(2) = 1 diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_cpl b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_cpl new file mode 100644 index 0000000000..e7f6c90a86 --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_cpl @@ -0,0 +1,20 @@ +!------------------------------------------------------------------------ +! Users should ONLY USE user_nl_cpl to change namelists variables +! for namelist variables in drv_in (except for the ones below) and +! any keyword/values in seq_maps.rc +! Users should add ALL user specific namelist and seq_maps.rc changes below +! using the following syntax +! namelist_var = new_namelist_value +! or +! mapname = new_map_name +! For example to change the default value of ocn2atm_fmapname to 'foo' use +! ocn2atm_fmapname = 'foo' +! +! Note that some namelist variables MAY NOT be changed in user_nl_cpl - +! they are defined in a $CASEROOT xml file and must be changed with +! xmlchange. +! +! For example, rather than set username to 'foo' in user_nl_cpl, call +! ./xmlchange USER=foo +!------------------------------------------------------------------------ +orb_iyear = 2018 diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_datm_streams new file mode 100644 index 0000000000..36f1e72b3a --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_datm_streams @@ -0,0 +1,39 @@ +!------------------------------------------------------------------------ +! This file is used to modify datm.streams.xml generated in $RUNDIR +! Entries should have the form +! :<= new stream_value> +! The following are accepted values for an assume streamname of foo +! foo:meshfile = character string +! foo:datafiles = comma separated string of full pathnames (e.g. file1,file2,file3...) +! foo:datavars = comma separated string of field pairs (e.g. foo foobar,foo2 foobar2...) +! foo:taxmode = one of [cycle, extend, limit] +! foo:tintalgo = one of [lower,upper,nearest,linear,coszen] +! foo:readmode = single (only suported mode right now) +! foo:mapalgo = one of [bilinear,redist,nn,consf,consd,none] +! foo:dtlimit = real (1.5 is default) +! foo:year_first = integer +! foo:year_last = integer +! foo:year_align = integer +! foo:vectors = one of [none,u:v] +! foo:lev_dimname: = one of [null,name of level dimenion name] +! foo:offset = integer +! As an example: +! foo:year_first = 1950 +! would change the stream year_first stream_entry to 1950 for the foo stream block +!------------------------------------------------------------------------ +presaero.SSP3-7.0:datafiles = $DIN_LOC_ROOT/atm/cam/chem/trop_mozart_aero/aero/aerodep_clm_SSP370_b.e21.BWSSP370cmip6.f09_g17.CMIP6-SSP3-7.0-WACCM.001_2018-2030_monthly_0.9x1.25_c210826.nc +presaero.SSP3-7.0:year_first=2018 +presaero.SSP3-7.0:year_last=2022 +presaero.SSP3-7.0:year_align=2018 +presaero.SSP3-7.0:dtlimit=30 + +presndep.SSP3-7.0:datafiles = $DIN_LOC_ROOT/lnd/clm2/ndepdata/fndep_clm_f09_g17.CMIP6-SSP3-7.0-WACCM_2018-2030_monthly_c210826.nc +presndep.SSP3-7.0:year_first=2018 +presndep.SSP3-7.0:year_last=2022 +presndep.SSP3-7.0:year_align=2018 +presndep.SSP3-7.0:dtlimit=30 + +preso3.SSP3-7.0:year_first=2018 +preso3.SSP3-7.0:year_last=2022 +preso3.SSP3-7.0:year_align=2018 +preso3.SSP3-7.0:dtlimit=30 From ee011faf27a382f9c7f2a5c6f664ff2f2638b68b Mon Sep 17 00:00:00 2001 From: adrifoster Date: Tue, 10 Jan 2023 08:22:31 -0700 Subject: [PATCH 112/332] create fates-specific calls --- tools/site_and_regional/neon_surf_wrapper.py | 28 +++-- .../neon_surf_wrapper_fates.py | 110 ------------------ 2 files changed, 20 insertions(+), 118 deletions(-) delete mode 100755 tools/site_and_regional/neon_surf_wrapper_fates.py diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index e02ee2bd53..d8e11415ff 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -33,7 +33,7 @@ import subprocess import pandas as pd -#import tqdm as tqdm + @@ -52,6 +52,12 @@ def get_parser(): dest="verbose", default=False) + parser.add_argument('-f', '--fates', + help='Create and/or modify surface data file for FATES. ', + action="store_true", + dest="fates", + default=False) + return parser @@ -98,13 +104,19 @@ def main(): pft = row['pft'] clmsite = "1x1_NEON_"+site print ("Now processing site :", site) - command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite,'--dompft',str(pft),'--crop', - '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - execute(command) - - command = ['./modify_singlept_site_neon.py','--neon_site',site, '--surf_dir', - 'subset_data_single_point'] - execute(command) + if args.fates: + + command1 = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite, + '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] + command2 = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + 'subset_data_single_point', '--fates'] + else: + command1 = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite,'--dompft',str(pft),'--crop', + '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] + command2 = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + 'subset_data_single_point'] + execute(command1) + execute(command2) if __name__ == "__main__": main() diff --git a/tools/site_and_regional/neon_surf_wrapper_fates.py b/tools/site_and_regional/neon_surf_wrapper_fates.py deleted file mode 100755 index 60003b857e..0000000000 --- a/tools/site_and_regional/neon_surf_wrapper_fates.py +++ /dev/null @@ -1,110 +0,0 @@ -#! /usr/bin/env python3 -""" -|------------------------------------------------------------------| -|--------------------- Instructions -----------------------------| -|------------------------------------------------------------------| -This script is a simple wrapper for neon sites that performs the -following: - 1) For neon sites, subset surface dataset from global dataset - (i.e. ./subset_data.py ) - 2) Download neon and update the created surface dataset - based on the downloaded neon data. - (i.e. modify_singlept_site_neon.py) - -Instructions for running using conda python environments: - -../../py_env_create -conda activate ctsm_py - -""" -# TODO -# Automatic downloading of missing files if they are missing -#-[ ] Download neon sites and dom pft file -#-[ ] Make sure verbose works for printing out commands running - -# Import libraries -from __future__ import print_function - -import os -import sys -import tqdm -import logging -import argparse -import subprocess - -import pandas as pd -#import tqdm as tqdm - - - -def get_parser(): - """ - Get parser object for this script. - """ - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - - parser.print_usage = parser.print_help - - parser.add_argument('-v','--verbose', - help='Verbose mode will print more information. ', - action="store_true", - dest="verbose", - default=False) - - - return parser - - -def execute(command): - """ - Function for running a command on shell. - Args: - command (str): - command that we want to run. - Raises: - Error with the return code from shell. - """ - print ('\n',' >> ',*command,'\n') - - try: - subprocess.check_call(command, stdout=open(os.devnull, "w"), stderr=subprocess.STDOUT) - - except subprocess.CalledProcessError as e: - #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) - #print (e.ouput) - print (e) - - - - - - -def main(): - - args = get_parser().parse_args() - - if args.verbose: - logging.basicConfig(level=logging.DEBUG) - - - neon_sites = pd.read_csv('neon_sites_dompft.csv') - - - for i, row in tqdm.tqdm(neon_sites.iterrows()): - lat = row['Lat'] - lon = row['Lon'] - site = row['Site'] - pft = row['pft'] - clmsite = "1x1_NEON_16PFT_"+site - print ("Now processing site :", site) - command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite, - '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - execute(command) - - command = ['./modify_singlept_site_neon.py','--neon_site',site, '--surf_dir', - 'subset_data_single_point'] - execute(command) - -if __name__ == "__main__": - main() From f1137fd74348359ff70c7952f550b1812d3e6217 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Tue, 10 Jan 2023 08:27:18 -0700 Subject: [PATCH 113/332] create fates-specific calls --- .../modify_singlept_site_neon.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index eabf5930f5..49f10acc2f 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -174,6 +174,15 @@ def get_parser(): default=False, ) + parser.add_argument( + "-f", + "--fates", + help="Modify FATES-specific surface data files (i.e. 16-PFT version", + action="store_true", + dest="fates", + default=False, + ) + return parser @@ -241,7 +250,7 @@ def get_neon(neon_dir, site_name): return neon_file -def find_surffile(surf_dir, site_name): +def find_surffile(surf_dir, site_name, fates): """ Function for finding and choosing surface file for a neon site. @@ -252,6 +261,7 @@ def find_surffile(surf_dir, site_name): Args: surf_dir (str): directory of single point surface data site_name (str): 4 letter neon site name + fates (bool): if true, use 16-PFT version of surface data file Raises: Error if the surface data for the site is not created @@ -260,7 +270,11 @@ def find_surffile(surf_dir, site_name): surf_file (str): name of the surface dataset file """ - sf_name = "surfdata_1x1_NEON_16PFT_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" + if fates: + sf_name = "surfdata_1x1_NEON_16PFT_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" + else: + sf_name = "surfdata_1x1_NEON_" + site_name + "*hist_78pfts_CMIP6_simyr2000_*.nc" + print (os.path.join(surf_dir , sf_name)) surf_file = sorted(glob.glob(os.path.join(surf_dir , sf_name))) @@ -533,12 +547,12 @@ def main(): # -- Look for surface data surf_dir = args.surf_dir - surf_file = find_surffile(surf_dir, site_name) + surf_file = find_surffile(surf_dir, site_name, args.fates) # -- directory structure current_dir = os.getcwd() parent_dir = os.path.dirname(current_dir) - clone_dir = os.path.abspath(os.path.join(__file__, "../../..")) + clone_dir = os.path.abspath(os.path.join(__file__, "../../../..")) neon_dir = os.path.join(clone_dir, "neon_surffiles") print("Present Directory", current_dir) From 057abd1311e825176a86984371e04db510ff1d8f Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Tue, 10 Jan 2023 09:35:29 -0700 Subject: [PATCH 114/332] update surface data name --- cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm index 688e651382..eac74ea00a 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm @@ -19,10 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/FATES/surfdata_1x1_NEON_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230110.nc" - -stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' -stream_meshfile_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/ESMF_MESH.Li_2016.360x720.NEONarea_cdf5_c221104.nc' +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/FATES/surfdata_1x1_NEON_16PFT_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230110.nc" ! h1 output stream hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI', From 187ef47ab2fc4971bed08ac0c4f94e16f22757a2 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Tue, 10 Jan 2023 10:26:54 -0700 Subject: [PATCH 115/332] change history names --- cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm index eac74ea00a..c842546ec7 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm @@ -22,7 +22,7 @@ flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but wi fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/FATES/surfdata_1x1_NEON_16PFT_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230110.nc" ! h1 output stream -hist_fincl2 = 'AR','ELAI','FCEV','FCTR','FGEV','FIRA','FSA','FSH','GPP','H2OSOI', - 'HR','SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FV','NET_NMIN_vr' +hist_fincl2 = 'FATES_AUTORESP','FCEV','FCTR','FGEV','FIRA','FSA','FSH','FATES_GPP','FATES_GPP_PF','H2OSOI', + 'SNOW_DEPTH','TBOT','TSOI','SOILC_vr','FATES_NPP','FATES_NPP_PF','FATES_VEGC','FATES_VEGC_PF' hist_mfilt(2) = 48 hist_nhtfrq(2) = 1 From 815ed9cd6e26fba1f730d8db0ef38e0d96089154 Mon Sep 17 00:00:00 2001 From: will wieder Date: Tue, 10 Jan 2023 20:05:55 -0700 Subject: [PATCH 116/332] Update neon_sites_dompft.csv --- tools/site_and_regional/neon_sites_dompft.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/neon_sites_dompft.csv b/tools/site_and_regional/neon_sites_dompft.csv index e67d67375b..dce0a6283d 100644 --- a/tools/site_and_regional/neon_sites_dompft.csv +++ b/tools/site_and_regional/neon_sites_dompft.csv @@ -34,7 +34,7 @@ 33,NIWO,13, 40.05236, -105.58324,12,2018,2021 34,JORN,14, 32.59052, -106.84377,14,2018,2021 35,SRER,14, 31.91068, -110.83549,9,2018,2021 -36,ONAQ,15, 35.68839, -83.50185,9,2018,2019 +36,ONAQ,15, 40.17760, -112.45245,9,2018,2019 37,ABBY,16,45.762378,-122.329672,1,2018,2021 38,WREF,16, 45.81637, -121.95838,1,2019,2021 39,SJER,17,37.107117, -119.73300,13,2019,2021 From 490fd344337258e754ae9f77b2cd598cd27ac494 Mon Sep 17 00:00:00 2001 From: will wieder Date: Tue, 10 Jan 2023 21:31:54 -0700 Subject: [PATCH 117/332] Update usermods shell commands --- cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 43dab69998..511813398b 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -1,7 +1,7 @@ #!/bin/bash ./xmlchange NEONSITE=ONAQ -./xmlchange PTS_LON=276.49815 -./xmlchange PTS_LAT=35.68839 +./xmlchange PTS_LON=247.54755 +./xmlchange PTS_LAT=40.17760 ./xmlchange DATM_YR_END=2019 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then From a374044a1a7cf191e53aae876cc9274332d13606 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 11 Jan 2023 07:46:38 -0700 Subject: [PATCH 118/332] Update user_nl_clm only needed for ONAQ, but done for all sites for consistency --- cime_config/usermods_dirs/NEON/defaults/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm index 29e50431ce..ddf627b04e 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_1x1_NEON_${NEONSITE}_hist_78pfts_CMIP6_simyr2000_c221111.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/surfdata_1x1_NEON_${NEONSITE}_hist_78pfts_CMIP6_simyr2000_c230111.nc" stream_fldfilename_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/clmforc.Li_2016_climo1995-2013.360x720.lnfm_Total_NEONarea_c210625.nc' stream_meshfile_lightng = '$DIN_LOC_ROOT/atm/datm7/NASA_LIS/ESMF_MESH.Li_2016.360x720.NEONarea_cdf5_c221104.nc' From 314191624b6d709a0a009b80b73483bc2ce1750b Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 11 Jan 2023 09:50:18 -0700 Subject: [PATCH 119/332] Update neon_sites_dompft.csv additional lat-lon changes to BLAN, UNDE, ORNL --- tools/site_and_regional/neon_sites_dompft.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/site_and_regional/neon_sites_dompft.csv b/tools/site_and_regional/neon_sites_dompft.csv index dce0a6283d..8fb9600f0f 100644 --- a/tools/site_and_regional/neon_sites_dompft.csv +++ b/tools/site_and_regional/neon_sites_dompft.csv @@ -1,7 +1,7 @@ ,Site,Domain,Lat,Lon,pft,start_year,end_year 1,BART, 1, 44.06516, -71.28834,7,2018,2021 2,HARV, 1, 42.53562, -72.17562,7,2018,2021 -3,BLAN, 2, 39.06044, -78.07115,7,2018,2021 +3,BLAN, 2, 39.033698, -78.041788,7,2018,2021 4,SCBI, 2, 38.89209, -78.13764,7,2018,2021 5,SERC, 2, 38.89124, -76.55884,7,2018,2021 6,DSNY, 3, 28.12919, -81.43394,14,2018,2021 @@ -11,13 +11,13 @@ 10,LAJA, 4, 18.02184, -67.07608,14,2019,2021 11,STEI, 5, 45.50760, -89.58880,7,2018,2021 12,TREE, 5, 45.49266, -89.58748,7,2018,2021 -13,UNDE, 5, 46.14103, -89.32210,7,2018,2021 +13,UNDE, 5, 46.23391, -89.537254,7,2018,2021 14,KONA, 6, 39.10828, -96.61044,19,2018,2021 15,KONZ, 6, 39.10070, -96.56227,14,2018,2021 16,UKFS, 6, 39.04168, -95.20495,7,2018,2021 17,GRSM, 7, 35.68839, -83.50185,7,2018,2021 18,MLBS, 7, 37.37783, -80.52425,7,2018,2019 -19,ORNL, 7, 35.57525, -84.16581,7,2018,2021 +19,ORNL, 7, 35.964128, -84.282588,7,2018,2021 20,DELA, 8, 32.54092, -87.80341,7,2018,2021 21,LENO, 8, 31.85310, -88.16103,7,2021,2021 22,TALL, 8, 32.95106, -87.39410,1,2018,2021 From c6bce1e4485634838c9c6085e9158850ff2f8bd0 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 11 Jan 2023 09:51:49 -0700 Subject: [PATCH 120/332] Update BLAN shell commands --- cime_config/usermods_dirs/NEON/BLAN/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/BLAN/shell_commands b/cime_config/usermods_dirs/NEON/BLAN/shell_commands index cb093d806a..0508481981 100644 --- a/cime_config/usermods_dirs/NEON/BLAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/BLAN/shell_commands @@ -1,3 +1,3 @@ ./xmlchange NEONSITE=BLAN -./xmlchange PTS_LON=281.92885 -./xmlchange PTS_LAT=39.06044 +./xmlchange PTS_LON=281.958212 +./xmlchange PTS_LAT=39.033698 From 09cfaa659347186a5e5b49b9fcaf4c56c9fa094f Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 11 Jan 2023 09:52:52 -0700 Subject: [PATCH 121/332] Update ORNL shell commands --- cime_config/usermods_dirs/NEON/ORNL/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/ORNL/shell_commands b/cime_config/usermods_dirs/NEON/ORNL/shell_commands index 264d451753..5708d3dec5 100644 --- a/cime_config/usermods_dirs/NEON/ORNL/shell_commands +++ b/cime_config/usermods_dirs/NEON/ORNL/shell_commands @@ -1,3 +1,3 @@ ./xmlchange NEONSITE=ORNL -./xmlchange PTS_LON=275.83419000000004 -./xmlchange PTS_LAT=35.57525 +./xmlchange PTS_LON=275.717412 +./xmlchange PTS_LAT=35.964128 From dc101f305c27395eea8993166199f24ad69a1d63 Mon Sep 17 00:00:00 2001 From: will wieder Date: Wed, 11 Jan 2023 09:53:40 -0700 Subject: [PATCH 122/332] Update UNDE shell commands --- cime_config/usermods_dirs/NEON/UNDE/shell_commands | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/UNDE/shell_commands b/cime_config/usermods_dirs/NEON/UNDE/shell_commands index 79688e0a8f..f810e4a76b 100644 --- a/cime_config/usermods_dirs/NEON/UNDE/shell_commands +++ b/cime_config/usermods_dirs/NEON/UNDE/shell_commands @@ -1,3 +1,3 @@ ./xmlchange NEONSITE=UNDE -./xmlchange PTS_LON=270.6779 -./xmlchange PTS_LAT=46.14103 +./xmlchange PTS_LON=270.462746 +./xmlchange PTS_LAT=46.23391 From de831f77e066263c59d5965c983d25562183b70b Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Thu, 12 Jan 2023 14:59:37 -0700 Subject: [PATCH 123/332] adding new updates to FATES side --- cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands | 4 ++-- cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands | 4 ++-- cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands | 4 ++-- cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands b/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands index cb093d806a..0508481981 100644 --- a/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/BLAN/shell_commands @@ -1,3 +1,3 @@ ./xmlchange NEONSITE=BLAN -./xmlchange PTS_LON=281.92885 -./xmlchange PTS_LAT=39.06044 +./xmlchange PTS_LON=281.958212 +./xmlchange PTS_LAT=39.033698 diff --git a/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands index 43dab69998..511813398b 100644 --- a/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/ONAQ/shell_commands @@ -1,7 +1,7 @@ #!/bin/bash ./xmlchange NEONSITE=ONAQ -./xmlchange PTS_LON=276.49815 -./xmlchange PTS_LAT=35.68839 +./xmlchange PTS_LON=247.54755 +./xmlchange PTS_LAT=40.17760 ./xmlchange DATM_YR_END=2019 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then diff --git a/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands b/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands index 264d451753..5708d3dec5 100644 --- a/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/ORNL/shell_commands @@ -1,3 +1,3 @@ ./xmlchange NEONSITE=ORNL -./xmlchange PTS_LON=275.83419000000004 -./xmlchange PTS_LAT=35.57525 +./xmlchange PTS_LON=275.717412 +./xmlchange PTS_LAT=35.964128 diff --git a/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands b/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands index 79688e0a8f..f810e4a76b 100644 --- a/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/UNDE/shell_commands @@ -1,3 +1,3 @@ ./xmlchange NEONSITE=UNDE -./xmlchange PTS_LON=270.6779 -./xmlchange PTS_LAT=46.14103 +./xmlchange PTS_LON=270.462746 +./xmlchange PTS_LAT=46.23391 From 3fbeeaf6226eb0818ed785a5855a1642737599eb Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Thu, 12 Jan 2023 15:17:00 -0700 Subject: [PATCH 124/332] add packaging to ctsm_py environment --- python/conda_env_ctsm_py.txt | 1 + python/conda_env_ctsm_py_cgd.txt | 1 + python/conda_env_ctsm_py_latest.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/python/conda_env_ctsm_py.txt b/python/conda_env_ctsm_py.txt index d757ae1782..4768d0ed62 100644 --- a/python/conda_env_ctsm_py.txt +++ b/python/conda_env_ctsm_py.txt @@ -16,6 +16,7 @@ tqdm scipy netcdf4 requests +packaging numpy=1.18.5 xarray=0.16.2 pylint=2.8.3 diff --git a/python/conda_env_ctsm_py_cgd.txt b/python/conda_env_ctsm_py_cgd.txt index e7ee4af8ab..f3cc1b3201 100644 --- a/python/conda_env_ctsm_py_cgd.txt +++ b/python/conda_env_ctsm_py_cgd.txt @@ -18,6 +18,7 @@ tqdm scipy netcdf4 requests +packaging numpy=1.18.5 xarray=0.16.2 pylint=2.8.3 diff --git a/python/conda_env_ctsm_py_latest.txt b/python/conda_env_ctsm_py_latest.txt index 2dc2ed518d..052fbf4fef 100644 --- a/python/conda_env_ctsm_py_latest.txt +++ b/python/conda_env_ctsm_py_latest.txt @@ -5,6 +5,7 @@ tqdm>=4.64.1 scipy netcdf4 requests +packaging numpy>=1.23.0 xarray>=2022.3.0 pylint>=2.8.3,<2.9.0 # Once, you move off of 2.8.3, make lint shows 2 errors, at 2.11.1 there are 2 more errors beyond that From 88afa24c8a83a57574ea5245a869f1233b953c73 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Thu, 12 Jan 2023 15:18:23 -0700 Subject: [PATCH 125/332] add back pandas version check --- tools/site_and_regional/modify_singlept_site_neon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 49f10acc2f..64e77eacd5 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -47,7 +47,7 @@ import numpy as np import pandas as pd import xarray as xr -#from packaging import version +from packaging import version from datetime import date from getpass import getuser @@ -535,9 +535,9 @@ def main(): logging.basicConfig(level=logging.DEBUG) # Check if pandas is a recent enough version - #pdvers = pd.__version__ - #if version.parse(pdvers) < version.parse("1.1.0"): - # sys.exit("The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", pdvers ) + pdvers = pd.__version__ + if version.parse(pdvers) < version.parse("1.1.0"): + sys.exit("The pandas version in your python environment is too old, update to a newer version of pandas (>=1.1.0): version=%s", pdvers ) file_time = check_neon_time() From b9437fe8acfa7cb1671620c651148d553cb7d54d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 13 Jan 2023 17:24:59 -0700 Subject: [PATCH 126/332] Add testing for 3D and 4D optional data, and get it working correctly --- .../ctsm/modify_input_files/modify_fsurdat.py | 12 +-- python/ctsm/test/test_sys_fsurdat_modifier.py | 81 ++++++++++++++++--- .../modify_fsurdat_opt_sections.cfg | 4 +- 3 files changed, 80 insertions(+), 17 deletions(-) diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index aac888ae39..599b8a6e84 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -293,15 +293,15 @@ def set_varlist(self, settings, cfg_path="unknown-config-file"): abort( "Variable " + var + " is of the wrong size. It should be = " + str(dim1) ) - for lev1 in range(dim1 - 1): + for lev1 in range(dim1): self.setvar_lev1(var, vallist[lev1], lev1_dim=lev1) elif len(self.file[var].dims) == 4: - dim1 = int(self.file.sizes[self.file[var].dims[0]]) - dim2 = int(self.file.sizes[self.file[var].dims[1]]) + dim_lev1 = int(self.file.sizes[self.file[var].dims[1]]) + dim_lev2 = int(self.file.sizes[self.file[var].dims[0]]) vallist = settings[var] - for lev1 in range(dim1 - 1): - for lev2 in range(dim2 - 1): - self.setvar_lev2(var, vallist[lev1], lev1_dim=lev1, lev2_dim=lev2) + for lev1 in range(dim_lev1): + for lev2 in range(dim_lev2): + self.setvar_lev2(var, vallist[lev2], lev1_dim=lev1, lev2_dim=lev2) else: abort( "Error: Variable " diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 7671e10692..fc5eb8d0e9 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -143,15 +143,78 @@ def test_opt_sections(self): fsurdat_modifier(parser) # Read the resultant output file and make sure the fields are changed as expected fsurdat_out_data = xr.open_dataset(outfile) - zero0d = np.zeros( (5,5) ) - one0d = np.ones( (5,5) ) - np.testing.assert_array_equal(fsurdat_out_data.PCT_NATVEG, zero0d ) - np.testing.assert_array_equal(fsurdat_out_data.PCT_CROP, zero0d ) - np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d ) - np.testing.assert_array_equal(fsurdat_out_data.PCT_WETLAND, zero0d ) - np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d ) - np.testing.assert_array_equal(fsurdat_out_data.PCT_GLACIER, zero0d ) - np.testing.assert_array_equal(fsurdat_out_data.LAKEDEPTH, one0d*200. ) + zero0d = np.zeros((5, 5)) + one0d = np.ones((5, 5)) + pct_urban = np.array( + [ + [ + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + ], + ] + ) + lev2_two = np.empty( (2,3,5,5) ) + lev2_two[0,:,:,:] = 200. + lev2_two[1,:,:,:] = 100. + lev2_five = np.empty( (5,3,5,5) ) + lev2_five[0,:,:,:] = 1. + lev2_five[1,:,:,:] = 2. + lev2_five[2,:,:,:] = 3. + lev2_five[3,:,:,:] = 4. + lev2_five[4,:,:,:] = 5. + lev1 = np.array( + [ + [ + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + [200.0, 200.0, 200.0, 200.0, 200.0], + ], + [ + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + [150.0, 150.0, 150.0, 150.0, 150.0], + ], + [ + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + [100.0, 100.0, 100.0, 100.0, 100.0], + ], + ] + ) + np.testing.assert_array_equal(fsurdat_out_data.PCT_NATVEG, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_CROP, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_WETLAND, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_LAKE, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_GLACIER, zero0d) + np.testing.assert_array_equal(fsurdat_out_data.PCT_URBAN, pct_urban) + np.testing.assert_array_equal(fsurdat_out_data.LAKEDEPTH, one0d * 200.0) + np.testing.assert_array_equal(fsurdat_out_data.T_BUILDING_MIN, lev1) + np.testing.assert_array_equal(fsurdat_out_data.ALB_ROOF_DIR, lev2_two) + np.testing.assert_array_equal(fsurdat_out_data.TK_ROOF, lev2_five) def test_cfg_file_DNE_fail(self): """ diff --git a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg index b9c006ce03..c068c5d851 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_opt_sections.cfg @@ -47,5 +47,5 @@ LAKEDEPTH = 200.00 CANYON_HWR = 200.00 150.0 100. HT_ROOF = 200.0 150.0 100. T_BUILDING_MIN = 200 150.0 100. -#ALB_ROOF_DIR = 200. 100. -#TK_ROOF = 1. 2. 3. 4. 5. +ALB_ROOF_DIR = 200. 100. +TK_ROOF = 1. 2. 3. 4. 5. From 8338b786b05a9e2160a3020b6e59154f5dcd4813 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 13 Jan 2023 17:25:55 -0700 Subject: [PATCH 127/332] Squashed 'manage_externals/' changes from a3b3a0373..82d3b247f MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 82d3b247f Merge pull request #178 from johnpaulalex/test_doc 3223f49ea Additional documentation of system tests - global variables, method descriptions. 45b7c01c3 Merge pull request #177 from jedwards4b/git_workflow ace90b2c2 try setting credentials this way f4d6aa933 try setting credentials this way 1d61a6944 use this to set git credentials 7f9d330e1 use this to set git credentials 5ac731b85 add tmate code 836847be7 get git workflow working dcd462d71 Merge pull request #176 from jedwards4b/add_github_testing 2d2479e9d Merge pull request #175 from johnpaulalex/fix 711a53fdf add github testing of prs and automatic tagging of main cfe0f888a fix typos 5665d6140 Fix broken checkout behavior introduced by PR #172. 27909e255 Merge pull request #173 from johnpaulalex/readall 00ad0440b Further tiny refactorings and docs of checkout API (no-op). Remove unused load_all param in _External.checkout(). Rename _External.checkout_externals() to checkout_subexternals(), to remove the ambiguity about whether the main external pointed to by the _External is itelf checked out (it is not) Clarify load_all documentation - it’s always recursive, but applies different criteria at each level. Rename variables in checkout.py (e.g. ext_description) to match the equivalent code in sourcetree.py. 2ea3d1a3a Merge pull request #172 from johnpaulalex/fixit 43bf8092c Merge pull request #171 from johnpaulalex/docstatus e6aa7d21e Merge pull request #170 from johnpaulalex/printdir adbd71557 On checkout, refresh locally installed optional packages regardless of whether -o is passed in. add074593 Comment tweaks, and fix 'ppath' typo 696527cb8 Document the format of various status dictionaries, and the various paths and path components within an _External. c677b9403 When processing an external, print out its path in addition to the base filename (to disambiguate all the externals.cfg's) 975d7fd5a Merge pull request #169 from johnpaulalex/docfix_branch 09709e36d Document _Externals.status(). The original comment was apparently copy-pasted from checkout(). 1d880e090 Merge pull request #167 from billsacks/fix_svn_on_windows 3510da848 Tweak a unit test to improve coverage eb7fc1368 Handle the possibility that the URL already ends with '/' 02ea87e3d Fix svn URLs on Windows b1c02ab54 Merge pull request #165 from gold2718/doc_fix 9f4be8c7b Add documentation about externals = None feature git-subtree-dir: manage_externals git-subtree-split: 82d3b247fc8fb3b70458bf2b4570ee6678754c4d --- .github/workflows/bumpversion.yml | 19 + .github/workflows/tests.yml | 30 ++ README.md | 5 + manic/checkout.py | 92 ++--- manic/externals_description.py | 19 +- manic/externals_status.py | 30 +- manic/repository_factory.py | 1 + manic/repository_svn.py | 9 +- manic/sourcetree.py | 275 ++++++++------ test/README.md | 38 +- test/test_sys_checkout.py | 591 ++++++++++++++++-------------- test/test_unit_repository_svn.py | 2 +- 12 files changed, 620 insertions(+), 491 deletions(-) create mode 100644 .github/workflows/bumpversion.yml create mode 100644 .github/workflows/tests.yml mode change 100644 => 100755 test/test_unit_repository_svn.py diff --git a/.github/workflows/bumpversion.yml b/.github/workflows/bumpversion.yml new file mode 100644 index 0000000000..f4dc9b7ca5 --- /dev/null +++ b/.github/workflows/bumpversion.yml @@ -0,0 +1,19 @@ +name: Bump version +on: + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + create_annotated_tag: true + default_bump: patch + dry_run: false + tag_prefix: manic- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..dd75b91b49 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,30 @@ +# This is a workflow to compile the cmeps source without cime +name: Test Manic + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + test-manic: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Test Manic + run: | + pushd test + git config --global user.email "devnull@example.com" + git config --global user.name "GITHUB tester" + git config --global protocol.file.allow always + make utest + make stest + popd + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/README.md b/README.md index c931c8e213..9475301b5d 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. externals description file pointed 'useful_library/sub-xternals.cfg', Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + Note that by default, `checkout_externals` will clone an external's + submodules. As a special case, the entry, `externals = None`, will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the `from_submodule` + configuration entry below). * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in diff --git a/manic/checkout.py b/manic/checkout.py index 2223b1f0d8..ac30f3a5d2 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -227,6 +227,12 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + Note that by default, checkout_externals will clone an external's + submodules. As a special case, the entry, "externals = None", will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the from_submodule + configuration entry below). + * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in this repository. Note that the section name (the entry in square @@ -332,7 +338,34 @@ def commandline_arguments(args=None): options = parser.parse_args() return options - +def _dirty_local_repo_msg(program_name, config_file): + return """The external repositories labeled with 'M' above are not in a clean state. +The following are four options for how to proceed: +(1) Go into each external that is not in a clean state and issue either a 'git status' or + an 'svn status' command (depending on whether the external is managed by git or + svn). Either revert or commit your changes so that all externals are in a clean + state. (To revert changes in git, follow the instructions given when you run 'git + status'.) (Note, though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. +(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually + update out-of-sync externals (labeled with 's' above) as described in the + configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' + commands to checkout the appropriate tags for each external, as given in + {config_file}.) +(3) You can also use {program_name} to manage most, but not all externals: You can specify + one or more externals to ignore using the '-x' or '--exclude' argument to + {program_name}. Excluding externals labeled with 'M' will allow {program_name} to + update the other, non-excluded externals. +(4) As a last resort, if you are confident that there is no work that needs to be saved + from a given external, you can remove that external (via "rm -rf [directory]") and + then rerun the {program_name} tool. This option is mainly useful as a workaround for + issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before rerunning the +{program_name} tool. +""".format(program_name=program_name, config_file=config_file) # --------------------------------------------------------------------- # # main @@ -363,19 +396,25 @@ def main(args): load_all = True root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components, exclude=args.exclude) + model_data = read_externals_description_file(root_dir, args.externals) + ext_description = create_externals_description( + model_data, components=args.components, exclude=args.exclude) for comp in args.components: - if comp not in external.keys(): + if comp not in ext_description.keys(): + # Note we can't print out the list of found externals because + # they were filtered in create_externals_description above. fatal_error( "No component {} found in {}".format( comp, args.externals)) - source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() + source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry) + if args.components: + components_str = 'specified components' + else: + components_str = 'required & optional components' + printlog('Checking local status of ' + components_str + ': ', end='') + tree_status = source_tree.status(print_progress=True) printlog('') if args.status: @@ -390,43 +429,8 @@ def main(args): for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are four options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either a 'git status' or - an 'svn status' command (depending on whether the external is managed by git or - svn). Either revert or commit your changes so that all externals are in a clean - state. (To revert changes in git, follow the instructions given when you run 'git - status'.) (Note, though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually - update out-of-sync externals (labeled with 's' above) as described in the - configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' - commands to checkout the appropriate tags for each external, as given in - {config_file}.) - -(3) You can also use {program_name} to manage most, but not all externals: You can specify - one or more externals to ignore using the '-x' or '--exclude' argument to - {program_name}. Excluding externals labeled with 'M' will allow {program_name} to - update the other, non-excluded externals. - -(4) As a last resort, if you are confident that there is no work that needs to be saved - from a given external, you can remove that external (via "rm -rf [directory]") and - then rerun the {program_name} tool. This option is mainly useful as a workaround for - issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before rerunning the -{program_name} tool. -""".format(program_name=program_name, config_file=args.externals) - printlog('-' * 70) - printlog(msg) + printlog(_dirty_local_repo_msg(program_name, args.externals)) printlog('-' * 70) else: if not args.components: diff --git a/manic/externals_description.py b/manic/externals_description.py index 6a54935935..f5615b6730 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) + printlog('Processing externals description file : {0} ({1})'.format(file_name, + root_dir)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): @@ -281,6 +282,10 @@ def read_gitmodules_file(root_dir, file_name): def create_externals_description( model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data + + components: list of component names to include, None to include all. If a + name isn't found, it is silently omitted from the return value. + exclude: list of component names to skip. """ externals_description = None if model_format == 'dict': @@ -357,8 +362,9 @@ class ExternalsDescription(dict): input value. """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' + # keywords defining the interface into the externals description data; these + # are brought together by the schema below. + EXTERNALS = 'externals' # path to externals file. BRANCH = 'branch' SUBMODULE = 'from_submodule' HASH = 'hash' @@ -384,6 +390,8 @@ class ExternalsDescription(dict): _V1_BRANCH = 'BRANCH' _V1_REQ_SOURCE = 'REQ_SOURCE' + # Dictionary keys are component names. The corresponding values are laid out + # according to this schema. _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', @@ -760,6 +768,8 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 @@ -783,6 +793,9 @@ def _remove_metadata(model_data): def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. + + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ def list_to_dict(input_list, convert_to_lower_case=True): """Convert a list of key-value pairs into a dictionary. diff --git a/manic/externals_status.py b/manic/externals_status.py index d3d238f289..4900e41254 100644 --- a/manic/externals_status.py +++ b/manic/externals_status.py @@ -29,16 +29,16 @@ class ExternalStatus(object): transactions (e.g. add, remove, rename, untracked files). """ - DEFAULT = '-' + # sync_state and clean_state can be one of the following: + DEFAULT = '-' # aka not set yet. UNKNOWN = '?' EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' + MODEL_MODIFIED = 's' # repo version != externals (sync_state only) + DIRTY = 'M' # repo is dirty (clean_state only) + STATUS_OK = ' ' # repo is clean/matches externals. STATUS_ERROR = '!' - # source types + # source_type can be one of the following: OPTIONAL = 'o' STANDALONE = 's' MANAGED = ' ' @@ -55,19 +55,21 @@ def __init__(self): def log_status_message(self, verbosity): """Write status message to the screen and log file """ - self._default_status_message() + printlog(self._default_status_message()) if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() + printlog(self._verbose_status_message()) if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() + printlog(self._dump_status_message()) + + def __repr__(self): + return self._default_status_message() def _default_status_message(self): """Return the default terse status message string """ - msg = '{sync}{clean}{src_type} {path}'.format( + return '{sync}{clean}{src_type} {path}'.format( sync=self.sync_state, clean=self.clean_state, src_type=self.source_type, path=self.path) - printlog(msg) def _verbose_status_message(self): """Return the verbose status message string @@ -82,14 +84,12 @@ def _verbose_status_message(self): if self.sync_state != self.STATUS_OK: sync_str = '{current} --> {expected}'.format( current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) + return ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) def _dump_status_message(self): """Return the dump status message string """ - msg = indent_string(self.status_output, 12) - printlog(msg) + return indent_string(self.status_output, 12) def safe_to_update(self): """Report if it is safe to update a repository. Safe is defined as: diff --git a/manic/repository_factory.py b/manic/repository_factory.py index 80a92a9d8a..18c73ffc4b 100644 --- a/manic/repository_factory.py +++ b/manic/repository_factory.py @@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. + Can return None (e.g. if protocol is 'externals_only'). """ protocol = repo_info[ExternalsDescription.PROTOCOL].lower() if protocol == 'git': diff --git a/manic/repository_svn.py b/manic/repository_svn.py index 408ed84676..922855d34e 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False): """ Repository.__init__(self, component_name, repo) self._ignore_ancestry = ignore_ancestry + if self._url.endswith('/'): + # there is already a '/' separator in the URL; no need to add another + url_sep = '' + else: + url_sep = '/' if self._branch: - self._url = os.path.join(self._url, self._branch) + self._url = self._url + url_sep + self._branch elif self._tag: - self._url = os.path.join(self._url, self._tag) + self._url = self._url + url_sep + self._tag else: msg = "DEV_ERROR in svn repository. Shouldn't be here!" fatal_error(msg) diff --git a/manic/sourcetree.py b/manic/sourcetree.py index 54de763c30..218b5febb4 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -19,7 +19,7 @@ class _External(object): """ - _External represents an external object inside a SourceTree + A single component hosted in an external repository (and any children). """ # pylint: disable=R0902 @@ -41,31 +41,40 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): """ self._name = name - self._repo = None - self._externals = EMPTY_STR + self._repo = None # Repository object. + + # Subcomponent externals file and data object, if any. + self._externals_path = EMPTY_STR # Can also be "none" self._externals_sourcetree = None - self._stat = ExternalStatus() + + self._stat = None # Populated in status() self._sparse = None # Parse the sub-elements - # _path : local path relative to the containing source tree + # _local_path : local path relative to the containing source tree, e.g. + # "components/mom" self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory + # _repo_dir_path : full repository directory, e.g. + # "/components/mom" repo_dir = os.path.join(root_dir, self._local_path) self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository + # _base_dir_path : base directory *containing* the repository, e.g. + # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) assert(os.path.join(self._base_dir_path, self._repo_dir_name) == self._repo_dir_path) self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] + + # Does this component have subcomponents aka an externals config? + self._externals_path = ext_description[ExternalsDescription.EXTERNALS] # Treat a .gitmodules file as a backup externals config - if not self._externals: + if not self._externals_path: if GitRepository.has_submodules(self._repo_dir_path): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME repo = create_repository( name, ext_description[ExternalsDescription.REPO], @@ -73,7 +82,8 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): if repo: self._repo = repo - if self._externals and (self._externals.lower() != 'none'): + # Recurse into subcomponents, if any. + if self._externals_path and (self._externals_path.lower() != 'none'): self._create_externals_sourcetree() def get_name(self): @@ -88,81 +98,88 @@ def get_local_path(self): """ return self._local_path - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. + def status(self, force=False, print_progress=False): """ + Returns status of this component and all subcomponents. - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED + Returns a dict mapping our local path (not component name!) to an + ExternalStatus dict. Any subcomponents will have their own top-level + path keys. Note the return value includes entries for this and all + subcomponents regardless of whether they are locally installed or not. - ext_stats = {} + Side-effect: If self._stat is empty or force is True, calculates _stat. + """ + calc_stat = force or not self._stat + + if calc_stat: + self._stat = ExternalStatus() + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_subexternals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_subexternals + self._stat.source_type = ExternalStatus.MANAGED + subcomponent_stats = {} if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() + if calc_stat: + # No local repository. + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() else: - if self._repo: + # Merge local repository state (e.g. clean/dirty) into self._stat. + if calc_stat and self._repo: self._repo.status(self._stat, self._repo_dir_path) - if self._externals and self._externals_sourcetree: - # we expect externals and they exist + # Status of subcomponents, if any. + if self._externals_path and self._externals_sourcetree: cwd = os.getcwd() - # SourceTree expects to be called from the correct + # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) + subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force, print_progress=print_progress) os.chdir(cwd) + # Merge our status + subcomponent statuses into one return dict keyed + # by component path. all_stats = {} # don't add the root component because we don't manage it # and can't provide useful info about it. if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so + # store the stats under the local_path, not comp name so # it will be sorted correctly all_stats[self._stat.path] = self._stat - if ext_stats: - all_stats.update(ext_stats) + if subcomponent_stats: + all_stats.update(subcomponent_stats) return all_stats - def checkout(self, verbosity, load_all): + def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. If the repo destination directory does not exist, checkout the correct branch or tag. - If load_all is True, also load all of the the externals sub-externals. + Does not check out sub-externals, see checkout_subexternals(). """ - if load_all: - pass # Make sure we are in correct location - if not os.path.exists(self._repo_dir_path): # repository directory doesn't exist. Need to check it # out, and for that we need the base_dir_path to exist @@ -174,6 +191,10 @@ def checkout(self, verbosity, load_all): self._base_dir_path) fatal_error(msg) + if not self._stat: + self.status() + assert self._stat + if self._stat.source_type != ExternalStatus.STANDALONE: if verbosity >= VERBOSITY_VERBOSE: # NOTE(bja, 2018-01) probably do not want to pass @@ -194,8 +215,10 @@ def checkout(self, verbosity, load_all): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object + def checkout_subexternals(self, verbosity, load_all): + """Recursively checkout the sub-externals for this component, if any. + + See load_all documentation in SourceTree.checkout(). """ if self.load_externals(): if self._externals_sourcetree: @@ -210,25 +233,26 @@ def checkout_externals(self, verbosity, load_all): self._externals_sourcetree.checkout(verbosity, load_all) def load_externals(self): - 'Return True iff an externals file should be loaded' + 'Return True iff an externals file exists (and therefore should be loaded)' load_ex = False if os.path.exists(self._repo_dir_path): - if self._externals: - if self._externals.lower() != 'none': + if self._externals_path: + if self._externals_path.lower() != 'none': load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals)) + self._externals_path)) return load_ex def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' - # Try recursive unless there is an externals entry - recursive = not self._externals + # Try recursive .gitmodules unless there is an externals entry + recursive = not self._externals_path return recursive def _create_externals_sourcetree(self): """ + Note this only creates an object, it doesn't write to the file system. """ if not os.path.exists(self._repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out @@ -239,29 +263,31 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) - if self._externals.lower() == 'none': + if self._externals_path.lower() == 'none': msg = ('Internal: Attempt to create source tree for ' 'externals = none in {}'.format(self._repo_dir_path)) fatal_error(msg) - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): if GitRepository.has_submodules(): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! msg = ('External externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) + self._externals_path, self._repo_dir_path)) fatal_error(msg) externals_root = self._repo_dir_path + # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, externals) + self._externals_path) + # ext_description is another dict-like object (see ExternalsDescription) + ext_description = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) class SourceTree(object): @@ -269,45 +295,48 @@ class SourceTree(object): SourceTree represents a group of managed externals """ - def __init__(self, root_dir, model, svn_ignore_ancestry=False): + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ - Build a SourceTree object from a model description + Build a SourceTree object from an ExternalDescription. """ self._root_dir = os.path.abspath(root_dir) - self._all_components = {} + self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + for comp in ext_description: + src = _External(self._root_dir, comp, ext_description[comp], + svn_ignore_ancestry) self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: + if ext_description[comp][ExternalsDescription.REQUIRED]: self._required_compnames.append(comp) - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components - - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... - - """ + def status(self, relative_path_base=LOCAL_PATH_INDICATOR, + force=False, print_progress=False): + """Return a dictionary of local path->ExternalStatus. + + Notes about the returned dictionary: + * It is keyed by local path (e.g. 'components/mom'), not by + component name (e.g. 'mom'). + * It contains top-level keys for all traversed components, whether + discovered by recursion or top-level. + * It contains entries for all components regardless of whether they + are locally installed or not, or required or optional. +x """ load_comps = self._all_components.keys() - summary = {} + summary = {} # Holds merged statuses from all components. for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() + if print_progress: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status(force=force, + print_progress=print_progress) + + # Returned status dictionary is keyed by local path; prepend + # relative_path_base if not already there. stat_final = {} for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. if stat[name].path.startswith(relative_path_base): - # use as is, without any changes to path stat_final[name] = stat[name] else: - # append relative_path_base to path and store under key = updated path modified_path = os.path.join(relative_path_base, stat[name].path) stat_final[modified_path] = stat[name] @@ -316,30 +345,48 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): return summary + def _find_installed_optional_components(self): + """Returns a list of installed optional component names, if any.""" + installed_comps = [] + for comp_name, ext in self._all_components.items(): + if comp_name in self._required_compnames: + continue + # Note that in practice we expect this status to be cached. + path_to_stat = ext.status() + if any(stat.sync_state != ExternalStatus.EMPTY + for stat in path_to_stat.values()): + installed_comps.append(comp_name) + return installed_comps + def checkout(self, verbosity, load_all, load_comp=None): """ - Checkout or update indicated components into the the configured - subdirs. + Checkout or update indicated components into the configured subdirs. - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. + If load_all is True, checkout all externals (required + optional), recursively. + If load_all is False and load_comp is set, checkout load_comp (and any required subexternals, plus any optional subexternals that are already checked out, recursively) + If load_all is False and load_comp is None, checkout all required externals, plus any optionals that are already checked out, recursively. """ - if verbosity >= VERBOSITY_VERBOSE: - printlog('Checking out externals: ') - else: - printlog('Checking out externals: ', end='') - if load_all: tmp_comps = self._all_components.keys() elif load_comp is not None: tmp_comps = [load_comp] else: - tmp_comps = self._required_compnames + local_optional_compnames = self._find_installed_optional_components() + tmp_comps = self._required_compnames + local_optional_compnames + if local_optional_compnames: + printlog('Found locally installed optional components: ' + + ', '.join(local_optional_compnames)) + + if verbosity >= VERBOSITY_VERBOSE: + printlog('Checking out externals: ') + else: + printlog('Checking out externals: ', end='') + # Sort by path so that if paths are nested the # parent repo is checked out first. load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path()) - # checkout the primary externals + + # checkout. for comp in load_comps: if verbosity < VERBOSITY_VERBOSE: printlog('{0}, '.format(comp), end='') @@ -347,7 +394,9 @@ def checkout(self, verbosity, load_all, load_comp=None): # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) - # now give each external an opportunitity to checkout it's externals. - self._all_components[comp].checkout_externals(verbosity, load_all) + # Does not recurse. + self._all_components[comp].checkout(verbosity) + # Recursively check out subexternals, if any. + self._all_components[comp].checkout_subexternals(verbosity, + load_all) printlog('') diff --git a/test/README.md b/test/README.md index 938a900eec..1e8f2eaa77 100644 --- a/test/README.md +++ b/test/README.md @@ -1,45 +1,25 @@ # Testing for checkout_externals -NOTE: Python2 is the supported runtime environment. Python3 compatibility is -in progress, complicated by the different proposed input methods -(yaml, xml, cfg/ini, json) and their different handling of strings -(unicode vs byte) in python2. Full python3 compatibility will be -possible once the number of possible input formats has been narrowed. - -## Setup development environment - -Development environments should be setup for python2 and python3: +## Unit tests ```SH cd checkout_externals/test - make python=python2 env - make python=python3 env + make utest ``` -## Unit tests - -Tests should be run for both python2 and python3. It is recommended -that you have seperate terminal windows open python2 and python3 -testing to avoid errors activating and deactivating environments. +## System tests ```SH cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + make stest ``` +Example to run a single test: ```SH - cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + cd checkout_externals + python -m unittest test.test_sys_checkout.TestSysCheckout.test_container_simple_required ``` -## System tests - -Not yet implemented. - ## Static analysis checkout_externals is difficult to test thoroughly because it relies @@ -51,9 +31,7 @@ regularly for automatic code formatting and linting. ```SH cd checkout_externals/test - . env_python2/bin/activate make lint - deactivate ``` The canonical formatting for the code is whatever autopep8 @@ -68,10 +46,8 @@ coverage, run the code coverage tool: ```SH cd checkout_externals/test - . env_python2/bin/activate make coverage open -a Firefox.app htmlcov/index.html - deactivate ``` diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index b13e96a17d..27b649bea4 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -66,27 +66,41 @@ # # --------------------------------------------------------------------- -# environment variable names -MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' -MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' -# directory names -TMP_REPO_DIR_NAME = 'tmp' -BARE_REPO_ROOT_NAME = 'repos' +# Module-wide root directory for all the per-test subdirs we'll create on +# the fly. +MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' # env var name +TMP_REPO_DIR_NAME = 'tmp' # subdir under $CWD + +# We clone these checked-in repositories on a per-test basis. The +# 'bare repo root' is the test/repos/ subdir that holds them all. +MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' # env var name +BARE_REPO_ROOT_NAME = 'repos' # subdir name + +# Subdirs under bare repo root, each holding a repository. CONTAINER_REPO_NAME = 'container.git' MIXED_REPO_NAME = 'mixed-cont-ext.git' SIMPLE_REPO_NAME = 'simple-ext.git' SIMPLE_FORK_NAME = 'simple-ext-fork.git' -SIMPLE_LOCAL_ONLY_NAME = '.' -ERROR_REPO_NAME = 'error' + + +# In each of the test repos, all externals live under this subdir. EXTERNALS_NAME = 'externals' -SUB_EXTERNALS_PATH = 'src' +SUB_EXTERNALS_PATH = 'src' # For mixed test repos, + +# For testing behavior with '.' instead of an explicit paths. +SIMPLE_LOCAL_ONLY_NAME = '.' + +# Filenames CFG_NAME = 'externals.cfg' CFG_SUB_NAME = 'sub-externals.cfg' -README_NAME = 'readme.txt' +README_NAME = 'readme.txt' # Arbitrary file to check in. + +# Arbirary test branch name REMOTE_BRANCH_FEATURE2 = 'feature2' -NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney'] +# Section names in container_nested_* test config. +NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney'] SVN_TEST_REPO = 'https://github.com/escomp/cesm' @@ -110,16 +124,18 @@ def setUpModule(): # pylint: disable=C0103 # create clean dir for this run os.mkdir(repo_root) # set into the environment so var will be expanded in externals - # filess when executables are run + # files when executables are run os.environ[MANIC_TEST_TMP_REPO_ROOT] = repo_root class GenerateExternalsDescriptionCfgV1(object): - """Class to provide building blocks to create - ExternalsDescriptionCfgV1 files. + """Building blocks to create ExternalsDescriptionCfgV1 files. - Includes predefined files used in tests. + Basic usage: create_config() multiple create_*(), then write_config(). + Optionally after that: write_with_*(). + Includes methods (like container_*()) to create predefined configs for the + checked-in repositories in test/repos/. """ def __init__(self): @@ -255,7 +271,7 @@ def mixed_simple_sub(self, dest_dir): self.write_config(dest_dir, filename=CFG_SUB_NAME) def write_config(self, dest_dir, filename=CFG_NAME): - """Write the configuration file to disk + """Write self._config to disk """ dest_path = os.path.join(dest_dir, filename) @@ -277,14 +293,17 @@ def create_metadata(self): self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, self._schema_version) - def create_section(self, repo_type, name, tag='', branch='', + def create_section(self, repo_path, name, tag='', branch='', ref_hash='', required=True, path=EXTERNALS_NAME, - externals='', repo_path=None, from_submodule=False, + externals='', repo_path_abs=None, from_submodule=False, sparse='', nested=False): # pylint: disable=too-many-branches - """Create a config section with autofilling some items and handling - optional items. + """Create a config ExternalsDescription section with the given name. + + Autofills some items and handles some optional items. + repo_path_abs overrides repo_path (which is relative to the bare repo) + path is a subdir under repo_path. """ # pylint: disable=R0913 self._config.add_section(name) @@ -300,7 +319,7 @@ def create_section(self, repo_type, name, tag='', branch='', # from_submodules is incompatible with some other options, turn them off if (from_submodule and - ((repo_path is not None) or tag or ref_hash or branch)): + ((repo_path_abs is not None) or tag or ref_hash or branch)): printlog('create_section: "from_submodule" is incompatible with ' '"repo_url", "tag", "hash", and "branch" options;\n' 'Ignoring those options for {}'.format(name)) @@ -309,10 +328,10 @@ def create_section(self, repo_type, name, tag='', branch='', ref_hash = '' branch = '' - if repo_path is not None: - repo_url = repo_path + if repo_path_abs is not None: + repo_url = repo_path_abs else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) if not from_submodule: self._config.set(name, ExternalsDescription.REPO_URL, repo_url) @@ -381,12 +400,11 @@ def create_svn_external(self, name, tag='', branch=''): @staticmethod def create_branch(dest_dir, repo_name, branch, with_commit=False): - """Update a repository branch, and potentially the remote. + """Create branch and optionally (with_commit) add a single commit. """ # pylint: disable=R0913 cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME) - repo_root = os.path.join(repo_root, repo_name) + repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) os.chdir(repo_root) cmd = ['git', 'checkout', '-b', branch, ] execute_subprocess(cmd) @@ -407,10 +425,11 @@ def create_commit(dest_dir, repo_name, local_tracking_branch=None): This is used to test sync state changes from local commits on detached heads and tracking branches. + NOTE(jpalex, 2023-01) This has nothing to do with self._config, + should probably live elsewhere. """ cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME) - repo_root = os.path.join(repo_root, repo_name) + repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) os.chdir(repo_root) if local_tracking_branch: cmd = ['git', 'checkout', '-b', local_tracking_branch, ] @@ -425,19 +444,21 @@ def create_commit(dest_dir, repo_name, local_tracking_branch=None): execute_subprocess(cmd) os.chdir(cwd) - def update_branch(self, dest_dir, name, branch, repo_type=None, - filename=CFG_NAME): - """Update a repository branch, and potentially the remote. + def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, + filename=CFG_NAME): + """Update fields in our config and write it to disk. + + name is the key of the ExternalsDescription in self._config to update. """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.BRANCH, branch) - if repo_type: - if repo_type == SIMPLE_LOCAL_ONLY_NAME: + if repo_path: + if repo_path == SIMPLE_LOCAL_ONLY_NAME: repo_url = SIMPLE_LOCAL_ONLY_NAME else: repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', - repo_type) + repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -448,7 +469,7 @@ def update_branch(self, dest_dir, name, branch, repo_type=None, self.write_config(dest_dir, filename) - def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): + def write_with_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): """Update a repository branch, and potentially the remote. """ # pylint: disable=R0913 @@ -462,7 +483,7 @@ def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): self.write_config(dest_dir, filename) - def update_tag(self, dest_dir, name, tag, repo_type=None, + def write_with_tag(self, dest_dir, name, tag, repo_path=None, filename=CFG_NAME, remove_branch=True): """Update a repository tag, and potentially the remote @@ -474,8 +495,8 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.TAG, tag) - if repo_type: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + if repo_path: + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -487,8 +508,8 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, self.write_config(dest_dir, filename) - def update_underspecify_branch_tag(self, dest_dir, name, - filename=CFG_NAME): + def write_without_branch_tag(self, dest_dir, name, + filename=CFG_NAME): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -506,7 +527,7 @@ def update_underspecify_branch_tag(self, dest_dir, name, self.write_config(dest_dir, filename) - def update_underspecify_remove_url(self, dest_dir, name, + def write_without_repo_url(self, dest_dir, name, filename=CFG_NAME): """Update a repository protocol, and potentially the remote """ @@ -519,15 +540,15 @@ def update_underspecify_remove_url(self, dest_dir, name, self.write_config(dest_dir, filename) - def update_protocol(self, dest_dir, name, protocol, repo_type=None, + def write_with_protocol(self, dest_dir, name, protocol, repo_path=None, filename=CFG_NAME): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.PROTOCOL, protocol) - if repo_type: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + if repo_path: + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) self.write_config(dest_dir, filename) @@ -543,6 +564,7 @@ class BaseTestSysCheckout(unittest.TestCase): # cryptic. # pylint: disable=invalid-name + # Command-line args for checkout_externals, used in execute_checkout_in_dir() status_args = ['--status'] checkout_args = [] optional_args = ['--optional'] @@ -575,9 +597,8 @@ def setUp(self): self._checkout = os.path.join(root_dir, 'checkout_externals') # directory where we have test repositories - test_dir = os.path.join(root_dir, 'test') - self._bare_root = os.path.join(test_dir, BARE_REPO_ROOT_NAME) - self._bare_root = os.path.abspath(self._bare_root) + self._bare_root = os.path.abspath( + os.path.join(root_dir, 'test', BARE_REPO_ROOT_NAME)) # set into the environment so var will be expanded in externals files os.environ[MANIC_TEST_BARE_REPO_ROOT] = self._bare_root @@ -596,16 +617,16 @@ def tearDown(self): # return to our common starting point os.chdir(self._return_dir) - def setup_test_repo(self, parent_repo_name, dest_dir_in=None): - """Setup the paths and clone the base test repo + def clone_test_repo(self, parent_repo_name, dest_dir_in=None): + """Clone repo under bare_root into dest_dir_in or local per-test-subdir. + Returns output dir. """ - # unique repo for this test - test_dir_name = self._test_id - print("Test repository name: {0}".format(test_dir_name)) - parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) if dest_dir_in is None: + # unique repo for this test + test_dir_name = self._test_id + print("Test repository name: {0}".format(test_dir_name)) dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], test_dir_name) else: @@ -638,7 +659,7 @@ def _add_file_to_repo(under_test_dir, filename, tracked): os.chdir(cwd) @staticmethod - def execute_cmd_in_dir(under_test_dir, args): + def execute_checkout_in_dir(dirname, args): """Extecute the checkout command in the appropriate repo dir with the specified additional args @@ -649,13 +670,13 @@ def execute_cmd_in_dir(under_test_dir, args): """ cwd = os.getcwd() checkout_path = os.path.abspath('{0}/../../checkout_externals') - os.chdir(under_test_dir) + os.chdir(dirname) cmdline = ['--externals', CFG_NAME, ] cmdline += args repo_root = 'MANIC_TEST_BARE_REPO_ROOT={root}'.format( root=os.environ[MANIC_TEST_BARE_REPO_ROOT]) manual_cmd = ('Test cmd:\npushd {cwd}; {env} {checkout} {args}'.format( - cwd=under_test_dir, env=repo_root, checkout=checkout_path, + cwd=dirname, env=repo_root, checkout=checkout_path, args=' '.join(cmdline))) printlog(manual_cmd) options = checkout.commandline_arguments(cmdline) @@ -667,6 +688,8 @@ def execute_cmd_in_dir(under_test_dir, args): # # Check results for generic perturbation of states # + # 'tree' is a dict with ExternalStatus as values. + # 'name' is a key into that dict. # ---------------------------------------------------------------- def _check_generic_empty_default_required(self, tree, name): self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) @@ -702,6 +725,7 @@ def _check_generic_ok_clean_optional(self, tree, name): # # Check results for individual named externals # + # 'tree' is a dict of string to ExternalStatus. # ---------------------------------------------------------------- def _check_simple_tag_empty(self, tree, directory=EXTERNALS_NAME): name = './{0}/simp_tag'.format(directory) @@ -1019,21 +1043,21 @@ def test_container_simple_required(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_pre_checkout(overall, tree) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # status clean checked out - overall, tree = self.execute_cmd_in_dir(under_test_dir, + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) self._check_container_simple_required_post_checkout(overall, tree) @@ -1049,23 +1073,23 @@ def test_container_nested_required(self): # create repo dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], self._test_id, "test"+str(n)) - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME, dest_dir_in=dest_dir) - self._generator.container_nested_required(under_test_dir, order) + self._generator.container_nested_required(cloned_repo_dir, order) # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_nested_required_pre_checkout(overall, tree, order) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_nested_required_checkout(overall, tree, order) # status clean checked out - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_nested_required_post_checkout(overall, tree, order) def test_container_simple_optional(self): @@ -1074,32 +1098,32 @@ def test_container_simple_optional(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_optional(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_optional(cloned_repo_dir) # check status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_pre_checkout(overall, tree) # checkout required - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_optional_checkout(overall, tree) # status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_post_checkout(overall, tree) # checkout optional - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.optional_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.optional_args) self._check_container_simple_optional_post_checkout(overall, tree) # status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_post_optional(overall, tree) def test_container_simple_verbose(self): @@ -1108,17 +1132,17 @@ def test_container_simple_verbose(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # check verbose status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.verbose_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_simple_dirty(self): @@ -1126,28 +1150,28 @@ def test_container_simple_dirty(self): and a dirty status exits gracefully. """ - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # add a file to the repo tracked = True - self._add_file_to_repo(under_test_dir, 'externals/simp_tag/tmp.txt', + self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', tracked) # checkout: pre-checkout status should be dirty, did not # modify working copy. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_optional_st_dirty(overall, tree) # verify status is still dirty - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_optional_st_dirty(overall, tree) def test_container_simple_untracked(self): @@ -1155,28 +1179,28 @@ def test_container_simple_untracked(self): is not considered 'dirty' and will attempt an update. """ - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # add a file to the repo tracked = False - self._add_file_to_repo(under_test_dir, 'externals/simp_tag/tmp.txt', + self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', tracked) # checkout: pre-checkout status should be clean, ignoring the # untracked file. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_post_checkout(overall, tree) # verify status is still clean - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_simple_detached_sync(self): @@ -1186,38 +1210,38 @@ def test_container_simple_detached_sync(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_pre_checkout(overall, tree) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # make a commit on the detached head of the tag and hash externals - self._generator.create_commit(under_test_dir, 'simp_tag') - self._generator.create_commit(under_test_dir, 'simp_hash') - self._generator.create_commit(under_test_dir, 'simp_branch') + self._generator.create_commit(cloned_repo_dir, 'simp_tag') + self._generator.create_commit(cloned_repo_dir, 'simp_hash') + self._generator.create_commit(cloned_repo_dir, 'simp_branch') # status of repo, branch, tag and hash should all be out of sync! - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_out_of_sync(overall, tree) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # same pre-checkout out of sync status self._check_container_simple_required_out_of_sync(overall, tree) # now status should be in-sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_remote_branch(self): @@ -1225,32 +1249,33 @@ def test_container_remote_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the same branch - self._generator.update_branch(under_test_dir, 'simp_branch', - REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) + self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', + branch=REMOTE_BRANCH_FEATURE2, + repo_path=SIMPLE_FORK_NAME) # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_sb_modified(overall, tree) # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_sb_modified(overall, tree) # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_remote_tag_same_branch(self): @@ -1261,33 +1286,33 @@ def test_container_remote_tag_same_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', 'forked-feature-v1', SIMPLE_FORK_NAME) # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_sb_modified(overall, tree) # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_sb_modified(overall, tree) # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_remote_tag_fetch_all(self): @@ -1299,33 +1324,33 @@ def test_container_remote_tag_fetch_all(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', 'abandoned-feature', SIMPLE_FORK_NAME) # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_sb_modified(overall, tree) # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_sb_modified(overall, tree) # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_preserve_dot(self): @@ -1334,40 +1359,41 @@ def test_container_preserve_dot(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_required_checkout(overall, tree) # update the config file to point to a different remote with # the same branch - self._generator.update_branch(under_test_dir, 'simp_branch', - REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) + self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', + branch=REMOTE_BRANCH_FEATURE2, + repo_path=SIMPLE_FORK_NAME) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) # update branch to point to a new branch that only exists in # the local fork - self._generator.create_branch(under_test_dir, 'simp_branch', - 'private-feature', with_commit=True) - self._generator.update_branch(under_test_dir, 'simp_branch', - 'private-feature', - SIMPLE_LOCAL_ONLY_NAME) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + self._generator.create_branch(cloned_repo_dir, repo_name='simp_branch', + branch='private-feature', with_commit=True) + self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', + branch='private-feature', + repo_path=SIMPLE_LOCAL_ONLY_NAME) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_required_post_checkout(overall, tree) def test_container_full(self): @@ -1379,98 +1405,99 @@ def test_container_full(self): """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_full(under_test_dir) + self._generator.container_full(cloned_repo_dir) # inital checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_full_pre_checkout(overall, tree) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_full_post_checkout(overall, tree) # Check existance of some files subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(under_test_dir, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) # update the mixed-use repo to point to different branch - self._generator.update_branch(under_test_dir, 'mixed_req', - 'new-feature', MIXED_REPO_NAME) + self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', + branch='new-feature', + repo_path=MIXED_REPO_NAME) # check status out of sync for mixed_req, but sub-externals # are still in sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_full_pre_checkout_ext_change(overall, tree) # run the checkout. Now the mixed use external and it's # sub-exterals should be changed. Returned status is # pre-checkout! - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_full_pre_checkout_ext_change(overall, tree) # check status out of sync for mixed_req, and sub-externals # are in sync. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_full_post_checkout(overall, tree) def test_container_component(self): """Verify that optional component checkout works """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_full(under_test_dir) + self._generator.container_full(cloned_repo_dir) # inital checkout, first try a nonexistant component argument noref checkout_args = ['simp_opt', 'noref'] checkout_args.extend(self.checkout_args) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) checkout_args = ['simp_opt'] checkout_args.extend(self.checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_component_post_checkout(overall, tree) checkout_args.append('simp_branch') - overall, tree = self.execute_cmd_in_dir(under_test_dir, - checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_component_post_checkout2(overall, tree) def test_container_exclude_component(self): """Verify that exclude component checkout works """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_full(under_test_dir) + self._generator.container_full(cloned_repo_dir) # inital checkout, exclude simp_opt checkout_args = ['--exclude', 'simp_opt'] checkout_args.extend(self.checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) checkout_args.append("--status") - overall, tree = self.execute_cmd_in_dir(under_test_dir, checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) self._check_container_component_post_checkout3(overall, tree) def test_mixed_simple(self): @@ -1480,22 +1507,22 @@ def test_mixed_simple(self): """ #import pdb; pdb.set_trace() # create repository - under_test_dir = self.setup_test_repo(MIXED_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(MIXED_REPO_NAME) # create top level externals file - self._generator.mixed_simple_base(under_test_dir) + self._generator.mixed_simple_base(cloned_repo_dir) # NOTE: sub-externals file is already in the repo so we can # switch branches during testing. Since this is a mixed-repo # serving as the top level container repo, we can't switch # during this test. # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_mixed_cont_simple_required_checkout(overall, tree) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_mixed_cont_simple_required_post_checkout(overall, tree) def test_container_sparse(self): @@ -1504,31 +1531,31 @@ def test_container_sparse(self): """ # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) # create the top level externals file - self._generator.container_sparse(under_test_dir) + self._generator.container_sparse(cloned_repo_dir) # inital checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_sparse_pre_checkout(overall, tree) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_sparse_post_checkout(overall, tree) # Check existance of some files subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(under_test_dir, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'readme.txt')) - self._check_file_exists(under_test_dir, os.path.join(subrepo_path, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) subrepo_path = os.path.join('externals', 'simp_sparse') - self._check_file_exists(under_test_dir, + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) @@ -1625,57 +1652,57 @@ def test_container_simple_svn(self): """ self.skip_if_no_svn_access() # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_svn(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_svn(cloned_repo_dir) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_svn_post_checkout(overall, tree) # update description file to make the tag into a branch and # trigger a switch - self._generator.update_svn_branch(under_test_dir, 'svn_tag', 'trunk') + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', 'trunk') # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) self._check_container_simple_svn_post_checkout(overall, tree) # add an untracked file to the repo tracked = False - self._add_file_to_repo(under_test_dir, + self._add_file_to_repo(cloned_repo_dir, 'externals/svn_branch/tmp.txt', tracked) # run a no-op checkout: pre-checkout status should be clean, # ignoring the untracked file. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_svn_post_checkout(overall, tree) # update description file to make the branch into a tag and # trigger a modified sync status - self._generator.update_svn_branch(under_test_dir, 'svn_tag', - 'tags/cesm2.0.beta07') + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', + 'tags/cesm2.0.beta07') # checkout: pre-checkout status should be clean and modified, # will modify working copy. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.checkout_args) self._check_container_simple_svn_sb_clean_st_mod(overall, tree) # verify status is still clean and unmodified, last # checkout modified the working dir state. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.verbose_args) + overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) self._check_container_simple_svn_post_checkout(overall, tree) class TestSubrepoCheckout(BaseTestSysCheckout): @@ -1707,7 +1734,7 @@ def setUp(self): self._test_id) self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) self._checkout_dir = 'repo_with_submodules' - check_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + check_dir = self.clone_test_repo(CONTAINER_REPO_NAME, dest_dir_in=self._repo_dir) self.assertTrue(self._repo_dir == check_dir) # Add the submodules @@ -1795,7 +1822,7 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, self._checkout_dir, branch=branch_name, path=name, externals=sub_externals, - repo_path=self._repo_dir) + repo_path_abs=self._repo_dir) self._generator.write_config(dest_dir, filename=filename) @@ -1804,11 +1831,11 @@ def idempotence_check(self, checkout_dir): checkout_externals --status does not cause errors""" cwd = os.getcwd() os.chdir(checkout_dir) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.status_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.status_args) self.assertTrue(overall == 0) os.chdir(cwd) @@ -1822,8 +1849,8 @@ def test_submodule_checkout_bare(self): simple_ext_fork_tag = "(tag1)" simple_ext_fork_status = " " self.create_externals_file(branch_name=self._bare_branch_name) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) @@ -1852,8 +1879,8 @@ def test_submodule_checkout_none(self): """ self.create_externals_file(branch_name=self._bare_branch_name, sub_externals="none") - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) @@ -1874,8 +1901,8 @@ def test_submodule_checkout_config(self): # pylint: disable=too-many-locals status_check = "-" # Not checked out as submodule self.create_externals_file(branch_name=self._config_branch_name, sub_externals=self._container_extern_name) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) + overall, _ = self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) self.assertTrue(overall == 0) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) @@ -1938,17 +1965,17 @@ def test_error_unknown_protocol(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_protocol(under_test_dir, 'simp_branch', + self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', 'this-protocol-does-not-exist') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_switch_protocol(self): """Verify that a runtime error is raised when the user switches @@ -1959,15 +1986,15 @@ def test_error_switch_protocol(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_protocol(under_test_dir, 'simp_branch', 'svn') + self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', 'svn') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_unknown_tag(self): """Verify that a runtime error is raised when the user specified tag @@ -1975,17 +2002,17 @@ def test_error_unknown_tag(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', 'this-tag-does-not-exist', SIMPLE_REPO_NAME) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_overspecify_tag_branch(self): """Verify that a runtime error is raised when the user specified both @@ -1993,18 +2020,18 @@ def test_error_overspecify_tag_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME, - remove_branch=False) + self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', + 'this-tag-does-not-exist', SIMPLE_REPO_NAME, + remove_branch=False) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_underspecify_tag_branch(self): """Verify that a runtime error is raised when the user specified @@ -2012,17 +2039,17 @@ def test_error_underspecify_tag_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_underspecify_branch_tag(under_test_dir, - 'simp_branch') + self._generator.write_without_branch_tag(cloned_repo_dir, + 'simp_branch') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_missing_url(self): """Verify that a runtime error is raised when the user specified @@ -2030,17 +2057,17 @@ def test_error_missing_url(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_underspecify_remove_url(under_test_dir, + self._generator.write_without_repo_url(cloned_repo_dir, 'simp_branch') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) if __name__ == '__main__': diff --git a/test/test_unit_repository_svn.py b/test/test_unit_repository_svn.py old mode 100644 new mode 100755 index 41b173bf3d..d9309df7f6 --- a/test/test_unit_repository_svn.py +++ b/test/test_unit_repository_svn.py @@ -60,7 +60,7 @@ def setUp(self): self._name = 'component' rdata = {ExternalsDescription.PROTOCOL: 'svn', ExternalsDescription.REPO_URL: - 'https://svn-ccsm-models.cgd.ucar.edu/', + 'https://svn-ccsm-models.cgd.ucar.edu', ExternalsDescription.TAG: 'mosart/trunk_tags/mosart1_0_26', } From d4fbadb1067e5e81eafe723d1578486eed2c6c2d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 13 Jan 2023 17:27:35 -0700 Subject: [PATCH 128/332] Run through black --- python/ctsm/test/test_sys_fsurdat_modifier.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index fc5eb8d0e9..526e5f5928 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -170,15 +170,15 @@ def test_opt_sections(self): ], ] ) - lev2_two = np.empty( (2,3,5,5) ) - lev2_two[0,:,:,:] = 200. - lev2_two[1,:,:,:] = 100. - lev2_five = np.empty( (5,3,5,5) ) - lev2_five[0,:,:,:] = 1. - lev2_five[1,:,:,:] = 2. - lev2_five[2,:,:,:] = 3. - lev2_five[3,:,:,:] = 4. - lev2_five[4,:,:,:] = 5. + lev2_two = np.empty((2, 3, 5, 5)) + lev2_two[0, :, :, :] = 200.0 + lev2_two[1, :, :, :] = 100.0 + lev2_five = np.empty((5, 3, 5, 5)) + lev2_five[0, :, :, :] = 1.0 + lev2_five[1, :, :, :] = 2.0 + lev2_five[2, :, :, :] = 3.0 + lev2_five[3, :, :, :] = 4.0 + lev2_five[4, :, :, :] = 5.0 lev1 = np.array( [ [ From e30022a93586007c6f93dc2095abf70b4ac0ab45 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 13 Jan 2023 18:04:31 -0700 Subject: [PATCH 129/332] Start adding a test for 1x1_mexicocity, it currently fails as expected, because the modify config file is not complete --- python/ctsm/test/test_sys_fsurdat_modifier.py | 36 +++++++++++++ .../modify_fsurdat_1x1mexicocity.cfg | 51 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 526e5f5928..0cd4514f90 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -216,6 +216,42 @@ def test_opt_sections(self): np.testing.assert_array_equal(fsurdat_out_data.ALB_ROOF_DIR, lev2_two) np.testing.assert_array_equal(fsurdat_out_data.TK_ROOF, lev2_five) + def test_1x1_mexicocity(self): + """ + Test that the mexicocity file is handled correctly + """ + self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_1x1mexicocity.cfg") + expectfile = os.path.join( + self._testinputs_path, + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc" + ) + outfile = os.path.join( + self._tempdir, + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc" + ) + infile = os.path.join( + self._testinputs_path, "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" + ) + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "-i", + infile, + "-o", + outfile, + ] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) + + # Read the resultant output file and make sure the fields are changed as expected + fsurdat_out_data = xr.open_dataset(outfile) + fsurdat_inp_data = xr.open_dataset(infile) + fsurdat_exp_data = xr.open_dataset(expectfile) + + self.assertFalse(fsurdat_out_data.equals(fsurdat_inp_data)) + # assert that fsurdat_out equals fsurdat_out_baseline + self.assertTrue(fsurdat_out_data.equals(fsurdat_exp_data)) + def test_cfg_file_DNE_fail(self): """ Test that if the config file does not exist that it gracefully fails diff --git a/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg b/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg new file mode 100644 index 0000000000..c068c5d851 --- /dev/null +++ b/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg @@ -0,0 +1,51 @@ +[modify_fsurdat_basic_options] + +idealized = False +process_subgrid_section = True +process_var_list_section = True +include_nonveg = True + +landmask_file = UNSET + +lat_dimname = lsmlat +lon_dimname = lsmlon + +dom_pft = UNSET + +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET +soil_color = UNSET +std_elev = UNSET +max_sat_area = UNSET + +lnd_lat_1 = -90 +lnd_lat_2 = 90 +lnd_lon_1 = 0 +lnd_lon_2 = 360 + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# Set to 100% urban for the test +PCT_NATVEG = 0.0 +PCT_CROP = 0.0 +PCT_LAKE = 0.0 +PCT_GLACIER = 0.0 +PCT_WETLAND = 0.0 +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 + + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# Set lake-depth to 200 for the test +LAKEDEPTH = 200.00 + +# Set soem urban multidimensional variables to 200 for the test +# If given as a single value set all to the value, if as list set by dimension and if array as an array +CANYON_HWR = 200.00 150.0 100. +HT_ROOF = 200.0 150.0 100. +T_BUILDING_MIN = 200 150.0 100. +ALB_ROOF_DIR = 200. 100. +TK_ROOF = 1. 2. 3. 4. 5. From 62b9a3fc723eb918cf8972228045e1e8ea52496d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 13:16:19 -0700 Subject: [PATCH 130/332] More work on 1x1_mexicocity test, add more variables, stest still fails, utest passes, lint and black clean --- python/ctsm/test/test_sys_fsurdat_modifier.py | 13 ++-- python/ctsm/test/test_unit_modify_fsurdat.py | 8 +-- .../modify_fsurdat_1x1mexicocity.cfg | 67 ++++++++++++++----- 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 0cd4514f90..223f678c3b 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -220,18 +220,21 @@ def test_1x1_mexicocity(self): """ Test that the mexicocity file is handled correctly """ - self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_1x1mexicocity.cfg") + self._cfg_file_path = os.path.join( + self._testinputs_path, "modify_fsurdat_1x1mexicocity.cfg" + ) expectfile = os.path.join( self._testinputs_path, - "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc" + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc", ) outfile = os.path.join( self._tempdir, - "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc" + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc", ) infile = os.path.join( - self._testinputs_path, "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" - ) + self._testinputs_path, + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc", + ) sys.argv = [ "fsurdat_modifier", self._cfg_file_path, diff --git a/python/ctsm/test/test_unit_modify_fsurdat.py b/python/ctsm/test/test_unit_modify_fsurdat.py index 5cbdefcc48..a075035b73 100755 --- a/python/ctsm/test/test_unit_modify_fsurdat.py +++ b/python/ctsm/test/test_unit_modify_fsurdat.py @@ -374,14 +374,8 @@ def test_getNotRectangle_latsOutOfBounds(self): latixy=latixy, ) - def test_set_varlist(self): - """Test the set_varlist method for all the dimensions that works""" - vallist = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] - settings = {"var_lev0": 100.0, "var_lev1": vallist, "var_lev2": vallist} - self.modify_fsurdat.set_varlist(settings) - def test_check_varlist_lists(self): - """Test the check_varlist method for lists""" + """Test the check_varlist method for list for dimensions that works""" lev1list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] lev2list = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] settings = {"var_lev1": lev1list, "var_lev2": lev2list} diff --git a/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg b/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg index c068c5d851..a4118a3255 100644 --- a/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg +++ b/python/ctsm/test/testinputs/modify_fsurdat_1x1mexicocity.cfg @@ -27,25 +27,58 @@ lnd_lon_2 = 360 # Section for subgrid_fractions [modify_fsurdat_subgrid_fractions] -# Set to 100% urban for the test -PCT_NATVEG = 0.0 -PCT_CROP = 0.0 -PCT_LAKE = 0.0 -PCT_GLACIER = 0.0 -PCT_WETLAND = 0.0 +# If subgrid_fractions = True this section will be enabled + # NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area PCT_URBAN = 100.0 0.0 0.0 - +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 # Section with a list of variables to prcoess [modify_fsurdat_variable_list] -# Set lake-depth to 200 for the test -LAKEDEPTH = 200.00 - -# Set soem urban multidimensional variables to 200 for the test -# If given as a single value set all to the value, if as list set by dimension and if array as an array -CANYON_HWR = 200.00 150.0 100. -HT_ROOF = 200.0 150.0 100. -T_BUILDING_MIN = 200 150.0 100. -ALB_ROOF_DIR = 200. 100. -TK_ROOF = 1. 2. 3. 4. 5. +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area + +# Variables on numurbl which is 3 +CANYON_HWR = 1.18 1.18 1.18 +EM_IMPROAD = 0.95 0.95 0.95 +EM_PERROAD = 0.95 0.95 0.95 +EM_ROOF = 0.9 0.9 0.9 +EM_WALL = 0.85 0.85 0.85 +HT_ROOF = 18.8 18.8 18.8 +THICK_ROOF = 0.185 0.185 0.185 +THICK_WALL = 0.45 0.45 0.45 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 9.4 9.4 9.4 +WTLUNIT_ROOF = 0.55 0.55 0.55 +WTROAD_PERV = 0.04 0.04 0.04 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 5 5 5 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.08 0.08 +ALB_IMPROAD_DIF = 0.08 0.08 +ALB_PERROAD_DIR = 0.08 0.08 +ALB_PERROAD_DIF = 0.08 0.08 +ALB_ROOF_DIR = 0.2 0.2 +ALB_ROOF_DIF = 0.2 0.2 +ALB_WALL_DIR = 0.25 0.25 +ALB_WALL_DIF = 0.25 0.25 + +# Variabls on nlevurb which is 5 +TK_ROOF = 0.20 0.93 0.93 0.03 0.16 +TK_WALL = 0.88 0.88 0.88 0.88 0.88 +TK_IMPROAD = 0.82 0.82 2.10 2.10 2.10 +CV_ROOF = 1760000.0 1500000.0 1500000.0 250000.0 870000.0 +CV_WALL = 1540000.0 1540000.0 1540000.0 1540000.0 1540000.0 +CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 From 9be40c61093e5be850c811b9f7639f35a64aa576 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 13:52:18 -0700 Subject: [PATCH 131/332] Set idealized to False for the subgrid and varlist tests, and to true for the test that ensures idealized variables don't conflict with varlist --- .../ctsm/test/test_unit_fsurdat_modifier.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 149ba5acac..d2f07d14b1 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -70,6 +70,8 @@ def test_read_subgrid(self): def test_read_subgrid_allglacier(self): """test a read of subgrid that's for all glacier""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "0. 0. 0.") self.config.set(section, "pct_lake", "0.") @@ -81,6 +83,8 @@ def test_read_subgrid_allglacier(self): def test_read_subgrid_allspecial(self): """test a read of subgrid that's all special landunits""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "0. 0. 0.") self.config.set(section, "pct_lake", "25.") @@ -92,6 +96,8 @@ def test_read_subgrid_allspecial(self): def test_read_subgrid_allurban(self): """test a read of subgrid that's all urban""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "100.0 0.0 0.0") self.config.set(section, "pct_lake", "0.") @@ -107,6 +113,8 @@ def test_read_var_list(self): def test_subgrid_outofrange(self): """test a read of subgrid that's out of range""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "101. 0. 0.") with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): @@ -114,6 +122,8 @@ def test_subgrid_outofrange(self): def test_subgrid_pct_urban_toosmall(self): """test a read of subgrid for PCT_URBAN that's an array too small""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "100. 0.") with self.assertRaisesRegex( @@ -123,6 +133,8 @@ def test_subgrid_pct_urban_toosmall(self): def test_subgrid_pct_urban_toobig(self): """test a read of subgrid for PCT_URBAN that's an array too big""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "100. 0. 0. 0.") with self.assertRaisesRegex( @@ -132,6 +144,8 @@ def test_subgrid_pct_urban_toobig(self): def test_subgrid_pct_urban_singlevalue(self): """test a read of subgrid for PCT_URBAN that's a single value""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "100.") with self.assertRaisesRegex( @@ -141,6 +155,8 @@ def test_subgrid_pct_urban_singlevalue(self): def test_subgrid_notsumtohundred(self): """test a read of subgrid that's doesn't sum to a hundred""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "0. 0. 0.") self.config.set(section, "pct_lake", "0.") @@ -155,13 +171,17 @@ def test_subgrid_notsumtohundred(self): def test_subgrid_badvar(self): """test a read of subgrid for a variable thats not in the list""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "badvariable", "100.") with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): read_subgrid(self.config, self.cfg_path) def test_varlist_varinidealized(self): - """test a read of varlist for a variable thats in the idealized list""" + """test a read of varlist for a variable thats in the idealized list, when idealized is on""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") section = "modify_fsurdat_variable_list" self.config.set(section, "PCT_SAND", "100.") with self.assertRaisesRegex( @@ -173,6 +193,8 @@ def test_varlist_varinidealized(self): def test_varlist_varinsubgrid(self): """test a read of varlist for a variable thats in the subgrid list""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "False") section = "modify_fsurdat_variable_list" self.config.set(section, "PCT_GLACIER", "100.") with self.assertRaisesRegex( From 16d94b0578c45cd93309496353843a57de0179d2 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 15:03:58 -0700 Subject: [PATCH 132/332] Add a check that makes sure that dom_pft and idealized aren't on at the same time, it fails as expected --- .../ctsm/test/test_unit_fsurdat_modifier.py | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index d2f07d14b1..f44878bb56 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -Unit tests for fsurdat_modifier subroutines: read_subgrid, read_varlist +Unit tests for fsurdat_modifier subroutines: """ import unittest @@ -11,14 +11,17 @@ import tempfile from configparser import ConfigParser +import xarray as xr from ctsm import unit_testing from ctsm.path_utils import path_to_ctsm_root from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process from ctsm.modify_input_files.fsurdat_modifier import read_subgrid +from ctsm.modify_input_files.fsurdat_modifier import read_opt_config from ctsm.modify_input_files.fsurdat_modifier import read_var_list from ctsm.modify_input_files.fsurdat_modifier import check_no_subgrid_section from ctsm.modify_input_files.fsurdat_modifier import check_no_varlist_section +from ctsm.modify_input_files.modify_fsurdat import ModifyFsurdat # Allow test names that pylint doesn't like; otherwise hard to make them # readable @@ -27,8 +30,10 @@ # pylint: disable=protected-access +# Allow all the instance attributes that we need +# pylint: disable=too-many-instance-attributes class TestFSurdatModifier(unittest.TestCase): - """Tests the read_subgrid and read_var_list methods""" + """Tests the fsurdat_modifier subroutines""" def setUp(self): """Setup for trying out the methods""" @@ -57,6 +62,17 @@ def setUp(self): self.cfg_path = str(parser.cfg_path) self.config = ConfigParser() self.config.read(self.cfg_path) + my_data = xr.open_dataset( self._fsurdat_in ) + self.modify_fsurdat = ModifyFsurdat( + my_data=my_data, + lon_1=0.0, + lon_2=360.0, + lat_1=90.0, + lat_2=90.0, + landmask_file=None, + lat_dimname=None, + lon_dimname=None, + ) def tearDown(self): """ @@ -64,6 +80,16 @@ def tearDown(self): """ shutil.rmtree(self._tempdir, ignore_errors=True) + def test_dom_pft_and_idealized_fails(self): + """test a simple read of subgrid""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + self.config.set(section, "dom_pft", "1") + with self.assertRaisesRegex( + SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" + ): + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + def test_read_subgrid(self): """test a simple read of subgrid""" read_subgrid(self.config, self.cfg_path) @@ -179,7 +205,8 @@ def test_subgrid_badvar(self): read_subgrid(self.config, self.cfg_path) def test_varlist_varinidealized(self): - """test a read of varlist for a variable thats in the idealized list, when idealized is on""" + """test a read of varlist for a variable thats in the idealized list, + when idealized is on""" section = "modify_fsurdat_basic_options" self.config.set(section, "idealized", "True") section = "modify_fsurdat_variable_list" From ac7c23e76940f48606b0d09d55f7cea8bcb5edda Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 17:25:13 -0700 Subject: [PATCH 133/332] Add some consistency checking tests that fail as expected --- .../ctsm/test/test_unit_fsurdat_modifier.py | 48 ++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index f44878bb56..585292277e 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -30,6 +30,8 @@ # pylint: disable=protected-access +# Allow as many public methods as needed... +# pylint: disable=too-many-public-methods # Allow all the instance attributes that we need # pylint: disable=too-many-instance-attributes class TestFSurdatModifier(unittest.TestCase): @@ -62,7 +64,7 @@ def setUp(self): self.cfg_path = str(parser.cfg_path) self.config = ConfigParser() self.config.read(self.cfg_path) - my_data = xr.open_dataset( self._fsurdat_in ) + my_data = xr.open_dataset(self._fsurdat_in) self.modify_fsurdat = ModifyFsurdat( my_data=my_data, lon_1=0.0, @@ -81,7 +83,7 @@ def tearDown(self): shutil.rmtree(self._tempdir, ignore_errors=True) def test_dom_pft_and_idealized_fails(self): - """test a simple read of subgrid""" + """test a that dom_pft and idealized fails gracefully""" section = "modify_fsurdat_basic_options" self.config.set(section, "idealized", "True") self.config.set(section, "dom_pft", "1") @@ -90,6 +92,48 @@ def test_dom_pft_and_idealized_fails(self): ): read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + def test_subgrid_and_idealized_fails(self): + """test that dom_pft and idealized fails gracefully""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + self.config.set(section, "process_subgrid_section", "True") + with self.assertRaisesRegex( + SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" + ): + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + + def test_optional_only_true_and_false(self): + """test that optional settings can only be true or false""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "dom_pft", "1") + varlist = ( + "idealized", + "include_nonveg", + "process_subgrid_section", + "process_var_list_section", + ) + for var in varlist: + self.config.set(section, var, "True") + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + for var in varlist: + self.config.set(section, var, "False") + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + self.config.set(section, "dom_pft", "UNSET") + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + for var in varlist: + with self.assertRaisesRegex(SystemExit, "must be either True or False"): + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + + def test_include_nonveg_and_idealized_fails(self): + """test a simple read of subgrid""" + section = "modify_fsurdat_basic_options" + self.config.set(section, "idealized", "True") + self.config.set(section, "include_nonveg", "True") + with self.assertRaisesRegex( + SystemExit, "idealized AND include_nonveg can NOT both be on, pick one or the other" + ): + read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + def test_read_subgrid(self): """test a simple read of subgrid""" read_subgrid(self.config, self.cfg_path) From be956b6d1fedb5e156f30314a8c769fdecbd5935 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 19:49:52 -0700 Subject: [PATCH 134/332] Add a subroutine to read the control variables, that can also do error checking --- .../modify_input_files/fsurdat_modifier.py | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 6653a4937d..e01877822d 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -363,6 +363,41 @@ def read_opt_config(modify_fsurdat, config, cfg_path, section): ) +def read_option_control(config, section, cfg_path): + """Read the option control section""" + # required but fallback values available for variables omitted + # entirely from the .cfg file + idealized = get_config_value( + config=config, + section=section, + item="idealized", + file_path=cfg_path, + convert_to_type=bool, + ) + process_subgrid = get_config_value( + config=config, + section=section, + item="process_subgrid_section", + file_path=cfg_path, + convert_to_type=bool, + ) + process_var_list = get_config_value( + config=config, + section=section, + item="process_var_list_section", + file_path=cfg_path, + convert_to_type=bool, + ) + include_nonveg = get_config_value( + config=config, + section=section, + item="include_nonveg", + file_path=cfg_path, + convert_to_type=bool, + ) + return (idealized, process_subgrid, process_var_list, include_nonveg) + + def fsurdat_modifier(parser): """Implementation of fsurdat_modifier command""" # read the .cfg (config) file @@ -401,37 +436,9 @@ def fsurdat_modifier(parser): errmsg = "Output file already exists: " + fsurdat_out abort(errmsg) - # required but fallback values available for variables omitted - # entirely from the .cfg file - idealized = get_config_value( - config=config, - section=section, - item="idealized", - file_path=cfg_path, - convert_to_type=bool, - ) - process_subgrid = get_config_value( - config=config, - section=section, - item="process_subgrid_section", - file_path=cfg_path, - convert_to_type=bool, + (idealized, process_subgrid, process_var_list, include_nonveg) = read_option_control( + config, section, cfg_path ) - process_var_list = get_config_value( - config=config, - section=section, - item="process_var_list_section", - file_path=cfg_path, - convert_to_type=bool, - ) - include_nonveg = get_config_value( - config=config, - section=section, - item="include_nonveg", - file_path=cfg_path, - convert_to_type=bool, - ) - lnd_lat_1 = get_config_value( config=config, section=section, From a23e03f6dc0871b66395151b05f5b8dffb7f782d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 21:22:57 -0700 Subject: [PATCH 135/332] Use read_option_control in the test --- python/ctsm/test/test_unit_fsurdat_modifier.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 585292277e..aafebbdf55 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -17,6 +17,7 @@ from ctsm.path_utils import path_to_ctsm_root from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process from ctsm.modify_input_files.fsurdat_modifier import read_subgrid +from ctsm.modify_input_files.fsurdat_modifier import read_option_control from ctsm.modify_input_files.fsurdat_modifier import read_opt_config from ctsm.modify_input_files.fsurdat_modifier import read_var_list from ctsm.modify_input_files.fsurdat_modifier import check_no_subgrid_section @@ -90,7 +91,7 @@ def test_dom_pft_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" ): - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) def test_subgrid_and_idealized_fails(self): """test that dom_pft and idealized fails gracefully""" @@ -100,7 +101,7 @@ def test_subgrid_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" ): - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) def test_optional_only_true_and_false(self): """test that optional settings can only be true or false""" @@ -114,15 +115,15 @@ def test_optional_only_true_and_false(self): ) for var in varlist: self.config.set(section, var, "True") - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) for var in varlist: self.config.set(section, var, "False") - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) self.config.set(section, "dom_pft", "UNSET") - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) for var in varlist: with self.assertRaisesRegex(SystemExit, "must be either True or False"): - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) def test_include_nonveg_and_idealized_fails(self): """test a simple read of subgrid""" @@ -132,7 +133,7 @@ def test_include_nonveg_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND include_nonveg can NOT both be on, pick one or the other" ): - read_opt_config(self.modify_fsurdat, self.config, self.cfg_path, section) + read_option_control(self.config, section, self.cfg_path) def test_read_subgrid(self): """test a simple read of subgrid""" From f28f37b3268c3ba4715c64188deeaebe250e64c0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 14 Jan 2023 22:38:29 -0700 Subject: [PATCH 136/332] Move read of dom_pft to read_option_control and get all the unit testing working --- .../modify_input_files/fsurdat_modifier.py | 41 +++++++++++-------- .../ctsm/test/test_unit_fsurdat_modifier.py | 28 ++++++++----- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index e01877822d..fc29a4605c 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -276,16 +276,6 @@ def modify_optional( def read_opt_config(modify_fsurdat, config, cfg_path, section): """read the optional parts of the main section config file""" # not required: user may set these in the .cfg file - max_pft = int(max(modify_fsurdat.file.lsmpft)) - dom_pft = get_config_value( - config=config, - section=section, - item="dom_pft", - file_path=cfg_path, - allowed_values=range(max_pft + 1), # integers from 0 to max_pft - convert_to_type=int, - can_be_unset=True, - ) lai = get_config_value( config=config, @@ -355,7 +345,6 @@ def read_opt_config(modify_fsurdat, config, cfg_path, section): max_sat_area, std_elev, soil_color, - dom_pft, lai, sai, hgt_top, @@ -363,7 +352,7 @@ def read_opt_config(modify_fsurdat, config, cfg_path, section): ) -def read_option_control(config, section, cfg_path): +def read_option_control(modify_fsurdat, config, section, cfg_path): """Read the option control section""" # required but fallback values available for variables omitted # entirely from the .cfg file @@ -395,7 +384,24 @@ def read_option_control(config, section, cfg_path): file_path=cfg_path, convert_to_type=bool, ) - return (idealized, process_subgrid, process_var_list, include_nonveg) + max_pft = int(max(modify_fsurdat.file.lsmpft)) + dom_pft = get_config_value( + config=config, + section=section, + item="dom_pft", + file_path=cfg_path, + allowed_values=range(max_pft + 1), # integers from 0 to max_pft + convert_to_type=int, + can_be_unset=True, + ) + if dom_pft is not None and idealized: + abort("idealized AND dom_pft can NOT both be on, pick one or the other") + if include_nonveg and idealized: + abort("idealized AND include_nonveg can NOT both be on, pick one or the other") + if process_subgrid and idealized: + abort("idealized AND process_subgrid_section can NOT both be on, pick one or the other") + + return (idealized, process_subgrid, process_var_list, include_nonveg, dom_pft) def fsurdat_modifier(parser): @@ -436,9 +442,6 @@ def fsurdat_modifier(parser): errmsg = "Output file already exists: " + fsurdat_out abort(errmsg) - (idealized, process_subgrid, process_var_list, include_nonveg) = read_option_control( - config, section, cfg_path - ) lnd_lat_1 = get_config_value( config=config, section=section, @@ -495,12 +498,16 @@ def fsurdat_modifier(parser): lon_dimname, ) + # Read control information about the optional sections + (idealized, process_subgrid, process_var_list, include_nonveg, dom_pft) = read_option_control( + modify_fsurdat, config, section, cfg_path + ) + # Read parts that are optional ( max_sat_area, std_elev, soil_color, - dom_pft, lai, sai, hgt_top, diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index aafebbdf55..5befe584c3 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -18,7 +18,6 @@ from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process from ctsm.modify_input_files.fsurdat_modifier import read_subgrid from ctsm.modify_input_files.fsurdat_modifier import read_option_control -from ctsm.modify_input_files.fsurdat_modifier import read_opt_config from ctsm.modify_input_files.fsurdat_modifier import read_var_list from ctsm.modify_input_files.fsurdat_modifier import check_no_subgrid_section from ctsm.modify_input_files.fsurdat_modifier import check_no_varlist_section @@ -91,17 +90,20 @@ def test_dom_pft_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" ): - read_option_control(self.config, section, self.cfg_path) + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_subgrid_and_idealized_fails(self): """test that dom_pft and idealized fails gracefully""" section = "modify_fsurdat_basic_options" self.config.set(section, "idealized", "True") + self.config.set(section, "include_nonveg", "False") self.config.set(section, "process_subgrid_section", "True") + self.config.set(section, "dom_pft", "UNSET") with self.assertRaisesRegex( - SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" + SystemExit, + "idealized AND process_subgrid_section can NOT both be on, pick one or the other", ): - read_option_control(self.config, section, self.cfg_path) + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_optional_only_true_and_false(self): """test that optional settings can only be true or false""" @@ -115,15 +117,19 @@ def test_optional_only_true_and_false(self): ) for var in varlist: self.config.set(section, var, "True") - read_option_control(self.config, section, self.cfg_path) + self.config.set(section, "idealized", "False") + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) for var in varlist: self.config.set(section, var, "False") - read_option_control(self.config, section, self.cfg_path) + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) self.config.set(section, "dom_pft", "UNSET") - read_option_control(self.config, section, self.cfg_path) - for var in varlist: - with self.assertRaisesRegex(SystemExit, "must be either True or False"): - read_option_control(self.config, section, self.cfg_path) + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + var = "idealized" + self.config.set(section, var, "Thing") + with self.assertRaisesRegex( + SystemExit, "Non-boolean value found for .cfg file variable: " + var + ): + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_include_nonveg_and_idealized_fails(self): """test a simple read of subgrid""" @@ -133,7 +139,7 @@ def test_include_nonveg_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND include_nonveg can NOT both be on, pick one or the other" ): - read_option_control(self.config, section, self.cfg_path) + read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_read_subgrid(self): """test a simple read of subgrid""" From 3da338893d9924e6683568546ac07e7298da9451 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 15 Jan 2023 00:51:49 -0700 Subject: [PATCH 137/332] Add options to allow some options if they are flagged, and only abort if idealized variables are being used if idealized is on, now all tests pass, lint and black clean --- .../modify_input_files/fsurdat_modifier.py | 40 +++++++++++++++---- python/ctsm/test/test_sys_fsurdat_modifier.py | 7 +++- .../ctsm/test/test_unit_fsurdat_modifier.py | 8 ++-- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index fc29a4605c..992aa1323c 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -57,6 +57,20 @@ def fsurdat_modifier_arg_process(): type=str, help="The output surface dataset with the modifications. ", ) + parser.add_argument( + "--allow_ideal_and_include_non_veg", + required=False, + default=False, + action="store_true", + help="Allow both idealized and include_nonveg to be on at the same time. ", + ) + parser.add_argument( + "--allow_dom_pft_and_idealized", + required=False, + default=False, + action="store_true", + help="Allow both idealized and dom_pft to be on at the same time. ", + ) add_logging_args(parser) args = parser.parse_args() process_logging_args(args) @@ -150,7 +164,7 @@ def read_subgrid(config, cfg_path, numurbl=3): return subgrid_settings -def read_var_list(config): +def read_var_list(config, idealized=True): """Read the variable list section from the config file""" section = "modify_fsurdat_variable_list" if not config.has_section(section): @@ -190,7 +204,7 @@ def read_var_list(config): "ws", ] for var in var_list: - if ideal_list.count(var) != 0: + if idealized and ideal_list.count(var) != 0: abort( var + " is a special variable handled in the idealized section." @@ -352,7 +366,14 @@ def read_opt_config(modify_fsurdat, config, cfg_path, section): ) -def read_option_control(modify_fsurdat, config, section, cfg_path): +def read_option_control( + modify_fsurdat, + config, + section, + cfg_path, + allow_ideal_and_include_non_veg=False, + allow_dom_pft_and_idealized=False, +): """Read the option control section""" # required but fallback values available for variables omitted # entirely from the .cfg file @@ -394,9 +415,9 @@ def read_option_control(modify_fsurdat, config, section, cfg_path): convert_to_type=int, can_be_unset=True, ) - if dom_pft is not None and idealized: + if dom_pft is not None and idealized and not allow_dom_pft_and_idealized: abort("idealized AND dom_pft can NOT both be on, pick one or the other") - if include_nonveg and idealized: + if include_nonveg and idealized and not allow_ideal_and_include_non_veg: abort("idealized AND include_nonveg can NOT both be on, pick one or the other") if process_subgrid and idealized: abort("idealized AND process_subgrid_section can NOT both be on, pick one or the other") @@ -500,7 +521,12 @@ def fsurdat_modifier(parser): # Read control information about the optional sections (idealized, process_subgrid, process_var_list, include_nonveg, dom_pft) = read_option_control( - modify_fsurdat, config, section, cfg_path + modify_fsurdat, + config, + section, + cfg_path, + parser.allow_ideal_and_include_non_veg, + parser.allow_dom_pft_and_idealized, ) # Read parts that are optional @@ -540,7 +566,7 @@ def fsurdat_modifier(parser): check_no_subgrid_section(config) if process_var_list: - varlist = read_var_list(config) + varlist = read_var_list(config, idealized=idealized) update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) modify_fsurdat.set_varlist(update_list, cfg_path) else: diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 223f678c3b..cde9cac846 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -363,7 +363,12 @@ def test_allInfo(self): self._create_config_file_complete() # run the fsurdat_modifier tool - sys.argv = ["fsurdat_modifier", self._cfg_file_path] + sys.argv = [ + "fsurdat_modifier", + self._cfg_file_path, + "--allow_ideal_and_include_non_veg", + "--allow_dom_pft_and_idealized", + ] parser = fsurdat_modifier_arg_process() fsurdat_modifier(parser) # the critical piece of this test is that the above command diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 5befe584c3..fee4ec2ab3 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -186,7 +186,7 @@ def test_read_subgrid_allurban(self): def test_read_var_list(self): """test a simple read of var_list""" - read_var_list(self.config) + read_var_list(self.config, idealized=True) def test_subgrid_outofrange(self): """test a read of subgrid that's out of range""" @@ -267,7 +267,7 @@ def test_varlist_varinidealized(self): "is a special variable handled in the idealized section." + " This should NOT be handled in the variiable list section. Special idealized vars =", ): - read_var_list(self.config) + read_var_list(self.config, idealized=True) def test_varlist_varinsubgrid(self): """test a read of varlist for a variable thats in the subgrid list""" @@ -280,7 +280,7 @@ def test_varlist_varinsubgrid(self): "is a variable handled in the subgrid section." + " This should NOT be handled in the variiable list section. Subgrid vars =", ): - read_var_list(self.config) + read_var_list(self.config, idealized=False) def test_varlist_monthlyvar(self): """test a read of varlist for a variable thats one of the monthly @@ -293,7 +293,7 @@ def test_varlist_monthlyvar(self): + " This should NOT be handled in the variiable list section." + " Monthly vars handled this way =", ): - read_var_list(self.config) + read_var_list(self.config, idealized=False) def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" From 49889d4f3c97202acfe846213c45ac2fc45de1c4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 15 Jan 2023 09:58:17 -0700 Subject: [PATCH 138/332] Add test files for the system testing for creation of urban surfdata files using fsurdat_modifier --- ..._mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc | 3 +++ ...tyMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc create mode 100644 python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc diff --git a/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc new file mode 100644 index 0000000000..4bf009ebe6 --- /dev/null +++ b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef672e3ab2c237fd6f76afdd1dfa7d01263c01bd94bef2a8f37acfba3a9b6e36 +size 27300 diff --git a/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc new file mode 100644 index 0000000000..80a66c286f --- /dev/null +++ b/python/ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206_modified.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfc14c3f7e9efa4f0acc4caca22b616e09e04b2713ab1424d695492ac52327c +size 27772 From df014edbb2eca3fcaa87823c29ab12cf77531106 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 17:33:32 -0700 Subject: [PATCH 139/332] Add a check_args method for subset_data and some testing for it --- python/ctsm/subset_data.py | 60 ++++++++++++----------- python/ctsm/test/test_unit_subset_data.py | 11 +++-- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 8015fe1858..172ec2b97b 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -345,6 +345,37 @@ def get_parser(): return parser +def check_args(parser, args): + """Check the command line arguments""" + # --------------------------------- # + # print help and exit when no option is chosen + if args.run_type not in ("point", "region"): + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n Must supply a positional argument: 'point' or 'region'. + """ + ) + raise parser.error(err_msg) + + if not any( + [ + args.create_surfdata, + args.create_domain, + args.create_landuse, + args.create_datm, + ] + ): + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n Must supply one of: + \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n + """ + ) + raise parser.error(err_msg) + + def setup_user_mods(user_mods_dir, cesmroot): """ Sets up the user mods files and directories @@ -580,34 +611,7 @@ def main(): parser = get_parser() args = parser.parse_args() - # --------------------------------- # - # print help and exit when no option is chosen - if args.run_type not in ("point", "region"): - err_msg = textwrap.dedent( - """\ - \n ------------------------------------ - \n Must supply a positional argument: 'point' or 'region'. - """ - ) - raise args.error(err_msg) - - if not any( - [ - args.create_surfdata, - args.create_domain, - args.create_landuse, - args.create_datm, - ] - ): - err_msg = textwrap.dedent( - """\ - \n ------------------------------------ - \n Must supply one of: - \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n - """ - ) - raise args.error(err_msg) - + check_args(parser, args) # --------------------------------- # # process logging args (i.e. debug and verbose) process_logging_args(args) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index aa4c412bbb..9fa09a9662 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -17,7 +17,7 @@ # pylint: disable=wrong-import-position from ctsm import unit_testing -from ctsm.subset_data import get_parser, setup_files +from ctsm.subset_data import get_parser, setup_files, check_args from ctsm.path_utils import path_to_ctsm_root # pylint: disable=invalid-name @@ -29,10 +29,10 @@ class TestSubsetData(unittest.TestCase): """ def setUp(self): - sys.argv = ["subset_data", "point"] + sys.argv = ["subset_data", "point", "--create-surface"] DEFAULTS_FILE = "default_data.cfg" - parser = get_parser() - self.args = parser.parse_args() + self.parser = get_parser() + self.args = self.parser.parse_args() self.cesmroot = path_to_ctsm_root() self.defaults = configparser.ConfigParser() self.defaults.read(os.path.join(self.cesmroot, "tools/site_and_regional", DEFAULTS_FILE)) @@ -41,12 +41,14 @@ def test_inputdata_setup_files_basic(self): """ Test """ + check_args(self.parser, self.args) setup_files(self.args, self.defaults, self.cesmroot) def test_inputdata_setup_files_inputdata_dne(self): """ Test that inputdata directory does not exist """ + check_args(self.parser, self.args) self.defaults.set("main", "clmforcingindir", "/zztop") with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) @@ -55,6 +57,7 @@ def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad """ + check_args(self.parser, self.args) self.args.inputdatadir = "/zztop" with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) From c8dee88ed6d658596f3fc1c335685c0fff1ec717 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 18:10:41 -0700 Subject: [PATCH 140/332] Remove parser from args of check_args, and add two fail tests for check_args --- python/ctsm/subset_data.py | 8 ++++---- python/ctsm/test/test_unit_subset_data.py | 25 ++++++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 172ec2b97b..b6244ce2f0 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -345,7 +345,7 @@ def get_parser(): return parser -def check_args(parser, args): +def check_args(args): """Check the command line arguments""" # --------------------------------- # # print help and exit when no option is chosen @@ -356,7 +356,7 @@ def check_args(parser, args): \n Must supply a positional argument: 'point' or 'region'. """ ) - raise parser.error(err_msg) + raise argparse.ArgumentError(None, err_msg) if not any( [ @@ -373,7 +373,7 @@ def check_args(parser, args): \n --create-surface \n --create-landuse \n --create-datm \n --create-domain \n """ ) - raise parser.error(err_msg) + raise argparse.ArgumentError(None, err_msg) def setup_user_mods(user_mods_dir, cesmroot): @@ -611,7 +611,7 @@ def main(): parser = get_parser() args = parser.parse_args() - check_args(parser, args) + check_args(args) # --------------------------------- # # process logging args (i.e. debug and verbose) process_logging_args(args) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 9fa09a9662..8bdacab180 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -8,6 +8,7 @@ import unittest import configparser +import argparse import os import sys @@ -41,23 +42,41 @@ def test_inputdata_setup_files_basic(self): """ Test """ - check_args(self.parser, self.args) + check_args(self.args) setup_files(self.args, self.defaults, self.cesmroot) def test_inputdata_setup_files_inputdata_dne(self): """ Test that inputdata directory does not exist """ - check_args(self.parser, self.args) + check_args(self.args) self.defaults.set("main", "clmforcingindir", "/zztop") with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) + def test_check_args_nooutput(self): + """ + Test that check args aborts when no-output is asked for + """ + sys.argv = ["subset_data", "point"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex(argparse.ArgumentError, "Must supply one of"): + check_args(self.args) + + def test_check_args_notype(self): + """ + Test that check args aborts when no type is asked for + """ + sys.argv = ["subset_data"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex(argparse.ArgumentError, "Must supply a positional argument:"): + check_args(self.args) + def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad """ - check_args(self.parser, self.args) + check_args(self.args) self.args.inputdatadir = "/zztop" with self.assertRaisesRegex(SystemExit, "inputdata directory does not exist"): setup_files(self.args, self.defaults, self.cesmroot) From 0178ca7f2e7de81d7cb493c3b1b7ff526ef1bfad Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 18:23:59 -0700 Subject: [PATCH 141/332] Add a check that the entered defaults config file exista and a test for it --- python/ctsm/subset_data.py | 9 +++++++++ python/ctsm/test/test_unit_subset_data.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index b6244ce2f0..9fca0b4f86 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -375,6 +375,15 @@ def check_args(args): ) raise argparse.ArgumentError(None, err_msg) + if not os.path.exists( args.config_file ): + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n Entered default config file does not exist" + """ + ) + raise argparse.ArgumentError(None, err_msg) + def setup_user_mods(user_mods_dir, cesmroot): """ diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 8bdacab180..2286233490 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -72,6 +72,15 @@ def test_check_args_notype(self): with self.assertRaisesRegex(argparse.ArgumentError, "Must supply a positional argument:"): check_args(self.args) + def test_check_args_badconfig(self): + """ + Test that check args aborts when a config file is entered that doesn't exist + """ + sys.argv = ["subset_data", "point", "--create-surface", "--file", "zztop"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex(argparse.ArgumentError, "Entered default config file does not exist"): + check_args(self.args) + def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad From 50b5fd9e77f0a7c33d1ecc8c95f2da91c3d0df33 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 18:43:39 -0700 Subject: [PATCH 142/332] Add --out-surface option so you can specify the output surface dataset name, add a simple check for using it --- python/ctsm/subset_data.py | 12 +++++++++++- python/ctsm/test/test_unit_subset_data.py | 13 ++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 9fca0b4f86..26d4afb80c 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -309,6 +309,16 @@ def get_parser(): type=str, default="", ) + + subparser.add_argument( + "--out-surface", + help="Output surface dataset name \ + (if you want to override the default based on the current date). \n \ + (only valid if outputing a surface dataset)", + action="store", + dest="out_surface", + type=str, + ) cesmroot = path_to_ctsm_root() defaults_file = os.path.join(cesmroot, DEFAULTS_CONFIG) subparser.add_argument( @@ -375,7 +385,7 @@ def check_args(args): ) raise argparse.ArgumentError(None, err_msg) - if not os.path.exists( args.config_file ): + if not os.path.exists(args.config_file): err_msg = textwrap.dedent( """\ \n ------------------------------------ diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 2286233490..85d1f3c838 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -78,9 +78,20 @@ def test_check_args_badconfig(self): """ sys.argv = ["subset_data", "point", "--create-surface", "--file", "zztop"] self.args = self.parser.parse_args() - with self.assertRaisesRegex(argparse.ArgumentError, "Entered default config file does not exist"): + with self.assertRaisesRegex( + argparse.ArgumentError, "Entered default config file does not exist" + ): check_args(self.args) + def test_check_args_outsurfdat_provided(self): + """ + Test that check args allows an output surface dataset to be specified + when create-surface is on + """ + sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", "outputsurface.nc"] + self.args = self.parser.parse_args() + check_args(self.args) + def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad From a41831c35847b6b7442c7929784838871e3a8226 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 18:57:08 -0700 Subject: [PATCH 143/332] Add a check that out-surface can not be given if create-surface is not set, and a test for it that passes --- python/ctsm/subset_data.py | 9 +++++++++ python/ctsm/test/test_unit_subset_data.py | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 26d4afb80c..7f0be28af6 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -394,6 +394,15 @@ def check_args(args): ) raise argparse.ArgumentError(None, err_msg) + if args.out_surface and not args.create_surfdata: + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n out-surface option is given without the --create-surface option" + """ + ) + raise argparse.ArgumentError(None, err_msg) + def setup_user_mods(user_mods_dir, cesmroot): """ diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 85d1f3c838..347e24d1cd 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -92,6 +92,18 @@ def test_check_args_outsurfdat_provided(self): self.args = self.parser.parse_args() check_args(self.args) + def test_check_args_outsurfdat_fails_without_create_surface(self): + """ + Test that check args does not allow an output surface dataset to be specified + when create-surface is not on + """ + sys.argv = ["subset_data", "point", "--create-landuse", "--out-surface", "outputsurface.nc"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex( + argparse.ArgumentError, "out-surface option is given without the --create-surface option" + ): + check_args(self.args) + def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad From 3b841e5f14a696553ea168f92b7ac7ece5e35c88 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 20:50:25 -0700 Subject: [PATCH 144/332] Add test to make sure if a file exists and overwrite isn't active that it fails --- python/ctsm/test/test_unit_subset_data.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 347e24d1cd..7d3b30c460 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -104,6 +104,19 @@ def test_check_args_outsurfdat_fails_without_create_surface(self): ): check_args(self.args) + def test_check_args_outsurfdat_fails_without_overwrite(self): + """ + Test that check args does not allow an output surface dataset to be specified + for an existing dataset without the overwrite option + """ + sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", "ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc"] + self.args = self.parser.parse_args() + with self.assertRaisesRegex( + argparse.ArgumentError, "out-surface filename exists and the overwrite option was not also selected", + ): + check_args(self.args) + + def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad From 8a46808709a4f524f58ceca7bc9672444108402c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 20:55:54 -0700 Subject: [PATCH 145/332] Add error check for out_surface that exists without overwrite, utest now passes --- python/ctsm/subset_data.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 7f0be28af6..21832fde63 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -403,6 +403,15 @@ def check_args(args): ) raise argparse.ArgumentError(None, err_msg) + if args.out_surface and os.path.exists(args.out_surface) and not args.overwrite: + err_msg = textwrap.dedent( + """\ + \n ------------------------------------ + \n out-surface filename exists and the overwrite option was not also selected" + """ + ) + raise argparse.ArgumentError(None, err_msg) + def setup_user_mods(user_mods_dir, cesmroot): """ From 53af6f9101d581d5c721dbe718e609429c93762d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 21:23:39 -0700 Subject: [PATCH 146/332] Have the defaults file read in the testinputs directory, run through black, check some of the values from setup_files --- python/ctsm/subset_data.py | 2 +- python/ctsm/test/test_unit_subset_data.py | 30 ++++++++++++++++---- python/ctsm/test/testinputs/default_data.cfg | 28 ++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 python/ctsm/test/testinputs/default_data.cfg diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 21832fde63..07f7e33cff 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -67,7 +67,7 @@ from ctsm.site_and_regional.regional_case import RegionalCase from ctsm.args_utils import plon_type, plat_type from ctsm.path_utils import path_to_ctsm_root -from ctsm.utils import abort +from ctsm.utils import abort, add_tag_to_filename # -- import ctsm logging flags from ctsm.ctsm_logging import ( diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 7d3b30c460..16c0b29b15 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -31,7 +31,7 @@ class TestSubsetData(unittest.TestCase): def setUp(self): sys.argv = ["subset_data", "point", "--create-surface"] - DEFAULTS_FILE = "default_data.cfg" + DEFAULTS_FILE = os.path.join(os.getcwd(), "ctsm/test/testinputs/default_data.cfg") self.parser = get_parser() self.args = self.parser.parse_args() self.cesmroot = path_to_ctsm_root() @@ -43,7 +43,17 @@ def test_inputdata_setup_files_basic(self): Test """ check_args(self.args) - setup_files(self.args, self.defaults, self.cesmroot) + files = setup_files(self.args, self.defaults, self.cesmroot) + self.assertEqual( + files["fsurf_in"], + "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + "fsurf_in filename not whats expected", + ) + self.assertEqual( + files["main_dir"], + "/glade/p/cesmdata/inputdata", + "main_dir directory not whats expected", + ) def test_inputdata_setup_files_inputdata_dne(self): """ @@ -100,7 +110,8 @@ def test_check_args_outsurfdat_fails_without_create_surface(self): sys.argv = ["subset_data", "point", "--create-landuse", "--out-surface", "outputsurface.nc"] self.args = self.parser.parse_args() with self.assertRaisesRegex( - argparse.ArgumentError, "out-surface option is given without the --create-surface option" + argparse.ArgumentError, + "out-surface option is given without the --create-surface option", ): check_args(self.args) @@ -109,14 +120,21 @@ def test_check_args_outsurfdat_fails_without_overwrite(self): Test that check args does not allow an output surface dataset to be specified for an existing dataset without the overwrite option """ - sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", "ctsm/test/testinputs/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc"] + sys.argv = [ + "subset_data", + "point", + "--create-surface", + "--out-surface", + "ctsm/test/testinputs/" + + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + ] self.args = self.parser.parse_args() with self.assertRaisesRegex( - argparse.ArgumentError, "out-surface filename exists and the overwrite option was not also selected", + argparse.ArgumentError, + "out-surface filename exists and the overwrite option was not also selected", ): check_args(self.args) - def test_inputdata_setup_files_bad_inputdata_arg(self): """ Test that inputdata directory provided on command line does not exist if it's bad diff --git a/python/ctsm/test/testinputs/default_data.cfg b/python/ctsm/test/testinputs/default_data.cfg new file mode 100644 index 0000000000..7e841dca54 --- /dev/null +++ b/python/ctsm/test/testinputs/default_data.cfg @@ -0,0 +1,28 @@ +[main] +clmforcingindir = /glade/p/cesmdata/inputdata + +[datm_gswp3] +dir = atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516 +domain = domain.lnd.360x720_gswp3.0v1.c170606.nc +solardir = Solar +precdir = Precip +tpqwdir = TPHWL +solartag = clmforc.GSWP3.c2011.0.5x0.5.Solr. +prectag = clmforc.GSWP3.c2011.0.5x0.5.Prec. +tpqwtag = clmforc.GSWP3.c2011.0.5x0.5.TPQWL. +solarname = CLMGSWP3v1.Solar +precname = CLMGSWP3v1.Precip +tpqwname = CLMGSWP3v1.TPQW + +[surfdat] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +surfdat_16pft = surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc +surfdat_78pft = surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr2000_c190214.nc + +[landuse] +dir = lnd/clm2/surfdata_map/release-clm5.0.18 +landuse_16pft = landuse.timeseries_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c190214.nc +landuse_78pft = landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1850-2015_c190214.nc + +[domain] +file = share/domains/domain.lnd.fv0.9x1.25_gx1v7.151020.nc From 685c2b1ce75ca35aa0520169f7c4287b686e9d1d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 21:27:46 -0700 Subject: [PATCH 147/332] Check for fsurf_out is None when out-surface not set and value given when set --- python/ctsm/test/test_unit_subset_data.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 16c0b29b15..f779c40cee 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -49,6 +49,11 @@ def test_inputdata_setup_files_basic(self): "surfdata_0.9x1.25_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", "fsurf_in filename not whats expected", ) + self.assertEqual( + files["fsurf_out"], + None, + "fsurf_out filename not whats expected", + ) self.assertEqual( files["main_dir"], "/glade/p/cesmdata/inputdata", @@ -101,6 +106,12 @@ def test_check_args_outsurfdat_provided(self): sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", "outputsurface.nc"] self.args = self.parser.parse_args() check_args(self.args) + files = setup_files(self.args, self.defaults, self.cesmroot) + self.assertEqual( + files["fsurf_out"], + "outputsurface.nc", + "fsurf_out filename not whats expected", + ) def test_check_args_outsurfdat_fails_without_create_surface(self): """ From 9f79c7b7c18864878babc9aa87e90a334d20378a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 21:33:47 -0700 Subject: [PATCH 148/332] Add fsurf_out to setup_files returned dictionary, set to None, unless args.out_surface is set --- python/ctsm/subset_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 07f7e33cff..15d0034438 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -67,7 +67,7 @@ from ctsm.site_and_regional.regional_case import RegionalCase from ctsm.args_utils import plon_type, plat_type from ctsm.path_utils import path_to_ctsm_root -from ctsm.utils import abort, add_tag_to_filename +from ctsm.utils import abort # -- import ctsm logging flags from ctsm.ctsm_logging import ( @@ -492,6 +492,10 @@ def setup_files(args, defaults, cesmroot): fsurf_in = defaults.get("surfdat", "surfdat_" + num_pft + "pft") fluse_in = defaults.get("landuse", "landuse_" + num_pft + "pft") + if args.out_surface: + fsurf_out = args.out_surface + else: + fsurf_out = None file_dict = { "main_dir": clmforcingindir, @@ -505,6 +509,7 @@ def setup_files(args, defaults, cesmroot): os.path.join(defaults.get("landuse", "dir")), ), "fsurf_in": fsurf_in, + "fsurf_out": fsurf_out, "fluse_in": fluse_in, "datm_tuple": DatmFiles( dir_input_datm, From eb7e94fea737a746ce8e7d83df698d99149f6aef Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 21:45:49 -0700 Subject: [PATCH 149/332] Specify the filename if out_surface is given, otherwise do things as currently done --- python/ctsm/site_and_regional/regional_case.py | 8 ++++++-- python/ctsm/site_and_regional/single_point_case.py | 7 +++++-- python/ctsm/subset_data.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/python/ctsm/site_and_regional/regional_case.py b/python/ctsm/site_and_regional/regional_case.py index 8d5d81e014..2a2dc66bce 100644 --- a/python/ctsm/site_and_regional/regional_case.py +++ b/python/ctsm/site_and_regional/regional_case.py @@ -145,7 +145,7 @@ def create_domain_at_reg(self, indir, file): f_in.close() f_out.close() - def create_surfdata_at_reg(self, indir, file, user_mods_dir): + def create_surfdata_at_reg(self, indir, file, user_mods_dir, specify_fsurf_out): """ Create surface data file for this RegionalCase class. """ @@ -154,7 +154,11 @@ def create_surfdata_at_reg(self, indir, file, user_mods_dir): # specify files fsurf_in = os.path.join(indir, file) - fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + if specify_fsurf_out is None: + fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + else: + fsurf_out = specify_fsurf_out + logger.info("fsurf_in: %s", fsurf_in) logger.info("fsurf_out: %s", os.path.join(self.out_dir, fsurf_out)) diff --git a/python/ctsm/site_and_regional/single_point_case.py b/python/ctsm/site_and_regional/single_point_case.py index 05b918d316..96544ff9c1 100644 --- a/python/ctsm/site_and_regional/single_point_case.py +++ b/python/ctsm/site_and_regional/single_point_case.py @@ -450,7 +450,7 @@ def modify_surfdata_atpoint(self, f_orig): return f_mod - def create_surfdata_at_point(self, indir, file, user_mods_dir): + def create_surfdata_at_point(self, indir, file, user_mods_dir, specify_fsurf_out): """ Create surface data file at a single point. """ @@ -464,7 +464,10 @@ def create_surfdata_at_point(self, indir, file, user_mods_dir): # specify file fsurf_in = os.path.join(indir, file) - fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + if specify_fsurf_out is None: + fsurf_out = add_tag_to_filename(fsurf_in, self.tag, replace_res=True) + else: + fsurf_out = specify_fsurf_out logger.info("fsurf_in: %s", fsurf_in) logger.info("fsurf_out: %s", os.path.join(self.out_dir, fsurf_out)) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index 15d0034438..a258c7b72e 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -569,7 +569,10 @@ def subset_point(args, file_dict: dict): # -- Create CTSM surface data file if single_point.create_surfdata: single_point.create_surfdata_at_point( - file_dict["fsurf_dir"], file_dict["fsurf_in"], args.user_mods_dir + file_dict["fsurf_dir"], + file_dict["fsurf_in"], + args.user_mods_dir, + specify_fsurf_out=file_dict["fsurf_out"], ) # -- Create CTSM transient landuse data file @@ -629,7 +632,10 @@ def subset_region(args, file_dict: dict): # -- Create CTSM surface data file if region.create_surfdata: region.create_surfdata_at_reg( - file_dict["fsurf_dir"], file_dict["fsurf_in"], args.user_mods_dir + file_dict["fsurf_dir"], + file_dict["fsurf_in"], + args.user_mods_dir, + specify_fsurf_out=file_dict["fsurf_out"], ) # -- Create CTSM transient landuse data file From a74ac52087a383ef672d96708b99207614c8658c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 22:01:56 -0700 Subject: [PATCH 150/332] Add capability to do the urban single point sites --- tools/mksurfdata_map/Makefile.data | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index caaef030e3..7d52fb230b 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -54,6 +54,9 @@ endif MKSURFDATA = $(BATCHJOBS) $(PWD)/mksurfdata.pl SUBSETDATA = $(PWD)/../site_and_regional/subset_data +MODIFYSURF = $(PWD)/../modify_input_files/fsurdat_modifier + +CDATE = $(shell date +%y%m%d) # subset_data options # @@ -71,9 +74,9 @@ SUBSETDATA_1X1_SMALL := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA \ # NOTE: The 1850 smallvilleIA site is constructed to start with 100% natural vegetation, so we can test transition to crops SUBSETDATA_1X1_SMALL1850 := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA --dompft 13 --pctpft 100 -SUBSETDATA_1X1_MEXICOCITY := --lat 19.5 --lon 260.5 --site 1x1_mexicocityMEX -SUBSETDATA_1X1_VANCOUVER := --lat 49.5 --lon 236.5 --site 1x1_vancouverCAN -SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha +SUBSETDATA_1X1_MEXICOCITY := --lat 19.5 --lon 260.5 --site 1x1_mexicocityMEX --out-surface surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc +SUBSETDATA_1X1_VANCOUVER := --lat 49.5 --lon 236.5 --site 1x1_vancouverCAN --out-surface surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc +SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha --out-surface surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc # f19 and f09 are standard resolutions, f10 is used for testing, f45 is used for FATES # ne30np4 is standard resolution for SE dycore in CAM, C96 is standard for fv3 dycore # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore @@ -260,15 +263,24 @@ crop-global-SSP5-8.5 : FORCE # urban : urban-present urban-alpha -urban-present : FORCE +urban-present : mexicocity vancouver + +mexicocity : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) + $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + +vancouver : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) + $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 urban-alpha : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) - + $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc # # landuse timeseries From 666d89ce34a8c61067df9a060707064afec8df5b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 18 Jan 2023 22:02:25 -0700 Subject: [PATCH 151/332] Add modify file for mexicocity --- .../modify_1x1_mexicocityMEX.cfg | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg diff --git a/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg new file mode 100644 index 0000000000..b138c75879 --- /dev/null +++ b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg @@ -0,0 +1,135 @@ +[modify_fsurdat_basic_options] + +lat_dimname = lsmlat +lon_dimname = lsmlon + +# idealized (bool) +# When user wants existing values in fsurdat to persist in all except the +# variables that they explicitly request to change, then set this to False. +# When user wants idealized representation of the land by resetting all +# fsurdat variables, some through this file and others by using hardwired +# defaults, then set this to True. Hardwired values are as follows: +# zbedrock = 10 +# SLOPE = 0 +# PFTDATA_MASK = 1 +# LANDFRAC_PFT = 1 +# PCT_NATVEG = 100 other landunits 0 +# PCT_SAND = 43 corresponds to loam +# PCT_CLAY = 18 corresponds to loam +# ORGANIC = 0 +idealized = False + +# sub +process_subgrid_section = True +process_var_list_section = True + +# Boundaries of user-defined rectangle (float) +# If lat_1 > lat_2, the code creates two rectangles, one in the north and +# one in the south. +# If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. +# Alternatively, user may specify a custom area in a .nc landmask_file +# below. If set, this will override the lat/lon settings. +# ----------------------------------- +# southernmost latitude for rectangle +lnd_lat_1 = -90 +# northernmost latitude for rectangle +lnd_lat_2 = 90 +# westernmost longitude for rectangle +lnd_lon_1 = 0 +# easternmost longitude for rectangle +lnd_lon_2 = 360 +# user-defined mask in a file, as alternative to setting lat/lon values +landmask_file = UNSET + +# PFT/CFT to be set to 100% according to user-defined mask. +# If idealized = True and dom_plant = UNSET, the latter defaults to 0 +# (bare soil). Valid values range from 0 to a max value (int) that one can +# obtain from the fsurdat_in file using ncdump (or method preferred by user). +# The max valid value will equal (lsmpft - 1) and will also equal the last +# value of cft(cft). +dom_pft = UNSET + +# LAI, SAI, HEIGHT_TOP, and HEIGHT_BOT values by month for dom_plant +# If dom_plant = 0, the next four default to 0 (space-delimited list +# of floats without brackets). +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET + +# SOIL_COLOR accepts integer values from 1 to 20 (see CTSM Technote for info). +# if idealized = True and soil_color = UNSET, soil_color = 15. +soil_color = UNSET + +# STD_ELEV (standard deviation of elevation) value (in meters) over the +# user_defined mask (float). +# if idealized = True and std_elev = UNSET, std_elev = 0. +std_elev = UNSET + +# FMAX (maximum fractional saturated area) value (fraction) over the +# user_defined mask (float). +# if idealized = True and max_sat_area = UNSET, max_sat_area = 0. +max_sat_area = UNSET + +# Set non-vegetation landunits to 0 (bool). +zero_nonveg = False + +include_nonveg = True + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area + +# Variables on numurbl which is 3 +CANYON_HWR = 1.18 1.18 1.18 +EM_IMPROAD = 0.95 0.95 0.95 +EM_PERROAD = 0.95 0.95 0.95 +EM_ROOF = 0.9 0.9 0.9 +EM_WALL = 0.85 0.85 0.85 +HT_ROOF = 18.8 18.8 18.8 +THICK_ROOF = 0.185 0.185 0.185 +THICK_WALL = 0.45 0.45 0.45 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 9.4 9.4 9.4 +WTLUNIT_ROOF = 0.55 0.55 0.55 +WTROAD_PERV = 0.04 0.04 0.04 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 5 5 5 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.08 0.08 +ALB_IMPROAD_DIF = 0.08 0.08 +ALB_PERROAD_DIR = 0.08 0.08 +ALB_PERROAD_DIF = 0.08 0.08 +ALB_ROOF_DIR = 0.2 0.2 +ALB_ROOF_DIF = 0.2 0.2 +ALB_WALL_DIR = 0.25 0.25 +ALB_WALL_DIF = 0.25 0.25 + +# Variabls on nlevurb which is 5 +TK_ROOF = 0.20 0.93 0.93 0.03 0.16 +TK_WALL = 0.88 0.88 0.88 0.88 0.88 +TK_IMPROAD = 0.82 0.82 2.10 2.10 2.10 +CV_ROOF = 1760000.0 1500000.0 1500000.0 250000.0 870000.0 +CV_WALL = 1540000.0 1540000.0 1540000.0 1540000.0 1540000.0 +CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 From e3a59fe3edada15b16c4ebe52d6ddb30e3e53c76 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 19 Jan 2023 16:13:09 -0700 Subject: [PATCH 152/332] A little more documentation comments on the modify fsurdat config files --- .../modify_1x1_mexicocityMEX.cfg | 23 +++++++++---------- .../modify_fsurdat_template.cfg | 2 +- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg index b138c75879..0e3c4c6ab6 100644 --- a/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg +++ b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg @@ -8,28 +8,22 @@ lon_dimname = lsmlon # variables that they explicitly request to change, then set this to False. # When user wants idealized representation of the land by resetting all # fsurdat variables, some through this file and others by using hardwired -# defaults, then set this to True. Hardwired values are as follows: -# zbedrock = 10 -# SLOPE = 0 -# PFTDATA_MASK = 1 -# LANDFRAC_PFT = 1 -# PCT_NATVEG = 100 other landunits 0 -# PCT_SAND = 43 corresponds to loam -# PCT_CLAY = 18 corresponds to loam -# ORGANIC = 0 +# defaults, then set this to True. idealized = False -# sub +# subgrid section to set the PCT_* variables process_subgrid_section = True +# Variable list section to set specific variable names process_var_list_section = True -# Boundaries of user-defined rectangle (float) +# Boundaries of user-defined rectangle to apply changes (float) # If lat_1 > lat_2, the code creates two rectangles, one in the north and # one in the south. # If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. # Alternatively, user may specify a custom area in a .nc landmask_file # below. If set, this will override the lat/lon settings. # ----------------------------------- +# (Use a grid that includes the entire globe as we are just setting a single point) # southernmost latitude for rectangle lnd_lat_1 = -90 # northernmost latitude for rectangle @@ -74,6 +68,7 @@ max_sat_area = UNSET # Set non-vegetation landunits to 0 (bool). zero_nonveg = False +# Include other land units besides vegetated include_nonveg = True # Section for subgrid_fractions @@ -90,10 +85,14 @@ PCT_LAKE = 0.0 # Section with a list of variables to prcoess [modify_fsurdat_variable_list] +# IMPORTANT NOTE: Config file strings are case inssentive! +# +# As such it will check for variable names both in lower case and upper case. +# # If variable_list = True this section will be enabled # Can't specify PFT as they are in dom_pft # Add variables on the file and assign a new value -# can't specify soil_color, max_sat_area +# can't specify soil_color, max_sat_area or other things that are above. # Variables on numurbl which is 3 CANYON_HWR = 1.18 1.18 1.18 diff --git a/tools/modify_input_files/modify_fsurdat_template.cfg b/tools/modify_input_files/modify_fsurdat_template.cfg index aebe42a4f8..3661784521 100644 --- a/tools/modify_input_files/modify_fsurdat_template.cfg +++ b/tools/modify_input_files/modify_fsurdat_template.cfg @@ -44,7 +44,7 @@ process_subgrid_section = False # Process the optional section that handles modifying an arbitrary list of variables process_var_list_section = False -# Boundaries of user-defined rectangle (float) +# Boundaries of user-defined rectangle to apply changes (float) # If lat_1 > lat_2, the code creates two rectangles, one in the north and # one in the south. # If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. From f933d51b00064f8f50d0a1c758744acbab7b0d93 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 19 Jan 2023 17:03:32 -0700 Subject: [PATCH 153/332] Add in the other two urban surface datasets --- tools/mksurfdata_map/Makefile.data | 1 - .../modify_1x1_urbanc_alpha.cfg | 134 ++++++++++++++++++ .../modify_1x1_vancouverCAN.cfg | 134 ++++++++++++++++++ 3 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg create mode 100644 tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 7d52fb230b..7bac3b8923 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -62,7 +62,6 @@ CDATE = $(shell date +%y%m%d) # SUBSETDATA_POINT = $(SUBSETDATA) point --overwrite --uniform-snowpack --cap-saturation --outdir . SUBSETDATA_POINT_ALLLU = $(SUBSETDATA_POINT) --include-nonveg -# NOTE: TODO: We need settings in subset_data to handle urban datasets see issue #1665 SUBSETDATA_POINT_URBAN = $(SUBSETDATA_POINT) --include-nonveg # Subset data sites... diff --git a/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg b/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg new file mode 100644 index 0000000000..3bb56917a3 --- /dev/null +++ b/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg @@ -0,0 +1,134 @@ +[modify_fsurdat_basic_options] + +lat_dimname = lsmlat +lon_dimname = lsmlon + +# idealized (bool) +# When user wants existing values in fsurdat to persist in all except the +# variables that they explicitly request to change, then set this to False. +# When user wants idealized representation of the land by resetting all +# fsurdat variables, some through this file and others by using hardwired +# defaults, then set this to True. +idealized = False + +# subgrid section to set the PCT_* variables +process_subgrid_section = True +# Variable list section to set specific variable names +process_var_list_section = True + +# Boundaries of user-defined rectangle to apply changes (float) +# If lat_1 > lat_2, the code creates two rectangles, one in the north and +# one in the south. +# If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. +# Alternatively, user may specify a custom area in a .nc landmask_file +# below. If set, this will override the lat/lon settings. +# ----------------------------------- +# (Use a grid that includes the entire globe as we are just setting a single point) +# southernmost latitude for rectangle +lnd_lat_1 = -90 +# northernmost latitude for rectangle +lnd_lat_2 = 90 +# westernmost longitude for rectangle +lnd_lon_1 = 0 +# easternmost longitude for rectangle +lnd_lon_2 = 360 +# user-defined mask in a file, as alternative to setting lat/lon values +landmask_file = UNSET + +# PFT/CFT to be set to 100% according to user-defined mask. +# If idealized = True and dom_plant = UNSET, the latter defaults to 0 +# (bare soil). Valid values range from 0 to a max value (int) that one can +# obtain from the fsurdat_in file using ncdump (or method preferred by user). +# The max valid value will equal (lsmpft - 1) and will also equal the last +# value of cft(cft). +dom_pft = UNSET + +# LAI, SAI, HEIGHT_TOP, and HEIGHT_BOT values by month for dom_plant +# If dom_plant = 0, the next four default to 0 (space-delimited list +# of floats without brackets). +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET + +# SOIL_COLOR accepts integer values from 1 to 20 (see CTSM Technote for info). +# if idealized = True and soil_color = UNSET, soil_color = 15. +soil_color = UNSET + +# STD_ELEV (standard deviation of elevation) value (in meters) over the +# user_defined mask (float). +# if idealized = True and std_elev = UNSET, std_elev = 0. +std_elev = UNSET + +# FMAX (maximum fractional saturated area) value (fraction) over the +# user_defined mask (float). +# if idealized = True and max_sat_area = UNSET, max_sat_area = 0. +max_sat_area = UNSET + +# Set non-vegetation landunits to 0 (bool). +zero_nonveg = False + +# Include other land units besides vegetated +include_nonveg = True + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# IMPORTANT NOTE: Config file strings are case inssentive! +# +# As such it will check for variable names both in lower case and upper case. +# +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area or other things that are above. + +# Variables on numurbl which is 3 +CANYON_HWR = 0.42 0.42 0.42 +EM_IMPROAD = 0.973 0.973 0.973 +EM_PERROAD = 0.973 0.973 0.973 +EM_ROOF = 0.973 0.973 0.973 +EM_WALL = 0.973 0.973 0.973 +HT_ROOF = 6.4 6.4 6.4 +THICK_ROOF = 0.1141 0.1141 0.1141 +THICK_WALL = 0.1489 0.1489 0.1489 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 3.2 3.2 3.2 +WTLUNIT_ROOF = 0.445 0.445 0.445 +WTROAD_PERV = 0.68 0.68 0.68 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 4 4 4 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.15 0.15 +ALB_IMPROAD_DIF = 0.15 0.15 +ALB_PERROAD_DIR = 0.15 0.15 +ALB_PERROAD_DIF = 0.15 0.15 +ALB_ROOF_DIR = 0.21 0.21 +ALB_ROOF_DIF = 0.21 0.21 +ALB_WALL_DIR = 0.21 0.21 +ALB_WALL_DIF = 0.21 0.21 + +# Variabls on nlevurb which is 5 +TK_ROOF = 6.530 0.025 0.230 0.160 0.00 +TK_WALL = 0.610 0.430 0.024 0.160 0.00 +TK_IMPROAD = 1.170 0.300 0.300 0.420 0.00 +CV_ROOF = 2070000.0 7100.0 1500000.0 670000.0 0.0 +CV_WALL = 1250000.0 1400000.0 1300.0 670000.0 0.0 +CV_IMPROAD = 1140000.0 1050000.0 1050000.0 1290000.0 0.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 diff --git a/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg b/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg new file mode 100644 index 0000000000..a8ef5e328b --- /dev/null +++ b/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg @@ -0,0 +1,134 @@ +[modify_fsurdat_basic_options] + +lat_dimname = lsmlat +lon_dimname = lsmlon + +# idealized (bool) +# When user wants existing values in fsurdat to persist in all except the +# variables that they explicitly request to change, then set this to False. +# When user wants idealized representation of the land by resetting all +# fsurdat variables, some through this file and others by using hardwired +# defaults, then set this to True. +idealized = False + +# subgrid section to set the PCT_* variables +process_subgrid_section = True +# Variable list section to set specific variable names +process_var_list_section = True + +# Boundaries of user-defined rectangle to apply changes (float) +# If lat_1 > lat_2, the code creates two rectangles, one in the north and +# one in the south. +# If lon_1 > lon_2, the rectangle wraps around the 0-degree meridian. +# Alternatively, user may specify a custom area in a .nc landmask_file +# below. If set, this will override the lat/lon settings. +# ----------------------------------- +# (Use a grid that includes the entire globe as we are just setting a single point) +# southernmost latitude for rectangle +lnd_lat_1 = -90 +# northernmost latitude for rectangle +lnd_lat_2 = 90 +# westernmost longitude for rectangle +lnd_lon_1 = 0 +# easternmost longitude for rectangle +lnd_lon_2 = 360 +# user-defined mask in a file, as alternative to setting lat/lon values +landmask_file = UNSET + +# PFT/CFT to be set to 100% according to user-defined mask. +# If idealized = True and dom_plant = UNSET, the latter defaults to 0 +# (bare soil). Valid values range from 0 to a max value (int) that one can +# obtain from the fsurdat_in file using ncdump (or method preferred by user). +# The max valid value will equal (lsmpft - 1) and will also equal the last +# value of cft(cft). +dom_pft = UNSET + +# LAI, SAI, HEIGHT_TOP, and HEIGHT_BOT values by month for dom_plant +# If dom_plant = 0, the next four default to 0 (space-delimited list +# of floats without brackets). +lai = UNSET +sai = UNSET +hgt_top = UNSET +hgt_bot = UNSET + +# SOIL_COLOR accepts integer values from 1 to 20 (see CTSM Technote for info). +# if idealized = True and soil_color = UNSET, soil_color = 15. +soil_color = UNSET + +# STD_ELEV (standard deviation of elevation) value (in meters) over the +# user_defined mask (float). +# if idealized = True and std_elev = UNSET, std_elev = 0. +std_elev = UNSET + +# FMAX (maximum fractional saturated area) value (fraction) over the +# user_defined mask (float). +# if idealized = True and max_sat_area = UNSET, max_sat_area = 0. +max_sat_area = UNSET + +# Set non-vegetation landunits to 0 (bool). +zero_nonveg = False + +# Include other land units besides vegetated +include_nonveg = True + +# Section for subgrid_fractions +[modify_fsurdat_subgrid_fractions] +# If subgrid_fractions = True this section will be enabled + +# NOTE: PCT_URBAN must be a list of three floats that sum to the total urban area +PCT_URBAN = 100.0 0.0 0.0 +PCT_CROP = 0.0 +PCT_NATVEG= 0.0 +PCT_GLACIER= 0.0 +PCT_WETLAND= 0.0 +PCT_LAKE = 0.0 + +# Section with a list of variables to prcoess +[modify_fsurdat_variable_list] +# IMPORTANT NOTE: Config file strings are case inssentive! +# +# As such it will check for variable names both in lower case and upper case. +# +# If variable_list = True this section will be enabled +# Can't specify PFT as they are in dom_pft +# Add variables on the file and assign a new value +# can't specify soil_color, max_sat_area or other things that are above. + +# Variables on numurbl which is 3 +CANYON_HWR = 0.39 0.39 0.39 +EM_IMPROAD = 0.95 0.95 0.95 +EM_PERROAD = 0.95 0.95 0.95 +EM_ROOF = 0.92 0.92 0.92 +EM_WALL = 0.90 0.90 0.90 +HT_ROOF = 5.8 5.8 5.8 +THICK_ROOF = 0.070 0.070 0.070 +THICK_WALL = 0.20 0.20 0.20 +T_BUILDING_MIN = 200.0 200.0 200.0 +WIND_HGT_CANYON = 2.9 2.9 2.9 +WTLUNIT_ROOF = 0.51 0.51 0.51 +WTROAD_PERV = 0.11 0.11 0.11 +# NOTE: This variable is integer rather than float +NLEV_IMPROAD = 5 5 5 + +# Variables on numrad which is 2 +ALB_IMPROAD_DIR = 0.08 0.08 +ALB_IMPROAD_DIF = 0.08 0.08 +ALB_PERROAD_DIR = 0.08 0.08 +ALB_PERROAD_DIF = 0.08 0.08 +ALB_ROOF_DIR = 0.12 0.12 +ALB_ROOF_DIF = 0.12 0.12 +ALB_WALL_DIR = 0.50 0.50 +ALB_WALL_DIF = 0.50 0.50 + +# Variabls on nlevurb which is 5 +TK_ROOF = 1.40 1.40 1.40 0.03 1.51 +TK_WALL = 1.51 1.51 0.67 0.67 1.51 +TK_IMPROAD = 0.82 0.82 2.10 2.10 2.10 +CV_ROOF = 1760000.0 1760000.0 1760000.0 40000.0 2210000.0 +CV_WALL = 2110000.0 2110000.0 1000000.0 1000000.0 2110000.0 +CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 + +# Natural and Crop PFT's don't really need to be set, since they have zero area, but +# it looks better to do so +PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +PCT_CFT = 100. 0.0 From 3d2576944c1c1dd342d782d99c60c24643b48a03 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Fri, 20 Jan 2023 08:59:57 -0700 Subject: [PATCH 154/332] updates to surfwrapper --- tools/site_and_regional/neon_surf_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index d8e11415ff..26827ae53c 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -109,7 +109,7 @@ def main(): command1 = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite, '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] command2 = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', - 'subset_data_single_point', '--fates'] + 'subset_data_single_point', '--16pft'] else: command1 = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite,'--dompft',str(pft),'--crop', '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] From ba597a56027382595fc50c0d33cae2be17fb3042 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 09:04:02 -0700 Subject: [PATCH 155/332] add back space in subset_data --- tools/site_and_regional/subset_data | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/site_and_regional/subset_data b/tools/site_and_regional/subset_data index ad8e60e053..6477b3034d 100755 --- a/tools/site_and_regional/subset_data +++ b/tools/site_and_regional/subset_data @@ -4,6 +4,7 @@ This is a just top-level skeleton script that calls subset_data.py. The original code (subset_data.py) is located under python/ctsm folder. + For full instructions on how to run the code and different options, please check python/ctsm/subset_data.py file. This script extracts domain files, surface dataset, and DATM files From 3f4e88018dcd13824f71ba8dfdaac6ef96b52ef4 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 09:25:16 -0700 Subject: [PATCH 156/332] update flags --- .../modify_singlept_site_neon.py | 15 +++--- tools/site_and_regional/neon_surf_wrapper.py | 52 ++++++++++++++----- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 64e77eacd5..9bb08f7e50 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -175,11 +175,10 @@ def get_parser(): ) parser.add_argument( - "-f", - "--fates", - help="Modify FATES-specific surface data files (i.e. 16-PFT version", + "--16pft", + help="Modify 16-pft surface data files (e.g. for a FATES run)", action="store_true", - dest="fates", + dest="pft_16", default=False, ) @@ -250,7 +249,7 @@ def get_neon(neon_dir, site_name): return neon_file -def find_surffile(surf_dir, site_name, fates): +def find_surffile(surf_dir, site_name, pft_16): """ Function for finding and choosing surface file for a neon site. @@ -261,7 +260,7 @@ def find_surffile(surf_dir, site_name, fates): Args: surf_dir (str): directory of single point surface data site_name (str): 4 letter neon site name - fates (bool): if true, use 16-PFT version of surface data file + pft_16 (bool): if true, use 16-PFT version of surface data file Raises: Error if the surface data for the site is not created @@ -270,8 +269,8 @@ def find_surffile(surf_dir, site_name, fates): surf_file (str): name of the surface dataset file """ - if fates: - sf_name = "surfdata_1x1_NEON_16PFT_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" + if pft_16: + sf_name = "surfdata_1x1_NEON_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" else: sf_name = "surfdata_1x1_NEON_" + site_name + "*hist_78pfts_CMIP6_simyr2000_*.nc" diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index 26827ae53c..4eb8da3536 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -52,10 +52,16 @@ def get_parser(): dest="verbose", default=False) - parser.add_argument('-f', '--fates', - help='Create and/or modify surface data file for FATES. ', + parser.add_argument('--16pft', + help='Create and/or modify 16-PFT surface datasets (e.g. for a FATES run) ', action="store_true", - dest="fates", + dest="pft_16", + default=False) + + parser.add_argument('-m', '--mixed', + help='Do not overwrite surface dataset to be just one dominant PFT at 100%', + action="store_true", + dest="mixed", default=False) @@ -104,19 +110,41 @@ def main(): pft = row['pft'] clmsite = "1x1_NEON_"+site print ("Now processing site :", site) - if args.fates: - command1 = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite, - '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - command2 = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + if args.mixed and args.pft_16: + # don't set crop flag + # don't set a dominant pft + subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), + '--site',clmsite, '--create-surface','--uniform-snowpack', + '--cap-saturation','--verbose','--overwrite'] + modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', 'subset_data_single_point', '--16pft'] + elif args.pft_16: + # don't set crop flag + # set dominant pft + subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), + '--site',clmsite,'--dompft',str(pft),'--create-surface', + '--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] + modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + 'subset_data_single_point', '--16pft'] + elif args.mixed: + # set crop flag + # don't set dominant pft + subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), + '--site',clmsite,'--crop','--create-surface', + '--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] + modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + 'subset_data_single_point'] else: - command1 = ['./subset_data','point','--lat',str(lat),'--lon',str(lon),'--site',clmsite,'--dompft',str(pft),'--crop', - '--create-surface','--uniform-snowpack','--cap-saturation','--verbose','--overwrite'] - command2 = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', + # set crop flag + # set dominant pft + subset_command = ['./subset_data', 'point', '--lat', str(lat), '--lon', str(lon), + '--site', clmsite,'--crop', '--dompft', str(pft), '--create-surface', + '--uniform-snowpack', '--cap-saturation', '--verbose', '--overwrite'] + modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', 'subset_data_single_point'] - execute(command1) - execute(command2) + execute(subset_command) + execute(modify_command) if __name__ == "__main__": main() From b26d33c55fa639a34f5ee864f12bb6b294473686 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 09:29:10 -0700 Subject: [PATCH 157/332] update notes --- tools/site_and_regional/neon_surf_wrapper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/site_and_regional/neon_surf_wrapper.py b/tools/site_and_regional/neon_surf_wrapper.py index 4eb8da3536..3271c72f08 100755 --- a/tools/site_and_regional/neon_surf_wrapper.py +++ b/tools/site_and_regional/neon_surf_wrapper.py @@ -112,6 +112,7 @@ def main(): print ("Now processing site :", site) if args.mixed and args.pft_16: + # use surface dataset with 16 pfts, and don't overwrite with 100% 1 dominant PFT # don't set crop flag # don't set a dominant pft subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), @@ -120,6 +121,7 @@ def main(): modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', 'subset_data_single_point', '--16pft'] elif args.pft_16: + # use surface dataset with 16 pfts, but overwrite to 100% 1 dominant PFT # don't set crop flag # set dominant pft subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), @@ -128,6 +130,8 @@ def main(): modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', 'subset_data_single_point', '--16pft'] elif args.mixed: + # use surface dataset with 78 pfts, and don't overwrite with 100% 1 dominant PFT + # NOTE: FATES will currently not run with a 78-PFT surface dataset # set crop flag # don't set dominant pft subset_command = ['./subset_data','point','--lat',str(lat),'--lon',str(lon), @@ -136,6 +140,8 @@ def main(): modify_command = ['./modify_singlept_site_neon.py', '--neon_site', site, '--surf_dir', 'subset_data_single_point'] else: + # use surface dataset with 78 pfts, and overwrite to 100% 1 dominant PFT + # NOTE: FATES will currently not run with a 78-PFT surface dataset # set crop flag # set dominant pft subset_command = ['./subset_data', 'point', '--lat', str(lat), '--lon', str(lon), From ab5f0c94723c3624f5467a5795120d62299cfc72 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 09:35:18 -0700 Subject: [PATCH 158/332] update file name --- tools/site_and_regional/modify_singlept_site_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 9bb08f7e50..5dc56ae0b2 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -272,7 +272,7 @@ def find_surffile(surf_dir, site_name, pft_16): if pft_16: sf_name = "surfdata_1x1_NEON_"+site_name+"*hist_16pfts_Irrig_CMIP6_simyr2000_*.nc" else: - sf_name = "surfdata_1x1_NEON_" + site_name + "*hist_78pfts_CMIP6_simyr2000_*.nc" + sf_name = "surfdata_1x1_NEON_" +site_name+"*hist_78pfts_CMIP6_simyr2000_*.nc" print (os.path.join(surf_dir , sf_name)) surf_file = sorted(glob.glob(os.path.join(surf_dir , sf_name))) From d22eb008e2d457df918790c5a50dc10f1bda05a6 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 09:37:02 -0700 Subject: [PATCH 159/332] remove fates argument --- tools/site_and_regional/modify_singlept_site_neon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/site_and_regional/modify_singlept_site_neon.py b/tools/site_and_regional/modify_singlept_site_neon.py index 5dc56ae0b2..e135760a48 100755 --- a/tools/site_and_regional/modify_singlept_site_neon.py +++ b/tools/site_and_regional/modify_singlept_site_neon.py @@ -546,7 +546,7 @@ def main(): # -- Look for surface data surf_dir = args.surf_dir - surf_file = find_surffile(surf_dir, site_name, args.fates) + surf_file = find_surffile(surf_dir, site_name, args.pft_16) # -- directory structure current_dir = os.getcwd() From 207091a32c9b794d6b80ca50bd873592db48678e Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 10:52:28 -0700 Subject: [PATCH 160/332] add NEON test, add 1pt fates compset alias --- cime_config/config_compsets.xml | 10 ++++++++++ cime_config/testdefs/testlist_clm.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 6a0ac8b3f6..61931c94ba 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -41,11 +41,21 @@ 2000_DATM%1PT_CLM51%BGC_SICE_SOCN_SROF_SGLC_SWAV + + I1PtClm51Fates + 2000_DATM%1PT_CLM51%FATES_SICE_SOCN_SROF_SGLC_SWAV + + IHist1PtClm51Bgc HIST_DATM%1PT_CLM51%BGC_SICE_SOCN_SROF_SGLC_SWAV + + IHist1PtClm51Fates + HIST_DATM%1PT_CLM51%FATES_SICE_SOCN_SROF_SGLC_SWAV + + I1PtClm51SpRs 2000_DATM%1PT_CLM51%SP_SICE_SOCN_SROF_SGLC_SWAV diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index ffc56de0c3..e08a578abb 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1552,6 +1552,16 @@ + + + + + + + + + + From 02612b0eaff023c3c78b4d7e7c171a874dd213b3 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 12:20:21 -0700 Subject: [PATCH 161/332] exit if trying to run KONA or STER with FATES --- cime_config/testdefs/testlist_clm.xml | 2 +- cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands | 7 +++---- cime_config/usermods_dirs/NEON/FATES/STER/shell_commands | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index e08a578abb..05053fc5d1 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1557,7 +1557,7 @@ - + diff --git a/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands b/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands index 66f274dd36..316509046c 100644 --- a/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands @@ -1,7 +1,6 @@ ./xmlchange NEONSITE=KONA ./xmlchange PTS_LON=263.38956 ./xmlchange PTS_LAT=39.10828 -# Setup to run with prognostic crops for this site -# If you want to explicitly run in SP mode or add other -# options you'll need to add that after this... -./xmlchange CLM_BLDNML_OPTS="--bgc bgc --crop" +# We can't run crops with FATES yet +echo "Cannot run FATES at KONA because FATES does not yet have crops." +exit 1 diff --git a/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands b/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands index 38b173c309..685eeb2529 100644 --- a/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands @@ -1,7 +1,6 @@ ./xmlchange NEONSITE=STER ./xmlchange PTS_LON=256.96992 ./xmlchange PTS_LAT=40.45984 -# Setup to run with prognostic crops for this site -# If you want to explicitly run in SP mode or add other -# # options you'll need to add that after this... -./xmlchange CLM_BLDNML_OPTS="--bgc bgc --crop" +# We can't run crops with FATES yet +echo "Cannot run FATES at STER because FATES does not yet have crops." +exit 1 From 79829404f5e615384d3c4cc151035c3bfcb799e7 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 20 Jan 2023 12:44:16 -0700 Subject: [PATCH 162/332] update surface file name --- cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm index c842546ec7..2a6b4d76d5 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/user_nl_clm @@ -19,7 +19,7 @@ !---------------------------------------------------------------------------------- flanduse_timeseries = ' ' ! This isn't needed for a non transient case, but will be once we start using transient compsets -fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/FATES/surfdata_1x1_NEON_16PFT_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230110.nc" +fsurdat = "$DIN_LOC_ROOT/lnd/clm2/surfdata_map/NEON/16PFT_mixed/surfdata_1x1_NEON_${NEONSITE}_hist_16pfts_Irrig_CMIP6_simyr2000_c230120.nc" ! h1 output stream hist_fincl2 = 'FATES_AUTORESP','FCEV','FCTR','FGEV','FIRA','FSA','FSH','FATES_GPP','FATES_GPP_PF','H2OSOI', From 85efdc530a8907ba70662db52cf77d7b68a6b5b4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 20 Jan 2023 12:55:16 -0700 Subject: [PATCH 163/332] Respond to @slevisconsulting review with changes asked for --- python/ctsm/modify_input_files/fsurdat_modifier.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 992aa1323c..39bd95986c 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -287,9 +287,10 @@ def modify_optional( logger.info("dom_pft complete") -def read_opt_config(modify_fsurdat, config, cfg_path, section): - """read the optional parts of the main section config file""" - # not required: user may set these in the .cfg file +def read_cfg_optional_basic_opts(modify_fsurdat, config, cfg_path, section): + """Read the optional parts of the main section of the config file. + The main section is called modify_fsurdat_basic_options. + Users may set these optional parts but are not required to do so.""" lai = get_config_value( config=config, @@ -538,7 +539,7 @@ def fsurdat_modifier(parser): sai, hgt_top, hgt_bot, - ) = read_opt_config(modify_fsurdat, config, cfg_path, section) + ) = read_cfg_optional_basic_opts(modify_fsurdat, config, cfg_path, section) # ------------------------------ # modify surface data properties # ------------------------------ From 3293ef432d1f3b4f4390f729d3b17d36da2aac1c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 20 Jan 2023 14:41:51 -0700 Subject: [PATCH 164/332] Get test that fails if overwrite not selected and surface file is named and the file exists --- python/ctsm/test/test_unit_subset_data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index f779c40cee..94a53e2dd2 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -131,13 +131,15 @@ def test_check_args_outsurfdat_fails_without_overwrite(self): Test that check args does not allow an output surface dataset to be specified for an existing dataset without the overwrite option """ + outfile = os.path.join( os.getcwd(), "ctsm/test/testinputs/", "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" ) + self.assertTrue( os.path.exists( outfile ), str(outfile)+" outfile should exist" ) + sys.argv = [ "subset_data", "point", "--create-surface", "--out-surface", - "ctsm/test/testinputs/" - + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc", + outfile ] self.args = self.parser.parse_args() with self.assertRaisesRegex( From a7d672e3d9f2668307ba99b192561d3fd45b1b77 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 20 Jan 2023 14:43:45 -0700 Subject: [PATCH 165/332] Run through black --- python/ctsm/test/test_unit_subset_data.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 94a53e2dd2..5d7a0f05b6 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -131,16 +131,14 @@ def test_check_args_outsurfdat_fails_without_overwrite(self): Test that check args does not allow an output surface dataset to be specified for an existing dataset without the overwrite option """ - outfile = os.path.join( os.getcwd(), "ctsm/test/testinputs/", "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc" ) - self.assertTrue( os.path.exists( outfile ), str(outfile)+" outfile should exist" ) - - sys.argv = [ - "subset_data", - "point", - "--create-surface", - "--out-surface", - outfile - ] + outfile = os.path.join( + os.getcwd(), + "ctsm/test/testinputs/", + "surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c221206.nc", + ) + self.assertTrue(os.path.exists(outfile), str(outfile) + " outfile should exist") + + sys.argv = ["subset_data", "point", "--create-surface", "--out-surface", outfile] self.args = self.parser.parse_args() with self.assertRaisesRegex( argparse.ArgumentError, From 040555c0eac09347821fb88fd30c1a791e3d3b70 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 20 Jan 2023 16:59:04 -0700 Subject: [PATCH 166/332] Change name of --file option to subset_data to --cfg-file option to make it more clear in response to review --- python/ctsm/subset_data.py | 2 +- python/ctsm/test/test_unit_subset_data.py | 2 +- tools/mksurfdata_map/Makefile.data | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/ctsm/subset_data.py b/python/ctsm/subset_data.py index a258c7b72e..4a3a5801f1 100644 --- a/python/ctsm/subset_data.py +++ b/python/ctsm/subset_data.py @@ -322,7 +322,7 @@ def get_parser(): cesmroot = path_to_ctsm_root() defaults_file = os.path.join(cesmroot, DEFAULTS_CONFIG) subparser.add_argument( - "--file", + "--cfg-file", help="Default configure file to use for default filenames.", action="store", dest="config_file", diff --git a/python/ctsm/test/test_unit_subset_data.py b/python/ctsm/test/test_unit_subset_data.py index 5d7a0f05b6..fd9aef631d 100755 --- a/python/ctsm/test/test_unit_subset_data.py +++ b/python/ctsm/test/test_unit_subset_data.py @@ -91,7 +91,7 @@ def test_check_args_badconfig(self): """ Test that check args aborts when a config file is entered that doesn't exist """ - sys.argv = ["subset_data", "point", "--create-surface", "--file", "zztop"] + sys.argv = ["subset_data", "point", "--create-surface", "--cfg-file", "zztop"] self.args = self.parser.parse_args() with self.assertRaisesRegex( argparse.ArgumentError, "Entered default config file does not exist" diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 7bac3b8923..ea17856d78 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -159,7 +159,7 @@ crop-tropics-present : brazil-tropics-present crop-tropics-historical : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --file default_data_1850.cfg --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --cfg-file default_data_1850.cfg --crop crop-tropics-transient : FORCE $(SUBSETDATA_POINT_ALLLU) --create-landuse $(SUBSETDATA_1X1_BRAZIL) --crop @@ -182,7 +182,7 @@ crop-numa-present : FORCE $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --crop crop-numa-historical : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --file default_data_1850.cfg --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --cfg-file default_data_1850.cfg --crop crop-smallville : FORCE $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL) --crop @@ -198,7 +198,7 @@ crop-global-present-ne120np4 : FORCE # adds crop (to make sure that it works properly to add crop in a grid cell # where there used to be no crop). crop-smallville-historical : FORCE - $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL1850) --file default_data_1850.cfg --crop + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL1850) --cfg-file default_data_1850.cfg --crop # Setup the historical case for SSP5-8.5 so that historical can be used to go into the future. crop-global-historical : FORCE From dacc5a563f4e7c6d951002a03102921707cdb4c0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 15:41:39 -0700 Subject: [PATCH 167/332] Add silent option resolving #1941 --- python/ctsm/ctsm_logging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index ff51c6d8f2..e14ec2754c 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -68,6 +68,7 @@ def add_logging_args(parser): logging_level.add_argument( "-v", "--verbose", action="store_true", help="Output extra logging info" ) + logging_level.add_argument("--silent", action="store_true", help="Only output errors") logging_level.add_argument( "--debug", @@ -84,6 +85,8 @@ def process_logging_args(args): root_logger.setLevel(logging.DEBUG) elif args.verbose: root_logger.setLevel(logging.INFO) + elif args.silent: + root_logger.setLevel(logging.ERROR) else: root_logger.setLevel(logging.WARNING) From 53f3b7aa102af1803b88bb0aa06afd0001b7f970 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 17:24:39 -0700 Subject: [PATCH 168/332] Update python design doc --- doc/design/python_script_user_interface.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design/python_script_user_interface.rst b/doc/design/python_script_user_interface.rst index 4dd0c9c546..3ad6a4d2cf 100644 --- a/doc/design/python_script_user_interface.rst +++ b/doc/design/python_script_user_interface.rst @@ -40,8 +40,8 @@ Options that are more than a single character should be formatted as ``--some-va 2. Standard options: verbose, silent, help and debug: -Scripts should support ``--verbose`` / ``-v`` and ``--debug`` options. See comments at the top of ``ctsm_logging.py`` for details. -Also the options silent and help are recommended as well. +Scripts should support ``--verbose`` / ``-v``, ''--silent'', and ``--debug`` options. See comments at the top of ``ctsm_logging.py`` for details. +Also the help option is highly recommended as well. 3. Value flags: From 2d607d90cb34e1ab5683e8b05eca48e3e66821cb Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 19:28:33 -0700 Subject: [PATCH 169/332] Remove the single point mksurfdata tests as we are now using subset_data --- test/tools/input_tests_master | 11 ----------- test/tools/tests_posttag_hobart_nompi | 1 - test/tools/tests_posttag_izumi_nompi | 1 - test/tools/tests_posttag_nompi_regression | 4 ---- test/tools/tests_pretag_cheyenne_nompi | 3 --- 5 files changed, 20 deletions(-) diff --git a/test/tools/input_tests_master b/test/tools/input_tests_master index f3e46d50b5..63e4e5173f 100644 --- a/test/tools/input_tests_master +++ b/test/tools/input_tests_master @@ -24,15 +24,6 @@ bli58 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_10x15_crp_1850-200 smi64 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_5x5_amazon_hirespft_2005^tools__ds bli64 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_5x5_amazon_hirespft_2005^tools__ds -smi74 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850-2000^tools__ds -bli74 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850-2000^tools__ds -smi78 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850^tools__ds -bli78 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_brazil_1850^tools__ds -smiT4 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_2000^tools__ds -bliT4 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_2000^tools__ds -smiT2 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100^tools__s -bliT2 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100^tools__s - sm0a1 TSMscript_tools.sh site_and_regional run_neon.py run_neon_OSBS bl0a1 TBLscript_tools.sh site_and_regional run_neon.py run_neon_OSBS @@ -54,5 +45,3 @@ smi#2 TSMscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_ne30np4 bli#2 TBLscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_ne30np4 smi59 TSMscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_if10 bli59 TBLscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_if10 -smi79 TSMscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_i1x1_brazil -bli79 TBLscript_tools.sh mkmapdata mkmapdata.sh mkmapdata_i1x1_brazil diff --git a/test/tools/tests_posttag_hobart_nompi b/test/tools/tests_posttag_hobart_nompi index 9f07863e4d..d3cbccecdf 100644 --- a/test/tools/tests_posttag_hobart_nompi +++ b/test/tools/tests_posttag_hobart_nompi @@ -1,4 +1,3 @@ smc#4 blc#4 smi54 bli54 smi57 bli57 -smiT4 bliT4 diff --git a/test/tools/tests_posttag_izumi_nompi b/test/tools/tests_posttag_izumi_nompi index 62687a7e3d..3e84fb8459 100644 --- a/test/tools/tests_posttag_izumi_nompi +++ b/test/tools/tests_posttag_izumi_nompi @@ -1,3 +1,2 @@ smi54 bli54 smi57 bli57 -smiT4 bliT4 diff --git a/test/tools/tests_posttag_nompi_regression b/test/tools/tests_posttag_nompi_regression index 1395aebe11..b665409c51 100644 --- a/test/tools/tests_posttag_nompi_regression +++ b/test/tools/tests_posttag_nompi_regression @@ -5,7 +5,3 @@ smi53 bli53 smi54 bli54 smi57 bli57 smi58 bli58 -smi74 bli74 -smi78 bli78 -smiT4 bliT4 -smiT2 bliT2 diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi index 19e96594bf..6280f33824 100644 --- a/test/tools/tests_pretag_cheyenne_nompi +++ b/test/tools/tests_pretag_cheyenne_nompi @@ -10,6 +10,3 @@ smi64 bli64 smi54 bli54 smi57 bli57 smi58 bli58 -smi74 bli74 -smiT4 bliT4 -smiT2 bliT2 From 02117d6ad233cb08ed8df530e7c8746c72a40193 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 19:33:30 -0700 Subject: [PATCH 170/332] Remove the single point mksurfdata and mkmapdata test files --- test/tools/nl_files/mkmapdata_i1x1_brazil | 1 - test/tools/nl_files/mksrfdt_1x1_brazil_1850 | 1 - test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 | 1 - test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 | 1 - test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 | 1 - test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 | 1 - 6 files changed, 6 deletions(-) delete mode 100644 test/tools/nl_files/mkmapdata_i1x1_brazil delete mode 100644 test/tools/nl_files/mksrfdt_1x1_brazil_1850 delete mode 100644 test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 delete mode 100644 test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 delete mode 100644 test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 delete mode 100644 test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 diff --git a/test/tools/nl_files/mkmapdata_i1x1_brazil b/test/tools/nl_files/mkmapdata_i1x1_brazil deleted file mode 100644 index bb54d468dc..0000000000 --- a/test/tools/nl_files/mkmapdata_i1x1_brazil +++ /dev/null @@ -1 +0,0 @@ --t regional -r 1x1_brazil --fast diff --git a/test/tools/nl_files/mksrfdt_1x1_brazil_1850 b/test/tools/nl_files/mksrfdt_1x1_brazil_1850 deleted file mode 100644 index 2330bd082e..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_brazil_1850 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_brazil -y 1850-2000 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 b/test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 deleted file mode 100644 index 2330bd082e..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_brazil_1850-2000 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_brazil -y 1850-2000 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 b/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 deleted file mode 100644 index 03304f81eb..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_2000 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_numaIA -y 2000 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 b/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 deleted file mode 100644 index ed83434075..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_numaIA_crp_SSP5-8.5_1850-2100 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_numaIA -y 1850-2100 -ssp_rcp SSP5-8.5 -exedir EXEDIR diff --git a/test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 b/test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 deleted file mode 100644 index a446e82fcd..0000000000 --- a/test/tools/nl_files/mksrfdt_1x1_vancouverCAN_2000 +++ /dev/null @@ -1 +0,0 @@ --l CSMDATA -r 1x1_vancouverCAN -no-crop -y 2000 -exedir EXEDIR From df5e517b254e4554994741ba489a144739ed6f34 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 20:39:45 -0700 Subject: [PATCH 171/332] Remove another test that was removed from the master list --- test/tools/tests_pretag_cheyenne_nompi | 1 - 1 file changed, 1 deletion(-) diff --git a/test/tools/tests_pretag_cheyenne_nompi b/test/tools/tests_pretag_cheyenne_nompi index 6280f33824..6ce4972915 100644 --- a/test/tools/tests_pretag_cheyenne_nompi +++ b/test/tools/tests_pretag_cheyenne_nompi @@ -1,4 +1,3 @@ -smi79 bli79 smc#4 blc#4 smg54 blg54 smba1 blba1 From 484f49050ab12291387278de807e768e1c9f5237 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 22:17:34 -0700 Subject: [PATCH 172/332] Fix for #1942 so that conda environment is just activated, but not set --- test/tools/test_driver.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/tools/test_driver.sh b/test/tools/test_driver.sh index ce501f980a..6a6749d493 100755 --- a/test/tools/test_driver.sh +++ b/test/tools/test_driver.sh @@ -373,8 +373,11 @@ else fi # Setup conda environement -\$CLM_ROOT/py_env_create conda activate ctsm_py +if [ \$? -ne 0 ]; then + echo "ERROR: Trouble activating the ctsm_py conda environment, be sure it's setup with \$CLM_ROOT/py_env_create, then rerun" + exit 4 +fi ##output files clm_log=\${initdir}/td.\${JOBID}.log From ce303459b68b9ff418cc25c1ab214c68ce0a8bae Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 22:27:51 -0700 Subject: [PATCH 173/332] Update MOSART so that it sends negative flow to river outlets by default --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 2df854055e..7a4eeb4a6e 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -23,7 +23,7 @@ required = True local_path = components/mosart protocol = git repo_url = https://github.com/ESCOMP/MOSART -tag = mosart1_0_47 +tag = mosart1_0_48 required = True [mizuRoute] From fedd4a65e1f8b4952a046e5cc6e788214e3317aa Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 21 Jan 2023 22:33:10 -0700 Subject: [PATCH 174/332] Add note that \$RM uses force option (\-f) so it does not generate an error if the file does not exist --- tools/mksurfdata_map/Makefile.data | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index ea17856d78..5c2efebca6 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -266,11 +266,13 @@ urban-present : mexicocity vancouver mexicocity : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) + # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc vancouver : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) + # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc @@ -278,6 +280,7 @@ vancouver : FORCE # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 urban-alpha : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) + # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc From 8a96cc71654722f6234c2773fe45fc16ec5f9450 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 22 Jan 2023 13:07:36 -0700 Subject: [PATCH 175/332] Turn on silent for subset_data, move comment about remove so it isn't echoed when the urban datasets are made --- tools/mksurfdata_map/Makefile.data | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 5c2efebca6..89a238ef64 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -60,7 +60,7 @@ CDATE = $(shell date +%y%m%d) # subset_data options # -SUBSETDATA_POINT = $(SUBSETDATA) point --overwrite --uniform-snowpack --cap-saturation --outdir . +SUBSETDATA_POINT = $(SUBSETDATA) point --silent --overwrite --uniform-snowpack --cap-saturation --outdir . SUBSETDATA_POINT_ALLLU = $(SUBSETDATA_POINT) --include-nonveg SUBSETDATA_POINT_URBAN = $(SUBSETDATA_POINT) --include-nonveg @@ -264,23 +264,23 @@ urban : urban-present urban-alpha urban-present : mexicocity vancouver +# NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist mexicocity : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) - # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc +# NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist vancouver : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) - # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 +# NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist urban-alpha : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) - # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc From 2aa7566eb8c0635025d1f40164d4bc6db60e42dd Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 22 Jan 2023 20:29:22 -0700 Subject: [PATCH 176/332] Remove the temporary files for the urban cases --- tools/mksurfdata_map/Makefile.data | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 89a238ef64..4d7ecb621e 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -269,12 +269,14 @@ mexicocity : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist vancouver : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 @@ -283,6 +285,7 @@ urban-alpha : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc # # landuse timeseries From 7ada54fe1b163a147982cf2184520210c4c0e000 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 22 Jan 2023 21:02:18 -0700 Subject: [PATCH 177/332] Add some info level logger output that will be turned on with the verbose option --- .../modify_input_files/fsurdat_modifier.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 39bd95986c..d1ed9e7967 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -385,6 +385,10 @@ def read_option_control( file_path=cfg_path, convert_to_type=bool, ) + if idealized: + logger.info("idealized option is on") + else: + logger.info("idealized option is off") process_subgrid = get_config_value( config=config, section=section, @@ -392,6 +396,10 @@ def read_option_control( file_path=cfg_path, convert_to_type=bool, ) + if process_subgrid: + logger.info("process_subgrid_section option is on") + else: + logger.info("process_subgrid_section option is off") process_var_list = get_config_value( config=config, section=section, @@ -399,6 +407,10 @@ def read_option_control( file_path=cfg_path, convert_to_type=bool, ) + if process_var_list: + logger.info("process_var_list_section option is on") + else: + logger.info("process_var_list_section option is off") include_nonveg = get_config_value( config=config, section=section, @@ -406,6 +418,10 @@ def read_option_control( file_path=cfg_path, convert_to_type=bool, ) + if include_nonveg: + logger.info("include_nonveg option is on") + else: + logger.info("include_nonveg option is off") max_pft = int(max(modify_fsurdat.file.lsmpft)) dom_pft = get_config_value( config=config, @@ -416,6 +432,10 @@ def read_option_control( convert_to_type=int, can_be_unset=True, ) + if dom_pft: + logger.info("dom_pft option is on and = %s", str(dom_pft)) + else: + logger.info("dom_pft option is off") if dom_pft is not None and idealized and not allow_dom_pft_and_idealized: abort("idealized AND dom_pft can NOT both be on, pick one or the other") if include_nonveg and idealized and not allow_ideal_and_include_non_veg: @@ -563,6 +583,7 @@ def fsurdat_modifier(parser): if process_subgrid: subgrid = read_subgrid(config, cfg_path, numurbl=modify_fsurdat.get_urb_dens()) modify_fsurdat.set_varlist(subgrid, cfg_path) + logger.info("process_subgrid is complete") else: check_no_subgrid_section(config) @@ -570,6 +591,7 @@ def fsurdat_modifier(parser): varlist = read_var_list(config, idealized=idealized) update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) modify_fsurdat.set_varlist(update_list, cfg_path) + logger.info("process_var_list is complete") else: check_no_varlist_section(config) From 59d2dfff3db7f7bd19fa33d566a5676fa537d597 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 22 Jan 2023 21:30:13 -0700 Subject: [PATCH 178/332] Replace the single point surface datasets with the newly generated ones --- bld/namelist_files/namelist_defaults_ctsm.xml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 69643041a4..ca262e21c0 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1058,7 +1058,7 @@ lnd/clm2/surfdata_map/surfdata_0.125nldas2_hist_16pfts_Irrig_CMIP6_simyr2005_c19 lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_5x5_amazon_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr2000_c190214.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr2000_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr2000_c230122.nc -lnd/clm2/surfdata_map/ctsm1.0.dev094-2-g633be0eb/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c200521.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230122.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc @@ -1113,11 +1113,11 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc @@ -1133,7 +1133,7 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_16pfts_Irrig_CMIP6_s lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850_c230122.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850_c190303.nc @@ -1161,12 +1161,12 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_78pfts_CMIP6_simyr18 lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c230122.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c230122.nc -lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c190214.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230122.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc From c92edb8adb26019e6d50fd39adc1d2dc93e8a2ff Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 23 Jan 2023 02:25:20 -0700 Subject: [PATCH 179/332] Fix issues with 1x1_brazil fsurdat/landuse.timeseries files --- bld/namelist_files/namelist_defaults_ctsm.xml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index ca262e21c0..41fd39b884 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1055,10 +1055,11 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_16pfts_Irrig_CMIP6 lnd/clm2/surfdata_map/surfdata_0.125nldas2_hist_16pfts_Irrig_CMIP6_simyr2005_c190412.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr2000_c230122.nc + lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_5x5_amazon_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr2000_c190214.nc -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr2000_c230122.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr2000_c230122.nc lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230122.nc @@ -1132,9 +1133,6 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_16pfts_Irrig_CMIP6_s lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850_c230122.nc - lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850_c190303.nc @@ -1160,12 +1158,12 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1.9x2.5_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_78pfts_CMIP6_simyr1850_c190214.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c230122.nc lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c230122.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230122.nc @@ -1210,8 +1208,6 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/surfdata_map/release-clm5.0.18/landuse.timeseries_48x96_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c190214.nc -lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c170824.nc lnd/clm2/surfdata_map/landuse.timeseries_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850-2015_c170824.nc @@ -1232,7 +1228,7 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_48x96_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_brazil_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc +>lnd/clm2/surfdata_map/ctsm5.1.dev116/landuse.timeseries_1x1_brazil_hist_78pfts_CMIP6_simyr1850-2015_c230122.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc From 6198e3cd13b8d9199e6ff44e1d37a86e15dc7c55 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Mon, 23 Jan 2023 10:14:33 -0700 Subject: [PATCH 180/332] remove KONA and STER FATES usermods, add README --- .../usermods_dirs/NEON/FATES/KONA/include_user_mods | 1 - .../usermods_dirs/NEON/FATES/KONA/shell_commands | 6 ------ cime_config/usermods_dirs/NEON/FATES/README.md | 10 ++++++++++ .../usermods_dirs/NEON/FATES/STER/include_user_mods | 1 - .../usermods_dirs/NEON/FATES/STER/shell_commands | 6 ------ 5 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods delete mode 100644 cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands create mode 100644 cime_config/usermods_dirs/NEON/FATES/README.md delete mode 100644 cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods delete mode 100644 cime_config/usermods_dirs/NEON/FATES/STER/shell_commands diff --git a/cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods deleted file mode 100644 index b152996d95..0000000000 --- a/cime_config/usermods_dirs/NEON/FATES/KONA/include_user_mods +++ /dev/null @@ -1 +0,0 @@ -../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands b/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands deleted file mode 100644 index 316509046c..0000000000 --- a/cime_config/usermods_dirs/NEON/FATES/KONA/shell_commands +++ /dev/null @@ -1,6 +0,0 @@ -./xmlchange NEONSITE=KONA -./xmlchange PTS_LON=263.38956 -./xmlchange PTS_LAT=39.10828 -# We can't run crops with FATES yet -echo "Cannot run FATES at KONA because FATES does not yet have crops." -exit 1 diff --git a/cime_config/usermods_dirs/NEON/FATES/README.md b/cime_config/usermods_dirs/NEON/FATES/README.md new file mode 100644 index 0000000000..dcfcfdf9af --- /dev/null +++ b/cime_config/usermods_dirs/NEON/FATES/README.md @@ -0,0 +1,10 @@ +# NEON user mods directories for FATES runs + +Use these user mods as you would any other user_mods, e.g.: + +`./create_newcase --case FATES_ABBY_test --res CLM_USRDAT --compset I1PtClm51Fates --run-unsupported --user-mods-dir /glade/work/$user/CTSM/cime_config/usermods_dirs/NEON/FATES/ABBY` + +## Note on crop sites KONA and STER + +Currently you cannot run FATES at these sites because FATES does not have crops as of yet. We will add these sites back once this capability is available. + diff --git a/cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods b/cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods deleted file mode 100644 index b152996d95..0000000000 --- a/cime_config/usermods_dirs/NEON/FATES/STER/include_user_mods +++ /dev/null @@ -1 +0,0 @@ -../defaults diff --git a/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands b/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands deleted file mode 100644 index 685eeb2529..0000000000 --- a/cime_config/usermods_dirs/NEON/FATES/STER/shell_commands +++ /dev/null @@ -1,6 +0,0 @@ -./xmlchange NEONSITE=STER -./xmlchange PTS_LON=256.96992 -./xmlchange PTS_LAT=40.45984 -# We can't run crops with FATES yet -echo "Cannot run FATES at STER because FATES does not yet have crops." -exit 1 From f49a75585ddc02e2a1288b92d6bc2ea4d81d450c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 23 Jan 2023 13:17:36 -0700 Subject: [PATCH 181/332] Always use --crop for the subset options, so that the 78pft dataset is always used, which will make it easier to remove the use of 16pft versions which we do want to do --- tools/mksurfdata_map/Makefile.data | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 4d7ecb621e..983dd4db9e 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -60,7 +60,7 @@ CDATE = $(shell date +%y%m%d) # subset_data options # -SUBSETDATA_POINT = $(SUBSETDATA) point --silent --overwrite --uniform-snowpack --cap-saturation --outdir . +SUBSETDATA_POINT = $(SUBSETDATA) point --silent --overwrite --uniform-snowpack --cap-saturation --crop --outdir . SUBSETDATA_POINT_ALLLU = $(SUBSETDATA_POINT) --include-nonveg SUBSETDATA_POINT_URBAN = $(SUBSETDATA_POINT) --include-nonveg @@ -73,9 +73,9 @@ SUBSETDATA_1X1_SMALL := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA \ # NOTE: The 1850 smallvilleIA site is constructed to start with 100% natural vegetation, so we can test transition to crops SUBSETDATA_1X1_SMALL1850 := --lat 40.6878 --lon 267.0228 --site 1x1_smallvilleIA --dompft 13 --pctpft 100 -SUBSETDATA_1X1_MEXICOCITY := --lat 19.5 --lon 260.5 --site 1x1_mexicocityMEX --out-surface surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc -SUBSETDATA_1X1_VANCOUVER := --lat 49.5 --lon 236.5 --site 1x1_vancouverCAN --out-surface surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc -SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha --out-surface surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc +SUBSETDATA_1X1_MEXICOCITY := --lat 19.5 --lon 260.5 --site 1x1_mexicocityMEX --out-surface surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc +SUBSETDATA_1X1_VANCOUVER := --lat 49.5 --lon 236.5 --site 1x1_vancouverCAN --out-surface surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc +SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha --out-surface surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc # f19 and f09 are standard resolutions, f10 is used for testing, f45 is used for FATES # ne30np4 is standard resolution for SE dycore in CAM, C96 is standard for fv3 dycore # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore @@ -155,14 +155,14 @@ crop-tropics-present : brazil-tropics-present $(MKSURFDATA) -glc_nec 10 -y 2000 -res 5x5_amazon $(BACKGROUND) 1x1_brazil-tropics-present : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) crop-tropics-historical : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --cfg-file default_data_1850.cfg --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_BRAZIL) --cfg-file default_data_1850.cfg crop-tropics-transient : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-landuse $(SUBSETDATA_1X1_BRAZIL) --crop + $(SUBSETDATA_POINT_ALLLU) --create-landuse $(SUBSETDATA_1X1_BRAZIL) # # crop @@ -179,13 +179,13 @@ crop-global-present-f05 : FORCE $(MKSURFDATA) -glc_nec 10 -y 1850,2000 -res 0.47x0.63 $(BACKGROUND) crop-numa-present : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) crop-numa-historical : FORCE - $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --cfg-file default_data_1850.cfg --crop + $(SUBSETDATA_POINT_ALLLU) --create-surface $(SUBSETDATA_1X1_NUMAIA) --cfg-file default_data_1850.cfg crop-smallville : FORCE - $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL) --crop + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL) crop-global-present-ne16np4 : FORCE $(MKSURFDATA) -glc_nec 10 -y 2000 -res ne16np4 $(BACKGROUND) @@ -198,7 +198,7 @@ crop-global-present-ne120np4 : FORCE # adds crop (to make sure that it works properly to add crop in a grid cell # where there used to be no crop). crop-smallville-historical : FORCE - $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL1850) --cfg-file default_data_1850.cfg --crop + $(SUBSETDATA_POINT) --create-surface $(SUBSETDATA_1X1_SMALL1850) --cfg-file default_data_1850.cfg # Setup the historical case for SSP5-8.5 so that historical can be used to go into the future. crop-global-historical : FORCE @@ -267,25 +267,25 @@ urban-present : mexicocity vancouver # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist mexicocity : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) - $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc - $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc - $(RM) surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000.nc + $(RM) surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist vancouver : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) - $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc - $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc - $(RM) surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000.nc + $(RM) surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 # NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist urban-alpha : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) - $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc - $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c$(CDATE).nc - $(RM) surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000.nc + $(RM) surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc + $(RM) surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc # # landuse timeseries From 9b962c05bf45e5888b54dd116ad93b67b83df35f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 23 Jan 2023 13:31:43 -0700 Subject: [PATCH 182/332] Increase size of PCT_CFT array to have 64 crop CFT's so that we can use the 78pft surface datasets --- tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg | 2 +- tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg | 2 +- tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg index 0e3c4c6ab6..5e9de7968b 100644 --- a/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg +++ b/tools/mksurfdata_map/modify_1x1_mexicocityMEX.cfg @@ -131,4 +131,4 @@ CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 # Natural and Crop PFT's don't really need to be set, since they have zero area, but # it looks better to do so PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -PCT_CFT = 100. 0.0 +PCT_CFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg b/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg index 3bb56917a3..38b8ce6372 100644 --- a/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg +++ b/tools/mksurfdata_map/modify_1x1_urbanc_alpha.cfg @@ -131,4 +131,4 @@ CV_IMPROAD = 1140000.0 1050000.0 1050000.0 1290000.0 0.0 # Natural and Crop PFT's don't really need to be set, since they have zero area, but # it looks better to do so PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -PCT_CFT = 100. 0.0 +PCT_CFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 diff --git a/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg b/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg index a8ef5e328b..abc5df16b1 100644 --- a/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg +++ b/tools/mksurfdata_map/modify_1x1_vancouverCAN.cfg @@ -131,4 +131,4 @@ CV_IMPROAD = 1740000.0 1740000.0 2000000.0 2000000.0 2000000.0 # Natural and Crop PFT's don't really need to be set, since they have zero area, but # it looks better to do so PCT_NAT_PFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -PCT_CFT = 100. 0.0 +PCT_CFT = 100. 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 From 5d108b4b6afe844b808befc6f05c4d767b0cf0d7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 23 Jan 2023 15:57:35 -0700 Subject: [PATCH 183/332] Update single point datasets --- bld/namelist_files/namelist_defaults_ctsm.xml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 41fd39b884..1afcb95ec1 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1056,7 +1056,7 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_16pfts_Irrig_CMIP6 lnd/clm2/surfdata_map/surfdata_0.125nldas2_hist_16pfts_Irrig_CMIP6_simyr2005_c190412.nc -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr2000_c230122.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr2000_c230123.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_5x5_amazon_hist_16pfts_Irrig_CMIP6_simyr2000_c190214.nc @@ -1086,10 +1086,10 @@ lnd/clm2/surfdata_map/release-clm5.0.24/surfdata_0.125x0.125_hist_78pfts_CMIP6_s lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_78pfts_CMIP6_simyr2000_c190214.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr2000_c190214.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr2000_c230122.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230122.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr2000_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc @@ -1113,12 +1113,12 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.ARCTIC.ne30x4_hist_78pft lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts_CMIP6_simyr2000_c200426.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_vancouverCAN_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_mexicocityMEX_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_urbanc_alpha_hist_16pfts_Irrig_CMIP6_simyr2000_c230122.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c230123.nc @@ -1158,13 +1158,13 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1.9x2.5_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_78pfts_CMIP6_simyr1850_c190214.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_78pfts_CMIP6_simyr1850_c190214.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c230122.nc - -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c230122.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr1850_c230123.nc + +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr1850_c230123.nc -lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230122.nc +lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1228,7 +1228,7 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_48x96_hist_78pfts_CMIP6_simyr1850-2015_c170824.nc lnd/clm2/surfdata_map/ctsm5.1.dev116/landuse.timeseries_1x1_brazil_hist_78pfts_CMIP6_simyr1850-2015_c230122.nc +>lnd/clm2/surfdata_map/ctsm5.1.dev116/landuse.timeseries_1x1_brazil_hist_78pfts_CMIP6_simyr1850-2015_c230123.nc lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc From edfde60b268b58b1382b1ba130347c3a04b10683 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 23 Jan 2023 15:58:32 -0700 Subject: [PATCH 184/332] So that resolution isn't done twice, explicitly add f09 resolution to list that also has specific resolutions --- bld/unit_testers/build-namelist_test.pl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 784d3b5e0c..f270cbfea4 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -319,22 +319,22 @@ sub cat_and_create_namelistinfile { foreach my $driver ( "mct", "nuopc" ) { print " For $driver driver\n\n"; # configuration, structure, irrigate, verbose, clm_demand, ssp_rcp, test, sim_year, use_case - foreach my $options ( "-configuration nwp", - "-structure fast", - "-namelist '&a irrigate=.true./'", "-verbose", "-ssp_rcp SSP1-2.6", "-test", "-sim_year 1850", - "-namelist '&a use_lai_streams=.true.,use_soil_moisture_streams=.true./'", - "-use_case 1850_control", + foreach my $options ( "-res 0.9x1.25 -configuration nwp", + "-res 0.9x1.25 -structure fast", + "-res 0.9x1.25 -namelist '&a irrigate=.true./'", "-res 0.9x1.25 -verbose", "-res 0.9x1.25 -ssp_rcp SSP1-2.6", "-res 0.9x1.25 -test", "-res 0.9x1.25 -sim_year 1850", + "-res 0.9x1.25 -namelist '&a use_lai_streams=.true.,use_soil_moisture_streams=.true./'", + "-res 0.9x1.25 -use_case 1850_control", "-res 1x1pt_US-UMB -clm_usr_name 1x1pt_US-UMB -namelist '&a fsurdat=\"/dev/null\"/'", "-res 1x1_brazil", - "-clm_start_type startup", "-namelist '&a irrigate=.false./' -crop -bgc bgc", - "-envxml_dir . -infile myuser_nl_clm", - "-ignore_ic_date -clm_start_type branch -namelist '&a nrevsn=\"thing.nc\"/' -bgc bgc -crop", - "-clm_start_type branch -namelist '&a nrevsn=\"thing.nc\",use_init_interp=T/'", - "-ignore_ic_date -clm_start_type startup -namelist '&a finidat=\"thing.nc\"/' -bgc bgc -crop", + "-res 0.9x1.25 -clm_start_type startup", "-namelist '&a irrigate=.false./' -crop -bgc bgc", + "-res 0.9x1.25 -infile myuser_nl_clm", + "-res 0.9x1.25 -ignore_ic_date -clm_start_type branch -namelist '&a nrevsn=\"thing.nc\"/' -bgc bgc -crop", + "-res 0.9x1.25 -clm_start_type branch -namelist '&a nrevsn=\"thing.nc\",use_init_interp=T/'", + "-res 0.9x1.25 -ignore_ic_date -clm_start_type startup -namelist '&a finidat=\"thing.nc\"/' -bgc bgc -crop", ) { my $file = $startfile; &make_env_run(); - my $base_options = "-res 0.9x1.25 -envxml_dir . -driver $driver"; + my $base_options = "-envxml_dir . -driver $driver"; if ( $driver eq "mct" ) { $base_options = "$base_options -lnd_frac $DOMFILE"; } else { From 42709c251e3561e1b2d2ce9e437c008d67aefc21 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 24 Jan 2023 14:44:43 -0700 Subject: [PATCH 185/332] Add a test for the --overwrite option which isn't added yet, so the test fails --- python/ctsm/test/test_sys_fsurdat_modifier.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index cde9cac846..c69a876df3 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -80,6 +80,11 @@ def test_short_config(self): fsurdat_out = ( "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" ) + # Run it again with the overwrite option so that it will overwrite the file just created + sys.argv = ["fsurdat_modifier", self._cfg_file_path, "--overwrite"] + parser = fsurdat_modifier_arg_process() + fsurdat_modifier(parser) + # Cleanup os.remove(fsurdat_out) def test_short_infile_both_cmdline_and_cfg(self): @@ -279,6 +284,7 @@ def test_input_fsurdat_DNE_fail(self): def test_output_fsurdat_EXISTS_fail(self): """ Test that if the output fsurdat file does exist that it gracefully fails + without --overwrite option """ self._cfg_file_path = os.path.join( self._testinputs_path, "modify_fsurdat_short_nofiles.cfg" From 6fea524c4373bfd6717e595cf46f960a7d18917e Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Tue, 24 Jan 2023 15:16:37 -0700 Subject: [PATCH 186/332] Rm .github/workflows/python_testing.yml to unblock merging this PR --- .github/workflows/python_testing.yml | 31 ---------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .github/workflows/python_testing.yml diff --git a/.github/workflows/python_testing.yml b/.github/workflows/python_testing.yml deleted file mode 100644 index f9234acc2f..0000000000 --- a/.github/workflows/python_testing.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: python testing on push and PR -# -# Run the python directory testing on push and pull request -# -on: [push, pull_request] - -jobs: - pytest: - runs-on: ubuntu-latest - steps: - # Checkout the code - - uses: actions/checkout@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: ctsm_pylib - environment-file: python/conda_env_ctsm_py.txt - auto-activate-base: false - channels: conda-forge - - name: manage_exernals and LFS - run: | - git lfs install - manage_externals/checkout_externals - - name: chdirectory python - run: | - cd python - - name: Run python tests - run: | - make test - - name: Run pylint - run: | - make lint From 050f110f94d4c7ec700d8ac4271db4d59d67f3b2 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 24 Jan 2023 16:53:59 -0700 Subject: [PATCH 187/332] Handle overwrite option and get the test working --- python/ctsm/modify_input_files/fsurdat_modifier.py | 14 ++++++++++++-- python/ctsm/test/test_sys_fsurdat_modifier.py | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index d1ed9e7967..747b8bc999 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -71,6 +71,13 @@ def fsurdat_modifier_arg_process(): action="store_true", help="Allow both idealized and dom_pft to be on at the same time. ", ) + parser.add_argument( + "--overwrite", + required=False, + default=False, + action="store_true", + help="Overwrite the output file if it already exists. ", + ) add_logging_args(parser) args = parser.parse_args() process_logging_args(args) @@ -481,8 +488,11 @@ def fsurdat_modifier(parser): # If output file exists, abort before starting work if os.path.exists(fsurdat_out): - errmsg = "Output file already exists: " + fsurdat_out - abort(errmsg) + if not parser.overwrite: + errmsg = "Output file already exists: " + fsurdat_out + abort(errmsg) + else: + logger.warning("Output file already exists, but the overwrite option was selected so the file will be overwritten.") lnd_lat_1 = get_config_value( config=config, diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index c69a876df3..529800c3f7 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -76,10 +76,11 @@ def test_short_config(self): self._cfg_file_path = os.path.join(self._testinputs_path, "modify_fsurdat_short.cfg") sys.argv = ["fsurdat_modifier", self._cfg_file_path] parser = fsurdat_modifier_arg_process() - fsurdat_modifier(parser) fsurdat_out = ( "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" ) + os.remove(fsurdat_out) + fsurdat_modifier(parser) # Run it again with the overwrite option so that it will overwrite the file just created sys.argv = ["fsurdat_modifier", self._cfg_file_path, "--overwrite"] parser = fsurdat_modifier_arg_process() From 5c2b5b6cc6dc54b8c32a8919aaa7a5e7242b0946 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 24 Jan 2023 16:59:50 -0700 Subject: [PATCH 188/332] Turn options needed for the system test so that it will work --- cime_config/SystemTests/fsurdatmodifyctsm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 899ab1aead..99a653129e 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -40,8 +40,11 @@ def __init__(self, case): self._cfg_file_path = os.path.join(self._get_caseroot(), 'modify_fsurdat.cfg') + logger.info(" create config file to modify") self._create_config_file() + logger.info(" run modify_fsurdat") self._run_modify_fsurdat() + logger.info(" modify user_nl files") self._modify_user_nl() with open('done_FSURDATMODIFYCTSM_setup.txt', 'w') as fp: pass @@ -68,7 +71,7 @@ def _run_modify_fsurdat(self): # Need to specify a specific python version that has the required # dependencies python_path = _get_python_path() - subprocess.check_call([python_path, tool_path, self._cfg_file_path]) + subprocess.check_call([python_path, tool_path, self._cfg_file_path, "--verbose", "--overwrite", "--allow_ideal_and_include_non_veg"]) def _modify_user_nl(self): append_to_user_nl_files(caseroot = self._get_caseroot(), From 22fe11f95636735b8206bc045c8947d47c36451f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 24 Jan 2023 17:11:24 -0700 Subject: [PATCH 189/332] Replace the RM with the use of the overwrite option for fsurdat_modifier --- tools/mksurfdata_map/Makefile.data | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index 983dd4db9e..c1df608436 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -54,7 +54,7 @@ endif MKSURFDATA = $(BATCHJOBS) $(PWD)/mksurfdata.pl SUBSETDATA = $(PWD)/../site_and_regional/subset_data -MODIFYSURF = $(PWD)/../modify_input_files/fsurdat_modifier +MODIFYSURF = $(PWD)/../modify_input_files/fsurdat_modifier --overwrite CDATE = $(shell date +%y%m%d) @@ -264,26 +264,20 @@ urban : urban-present urban-alpha urban-present : mexicocity vancouver -# NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist mexicocity : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_MEXICOCITY) - $(RM) surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_mexicocityMEX.cfg -i surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc $(RM) surfdata_1x1_mexicocityMEX_hist_78pfts_CMIP6_simyr2000.nc -# NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist vancouver : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_VANCOUVER) - $(RM) surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_vancouverCAN.cfg -i surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc $(RM) surfdata_1x1_vancouverCAN_hist_78pfts_CMIP6_simyr2000.nc # NOTE(bja, 2015-01) skip abort on invalid data necessary as of 2015-01. See # /glade/p/cesm/cseg/inputdata/lnd/clm2/surfdata_map/README_c141219 -# NOTE: $(RM) by default uses rm -f which doesn't generate an error if the file doesn't exist urban-alpha : FORCE $(SUBSETDATA_POINT_URBAN) --create-surface $(SUBSETDATA_1X1_URBALPHA) - $(RM) surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc $(MODIFYSURF) modify_1x1_urbanc_alpha.cfg -i surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc -o surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000_c$(CDATE).nc $(RM) surfdata_1x1_urbanc_alpha_hist_78pfts_CMIP6_simyr2000.nc From b9ab8e0a9d173231a5b4562604ba91c3f49cc648 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 24 Jan 2023 23:52:04 -0700 Subject: [PATCH 190/332] Spin off read_cfg_required_basic_opts to make lint happy --- .../modify_input_files/fsurdat_modifier.py | 98 +++++++++++-------- 1 file changed, 57 insertions(+), 41 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 747b8bc999..d3a39f25a2 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -453,47 +453,8 @@ def read_option_control( return (idealized, process_subgrid, process_var_list, include_nonveg, dom_pft) -def fsurdat_modifier(parser): - """Implementation of fsurdat_modifier command""" - # read the .cfg (config) file - cfg_path = str(parser.cfg_path) - config = ConfigParser() - config.read(cfg_path) - section = "modify_fsurdat_basic_options" - if not config.has_section(section): - abort("Config file does not have the expected section: " + section) - - if parser.fsurdat_in == "UNSET": - # required: user must set these in the .cfg file - fsurdat_in = get_config_value( - config=config, section=section, item="fsurdat_in", file_path=cfg_path - ) - else: - if config.has_option(section=section, option="fsurdat_in"): - abort("fsurdat_in is specified in both the command line and the config file, pick one") - fsurdat_in = str(parser.fsurdat_in) - - # Error checking of input file - if not os.path.exists(fsurdat_in): - abort("Input fsurdat_in file does NOT exist: " + str(fsurdat_in)) - - if parser.fsurdat_out == "UNSET": - fsurdat_out = get_config_value( - config=config, section=section, item="fsurdat_out", file_path=cfg_path - ) - else: - if config.has_option(section=section, option="fsurdat_out"): - abort("fsurdat_out is specified in both the command line and the config file, pick one") - fsurdat_out = str(parser.fsurdat_out) - - # If output file exists, abort before starting work - if os.path.exists(fsurdat_out): - if not parser.overwrite: - errmsg = "Output file already exists: " + fsurdat_out - abort(errmsg) - else: - logger.warning("Output file already exists, but the overwrite option was selected so the file will be overwritten.") - +def read_cfg_required_basic_opts(config, section, cfg_path): + """Read the required part of the control section""" lnd_lat_1 = get_config_value( config=config, section=section, @@ -537,7 +498,62 @@ def fsurdat_modifier(parser): lon_dimname = get_config_value( config=config, section=section, item="lon_dimname", file_path=cfg_path, can_be_unset=True ) + return (lnd_lat_1, lnd_lat_2, lnd_lon_1, lnd_lon_2, landmask_file, lat_dimname, lon_dimname) + +def fsurdat_modifier(parser): + """Implementation of fsurdat_modifier command""" + # read the .cfg (config) file + cfg_path = str(parser.cfg_path) + config = ConfigParser() + config.read(cfg_path) + section = "modify_fsurdat_basic_options" + if not config.has_section(section): + abort("Config file does not have the expected section: " + section) + + if parser.fsurdat_in == "UNSET": + # required: user must set these in the .cfg file + fsurdat_in = get_config_value( + config=config, section=section, item="fsurdat_in", file_path=cfg_path + ) + else: + if config.has_option(section=section, option="fsurdat_in"): + abort("fsurdat_in is specified in both the command line and the config file, pick one") + fsurdat_in = str(parser.fsurdat_in) + + # Error checking of input file + if not os.path.exists(fsurdat_in): + abort("Input fsurdat_in file does NOT exist: " + str(fsurdat_in)) + + if parser.fsurdat_out == "UNSET": + fsurdat_out = get_config_value( + config=config, section=section, item="fsurdat_out", file_path=cfg_path + ) + else: + if config.has_option(section=section, option="fsurdat_out"): + abort("fsurdat_out is specified in both the command line and the config file, pick one") + fsurdat_out = str(parser.fsurdat_out) + + # If output file exists, abort before starting work + if os.path.exists(fsurdat_out): + if not parser.overwrite: + errmsg = "Output file already exists: " + fsurdat_out + abort(errmsg) + else: + warnmsg = ( + "Output file already exists" + + ", but the overwrite option was selected so the file will be overwritten." + ) + logger.warning(warnmsg) + ( + lnd_lat_1, + lnd_lat_2, + lnd_lon_1, + lnd_lon_2, + landmask_file, + lat_dimname, + lon_dimname, + ) = read_cfg_required_basic_opts(config, section, cfg_path) # Create ModifyFsurdat object modify_fsurdat = ModifyFsurdat.init_from_file( fsurdat_in, From f4e9ea82935a3bdb7dda6be74438b03e658af692 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 24 Jan 2023 23:53:08 -0700 Subject: [PATCH 191/332] Check for existance before removing in the test, it will only be removed if the test previously failed --- python/ctsm/test/test_sys_fsurdat_modifier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 529800c3f7..cee7a51d36 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -79,7 +79,8 @@ def test_short_config(self): fsurdat_out = ( "ctsm/test/testinputs/surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214_out.nc" ) - os.remove(fsurdat_out) + if os.path.exists(fsurdat_out): + os.remove(fsurdat_out) fsurdat_modifier(parser) # Run it again with the overwrite option so that it will overwrite the file just created sys.argv = ["fsurdat_modifier", self._cfg_file_path, "--overwrite"] From c04e89f8b92787ce0afcfb94e942917d6b623693 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 25 Jan 2023 00:00:26 -0700 Subject: [PATCH 192/332] Make sure the read_ subroutines that are reading the config file consistently start with read_cfg_ --- .../modify_input_files/fsurdat_modifier.py | 18 ++++--- .../ctsm/test/test_unit_fsurdat_modifier.py | 50 +++++++++---------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index d3a39f25a2..75fbbdba12 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -116,7 +116,7 @@ def check_range(var, section, value, minval, maxval): abort("Variable " + var + " in " + section + " is out of range of 0 to 100 = " + str(value)) -def read_subgrid(config, cfg_path, numurbl=3): +def read_cfg_subgrid(config, cfg_path, numurbl=3): """Read the subgrid fraction section from the config file""" section = "modify_fsurdat_subgrid_fractions" if not config.has_section(section): @@ -171,7 +171,7 @@ def read_subgrid(config, cfg_path, numurbl=3): return subgrid_settings -def read_var_list(config, idealized=True): +def read_cfg_var_list(config, idealized=True): """Read the variable list section from the config file""" section = "modify_fsurdat_variable_list" if not config.has_section(section): @@ -374,7 +374,7 @@ def read_cfg_optional_basic_opts(modify_fsurdat, config, cfg_path, section): ) -def read_option_control( +def read_cfg_option_control( modify_fsurdat, config, section, @@ -567,7 +567,13 @@ def fsurdat_modifier(parser): ) # Read control information about the optional sections - (idealized, process_subgrid, process_var_list, include_nonveg, dom_pft) = read_option_control( + ( + idealized, + process_subgrid, + process_var_list, + include_nonveg, + dom_pft, + ) = read_cfg_option_control( modify_fsurdat, config, section, @@ -607,14 +613,14 @@ def fsurdat_modifier(parser): # Handle optional sections # if process_subgrid: - subgrid = read_subgrid(config, cfg_path, numurbl=modify_fsurdat.get_urb_dens()) + subgrid = read_cfg_subgrid(config, cfg_path, numurbl=modify_fsurdat.get_urb_dens()) modify_fsurdat.set_varlist(subgrid, cfg_path) logger.info("process_subgrid is complete") else: check_no_subgrid_section(config) if process_var_list: - varlist = read_var_list(config, idealized=idealized) + varlist = read_cfg_var_list(config, idealized=idealized) update_list = modify_fsurdat.check_varlist(varlist, allow_uppercase_vars=True) modify_fsurdat.set_varlist(update_list, cfg_path) logger.info("process_var_list is complete") diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index fee4ec2ab3..0c7a244584 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -16,9 +16,9 @@ from ctsm import unit_testing from ctsm.path_utils import path_to_ctsm_root from ctsm.modify_input_files.fsurdat_modifier import fsurdat_modifier_arg_process -from ctsm.modify_input_files.fsurdat_modifier import read_subgrid -from ctsm.modify_input_files.fsurdat_modifier import read_option_control -from ctsm.modify_input_files.fsurdat_modifier import read_var_list +from ctsm.modify_input_files.fsurdat_modifier import read_cfg_subgrid +from ctsm.modify_input_files.fsurdat_modifier import read_cfg_option_control +from ctsm.modify_input_files.fsurdat_modifier import read_cfg_var_list from ctsm.modify_input_files.fsurdat_modifier import check_no_subgrid_section from ctsm.modify_input_files.fsurdat_modifier import check_no_varlist_section from ctsm.modify_input_files.modify_fsurdat import ModifyFsurdat @@ -90,7 +90,7 @@ def test_dom_pft_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" ): - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_subgrid_and_idealized_fails(self): """test that dom_pft and idealized fails gracefully""" @@ -103,7 +103,7 @@ def test_subgrid_and_idealized_fails(self): SystemExit, "idealized AND process_subgrid_section can NOT both be on, pick one or the other", ): - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_optional_only_true_and_false(self): """test that optional settings can only be true or false""" @@ -118,18 +118,18 @@ def test_optional_only_true_and_false(self): for var in varlist: self.config.set(section, var, "True") self.config.set(section, "idealized", "False") - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) for var in varlist: self.config.set(section, var, "False") - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) self.config.set(section, "dom_pft", "UNSET") - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) var = "idealized" self.config.set(section, var, "Thing") with self.assertRaisesRegex( SystemExit, "Non-boolean value found for .cfg file variable: " + var ): - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_include_nonveg_and_idealized_fails(self): """test a simple read of subgrid""" @@ -139,11 +139,11 @@ def test_include_nonveg_and_idealized_fails(self): with self.assertRaisesRegex( SystemExit, "idealized AND include_nonveg can NOT both be on, pick one or the other" ): - read_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) + read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) def test_read_subgrid(self): """test a simple read of subgrid""" - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_read_subgrid_allglacier(self): """test a read of subgrid that's for all glacier""" @@ -156,7 +156,7 @@ def test_read_subgrid_allglacier(self): self.config.set(section, "pct_glacier", "100.") self.config.set(section, "pct_natveg", "0.") self.config.set(section, "pct_crop", "0.") - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_read_subgrid_allspecial(self): """test a read of subgrid that's all special landunits""" @@ -169,7 +169,7 @@ def test_read_subgrid_allspecial(self): self.config.set(section, "pct_glacier", "40.") self.config.set(section, "pct_natveg", "0.") self.config.set(section, "pct_crop", "0.") - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_read_subgrid_allurban(self): """test a read of subgrid that's all urban""" @@ -182,11 +182,11 @@ def test_read_subgrid_allurban(self): self.config.set(section, "pct_glacier", "0.") self.config.set(section, "pct_natveg", "0.") self.config.set(section, "pct_crop", "0.") - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_read_var_list(self): """test a simple read of var_list""" - read_var_list(self.config, idealized=True) + read_cfg_var_list(self.config, idealized=True) def test_subgrid_outofrange(self): """test a read of subgrid that's out of range""" @@ -195,7 +195,7 @@ def test_subgrid_outofrange(self): section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "pct_urban", "101. 0. 0.") with self.assertRaisesRegex(SystemExit, "is out of range of 0 to 100 ="): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_subgrid_pct_urban_toosmall(self): """test a read of subgrid for PCT_URBAN that's an array too small""" @@ -206,7 +206,7 @@ def test_subgrid_pct_urban_toosmall(self): with self.assertRaisesRegex( SystemExit, "PCT_URBAN is not a list of the expected size of 3" ): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_subgrid_pct_urban_toobig(self): """test a read of subgrid for PCT_URBAN that's an array too big""" @@ -217,7 +217,7 @@ def test_subgrid_pct_urban_toobig(self): with self.assertRaisesRegex( SystemExit, "PCT_URBAN is not a list of the expected size of 3" ): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_subgrid_pct_urban_singlevalue(self): """test a read of subgrid for PCT_URBAN that's a single value""" @@ -228,7 +228,7 @@ def test_subgrid_pct_urban_singlevalue(self): with self.assertRaisesRegex( SystemExit, "PCT_URBAN is not a list of the expected size of 3" ): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_subgrid_notsumtohundred(self): """test a read of subgrid that's doesn't sum to a hundred""" @@ -244,7 +244,7 @@ def test_subgrid_notsumtohundred(self): with self.assertRaisesRegex( SystemExit, "PCT fractions in subgrid section do NOT sum to a hundred as they should" ): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_subgrid_badvar(self): """test a read of subgrid for a variable thats not in the list""" @@ -253,7 +253,7 @@ def test_subgrid_badvar(self): section = "modify_fsurdat_subgrid_fractions" self.config.set(section, "badvariable", "100.") with self.assertRaisesRegex(SystemExit, "is not a valid variable name. Valid vars ="): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_varlist_varinidealized(self): """test a read of varlist for a variable thats in the idealized list, @@ -267,7 +267,7 @@ def test_varlist_varinidealized(self): "is a special variable handled in the idealized section." + " This should NOT be handled in the variiable list section. Special idealized vars =", ): - read_var_list(self.config, idealized=True) + read_cfg_var_list(self.config, idealized=True) def test_varlist_varinsubgrid(self): """test a read of varlist for a variable thats in the subgrid list""" @@ -280,7 +280,7 @@ def test_varlist_varinsubgrid(self): "is a variable handled in the subgrid section." + " This should NOT be handled in the variiable list section. Subgrid vars =", ): - read_var_list(self.config, idealized=False) + read_cfg_var_list(self.config, idealized=False) def test_varlist_monthlyvar(self): """test a read of varlist for a variable thats one of the monthly @@ -293,14 +293,14 @@ def test_varlist_monthlyvar(self): + " This should NOT be handled in the variiable list section." + " Monthly vars handled this way =", ): - read_var_list(self.config, idealized=False) + read_cfg_var_list(self.config, idealized=False) def test_subgrid_remove(self): """test a read of subgrid when it's section has been removed""" section = "modify_fsurdat_subgrid_fractions" self.config.remove_section(section) with self.assertRaisesRegex(SystemExit, "Config file does not have the expected section"): - read_subgrid(self.config, self.cfg_path) + read_cfg_subgrid(self.config, self.cfg_path) def test_subgrid_not_thereifoff(self): """test that a graceful error happens if subgrid section is off, From 6008d28a920d28a7f92592aae197dce768451035 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 26 Jan 2023 02:17:40 -0700 Subject: [PATCH 193/332] Update change files --- doc/ChangeLog | 157 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 158 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 49f49c235f..b1f5fdf975 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,161 @@ =============================================================== +Tag name: ctsm5.1.dev116 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Thu Jan 26 02:17:27 MST 2023 +One-line Summary: Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files + +Purpose and description of changes +---------------------------------- + +Change zetamaxstable to 2 when biomass-heat-storage is on. This changes simulation answers once they +run long enough to exceed that threshold. + +Also fix an issue with maintence respiration (MR) for BGC simulations. This changes answers for most BGC cases once +they run long enough. Live course MR wasn't included. + +Make the default for MOSART to send negative flow to river outlets. Also fix an issue with this mode. + +Bring in new surface datasets for the single point sites. We now make these sites using subset_data rather +than mksurfdata. + +Some new capability to the subset_data and modify_fsurdat tools. + +subset_data add options: +--out-surface -- To name the surface dataset on the command line rather than based on the current date +--cfg-file ----- Enter the default configure file to use rather than assume a fixed one + +modify_fsurdat add options: +--fsurdat_in -- to input on command line rather than config file +--fsurdat_out -- to input on command line rather than config file +--allow_ideal_and_include_non_veg -- to allow idealized and include_non_veg at the same time +--allow_dom_pft_and_idealized -- to allow dom_pft and idealized at the same time +--overwrite -- allow output file to be overwritten +config file options: +process_subgrid_section -- Read in an optional section to set the PCT_* fractions +process_var_list_section - Read in an optional section to set any variable on the file + +Add --silent option to python tools. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[X] clm5_1 + +[x] clm5_0 + +[ ] ctsm5_0-nwp + +[x] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): + Fixes #1676 -- live coarse maintenance respiration is not included in the root respiration + Fixes #1674 -- Change mksurfdata_map/mksurfdata_esmf Makefile to build single-point datasets using subset_data + Fixes #1809 -- Add ability to name a different default config file for subset_data + Fixes #1941 -- Add --silent option to ctsm_logging python infrastructure + Fixes #1942 -- Move py_env_create outside of tools test driver, as fails on compute nodes on cheyenne + Fixes #1924 -- Some updates to fsurdat_modifier script + Fixes #1690 -- Set and use zetamaxstable for BHS cases + Fixes #1689 -- Set zetamaxstable to 2 consistently for BHS + +Externals issues fixed (include issue #): + MOSART #58 Make negative and direct_to_outlet the default option + MOSART #56 Some issues with direct_to_outlet + +Notes of particular relevance for users +--------------------------------------- + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): tools + Just to the subset_data and fsurdat_modifier tools as outlined above + +Changes made to namelist defaults (e.g., changed parameter values): + zetamaxstable now 2.0 is use_biomass_heat_storage is on + +Changes to the datasets (e.g., parameter, surface or initial files): Single point fsurdat + 1x1_brazil, 1x1_numaIA, 1x1_smallvilleIA, 1x1_vancouverCAN, 1x1_mexicocityMEX, 1x1_urbanc_alpha + surface datasets and 1x1_brazil landuse.timeseries + +Notes of particular relevance for developers: +--------------------------------------------- + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + tools test now just activates the ctsm_py conda environment rather than creating it + There are seperate configure files for each urban surface dataset for modify_fsurdat + There is a 1850 configure file for subset_data + +Changes to tests or testing: + fsurdatmodifyctsm.py script adjusted to compensate for changes + python directory changes include unit and system tests to support updates + Single point mksurdata_map tests were removed + + +Testing summary: regular tools +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - PASS + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - PASS + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + cheyenne - PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + +Answer changes +-------------- + +Changes answers relative to baseline: + + Summarize any changes to answers, i.e., + - what code configurations: clm5_1, supported single point resolutions + - what platforms/compilers: All + - nature of change: adjusted climate + clm5_1 changes because zetmaxstable set to 2.0. If a simulation runs long enough + this max will be hit and it will change answers once it does. But if stability + doesn't hit the max answers can be identical. + clm5_0 and clm4_5 also change if biomass heat storage is turned on + single point resolutions (i.e. 1x1_smallvilleIA, 1x1_brazil, 1x1_mexicocityMEX) have differences + maintenence respiration + + If this tag changes climate describe the run(s) done to evaluate the new + climate (put details of the simulations in the experiment database) + Keith Oleson ran experiments with changing zetamaxstable some slides showing this are here: + https://docs.google.com/presentation/d/1u6ycr7F97QYYRcRfEdD9yIxH75diUx2r + + +Other details +------------- + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): mosart + mosart updated to mosart1_0_48 + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + #1812 -- Get single point surface datasets from subset_data rather than mksurfdata + #1802 -- Make zetamaxstable consistently 2.0 when BHS on + #1915 -- Fix issue #1864 in release documentation + Update manage_externals (direct push to main-dev) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev115 Originator(s): rgknox (Ryan Knox) Date: Fri Dec 2 15:45:32 MST 2022 diff --git a/doc/ChangeSum b/doc/ChangeSum index 1d42a3522d..3042b38a2a 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics ctsm5.1.dev114 multiple 11/19/2022 Some NEON updates fixing AG sites, update MOSART, small fixes ctsm5.1.dev113 multiple 10/28/2022 Fix some compsets; add only clauses for ESMF use statements From a1130ef8a82e1bfb6ae5df7a4e57871737ac1085 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Thu, 26 Jan 2023 10:19:58 -0700 Subject: [PATCH 194/332] add new test to fates list --- cime_config/testdefs/testlist_clm.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 05053fc5d1..86ed45f09e 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1555,6 +1555,7 @@ + From aad7de1bf1b6c98d8e52c1d88ac62389e6cbf74a Mon Sep 17 00:00:00 2001 From: adrifoster Date: Thu, 26 Jan 2023 14:16:41 -0700 Subject: [PATCH 195/332] add fire_emis and fates test and check --- bld/CLMBuildNamelist.pm | 4 ++++ bld/unit_testers/build-namelist_test.pl | 5 +++++ cime_config/testdefs/testlist_clm.xml | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index f046172f7e..05631edc23 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -3683,6 +3683,10 @@ sub setup_logic_fire_emis { my ($opts, $nl_flags, $definition, $defaults, $nl) = @_; if ($opts->{'fire_emis'} ) { + if ( &value_is_true( $nl_flags->{'use_fates'} ) ) { + $log->warning("Fire emission can NOT be on when FATES is also on.\n" . + " DON'T use the '-fire_emis' option when '-bgc fates' is activated"); + } add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fire_emis_factors_file'); add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fire_emis_specifier'); } else { diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index f270cbfea4..c8b7538d91 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -1006,6 +1006,11 @@ sub cat_and_create_namelistinfile { GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm4_5", }, + "useFIREEMISSwithFATES" =>{ options=>"-bgc fates -envxml_dir . -fire_emis", + namelst=>"", + GLC_TWO_WAY_COUPLING=>"FALSE", + phys=>"clm4_5", + }, "useDRYDEPwithFATES" =>{ options=>"--bgc fates --envxml_dir . --no-megan --drydep", namelst=>"", GLC_TWO_WAY_COUPLING=>"FALSE", diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 86ed45f09e..4d074eae11 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1552,7 +1552,7 @@ - + From 435691e7b35d3b095a7a05de29f24f9ca53c432b Mon Sep 17 00:00:00 2001 From: adrifoster Date: Thu, 26 Jan 2023 14:43:51 -0700 Subject: [PATCH 196/332] update test number --- bld/unit_testers/build-namelist_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index c8b7538d91..ffb01fe02c 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,7 +163,7 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1846; +my $ntests = 1847; if ( defined($opts{'compare'}) ) { $ntests += 1254; } From ef82a689741b80ae3738b776d6224e6925e6ac39 Mon Sep 17 00:00:00 2001 From: Adrianna Foster Date: Fri, 27 Jan 2023 15:36:05 -0700 Subject: [PATCH 197/332] add --no-megan to build-namelist test --- bld/unit_testers/build-namelist_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index ffb01fe02c..588c46c9e5 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -1006,7 +1006,7 @@ sub cat_and_create_namelistinfile { GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm4_5", }, - "useFIREEMISSwithFATES" =>{ options=>"-bgc fates -envxml_dir . -fire_emis", + "useFIREEMISwithFATES" =>{ options=>"-bgc fates -envxml_dir . -fire_emis --no-megan", namelst=>"", GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm4_5", From 72e252ac73f48be0efaaa2be40fad15134602e24 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Fri, 27 Jan 2023 15:38:11 -0700 Subject: [PATCH 198/332] add test to expected fails --- cime_config/testdefs/ExpectedTestFails.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 90c45ef919..3affa5be65 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -80,5 +80,13 @@ FATES#701 - + + + + FAIL + FATES#701 + + + + From 0e09a64fc7f6d9239e1ebdba9c41e54dfe7be374 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 31 Jan 2023 16:39:46 -0700 Subject: [PATCH 199/332] Add tests for the Gross unrepresented LU change option --- .../clm/f09_dec1990Start_GU_LULCC/include_user_mods | 1 + .../testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands | 2 ++ .../testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm | 3 +++ .../clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods | 2 ++ 4 files changed, 8 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/include_user_mods create mode 100755 cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands create mode 100644 cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm create mode 100644 cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/include_user_mods new file mode 100644 index 0000000000..fe0e18cf88 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/include_user_mods @@ -0,0 +1 @@ +../default diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands new file mode 100755 index 0000000000..4e10e7086a --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands @@ -0,0 +1,2 @@ +./xmlchange RUN_STARTDATE=1990-12-25 +./xmlchange CLM_BLDNML_OPTS=-ignore_warnings --append diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm new file mode 100644 index 0000000000..6a666f2684 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm @@ -0,0 +1,3 @@ + flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1700-2021_c220825.nc' + fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr1700_c220825.nc' + do_grossunrep = .true. diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods new file mode 100644 index 0000000000..24f76fbb9f --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods @@ -0,0 +1,2 @@ +../../f09_dec1990Start_GU_LULCC +../monthly From 0a087c5495283171450842d072ddbbf9869bfb6a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 31 Jan 2023 16:42:33 -0700 Subject: [PATCH 200/332] Add tests to test list for Gross unrepresented land use change --- cime_config/testdefs/testlist_clm.xml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 8906c6c4bd..6cfe703c6d 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -317,11 +317,32 @@ - + + + + + + + + + + + + - + + + + + + + + + + + From 29f9b5971fb080318aabd0a78f132764aa2ef48d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 31 Jan 2023 16:49:28 -0700 Subject: [PATCH 201/332] Add some comments and fix directory name --- .../testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands | 1 + .../testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm | 2 ++ .../clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands index 4e10e7086a..972cd25f5c 100755 --- a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands @@ -1,2 +1,3 @@ ./xmlchange RUN_STARTDATE=1990-12-25 +# Ignore warnings because we are using crop, but starting from a different date than the initial conditions were for ./xmlchange CLM_BLDNML_OPTS=-ignore_warnings --append diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm index 6a666f2684..efb3212d5f 100644 --- a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/user_nl_clm @@ -1,3 +1,5 @@ + ! Specify a dataset that has non-zero Gross Unrepresented Land Use change fields on it + ! And turn it on flanduse_timeseries = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_0.9x1.25_hist_78pfts_CMIP6_simyr1700-2021_c220825.nc' fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_0.9x1.25_hist_78pfts_CMIP6_simyr1700_c220825.nc' do_grossunrep = .true. diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods index 24f76fbb9f..b10682f861 100644 --- a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods @@ -1,2 +1,2 @@ -../../f09_dec1990Start_GU_LULCC +../f09_dec1990Start_GU_LULCC ../monthly From 47d61a61ba7d264336a5d0a16dea355a5002ba7f Mon Sep 17 00:00:00 2001 From: adrifoster Date: Wed, 1 Feb 2023 08:18:47 -0700 Subject: [PATCH 202/332] add PRT2 test to expected fails --- cime_config/testdefs/ExpectedTestFails.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 3affa5be65..6ddfea3437 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -88,5 +88,22 @@ + + + PEND + #1045 + + + + + + FAIL + FATES#983 + This job should time out on izumi, seems to be hanging on history output. + + + + + From 8f6c03e8e4a54b9f90faba791684489a3fb432ce Mon Sep 17 00:00:00 2001 From: adrifoster Date: Wed, 1 Feb 2023 11:51:18 -0700 Subject: [PATCH 203/332] add back space --- tools/site_and_regional/subset_data | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/site_and_regional/subset_data b/tools/site_and_regional/subset_data index 6477b3034d..d19525c97d 100755 --- a/tools/site_and_regional/subset_data +++ b/tools/site_and_regional/subset_data @@ -9,6 +9,7 @@ For full instructions on how to run the code and different options, please check python/ctsm/subset_data.py file. This script extracts domain files, surface dataset, and DATM files at either a single point or a region using the global dataset. + To run this script the following packages are required: - numpy - xarray From 48a38b6a323d4e53cfe528c97c67ade1f716b3f0 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Wed, 1 Feb 2023 13:56:02 -0700 Subject: [PATCH 204/332] update changelog --- doc/ChangeLog | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 117 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index b1f5fdf975..5a90817673 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,120 @@ =============================================================== +Tag name: ctsm5.1.dev117 +Originator(s): afoster (Adrianna Foster) +Date: Wed Feb 1 13:36:27 MST 2023 +One-line Summary: Updates to facilitate running FATES at NEON sites + +Purpose and description of changes +---------------------------------- + +Small updates to facilitate creation, modification, and +use of FATES-usable (i.e. 16-PFT) NEON surface data files and +user-mods for FATES NEON cases. + +Updated neon_surf_wrapper.py and modify_singlept_site_neon.py to include a +--16pft argument that will create and/or modify the 16-PFT versions of the +surface datasets, as well as a --mixed flag to the neon_surf_wrapper.py +which tells subset_data to not overwrite the surfae dataset to be just 100% +one PFT. + +Also corrects lat-lon being used for ONAQ NEON site and updates the surface +datasets for all NEON sites. + +Also adds a check to ensure that fire emission (-fire_emis) is not on if FATES +is being run. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): +- Partially addresses ESCOMP/CTSM#1609(Get FATES working for NEON) + +Known bugs introduced in this tag (include issue #): +#1949 - Duplication problems in user_mods + +Known bugs found since the previous tag (include issue #): +#1948 - FATES and 78PFT surface datasets +FATES#983 - PRT2 test failing on izumi + + +Notes of particular relevance for users +--------------------------------------- + +Changes to the datasets (e.g., parameter, surface or initial files): +updated surface datasets for all NEON sites for big-leaf (78-PFT) and FATES (16-PFT) + + +Notes of particular relevance for developers: +--------------------------------------------- + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): +Files changed in cime_config/usermods_dirs/NEON for big-leaf CLM must also +be changed in similar files in cime_config/usermods_dirs/NEON/FATES, as these are +currently duplicated. This duplication should be fixed at a later date. + +Changes to tests or testing: +Added a test in bld/unit_testers/build-namelist_test.pl to check that FATES and +fire emission cannot be on at the same time. + + +Testing summary: +---------------- + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- OK + izumi ------- OK + + +Answer changes +-------------- + +Changes answers relative to baseline: + +Only for NEON sites. + + + Summarize any changes to answers, i.e., + - what code configurations: All configurations at NEON sites + - what platforms/compilers: All platforms when running NEON sites + - nature of change: updated surface datasets + + +Other details +------------- +#1933 - Update neon_sites_dompft.csv +#1932 - NEON FATES capabilities + + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev116 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) Date: Thu Jan 26 02:17:27 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 3042b38a2a..21f9dd70b9 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev117 afoster 02/01/2023 Updates to facilitate running FATES at NEON sites ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics ctsm5.1.dev114 multiple 11/19/2022 Some NEON updates fixing AG sites, update MOSART, small fixes From eb29c80c1274d8307d79968b2a6a91df475c5820 Mon Sep 17 00:00:00 2001 From: adrifoster Date: Wed, 1 Feb 2023 13:56:54 -0700 Subject: [PATCH 205/332] update changelog date --- doc/ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 5a90817673..1cc7768243 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev117 Originator(s): afoster (Adrianna Foster) -Date: Wed Feb 1 13:36:27 MST 2023 +Date: Wed Feb 1 13:56:41 MST 2023 One-line Summary: Updates to facilitate running FATES at NEON sites Purpose and description of changes From 0e83df379ffd68600514cc51a1358e51f602f30c Mon Sep 17 00:00:00 2001 From: adrifoster Date: Thu, 2 Feb 2023 10:34:35 -0700 Subject: [PATCH 206/332] update changelog date --- doc/ChangeLog | 2 +- doc/ChangeSum | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 1cc7768243..ec6ca43ce2 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev117 Originator(s): afoster (Adrianna Foster) -Date: Wed Feb 1 13:56:41 MST 2023 +Date: Thu Feb 2 10:34:23 MST 2023 One-line Summary: Updates to facilitate running FATES at NEON sites Purpose and description of changes diff --git a/doc/ChangeSum b/doc/ChangeSum index 21f9dd70b9..ac424326b0 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev117 afoster 02/01/2023 Updates to facilitate running FATES at NEON sites + ctsm5.1.dev117 afoster 02/02/2023 Updates to facilitate running FATES at NEON sites ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics ctsm5.1.dev114 multiple 11/19/2022 Some NEON updates fixing AG sites, update MOSART, small fixes From c96735b53ae9816e2de88b667f588a70b69cbeed Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Thu, 2 Feb 2023 16:25:56 -0700 Subject: [PATCH 207/332] Updating new tests, mainly with longer walltimes --- cime_config/testdefs/testlist_clm.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 6cfe703c6d..4dfddb4a11 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -320,7 +320,7 @@ - + @@ -331,17 +331,17 @@ - + - + - + From 07e8462cba9351cf4cc63968326b653509bc24dc Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 3 Feb 2023 16:25:26 -0700 Subject: [PATCH 208/332] Rm override allow_ideal_and_include_non_veg & corresponding restriction --- python/ctsm/modify_input_files/fsurdat_modifier.py | 11 ----------- python/ctsm/test/test_sys_fsurdat_modifier.py | 1 - 2 files changed, 12 deletions(-) diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index 75fbbdba12..f5543a9506 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -57,13 +57,6 @@ def fsurdat_modifier_arg_process(): type=str, help="The output surface dataset with the modifications. ", ) - parser.add_argument( - "--allow_ideal_and_include_non_veg", - required=False, - default=False, - action="store_true", - help="Allow both idealized and include_nonveg to be on at the same time. ", - ) parser.add_argument( "--allow_dom_pft_and_idealized", required=False, @@ -379,7 +372,6 @@ def read_cfg_option_control( config, section, cfg_path, - allow_ideal_and_include_non_veg=False, allow_dom_pft_and_idealized=False, ): """Read the option control section""" @@ -445,8 +437,6 @@ def read_cfg_option_control( logger.info("dom_pft option is off") if dom_pft is not None and idealized and not allow_dom_pft_and_idealized: abort("idealized AND dom_pft can NOT both be on, pick one or the other") - if include_nonveg and idealized and not allow_ideal_and_include_non_veg: - abort("idealized AND include_nonveg can NOT both be on, pick one or the other") if process_subgrid and idealized: abort("idealized AND process_subgrid_section can NOT both be on, pick one or the other") @@ -578,7 +568,6 @@ def fsurdat_modifier(parser): config, section, cfg_path, - parser.allow_ideal_and_include_non_veg, parser.allow_dom_pft_and_idealized, ) diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index cee7a51d36..b4598fae20 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -374,7 +374,6 @@ def test_allInfo(self): sys.argv = [ "fsurdat_modifier", self._cfg_file_path, - "--allow_ideal_and_include_non_veg", "--allow_dom_pft_and_idealized", ] parser = fsurdat_modifier_arg_process() From 662bf9295db86424df4dc29e70bc2057b24339d1 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 3 Feb 2023 17:06:02 -0700 Subject: [PATCH 209/332] First draft of ChangeLog --- doc/ChangeLog | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 80 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index ec6ca43ce2..0804cd8d4a 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,83 @@ =============================================================== +Tag name: ctsm5.1.dev118 +Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310) +Date: Fri Feb 3 16:49:44 MST 2023 +One-line Summary: Use conda environment rather than ncar_pylib with the fsurdat_modifier system test + +Purpose and description of changes +---------------------------------- + + ncar_pylib is going away soon. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): + The fsurdat_modifier system test that we're discussing in this tag and + pull request (PR #1798) + FSURDATMODIFYCTSM_D_Mmpi-serial_Ld1.5x5_amazon.I2000Clm50SpRs.cheyenne_intel + stopped working when I updated the PR to dev117. + The test worked in the PR when I was still in dev115. + The test worked in vanilla dev117. + I fixed the test by removing a restriction added in dev116 and the + corresponding override --allow_ideal_and_include_non_veg. + + PENDING approval by Erik: I would like to remove another restriction added + in dev116 and the corresponding override --allow_dom_pft_and_idealized. + +Testing summary: +---------------- + +[Remove any lines that don't apply.] + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + [If python code has changed and you are NOT running aux_clm (e.g., because the only changes are in python + code) then also run the clm_pymods test suite; this is a small subset of aux_clm that runs the system + tests impacted by python changes. The best way to do this, if you expect no changes from the last tag in + either model output or namelists, is: create sym links pointing to the last tag's baseline directory, + named with the upcoming tag; then run the clm_pymods test suite comparing against these baselines but NOT + doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to + separately run the clm_pymods test suite, and you can remove the following line.] + + clm_pymods test suite on cheyenne - PASS + + any other testing (give details below): + + make all (in /python directory) - PASS + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + +Answer changes +-------------- + +Changes answers relative to baseline: NO + +Other details +------------- +Pull Requests that document the changes (include PR ids): + https://github.com/ESCOMP/ctsm/pull/1798 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev117 Originator(s): afoster (Adrianna Foster) Date: Thu Feb 2 10:34:23 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index ac424326b0..175cfdc588 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev118 slevis 02/03/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test ctsm5.1.dev117 afoster 02/02/2023 Updates to facilitate running FATES at NEON sites ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics From 6699c0dc869d40cd54d1e3b3cf9249f9d8c63b44 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Fri, 3 Feb 2023 17:57:25 -0700 Subject: [PATCH 210/332] Rm override allow_dom_pft_and_idealized & corresponding restriction Update ChangeLog and python tests --- doc/ChangeLog | 13 +++++---- .../modify_input_files/fsurdat_modifier.py | 23 +++++---------- .../ctsm/modify_input_files/modify_fsurdat.py | 2 ++ python/ctsm/test/test_sys_fsurdat_modifier.py | 1 - .../ctsm/test/test_unit_fsurdat_modifier.py | 28 +++---------------- 5 files changed, 20 insertions(+), 47 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 0804cd8d4a..b75f939799 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,13 +1,13 @@ =============================================================== Tag name: ctsm5.1.dev118 Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310) -Date: Fri Feb 3 16:49:44 MST 2023 +Date: Fri Feb 3 17:52:37 MST 2023 One-line Summary: Use conda environment rather than ncar_pylib with the fsurdat_modifier system test Purpose and description of changes ---------------------------------- - ncar_pylib is going away soon. + Reason: ncar_pylib is going away soon. Significant changes to scientifically-supported configurations -------------------------------------------------------------- @@ -31,16 +31,17 @@ Bugs fixed or introduced CTSM issues fixed (include CTSM Issue #): The fsurdat_modifier system test that we're discussing in this tag and - pull request (PR #1798) + corresponding pull request (PR #1798) FSURDATMODIFYCTSM_D_Mmpi-serial_Ld1.5x5_amazon.I2000Clm50SpRs.cheyenne_intel stopped working when I updated the PR to dev117. The test worked in the PR when I was still in dev115. The test worked in vanilla dev117. - I fixed the test by removing a restriction added in dev116 and the + I fixed the failure by removing a restriction added in dev116 and the corresponding override --allow_ideal_and_include_non_veg. - PENDING approval by Erik: I would like to remove another restriction added - in dev116 and the corresponding override --allow_dom_pft_and_idealized. + I am removing another restriction added in dev116 and the corresponding + override --allow_dom_pft_and_idealized. This one didn't cause a test to fail + but unnecessarily restricted usage of the fsurdat_modifier tool. Testing summary: ---------------- diff --git a/python/ctsm/modify_input_files/fsurdat_modifier.py b/python/ctsm/modify_input_files/fsurdat_modifier.py index f5543a9506..492fa74230 100644 --- a/python/ctsm/modify_input_files/fsurdat_modifier.py +++ b/python/ctsm/modify_input_files/fsurdat_modifier.py @@ -57,13 +57,6 @@ def fsurdat_modifier_arg_process(): type=str, help="The output surface dataset with the modifications. ", ) - parser.add_argument( - "--allow_dom_pft_and_idealized", - required=False, - default=False, - action="store_true", - help="Allow both idealized and dom_pft to be on at the same time. ", - ) parser.add_argument( "--overwrite", required=False, @@ -208,7 +201,7 @@ def read_cfg_var_list(config, idealized=True): abort( var + " is a special variable handled in the idealized section." - + " This should NOT be handled in the variiable list section." + + " This should NOT be handled in the variable list section." + " Special idealized vars =" + str(ideal_list) ) @@ -216,7 +209,7 @@ def read_cfg_var_list(config, idealized=True): abort( var + " is a variable handled in the subgrid section." - + " This should NOT be handled in the variiable list section." + + " This should NOT be handled in the variable list section." + " Subgrid vars =" + str(subgrid_list) ) @@ -224,7 +217,7 @@ def read_cfg_var_list(config, idealized=True): abort( var + " is a variable handled as part of the dom_pft handling." - + " This should NOT be handled in the variiable list section." + + " This should NOT be handled in the variable list section." + " Monthly vars handled this way =" + str(monthly_list) ) @@ -259,6 +252,8 @@ def modify_optional( if idealized: modify_fsurdat.set_idealized() # set 2D variables # set 3D and 4D variables pertaining to natural vegetation + # to default values here; allow override values with the later call + # to set_dom_pft modify_fsurdat.set_dom_pft(dom_pft=0, lai=[], sai=[], hgt_top=[], hgt_bot=[]) logger.info("idealized complete") @@ -278,8 +273,8 @@ def modify_optional( modify_fsurdat.zero_nonveg() logger.info("zero_nonveg complete") - # set_dom_pft follows zero_nonveg because it modifies PCT_NATVEG - # and PCT_CROP in the user-defined rectangle + # set_dom_pft follows idealized and zero_nonveg because it modifies + # PCT_NATVEG and PCT_CROP in the user-defined rectangle if dom_pft is not None: modify_fsurdat.set_dom_pft( dom_pft=dom_pft, lai=lai, sai=sai, hgt_top=hgt_top, hgt_bot=hgt_bot @@ -372,7 +367,6 @@ def read_cfg_option_control( config, section, cfg_path, - allow_dom_pft_and_idealized=False, ): """Read the option control section""" # required but fallback values available for variables omitted @@ -435,8 +429,6 @@ def read_cfg_option_control( logger.info("dom_pft option is on and = %s", str(dom_pft)) else: logger.info("dom_pft option is off") - if dom_pft is not None and idealized and not allow_dom_pft_and_idealized: - abort("idealized AND dom_pft can NOT both be on, pick one or the other") if process_subgrid and idealized: abort("idealized AND process_subgrid_section can NOT both be on, pick one or the other") @@ -568,7 +560,6 @@ def fsurdat_modifier(parser): config, section, cfg_path, - parser.allow_dom_pft_and_idealized, ) # Read parts that are optional diff --git a/python/ctsm/modify_input_files/modify_fsurdat.py b/python/ctsm/modify_input_files/modify_fsurdat.py index 599b8a6e84..9b3760e303 100644 --- a/python/ctsm/modify_input_files/modify_fsurdat.py +++ b/python/ctsm/modify_input_files/modify_fsurdat.py @@ -180,6 +180,8 @@ def set_dom_pft(self, dom_pft, lai, sai, hgt_top, hgt_bot): --------- dom_pft: (int) User's entry of PFT/CFT to be set to 100% everywhere + If user left this UNSET in the configure file, then it + will default to 0 (bare ground). lai: (float) User's entry of MONTHLY_LAI for their dom_pft sai: diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index b4598fae20..72d38732cf 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -374,7 +374,6 @@ def test_allInfo(self): sys.argv = [ "fsurdat_modifier", self._cfg_file_path, - "--allow_dom_pft_and_idealized", ] parser = fsurdat_modifier_arg_process() fsurdat_modifier(parser) diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 0c7a244584..0ea862a8e4 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -82,18 +82,8 @@ def tearDown(self): """ shutil.rmtree(self._tempdir, ignore_errors=True) - def test_dom_pft_and_idealized_fails(self): - """test a that dom_pft and idealized fails gracefully""" - section = "modify_fsurdat_basic_options" - self.config.set(section, "idealized", "True") - self.config.set(section, "dom_pft", "1") - with self.assertRaisesRegex( - SystemExit, "idealized AND dom_pft can NOT both be on, pick one or the other" - ): - read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) - def test_subgrid_and_idealized_fails(self): - """test that dom_pft and idealized fails gracefully""" + """test that subgrid and idealized fails gracefully""" section = "modify_fsurdat_basic_options" self.config.set(section, "idealized", "True") self.config.set(section, "include_nonveg", "False") @@ -131,16 +121,6 @@ def test_optional_only_true_and_false(self): ): read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) - def test_include_nonveg_and_idealized_fails(self): - """test a simple read of subgrid""" - section = "modify_fsurdat_basic_options" - self.config.set(section, "idealized", "True") - self.config.set(section, "include_nonveg", "True") - with self.assertRaisesRegex( - SystemExit, "idealized AND include_nonveg can NOT both be on, pick one or the other" - ): - read_cfg_option_control(self.modify_fsurdat, self.config, section, self.cfg_path) - def test_read_subgrid(self): """test a simple read of subgrid""" read_cfg_subgrid(self.config, self.cfg_path) @@ -265,7 +245,7 @@ def test_varlist_varinidealized(self): with self.assertRaisesRegex( SystemExit, "is a special variable handled in the idealized section." - + " This should NOT be handled in the variiable list section. Special idealized vars =", + + " This should NOT be handled in the variable list section. Special idealized vars =", ): read_cfg_var_list(self.config, idealized=True) @@ -278,7 +258,7 @@ def test_varlist_varinsubgrid(self): with self.assertRaisesRegex( SystemExit, "is a variable handled in the subgrid section." - + " This should NOT be handled in the variiable list section. Subgrid vars =", + + " This should NOT be handled in the variable list section. Subgrid vars =", ): read_cfg_var_list(self.config, idealized=False) @@ -290,7 +270,7 @@ def test_varlist_monthlyvar(self): with self.assertRaisesRegex( SystemExit, "is a variable handled as part of the dom_pft handling." - + " This should NOT be handled in the variiable list section." + + " This should NOT be handled in the variable list section." + " Monthly vars handled this way =", ): read_cfg_var_list(self.config, idealized=False) From f3f26fbf3d9c74e5d41b781b714c89d5c27e5dd0 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 5 Feb 2023 18:31:39 -0700 Subject: [PATCH 211/332] Update changelog --- doc/ChangeLog | 42 ++++++++++++++++++------------------------ doc/ChangeSum | 2 +- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index b75f939799..5870abdb23 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev118 Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310) -Date: Fri Feb 3 17:52:37 MST 2023 +Date: Sun Feb 5 18:31:29 MST 2023 One-line Summary: Use conda environment rather than ncar_pylib with the fsurdat_modifier system test Purpose and description of changes @@ -9,6 +9,19 @@ Purpose and description of changes Reason: ncar_pylib is going away soon. + The fsurdat_modifier system test that we're discussing in this tag and + corresponding pull request (PR #1798) + FSURDATMODIFYCTSM_D_Mmpi-serial_Ld1.5x5_amazon.I2000Clm50SpRs.cheyenne_intel + stopped working when I updated the PR to dev117. + The test worked in the PR when I was still in dev115. + The test worked in vanilla dev117. + I fixed the failure by removing a restriction added in dev116 and the + corresponding override --allow_ideal_and_include_non_veg. + + I am removing another restriction added in dev116 and the corresponding + override --allow_dom_pft_and_idealized. This one didn't cause a test to fail + but unnecessarily restricted usage of the fsurdat_modifier tool. + Significant changes to scientifically-supported configurations -------------------------------------------------------------- @@ -30,35 +43,16 @@ Bugs fixed or introduced ------------------------ CTSM issues fixed (include CTSM Issue #): - The fsurdat_modifier system test that we're discussing in this tag and - corresponding pull request (PR #1798) - FSURDATMODIFYCTSM_D_Mmpi-serial_Ld1.5x5_amazon.I2000Clm50SpRs.cheyenne_intel - stopped working when I updated the PR to dev117. - The test worked in the PR when I was still in dev115. - The test worked in vanilla dev117. - I fixed the failure by removing a restriction added in dev116 and the - corresponding override --allow_ideal_and_include_non_veg. - - I am removing another restriction added in dev116 and the corresponding - override --allow_dom_pft_and_idealized. This one didn't cause a test to fail - but unnecessarily restricted usage of the fsurdat_modifier tool. +Fixes #1786 -- ncar_pylib +Fixes #1925 -- replace ncar_pylib Testing summary: ---------------- -[Remove any lines that don't apply.] - [PASS means all tests PASS; OK means tests PASS other than expected fails.] - [If python code has changed and you are NOT running aux_clm (e.g., because the only changes are in python - code) then also run the clm_pymods test suite; this is a small subset of aux_clm that runs the system - tests impacted by python changes. The best way to do this, if you expect no changes from the last tag in - either model output or namelists, is: create sym links pointing to the last tag's baseline directory, - named with the upcoming tag; then run the clm_pymods test suite comparing against these baselines but NOT - doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to - separately run the clm_pymods test suite, and you can remove the following line.] - clm_pymods test suite on cheyenne - PASS + (softlinks created for baselines to previous tag since this is bit-for-bit) any other testing (give details below): @@ -75,7 +69,7 @@ Changes answers relative to baseline: NO Other details ------------- Pull Requests that document the changes (include PR ids): - https://github.com/ESCOMP/ctsm/pull/1798 + https://github.com/ESCOMP/ctsm/pull/1798 -- Have fsurdat_modifier system test use conda environment rather than ncar_pylib =============================================================== =============================================================== diff --git a/doc/ChangeSum b/doc/ChangeSum index 175cfdc588..215517275f 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev118 slevis 02/03/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test + ctsm5.1.dev118 slevis 02/05/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test ctsm5.1.dev117 afoster 02/02/2023 Updates to facilitate running FATES at NEON sites ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files ctsm5.1.dev115 rgknox 12/02/2022 API compatability with FATES V2 nutrient dynamics From 05fb17bff05aaf0544170ecbb41984bdbb875807 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 3 Feb 2023 14:01:49 -0700 Subject: [PATCH 212/332] Update the core team lists --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c56c0d4852..298e5adc04 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,8 @@ CTSM code management is provided primarily by: Software engineering team: - [Erik Kluzek](https://github.com/ekluzek) - [Bill Sacks](https://github.com/billsacks) -- [Mariana Vertenstein](https://github.com/mvertens) -- [Negin Sobhani](https://github.com/negin513) - [Sam Levis](https://github.com/slevisconsulting) +- [Adrianna Foster](https://github.com/adrifoster) Science team: - [Dave Lawrence](https://github.com/dlawrenncar) @@ -53,5 +52,7 @@ Science team: - [Danica Lombardozzi](https://github.com/danicalombardozzi) - [Keith Oleson](https://github.com/olyson) - [Sean Swenson](https://github.com/swensosc) -- [Mike Barlage](https://github.com/barlage) +- [Jackie Shuman](https://github.com/jkshuman) +- [Peter Lawrence](https://github.com/lawrencepj1) - [Rosie Fisher](https://github.com/rosiealice) +- Gordon Bonan From 463c8b9a7eb483cd599b93a2f9a1f68304b68517 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Sat, 11 Feb 2023 18:01:22 -0700 Subject: [PATCH 213/332] Updating new tests with longer walltimes again ERP_D_Ld10.f09_f09_mg17.IHistClm50BgcCrop.cheyenne_gnu.clm-f09_dec1990Start ran out of time at 40 minutes, so I bumped it up to 59 and it passed --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 4dfddb4a11..e602ac3597 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -331,7 +331,7 @@ - + From 989c87f6bdef2f4fa2b95df6f0c84a6ef649ccf3 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 21 Feb 2023 14:59:56 -0700 Subject: [PATCH 214/332] include PRISM precipitation data stream --- cime_config/config_component.xml | 7 +++++++ .../NEON/defaults/user_nl_datm_streams | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 555a0ce9b5..b5741e5b40 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -295,6 +295,13 @@ ABBY,BLAN,CPER,DEJU,GRSM,HEAL,KONA,LENO,NIWO,ONAQ,PUUM,SERC,SRER,TALL,TREE,WOOD, BARR,BONA,DCFS,DELA,GUAN,JERC,KONZ,MLBS,NOGP,ORNL,RMNP,SJER,STEI,TEAK,UKFS,WREF, BART,CLBJ,DSNY,HARV,JORN,LAJA,MOAB,OAES,OSBS,SCBI,SOAP,STER,TOOL,UNDE,YELL, + ABBY.PRECIP,BLAN.PRECIP,CPER.PRECIP,DEJU.PRECIP,GRSM.PRECIP,HEAL.PRECIP,KONA.PRECIP, + LENO.PRECIP,NIWO.PRECIP,ONAQ.PRECIP,PUUM.PRECIP,SERC.PRECIP,SRER.PRECIP,TALL.PRECIP, + TREE.PRECIP,WOOD.PRECIP,BARR.PRECIP,BONA.PRECIP,DCFS.PRECIP,DELA.PRECIP,GUAN.PRECIP, + JERC.PRECIP,KONZ.PRECIP,MLBS.PRECIP,NOGP.PRECIP,ORNL.PRECIP,RMNP.PRECIP,SJER.PRECIP, + STEI.PRECIP,TEAK.PRECIP,UKFS.PRECIP,WREF.PRECIP,BART.PRECIP,CLBJ.PRECIP,DSNY.PRECIP, + HARV.PRECIP,JORN.PRECIP,LAJA.PRECIP,MOAB.PRECIP,OAES.PRECIP,OSBS.PRECIP,SCBI.PRECIP, + SOAP.PRECIP,STER.PRECIP,TOOL.PRECIP,UNDE.PRECIP,YELL.PRECIP, run_component_ctsm diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index 36f1e72b3a..636ceb2ad8 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -37,3 +37,23 @@ preso3.SSP3-7.0:year_first=2018 preso3.SSP3-7.0:year_last=2022 preso3.SSP3-7.0:year_align=2018 preso3.SSP3-7.0:dtlimit=30 + +NEON.${NEONSITE}:datavars = \ + FSDS Faxa_swdn, \ + ZBOT Sa_z, \ + TBOT Sa_tbot, \ + WIND Sa_wind, \ + RH Sa_rh, \ + PSRF Sa_pbot, \ + FLDS Faxa_lwdn + +NEON.${NEONSITE}.PRECIP:datavars = \ + PRECIP Faxa_precn + +NEON.${NEONSITE}.PRECIP:datafiles = \ + /glade/u/home/tking/neon/prism_${NEONSITE}_2018.nc, \ + /glade/u/home/tking/neon/prism_${NEONSITE}_2019.nc, \ + /glade/u/home/tking/neon/prism_${NEONSITE}_2020.nc, \ + /glade/u/home/tking/neon/prism_${NEONSITE}_2021.nc, \ + /glade/u/home/tking/neon/prism_${NEONSITE}_2022.nc + From 22862978cbb8b269b4af80c27a13f59f2deab52a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 21 Feb 2023 16:46:51 -0700 Subject: [PATCH 215/332] remove unnecessary shell commands for gap filling --- cime_config/usermods_dirs/NEON/NIWO/shell_commands | 1 - 1 file changed, 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index a3e73ca343..65926f2188 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -1,4 +1,3 @@ ./xmlchange NEONSITE=NIWO ./xmlchange PTS_LON=254.41676 ./xmlchange PTS_LAT=40.05236 -./xmlchange DATM_YR_END=2018 From 8ef3f2882c56ce9275dd36b1da7e97eea4ba709b Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 22 Feb 2023 13:50:02 -0700 Subject: [PATCH 216/332] Use ERP_D_Ld10_P36x2 on cheyenne only; add ERP_D_Ld10 test on izumi These changes pertain to GU_LULCC tests in this commit. --- cime_config/testdefs/testlist_clm.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 59251da53e..34e22e7272 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -462,10 +462,18 @@ + + + + + + + + + - From 093c07506bf758ab207405f3a06c7bd3e719e967 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Wed, 22 Feb 2023 14:12:56 -0700 Subject: [PATCH 217/332] Draft ChangeLog and ChangeSum --- doc/ChangeLog | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 83 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 5870abdb23..6df1764ff4 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,86 @@ =============================================================== +Tag name: ctsm5.1.dev119 +Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310), ekluzek Erik Kluzek), lawrencepj1 (Peter Lawrence) +Date: Wed Feb 22 13:57:36 MST 2023 +One-line Summary: Allow gross unrepresented land use transitions (PR #309) + +Purpose and description of changes +---------------------------------- + + Get gross unrepresented land use transitions working in CLM5.1. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics +onfigurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer +hanges.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): + I didn't find a corresponding issue. + +Notes of particular relevance for users +--------------------------------------- +Changes made to namelist defaults (e.g., changed parameter values): + New namelist variable: do_grossunrep + +Changes to the datasets (e.g., parameter, surface or initial files): + Surface datasets may now contain non-zero gross unrepresented land use + transitions. + +Substantial timing or memory changes: +[e.g., check PFS test in the test suite and look at timings, if you +expect possible significant timing changes] + + +Notes of particular relevance for developers: +--------------------------------------------- +Changes to tests or testing: + Erik introduced new tests that can be identified by the GU_LULCC in + their names. I ran these with the test-suites and generated baselines + for them. + +Testing summary: +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + +Answer changes +-------------- + +Changes answers relative to baseline: NO (unless see next) + + Code configurations: do_grossunrep = .true. and surface dataset + includes non-zero gross unrepresented land use + transitions. I (slevis) have not investigated the + nature of the changes. + +Other details +------------- +Pull Requests that document the changes (include PR ids): + https://github.com/ESCOMP/ctsm/pull/309 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev118 Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310) Date: Sun Feb 5 18:31:29 MST 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 215517275f..66e229d06c 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev119 slevis 02/22/2023 Allow gross unrepresented land use transition (PR #309) ctsm5.1.dev118 slevis 02/05/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test ctsm5.1.dev117 afoster 02/02/2023 Updates to facilitate running FATES at NEON sites ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files From 55c6be2362d53d060e68e80b46c53ef7981d4ec5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 27 Feb 2023 14:55:10 -0700 Subject: [PATCH 218/332] Correct test number --- bld/unit_testers/build-namelist_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 1cccae0e29..1c6d951f0e 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,7 +163,7 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1847; +my $ntests = 1848; if ( defined($opts{'compare'}) ) { $ntests += 1254; } From 3f6c67af23ea0b6891ff8002f43369f7597e9f10 Mon Sep 17 00:00:00 2001 From: Samuel Levis Date: Mon, 27 Feb 2023 15:25:14 -0700 Subject: [PATCH 219/332] Returned hrv_xsmrpool_to_atm_patch to CNVegCarbonFluxType This was an unintentional line removal that Erik realized. I also reordered the error checks that Erik recommended in dynSubgridControlMod. --- src/biogeochem/CNVegCarbonFluxType.F90 | 1 + src/dyn_subgrid/dynSubgridControlMod.F90 | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index 585859beaa..7864007faf 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -4721,6 +4721,7 @@ subroutine Summary_carbonflux(this, & this%hrv_deadcrootc_to_litter_patch(p) + & this%hrv_deadcrootc_storage_to_litter_patch(p) + & this%hrv_deadcrootc_xfer_to_litter_patch(p) + & + this%hrv_xsmrpool_to_atm_patch(p) + & this%hrv_gresp_storage_to_litter_patch(p) + & this%hrv_gresp_xfer_to_litter_patch(p) diff --git a/src/dyn_subgrid/dynSubgridControlMod.F90 b/src/dyn_subgrid/dynSubgridControlMod.F90 index 12fce7b3f0..72e7229d0b 100644 --- a/src/dyn_subgrid/dynSubgridControlMod.F90 +++ b/src/dyn_subgrid/dynSubgridControlMod.F90 @@ -320,14 +320,17 @@ subroutine check_namelist_consistency end if if (dyn_subgrid_control_inst%do_grossunrep) then - if (.not. use_cn) then - write(iulog,*) 'ERROR: do_grossunrep can only be true if use_cn is true' - call endrun(msg=errMsg(sourcefile, __LINE__)) - end if + ! First check if use_fates. In this case the .not. use_cn error will not + ! appear. The .not. use_cn error will appea + ! if .not. use_fates and .not. use_cn. if (use_fates) then write(iulog,*) 'ERROR: do_grossunrep currently does not work with use_fates' call endrun(msg=errMsg(sourcefile, __LINE__)) end if + if (.not. use_cn) then + write(iulog,*) 'ERROR: do_grossunrep can only be true if use_cn is true' + call endrun(msg=errMsg(sourcefile, __LINE__)) + end if end if end subroutine check_namelist_consistency From 8ee4aeaee05810764a537ec177e472478de168a1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 27 Feb 2023 15:54:20 -0700 Subject: [PATCH 220/332] Add additional tests do new do_grossunrepresentedlanduse option --- bld/unit_testers/build-namelist_test.pl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 1c6d951f0e..8398e8359a 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,7 +163,7 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1848; +my $ntests = 1850; if ( defined($opts{'compare'}) ) { $ntests += 1254; } @@ -1066,6 +1066,16 @@ sub cat_and_create_namelistinfile { GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm5_0", }, + "dogrossandfates" =>{ options=>"-envxml_dir . -bgc fates -use_case 20thC_transient", + namelst=>"do_grossunrep=.true.", + GLC_TWO_WAY_COUPLING=>"FALSE", + phys=>"clm5_0", + }, + "dogrossandnottrans" =>{ options=>"-envxml_dir . -bgc fates -use_case 2000_control", + namelst=>"do_grossunrep=.true.", + GLC_TWO_WAY_COUPLING=>"FALSE", + phys=>"clm5_0", + }, "nocropwfert" =>{ options=>"-envxml_dir . -bgc sp -no-crop", namelst=>"use_fertilizer=T", GLC_TWO_WAY_COUPLING=>"FALSE", From 179de38028116b3a60b60c6d51dc83f90b10d559 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 27 Feb 2023 15:54:59 -0700 Subject: [PATCH 221/332] Change order of if statement so that FATES error will trigger rather than the CN one for it --- bld/CLMBuildNamelist.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm index e30b9b83f0..02d9799487 100755 --- a/bld/CLMBuildNamelist.pm +++ b/bld/CLMBuildNamelist.pm @@ -2866,12 +2866,12 @@ sub setup_logic_do_grossunrep { if (string_is_undef_or_empty($nl->get_value('flanduse_timeseries'))) { $cannot_be_true = "$var can only be set to true when running a transient case (flanduse_timeseries non-blank)"; } - elsif (!&value_is_true($nl->get_value('use_cn'))) { - $cannot_be_true = "$var can only be set to true when running with CN (use_cn = true)"; - } elsif (&value_is_true($nl->get_value('use_fates'))) { $cannot_be_true = "$var currently doesn't work with FATES"; } + elsif (!&value_is_true($nl->get_value('use_cn'))) { + $cannot_be_true = "$var can only be set to true when running with CN (use_cn = true)"; + } if ($cannot_be_true) { $default_val = ".false."; From 91bbef6c1da6ae2053487b359c3a5535adafa817 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 27 Feb 2023 17:17:28 -0700 Subject: [PATCH 222/332] Fix the new tests added in --- bld/unit_testers/build-namelist_test.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 8398e8359a..79add93d88 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -1061,17 +1061,17 @@ sub cat_and_create_namelistinfile { GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm5_0", }, - "dogrossandsp" =>{ options=>"-envxml_dir . -bgc sp -use_case 20thC_transient", + "dogrossandsp" =>{ options=>"--envxml_dir . --bgc sp --use_case 20thC_transient", namelst=>"do_grossunrep=.true.", GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm5_0", }, - "dogrossandfates" =>{ options=>"-envxml_dir . -bgc fates -use_case 20thC_transient", + "dogrossandfates" =>{ options=>"--envxml_dir . --bgc fates --use_case 20thC_transient --no-megan", namelst=>"do_grossunrep=.true.", GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm5_0", }, - "dogrossandnottrans" =>{ options=>"-envxml_dir . -bgc fates -use_case 2000_control", + "dogrossandnottrans" =>{ options=>"--envxml_dir . --bgc bgc --use_case 2000_control", namelst=>"do_grossunrep=.true.", GLC_TWO_WAY_COUPLING=>"FALSE", phys=>"clm5_0", From da8224fc171b1e2d46093f6315470759db87aea3 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 3 Mar 2023 16:09:23 -0700 Subject: [PATCH 223/332] Squashed 'manage_externals/' changes from 82d3b247f..7b6d92ef6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7b6d92ef6 Merge pull request #198 from johnpaulalex/gitdir 927ce3a98 Merge pull request #197 from johnpaulalex/testpath a04f1148f Merge pull request #196 from johnpaulalex/readmod d9c14bf25 Change the rest of the methods to use -C. Still some usage of getcwd in test_unit_repository_git. 332b10640 Fix incorrect logged path of checkout_externals in test_sys_checkout: it was basically the parent of the current directory, which varies throughout the test. (it called abspath with '{0}/../../', which adds arbitrary and not-interpolated subdir '{0}' to the path, then removes it and removes one more level). 932a7499b Remove printlog from read_gitmodules_file since read_externals_description_file() already has a nearly-the-same printlog (but add it to the other caller). 5d13719ed Merge pull request #195 from johnpaulalex/check_repo 423395449 Update utest to mock _git_remote_verbose in a new way, since it is now called via the GitRepository class rather than on the specific GitRepository instance. d7a42ae96 Check that desired repo was actually checked out. 71596bbc1 Merge pull request #194 from johnpaulalex/manic2 4c96e824e Make the MANIC_TEST_BARE_REPO_ROOT env var special - give it a constant for easy tracking, and automatically tear it down after each test. 259bfc04d test_sys_checkout: use actual paths in on-the-fly configs rather than MANIC_TEST_BARE_REPO_ROOT env var. This will make it easier to test (in the near future) that checkout_externals actually checked out the desired repo dir. 557bbd6eb Merge pull request #193 from johnpaulalex/manic 5314eede1 Remove MANIC_TEST_TMP_REPO_ROOT environment variable in favor of module-level variable. 345fc1e14 Merge pull request #191 from johnpaulalex/test_doc12 2117b843c test_sys_checkout: verify that basic by-tag/branch/hash tests actually take us to the correct git tag/branch/hash. 94d6e5f2b Merge pull request #190 from johnpaulalex/test_doc11 3ff33a6a8 Inline local-path-creation methods 47dea7f64 Merge pull request #189 from johnpaulalex/test_doc10 9ea75cbf8 Grab-bag of renamings: Remove redundant _NAME from repo constants, and consistently add _REPO suffix (This causes the majority of diffs). c0c847ec8 Merge pull request #188 from johnpaulalex/test_doc9 2dd5ce0f7 test_sys_checkout.py: only check for correct 'required' or 'optional' state in the test that exercises required vs optional behavior. Removed a lot of boilerplate. eb3085984 Merge pull request #187 from johnpaulalex/test_doc8 1832e1f84 test_sys_checkout: Simplify many tests to only use a single external. 8689d61ec Merge pull request #186 from johnpaulalex/test_doc7 fbee4253e Grab bag of test_sys_checkout cleanups: Doc inside of each test more clearly/consistently. TestSysCheckoutSVN didn’t get the inlining-of-helper-methods treatment, now it has that. Move various standalone repo helper methods (like create_branch) into a RepoUtils class. README.md was missing newlines when rendered as markdown. Doc the return value of checkout.main Fix test_container_exclude_component - it was looking for the wrong key (which is never present); now it looks for the correct key. f0ed44a6e Merge pull request #185 from johnpaulalex/test_doc6 a3d59f5f2 Merge pull request #184 from johnpaulalex/test_doc5 5329c8ba7 test_sys_checkout: Inline config generation functions that are only called once. 464f2c7a7 test_sys_checkout: Inline another layer (per-config-file checks). Rename the 4 methods that are used multiple times, to reflect what they do rather than what they're called. 8872c0df6 Merge pull request #183 from johnpaulalex/doc_test4 c045335f6 Merge pull request #182 from johnpaulalex/doc_test3 c583b956e Merge pull request #181 from johnpaulalex/doc_test2 e01cfe278 test_sys_checkout: less confusing handling of return values from checkout_externals. Specifically, when doing a checkout, don't return tree_status from _before_ the checkout. Make a new wrapper to call checkout_externals a second time, to calculate the new status after a checkout (very frequent pattern). 23286818c test_sys_checkout: Remove another layer (which generates test component names) c3717b6bc Merge pull request #180 from johnpaulalex/doc_test 36d7a4434 test_sys_checkout.py: remove one layer of functions (that check for local status enums). No-op. 2c4584bf7 More documentation about tests: * contents of test repositories (n a new README.md) * various constants in test_sys_checkout.py that point to those contents, and terminology like container/simple/mixed. * in each test method, the scenarios being tested. * The coupling between test methods. 55e74bd0a Merge pull request #179 from johnpaulalex/circ 66be84290 Remove circular dependency by making _External stop doing tricky things with sourcetrees. git-subtree-dir: manage_externals git-subtree-split: 7b6d92ef689e2f65733e27f8635ab91fb341356b --- manic/checkout.py | 8 +- manic/externals_description.py | 20 +- manic/externals_status.py | 4 +- manic/repository_git.py | 315 +++-- manic/sourcetree.py | 249 ++-- test/repos/README.md | 33 + test/test_sys_checkout.py | 2182 ++++++++++++++---------------- test/test_sys_repository_git.py | 58 +- test/test_unit_repository_git.py | 133 +- 9 files changed, 1458 insertions(+), 1544 deletions(-) create mode 100644 test/repos/README.md diff --git a/manic/checkout.py b/manic/checkout.py index ac30f3a5d2..3f5537adce 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -378,9 +378,9 @@ def main(args): the --all option is passed. Returns a tuple (overall_status, tree_status). overall_status is 0 - on success, non-zero on failure. tree_status gives the full status - *before* executing the checkout command - i.e., the status that it - used to determine if it's safe to proceed with the checkout. + on success, non-zero on failure. tree_status is a dict mapping local path + to ExternalStatus -- if no checkout is happening. If checkout is happening, tree_status + is None. """ if args.do_logging: logging.basicConfig(filename=LOG_FILE_NAME, @@ -438,6 +438,8 @@ def main(args): for comp in args.components: source_tree.checkout(args.verbose, load_all, load_comp=comp) printlog('') + # New tree status is unknown, don't return anything. + tree_status = None logging.info('%s completed without exceptions.', program_name) # NOTE(bja, 2017-11) tree status is used by the systems tests diff --git a/manic/externals_description.py b/manic/externals_description.py index f5615b6730..546e7fdcb4 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -88,7 +88,7 @@ def read_externals_description_file(root_dir, file_name): externals_description = None if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: - externals_description = read_gitmodules_file(root_dir, file_name) + externals_description = _read_gitmodules_file(root_dir, file_name) else: try: config = config_parser() @@ -151,9 +151,8 @@ def git_submodule_status(repo_dir): """Run the git submodule status command to obtain submodule hashes. """ # This function is here instead of GitRepository to avoid a dependency loop - cwd = os.getcwd() - os.chdir(repo_dir) - cmd = ['git', 'submodule', 'status'] + cmd = 'git -C {repo_dir} submodule status'.format( + repo_dir=repo_dir).split() git_output = execute_subprocess(cmd, output_to_caller=True) submodules = {} submods = git_output.split('\n') @@ -168,7 +167,6 @@ def git_submodule_status(repo_dir): submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} - os.chdir(cwd) return submodules def parse_submodules_desc_section(section_items, file_path): @@ -191,7 +189,7 @@ def parse_submodules_desc_section(section_items, file_path): return path, url -def read_gitmodules_file(root_dir, file_name): +def _read_gitmodules_file(root_dir, file_name): # pylint: disable=deprecated-method # Disabling this check because the method is only used for python2 # pylint: disable=too-many-locals @@ -203,12 +201,11 @@ def read_gitmodules_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing submodules description file : {0}'.format(file_name)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): msg = ('ERROR: submodules description file, "{0}", does not ' - 'exist at path:\n {1}'.format(file_name, file_path)) + 'exist in dir:\n {1}'.format(file_name, root_dir)) fatal_error(msg) submodules_description = None @@ -640,8 +637,11 @@ def _repo_config_from_submodule(self, field, submod_desc): ' Parent repo, "{1}" does not have submodules') fatal_error(msg.format(field, self._parent_repo.name())) - submod_file = read_gitmodules_file(repo_path, submod_file) - submod_desc = create_externals_description(submod_file) + printlog( + 'Processing submodules description file : {0} ({1})'.format( + submod_file, repo_path)) + submod_model_data= _read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_model_data) # Can we find our external? repo_url = None diff --git a/manic/externals_status.py b/manic/externals_status.py index 4900e41254..6bc29e9732 100644 --- a/manic/externals_status.py +++ b/manic/externals_status.py @@ -30,12 +30,12 @@ class ExternalStatus(object): """ # sync_state and clean_state can be one of the following: - DEFAULT = '-' # aka not set yet. + DEFAULT = '-' # not set yet (sync_state). clean_state can be this if sync_state is EMPTY. UNKNOWN = '?' EMPTY = 'e' MODEL_MODIFIED = 's' # repo version != externals (sync_state only) DIRTY = 'M' # repo is dirty (clean_state only) - STATUS_OK = ' ' # repo is clean/matches externals. + STATUS_OK = ' ' # repo is clean (clean_state) or matches externals version (sync_state) STATUS_ERROR = '!' # source_type can be one of the following: diff --git a/manic/repository_git.py b/manic/repository_git.py index 3a6a0f1716..adc666cc57 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -25,7 +25,7 @@ class GitRepository(Repository): * be isolated in separate functions with no application logic * of the form: - - cmd = ['git', ...] + - cmd = 'git -C {dirname} ...'.format(dirname=dirname).split() - value = execute_subprocess(cmd, output_to_caller={T|F}, status_to_caller={T|F}) - return value @@ -39,7 +39,7 @@ class GitRepository(Repository): def __init__(self, component_name, repo): """ - Parse repo (a XML element). + repo: ExternalsDescription. """ Repository.__init__(self, component_name, repo) self._gitmodules = None @@ -99,15 +99,13 @@ def submodules_file(self, repo_path=None): # # ---------------------------------------------------------------- def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): - """Prepare to execute the clone by managing directory location + """Clones repo_dir_name into base_dir_path. """ - cwd = os.getcwd() - os.chdir(base_dir_path) - self._git_clone(self._url, repo_dir_name, verbosity) - os.chdir(cwd) + self._git_clone(self._url, os.path.join(base_dir_path, repo_dir_name), + verbosity=verbosity) - def _current_ref(self): - """Determine the *name* associated with HEAD. + def _current_ref(self, dirname): + """Determine the *name* associated with HEAD at dirname. If we're on a tag, then returns the tag name; otherwise, returns the current hash. Returns an empty string if no reference can be @@ -119,21 +117,21 @@ def _current_ref(self): ref_found = False # If we're exactly at a tag, use that as the current ref - tag_found, tag_name = self._git_current_tag() + tag_found, tag_name = self._git_current_tag(dirname) if tag_found: current_ref = tag_name ref_found = True if not ref_found: # Otherwise, use current hash as the current ref - hash_found, hash_name = self._git_current_hash() + hash_found, hash_name = self._git_current_hash(dirname) if hash_found: current_ref = hash_name ref_found = True if ref_found: # If we're on a branch, include branch name in current ref - branch_found, branch_name = self._git_current_branch() + branch_found, branch_name = self._git_current_branch(dirname) if branch_found: current_ref = "{} (branch {})".format(current_ref, branch_name) else: @@ -184,17 +182,15 @@ def compare_refs(current_ref, expected_ref): status = ExternalStatus.MODEL_MODIFIED return status - cwd = os.getcwd() - os.chdir(repo_dir_path) - # get the full hash of the current commit - _, current_ref = self._git_current_hash() + _, current_ref = self._git_current_hash(repo_dir_path) if self._branch: if self._url == LOCAL_PATH_INDICATOR: expected_ref = self._branch else: - remote_name = self._determine_remote_name() + remote_name = self._remote_name_for_url(self._url, + repo_dir_path) if not remote_name: # git doesn't know about this remote. by definition # this is a modified state. @@ -211,7 +207,7 @@ def compare_refs(current_ref, expected_ref): fatal_error(msg) # record the *names* of the current and expected branches - stat.current_version = self._current_ref() + stat.current_version = self._current_ref(repo_dir_path) stat.expected_version = copy.deepcopy(expected_ref) if current_ref == EMPTY_STR: @@ -219,7 +215,7 @@ def compare_refs(current_ref, expected_ref): else: # get the underlying hash of the expected ref revparse_status, expected_ref_hash = self._git_revparse_commit( - expected_ref) + expected_ref, repo_dir_path) if revparse_status: # We failed to get the hash associated with # expected_ref. Maybe we should assign this to some special @@ -230,18 +226,13 @@ def compare_refs(current_ref, expected_ref): # compare the underlying hashes stat.sync_state = compare_refs(current_ref, expected_ref_hash) - os.chdir(cwd) - - def _determine_remote_name(self): - """Return the remote name. - - Note that this is for the *future* repo url and branch, not - the current working copy! + @classmethod + def _remote_name_for_url(cls, remote_url, dirname): + """Return the remote name matching remote_url (or None) """ - git_output = self._git_remote_verbose() + git_output = cls._git_remote_verbose(dirname) git_output = git_output.splitlines() - remote_name = '' for line in git_output: data = line.strip() if not data: @@ -249,10 +240,9 @@ def _determine_remote_name(self): data = data.split() name = data[0].strip() url = data[1].strip() - if self._url == url: - remote_name = name - break - return remote_name + if remote_url == url: + return name + return None def _create_remote_name(self): """The url specified in the externals description file was not known @@ -308,19 +298,16 @@ def _checkout_ref(self, repo_dir, verbosity, submodules): the repo's submodules """ # import pdb; pdb.set_trace() - cwd = os.getcwd() - os.chdir(repo_dir) if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity, submodules) + self._checkout_local_ref(verbosity, submodules, repo_dir) else: - self._checkout_external_ref(verbosity, submodules) + self._checkout_external_ref(verbosity, submodules, repo_dir) if self._sparse: self._sparse_checkout(repo_dir, verbosity) - os.chdir(cwd) - def _checkout_local_ref(self, verbosity, submodules): + def _checkout_local_ref(self, verbosity, submodules, dirname): """Checkout the reference considering the local repo only. Do not fetch any additional remotes or specify the remote when checkout out the ref. @@ -334,13 +321,18 @@ def _checkout_local_ref(self, verbosity, submodules): else: ref = self._hash - self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity, submodules) + self._check_for_valid_ref(ref, remote_name=None, + dirname=dirname) + self._git_checkout_ref(ref, verbosity, submodules, dirname) - def _checkout_external_ref(self, verbosity, submodules): - """Checkout the reference from a remote repository + def _checkout_external_ref(self, verbosity, submodules, dirname): + """Checkout the reference from a remote repository into dirname. if is True, recursively initialize and update - the repo's submodules + the repo's submodules. + Note that this results in a 'detached HEAD' state if checking out + a branch, because we check out the remote branch rather than the + local. See https://github.com/ESMCI/manage_externals/issues/34 for + more discussion. """ if self._tag: ref = self._tag @@ -349,44 +341,45 @@ def _checkout_external_ref(self, verbosity, submodules): else: ref = self._hash - remote_name = self._determine_remote_name() + remote_name = self._remote_name_for_url(self._url, dirname) if not remote_name: remote_name = self._create_remote_name() - self._git_remote_add(remote_name, self._url) - self._git_fetch(remote_name) + self._git_remote_add(remote_name, self._url, dirname) + self._git_fetch(remote_name, dirname) # NOTE(bja, 2018-03) we need to send separate ref and remote # name to check_for_vaild_ref, but the combined name to # checkout_ref! - self._check_for_valid_ref(ref, remote_name) + self._check_for_valid_ref(ref, remote_name, dirname) if self._branch: + # Prepend remote name to branch. This means we avoid various + # special cases if the local branch is not tracking the remote or + # cannot be trivially fast-forwarded to match; but, it also + # means we end up in a 'detached HEAD' state. ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity, submodules) + self._git_checkout_ref(ref, verbosity, submodules, dirname) def _sparse_checkout(self, repo_dir, verbosity): """Use git read-tree to thin the working tree.""" - cwd = os.getcwd() - - cmd = ['cp', self._sparse, os.path.join(repo_dir, - '.git/info/sparse-checkout')] + cmd = ['cp', os.path.join(repo_dir, self._sparse), + os.path.join(repo_dir, + '.git/info/sparse-checkout')] if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - os.chdir(repo_dir) - self._git_sparse_checkout(verbosity) + self._git_sparse_checkout(verbosity, repo_dir) - os.chdir(cwd) - - def _check_for_valid_ref(self, ref, remote_name=None): + def _check_for_valid_ref(self, ref, remote_name, dirname): """Try some basic sanity checks on the user supplied reference so we can provide a more useful error message than calledprocess error... + remote_name can be NOne """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) + is_tag = self._ref_is_tag(ref, dirname) + is_branch = self._ref_is_branch(ref, remote_name, dirname) + is_hash = self._ref_is_hash(ref, dirname) is_valid = is_tag or is_branch or is_hash if not is_valid: @@ -397,7 +390,8 @@ def _check_for_valid_ref(self, ref, remote_name=None): fatal_error(msg) if is_tag: - is_unique_tag, msg = self._is_unique_tag(ref, remote_name) + is_unique_tag, msg = self._is_unique_tag(ref, remote_name, + dirname) if not is_unique_tag: msg = ('In repo "{0}": tag "{1}" {2}'.format( self._name, self._tag, msg)) @@ -405,7 +399,7 @@ def _check_for_valid_ref(self, ref, remote_name=None): return is_valid - def _is_unique_tag(self, ref, remote_name): + def _is_unique_tag(self, ref, remote_name, dirname): """Verify that a reference is a valid tag and is unique (not a branch) Tags may be tag names, or SHA id's. It is also possible that a @@ -416,9 +410,9 @@ def _is_unique_tag(self, ref, remote_name): error! """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) + is_tag = self._ref_is_tag(ref, dirname) + is_branch = self._ref_is_branch(ref, remote_name, dirname) + is_hash = self._ref_is_hash(ref, dirname) msg = '' is_unique_tag = False @@ -449,7 +443,7 @@ def _is_unique_tag(self, ref, remote_name): return is_unique_tag, msg - def _ref_is_tag(self, ref): + def _ref_is_tag(self, ref, dirname): """Verify that a reference is a valid tag according to git. Note: values returned by git_showref_* and git_revparse are @@ -457,28 +451,30 @@ def _ref_is_tag(self, ref): error! """ is_tag = False - value = self._git_showref_tag(ref) + value = self._git_showref_tag(ref, dirname) if value == 0: is_tag = True return is_tag - def _ref_is_branch(self, ref, remote_name=None): + def _ref_is_branch(self, ref, remote_name, dirname): """Verify if a ref is any kind of branch (local, tracked remote, untracked remote). + remote_name can be None. """ local_branch = False remote_branch = False if remote_name: - remote_branch = self._ref_is_remote_branch(ref, remote_name) - local_branch = self._ref_is_local_branch(ref) + remote_branch = self._ref_is_remote_branch(ref, remote_name, + dirname) + local_branch = self._ref_is_local_branch(ref, dirname) is_branch = False if local_branch or remote_branch: is_branch = True return is_branch - def _ref_is_local_branch(self, ref): + def _ref_is_local_branch(self, ref, dirname): """Verify that a reference is a valid branch according to git. show-ref branch returns local branches that have been @@ -491,12 +487,12 @@ def _ref_is_local_branch(self, ref): """ is_branch = False - value = self._git_showref_branch(ref) + value = self._git_showref_branch(ref, dirname) if value == 0: is_branch = True return is_branch - def _ref_is_remote_branch(self, ref, remote_name): + def _ref_is_remote_branch(self, ref, remote_name, dirname): """Verify that a reference is a valid branch according to git. show-ref branch returns local branches that have been @@ -509,12 +505,12 @@ def _ref_is_remote_branch(self, ref, remote_name): """ is_branch = False - value = self._git_lsremote_branch(ref, remote_name) + value = self._git_lsremote_branch(ref, remote_name, dirname) if value == 0: is_branch = True return is_branch - def _ref_is_commit(self, ref): + def _ref_is_commit(self, ref, dirname): """Verify that a reference is a valid commit according to git. This could be a tag, branch, sha1 id, HEAD and potentially others... @@ -524,12 +520,12 @@ def _ref_is_commit(self, ref): error! """ is_commit = False - value, _ = self._git_revparse_commit(ref) + value, _ = self._git_revparse_commit(ref, dirname) if value == 0: is_commit = True return is_commit - def _ref_is_hash(self, ref): + def _ref_is_hash(self, ref, dirname): """Verify that a reference is a valid hash according to git. Git doesn't seem to provide an exact way to determine if user @@ -544,7 +540,7 @@ def _ref_is_hash(self, ref): """ is_hash = False - status, git_output = self._git_revparse_commit(ref) + status, git_output = self._git_revparse_commit(ref, dirname) if status == 0: if git_output.strip().startswith(ref): is_hash = True @@ -554,9 +550,7 @@ def _status_summary(self, stat, repo_dir_path): """Determine the clean/dirty status of a git repository """ - cwd = os.getcwd() - os.chdir(repo_dir_path) - git_output = self._git_status_porcelain_v1z() + git_output = self._git_status_porcelain_v1z(repo_dir_path) is_dirty = self._status_v1z_is_dirty(git_output) if is_dirty: stat.clean_state = ExternalStatus.DIRTY @@ -565,8 +559,7 @@ def _status_summary(self, stat, repo_dir_path): # Now save the verbose status output incase the user wants to # see it. - stat.status_output = self._git_status_verbose() - os.chdir(cwd) + stat.status_output = self._git_status_verbose(repo_dir_path) @staticmethod def _status_v1z_is_dirty(git_output): @@ -601,7 +594,7 @@ def _status_v1z_is_dirty(git_output): # # ---------------------------------------------------------------- @staticmethod - def _git_current_hash(): + def _git_current_hash(dirname): """Return the full hash of the currently checked-out version. Returns a tuple, (hash_found, hash), where hash_found is a @@ -609,21 +602,51 @@ def _git_current_hash(): could mean we're not in a git repository at all). (If hash_found is False, then hash is ''.) """ - status, git_output = GitRepository._git_revparse_commit("HEAD") + status, git_output = GitRepository._git_revparse_commit("HEAD", + dirname) hash_found = not status if not hash_found: git_output = '' return hash_found, git_output @staticmethod - def _git_current_branch(): - """Determines the name of the current branch. + def _git_current_remote_branch(dirname): + """Determines the name of the current remote branch, if any. + + if dir is None, uses the cwd. Returns a tuple, (branch_found, branch_name), where branch_found - is a logical specifying whether a branch name was found for + is a bool specifying whether a branch name was found for + HEAD. (If branch_found is False, then branch_name is ''). + branch_name is in the format '$remote/$branch', e.g. 'origin/foo'. + """ + branch_found = False + branch_name = '' + + cmd = 'git -C {dirname} log -n 1 --pretty=%d HEAD'.format( + dirname=dirname).split() + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + branch_found = 'HEAD,' in git_output + if branch_found: + # git_output is of the form " (HEAD, origin/blah)" + branch_name = git_output.split(',')[1].strip()[:-1] + return branch_found, branch_name + + @staticmethod + def _git_current_branch(dirname): + """Determines the name of the current local branch. + + Returns a tuple, (branch_found, branch_name), where branch_found + is a bool specifying whether a branch name was found for HEAD. (If branch_found is False, then branch_name is ''.) + Note that currently we check out the remote branch rather than + the local, so this command does not return the just-checked-out + branch. See _git_current_remote_branch. """ - cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] + cmd = 'git -C {dirname} symbolic-ref --short -q HEAD'.format( + dirname=dirname).split() status, git_output = execute_subprocess(cmd, output_to_caller=True, status_to_caller=True) @@ -635,15 +658,17 @@ def _git_current_branch(): return branch_found, git_output @staticmethod - def _git_current_tag(): + def _git_current_tag(dirname): """Determines the name tag corresponding to HEAD (if any). + if dirname is None, uses the cwd. + Returns a tuple, (tag_found, tag_name), where tag_found is a - logical specifying whether we found a tag name corresponding to + bool specifying whether we found a tag name corresponding to HEAD. (If tag_found is False, then tag_name is ''.) """ - # git describe --exact-match --tags HEAD - cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] + cmd = 'git -C {dirname} describe --exact-match --tags HEAD'.format( + dirname=dirname).split() status, git_output = execute_subprocess(cmd, output_to_caller=True, status_to_caller=True) @@ -655,53 +680,54 @@ def _git_current_tag(): return tag_found, git_output @staticmethod - def _git_showref_tag(ref): + def _git_showref_tag(ref, dirname): """Run git show-ref check if the user supplied ref is a tag. could also use git rev-parse --quiet --verify tagname^{tag} """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/tags/{0}'.format(ref), ] + cmd = ('git -C {dirname} show-ref --quiet --verify refs/tags/{ref}' + .format(dirname=dirname, ref=ref).split()) status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_showref_branch(ref): + def _git_showref_branch(ref, dirname): """Run git show-ref check if the user supplied ref is a local or tracked remote branch. """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/heads/{0}'.format(ref), ] + cmd = ('git -C {dirname} show-ref --quiet --verify refs/heads/{ref}' + .format(dirname=dirname, ref=ref).split()) status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_lsremote_branch(ref, remote_name): + def _git_lsremote_branch(ref, remote_name, dirname): """Run git ls-remote to check if the user supplied ref is a remote branch that is not being tracked """ - cmd = ['git', 'ls-remote', '--exit-code', '--heads', - remote_name, ref, ] + cmd = ('git -C {dirname} ls-remote --exit-code --heads ' + '{remote_name} {ref}').format( + dirname=dirname, remote_name=remote_name, ref=ref).split() status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_revparse_commit(ref): + def _git_revparse_commit(ref, dirname): """Run git rev-parse to detect if a reference is a SHA, HEAD or other valid commit. """ - cmd = ['git', 'rev-parse', '--quiet', '--verify', - '{0}^{1}'.format(ref, '{commit}'), ] + cmd = ('git -C {dirname} rev-parse --quiet --verify {ref}^{commit}' + .format(dirname=dirname, ref=ref, commit='{commit}').split()) status, git_output = execute_subprocess(cmd, status_to_caller=True, output_to_caller=True) git_output = git_output.strip() return status, git_output @staticmethod - def _git_status_porcelain_v1z(): + def _git_status_porcelain_v1z(dirname): """Run git status to obtain repository information. This is run with '--untracked=no' to ignore untracked files. @@ -710,36 +736,38 @@ def _git_status_porcelain_v1z(): between git versions or *user configuration*. """ - cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] + cmd = ('git -C {dirname} status --untracked-files=no --porcelain -z' + .format(dirname=dirname)).split() git_output = execute_subprocess(cmd, output_to_caller=True) return git_output @staticmethod - def _git_status_verbose(): + def _git_status_verbose(dirname): """Run the git status command to obtain repository information. """ - cmd = ['git', 'status'] + cmd = 'git -C {dirname} status'.format(dirname=dirname).split() git_output = execute_subprocess(cmd, output_to_caller=True) return git_output @staticmethod - def _git_remote_verbose(): + def _git_remote_verbose(dirname): """Run the git remote command to obtain repository information. + + Returned string is of the form: + myfork git@github.com:johnpaulalex/manage_externals_jp.git (fetch) + myfork git@github.com:johnpaulalex/manage_externals_jp.git (push) """ - cmd = ['git', 'remote', '--verbose'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output + cmd = 'git -C {dirname} remote --verbose'.format( + dirname=dirname).split() + return execute_subprocess(cmd, output_to_caller=True) @staticmethod - def has_submodules(repo_dir_path=None): - """Return True iff the repository at (or the current - directory if is None) has a '.gitmodules' file + def has_submodules(repo_dir_path): + """Return True iff the repository at has a + '.gitmodules' file """ - if repo_dir_path is None: - fname = ExternalsDescription.GIT_SUBMODULES_FILENAME - else: - fname = os.path.join(repo_dir_path, - ExternalsDescription.GIT_SUBMODULES_FILENAME) + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) return os.path.exists(fname) @@ -750,68 +778,71 @@ def has_submodules(repo_dir_path=None): # ---------------------------------------------------------------- @staticmethod def _git_clone(url, repo_dir_name, verbosity): - """Run git clone for the side effect of creating a repository. + """Clones url into repo_dir_name. """ - cmd = ['git', 'clone', '--quiet'] - subcmd = None - - cmd.extend([url, repo_dir_name]) + cmd = 'git clone --quiet {url} {repo_dir_name}'.format( + url=url, repo_dir_name=repo_dir_name).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - if subcmd is not None: - os.chdir(repo_dir_name) - execute_subprocess(subcmd) @staticmethod - def _git_remote_add(name, url): + def _git_remote_add(name, url, dirname): """Run the git remote command for the side effect of adding a remote """ - cmd = ['git', 'remote', 'add', name, url] + cmd = 'git -C {dirname} remote add {name} {url}'.format( + dirname=dirname, name=name, url=url).split() execute_subprocess(cmd) @staticmethod - def _git_fetch(remote_name): + def _git_fetch(remote_name, dirname): """Run the git fetch command for the side effect of updating the repo """ - cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] + cmd = 'git -C {dirname} fetch --quiet --tags {remote_name}'.format( + dirname=dirname, remote_name=remote_name).split() execute_subprocess(cmd) @staticmethod - def _git_checkout_ref(ref, verbosity, submodules): + def _git_checkout_ref(ref, verbosity, submodules, dirname): """Run the git checkout command for the side effect of updating the repo Param: ref is a reference to a local or remote object in the form 'origin/my_feature', or 'tag1'. """ - cmd = ['git', 'checkout', '--quiet', ref] + cmd = 'git -C {dirname} checkout --quiet {ref}'.format( + dirname=dirname, ref=ref).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) if submodules: - GitRepository._git_update_submodules(verbosity) + GitRepository._git_update_submodules(verbosity, dirname) @staticmethod - def _git_sparse_checkout(verbosity): + def _git_sparse_checkout(verbosity, dirname): """Configure repo via read-tree.""" - cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + cmd = 'git -C {dirname} config core.sparsecheckout true'.format( + dirname=dirname).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - cmd = ['git', 'read-tree', '-mu', 'HEAD'] + cmd = 'git -C {dirname} read-tree -mu HEAD'.format( + dirname=dirname).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) @staticmethod - def _git_update_submodules(verbosity): + def _git_update_submodules(verbosity, dirname): """Run git submodule update for the side effect of updating this repo's submodules. """ # First, verify that we have a .gitmodules file - if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): - cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if os.path.exists( + os.path.join(dirname, + ExternalsDescription.GIT_SUBMODULES_FILENAME)): + cmd = ('git -C {dirname} submodule update --init --recursive' + .format(dirname=dirname)).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) diff --git a/manic/sourcetree.py b/manic/sourcetree.py index 218b5febb4..cf2a5b7569 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -1,6 +1,6 @@ """ - -FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! +Classes to represent an externals config file (SourceTree) and the components +within it (_External). """ import errno @@ -20,71 +20,53 @@ class _External(object): """ A single component hosted in an external repository (and any children). - """ + The component may or may not be checked-out upon construction. + """ # pylint: disable=R0902 - def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): - """Parse an external description file into a dictionary of externals. + def __init__(self, root_dir, name, local_path, required, subexternals_path, + repo, svn_ignore_ancestry, subexternal_sourcetree): + """Create a single external component (checked out or not). Input: + root_dir : string - the (checked-out) parent repo's root dir. + local_path : string - this external's (checked-out) subdir relative + to root_dir, e.g. "components/mom" + repo: Repository - the repo object for this external. Can be None (e.g. if this external just refers to another external file). - root_dir : string - the root directory path where - 'local_path' is relative to. - - name : string - name of the ext_description object. may or may not - correspond to something in the path. + name : string - name of this external (as named by the parent + reference). May or may not correspond to something in the path. ext_description : dict - source ExternalsDescription object svn_ignore_ancestry : bool - use --ignore-externals with svn switch + subexternals_path: string - path to sub-externals config file, if any. Relative to local_path, or special value 'none'. + subexternal_sourcetree: SourceTree - corresponding to subexternals_path, if subexternals_path exists (it might not, if it is not checked out yet). """ self._name = name - self._repo = None # Repository object. - - # Subcomponent externals file and data object, if any. - self._externals_path = EMPTY_STR # Can also be "none" - self._externals_sourcetree = None + self._required = required self._stat = None # Populated in status() - self._sparse = None - # Parse the sub-elements - # _local_path : local path relative to the containing source tree, e.g. - # "components/mom" - self._local_path = ext_description[ExternalsDescription.PATH] + self._local_path = local_path # _repo_dir_path : full repository directory, e.g. # "/components/mom" - repo_dir = os.path.join(root_dir, self._local_path) + repo_dir = os.path.join(root_dir, local_path) self._repo_dir_path = os.path.abspath(repo_dir) # _base_dir_path : base directory *containing* the repository, e.g. # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = repo_dir_path # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) - assert(os.path.join(self._base_dir_path, self._repo_dir_name) - == self._repo_dir_path) - - self._required = ext_description[ExternalsDescription.REQUIRED] + self._repo = repo # Does this component have subcomponents aka an externals config? - self._externals_path = ext_description[ExternalsDescription.EXTERNALS] - # Treat a .gitmodules file as a backup externals config - if not self._externals_path: - if GitRepository.has_submodules(self._repo_dir_path): - self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME - - repo = create_repository( - name, ext_description[ExternalsDescription.REPO], - svn_ignore_ancestry=svn_ignore_ancestry) - if repo: - self._repo = repo - - # Recurse into subcomponents, if any. - if self._externals_path and (self._externals_path.lower() != 'none'): - self._create_externals_sourcetree() + self._subexternals_path = subexternals_path + self._subexternal_sourcetree = subexternal_sourcetree + def get_name(self): """ @@ -98,6 +80,15 @@ def get_local_path(self): """ return self._local_path + def get_repo_dir_path(self): + return self._repo_dir_path + + def get_subexternals_path(self): + return self._subexternals_path + + def get_repo(self): + return self._repo + def status(self, force=False, print_progress=False): """ Returns status of this component and all subcomponents. @@ -148,12 +139,12 @@ def status(self, force=False, print_progress=False): self._repo.status(self._stat, self._repo_dir_path) # Status of subcomponents, if any. - if self._externals_path and self._externals_sourcetree: + if self._subexternals_path and self._subexternal_sourcetree: cwd = os.getcwd() # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force, print_progress=print_progress) + subcomponent_stats = self._subexternal_sourcetree.status(self._local_path, force=force, print_progress=print_progress) os.chdir(cwd) # Merge our status + subcomponent statuses into one return dict keyed @@ -174,10 +165,10 @@ def status(self, force=False, print_progress=False): def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. + correct URL, correct branch or tag), and possibly updateit. If the repo destination directory does not exist, checkout the correct branch or tag. - Does not check out sub-externals, see checkout_subexternals(). + Does not check out sub-externals, see SourceTree.checkout(). """ # Make sure we are in correct location if not os.path.exists(self._repo_dir_path): @@ -215,98 +206,117 @@ def checkout(self, verbosity): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_subexternals(self, verbosity, load_all): - """Recursively checkout the sub-externals for this component, if any. - - See load_all documentation in SourceTree.checkout(). - """ - if self.load_externals(): - if self._externals_sourcetree: - # NOTE(bja, 2018-02): the subtree externals objects - # were created during initial status check. Updating - # the external may have changed which sub-externals - # are needed. We need to delete those objects and - # re-read the potentially modified externals - # description file. - self._externals_sourcetree = None - self._create_externals_sourcetree() - self._externals_sourcetree.checkout(verbosity, load_all) - - def load_externals(self): - 'Return True iff an externals file exists (and therefore should be loaded)' - load_ex = False - if os.path.exists(self._repo_dir_path): - if self._externals_path: - if self._externals_path.lower() != 'none': - load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals_path)) - - return load_ex + def replace_subexternal_sourcetree(self, sourcetree): + self._subexternal_sourcetree = sourcetree def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' # Try recursive .gitmodules unless there is an externals entry - recursive = not self._externals_path + recursive = not self._subexternals_path return recursive - def _create_externals_sourcetree(self): - """ - Note this only creates an object, it doesn't write to the file system. + +class SourceTree(object): + """ + SourceTree represents a group of managed externals. + + Those externals may not be checked out locally yet, they might only + have Repository objects pointing to their respective repositories. + """ + + @classmethod + def from_externals_file(cls, parent_repo_dir_path, parent_repo, + externals_path): + """Creates a SourceTree representing the given externals file. + + Looks up a git submodules file as an optional backup if there is no + externals file specified. + + Returns None if there is no externals file (i.e. it's None or 'none'), + or if the externals file hasn't been checked out yet. + + parent_repo_dir_path: parent repo root dir + parent_repo: parent repo. + externals_path: path to externals file, relative to parent_repo_dir_path. """ - if not os.path.exists(self._repo_dir_path): + if not os.path.exists(parent_repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out # yet, can't process the externals file. Assume we are # checking status before code is checkoud out and this # will be handled correctly later. - return + return None - cwd = os.getcwd() - os.chdir(self._repo_dir_path) - if self._externals_path.lower() == 'none': - msg = ('Internal: Attempt to create source tree for ' - 'externals = none in {}'.format(self._repo_dir_path)) - fatal_error(msg) + if externals_path.lower() == 'none': + # With explicit 'none', do not look for git submodules file. + return None - if not os.path.exists(self._externals_path): - if GitRepository.has_submodules(): - self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + cwd = os.getcwd() + os.chdir(parent_repo_dir_path) + + if not externals_path: + if GitRepository.has_submodules(parent_repo_dir_path): + externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + return None - if not os.path.exists(self._externals_path): - # NOTE(bja, 2017-10) this check is redundent with the one + if not os.path.exists(externals_path): + # NOTE(bja, 2017-10) this check is redundant with the one # in read_externals_description_file! - msg = ('External externals description file "{0}" ' + msg = ('Externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals_path, self._repo_dir_path)) + externals_path, parent_repo_dir_path)) fatal_error(msg) - externals_root = self._repo_dir_path + externals_root = parent_repo_dir_path # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals_path) + externals_path) # ext_description is another dict-like object (see ExternalsDescription) ext_description = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, ext_description) + parent_repo=parent_repo) + externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) - -class SourceTree(object): - """ - SourceTree represents a group of managed externals - """ - + return externals_sourcetree + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ Build a SourceTree object from an ExternalDescription. + + root_dir: the (checked-out) parent repo root dir. """ self._root_dir = os.path.abspath(root_dir) self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in ext_description: - src = _External(self._root_dir, comp, ext_description[comp], - svn_ignore_ancestry) + for comp, desc in ext_description.items(): + local_path = desc[ExternalsDescription.PATH] + required = desc[ExternalsDescription.REQUIRED] + repo_info = desc[ExternalsDescription.REPO] + subexternals_path = desc[ExternalsDescription.EXTERNALS] + + repo = create_repository(comp, + repo_info, + svn_ignore_ancestry=svn_ignore_ancestry) + + sourcetree = None + # Treat a .gitmodules file as a backup externals config + if not subexternals_path: + parent_repo_dir_path = os.path.abspath(os.path.join(root_dir, + local_path)) + if GitRepository.has_submodules(parent_repo_dir_path): + subexternals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + + # Might return None (if the subexternal isn't checked out yet, or subexternal is None or 'none') + subexternal_sourcetree = SourceTree.from_externals_file( + os.path.join(self._root_dir, local_path), + repo, + subexternals_path) + src = _External(self._root_dir, comp, local_path, required, + subexternals_path, repo, svn_ignore_ancestry, + subexternal_sourcetree) + self._all_components[comp] = src - if ext_description[comp][ExternalsDescription.REQUIRED]: + if required: self._required_compnames.append(comp) def status(self, relative_path_base=LOCAL_PATH_INDICATOR, @@ -353,8 +363,11 @@ def _find_installed_optional_components(self): continue # Note that in practice we expect this status to be cached. path_to_stat = ext.status() - if any(stat.sync_state != ExternalStatus.EMPTY - for stat in path_to_stat.values()): + + # If any part of this component exists locally, consider it + # installed and therefore eligible for updating. + if any(s.sync_state != ExternalStatus.EMPTY + for s in path_to_stat.values()): installed_comps.append(comp_name) return installed_comps @@ -376,6 +389,9 @@ def checkout(self, verbosity, load_all, load_comp=None): if local_optional_compnames: printlog('Found locally installed optional components: ' + ', '.join(local_optional_compnames)) + bad_compnames = set(local_optional_compnames) - set(self._all_components.keys()) + if bad_compnames: + printlog('Internal error: found locally installed components that are not in the global list of all components: ' + ','.join(bad_compnames)) if verbosity >= VERBOSITY_VERBOSE: printlog('Checking out externals: ') @@ -387,16 +403,23 @@ def checkout(self, verbosity, load_all, load_comp=None): load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path()) # checkout. - for comp in load_comps: + for comp_name in load_comps: if verbosity < VERBOSITY_VERBOSE: - printlog('{0}, '.format(comp), end='') + printlog('{0}, '.format(comp_name), end='') else: # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) + c = self._all_components[comp_name] # Does not recurse. - self._all_components[comp].checkout(verbosity) - # Recursively check out subexternals, if any. - self._all_components[comp].checkout_subexternals(verbosity, - load_all) + c.checkout(verbosity) + # Recursively check out subexternals, if any. Returns None + # if there's no subexternals path. + component_subexternal_sourcetree = SourceTree.from_externals_file( + c.get_repo_dir_path(), + c.get_repo(), + c.get_subexternals_path()) + c.replace_subexternal_sourcetree(component_subexternal_sourcetree) + if component_subexternal_sourcetree: + component_subexternal_sourcetree.checkout(verbosity, load_all) printlog('') diff --git a/test/repos/README.md b/test/repos/README.md new file mode 100644 index 0000000000..8a3502c35f --- /dev/null +++ b/test/repos/README.md @@ -0,0 +1,33 @@ +Git repositories for testing git-related behavior. For usage and terminology notes, see test/test_sys_checkout.py. + +To list files and view file contents at HEAD: +``` +cd +git ls-tree --full-tree -r --name-only HEAD +git cat-file -p HEAD: +``` + +File contents at a glance: +``` +container.git/ + readme.txt + +simple-ext.git/ + (has branches: feature2, feature3) + (has tags: tag1, tag2) + readme.txt + simple_subdir/subdir_file.txt + +simple-ext-fork.git/ + (has tags: abandoned-feature, forked-feature-v1, tag1) + (has branch: feature2) + readme.txt + +mixed-cont-ext.git/ + (has branch: new-feature) + readme.txt + sub-externals.cfg ('simp_branch' section refers to 'feature2' branch in simple-ext.git/ repo) + +error/ + (no git repo here, just a readme.txt in the clear) +``` diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 27b649bea4..ab4f77e88f 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -2,6 +2,14 @@ """Unit test driver for checkout_externals +Terminology: + * 'container': a repo that has externals + * 'simple': a repo that has no externals, but is referenced as an external by another repo. + * 'mixed': a repo that both has externals and is referenced as an external by another repo. + + * 'clean': the local repo matches the version in the externals and has no local modifications. + * 'empty': the external isn't checked out at all. + Note: this script assume the path to the manic and checkout_externals module is already in the python path. This is usually handled by the makefile. If you call it directly, you may need @@ -21,7 +29,6 @@ * Erase any existing repos at the begining of the module in setUpModule. - """ # NOTE(bja, 2017-11) pylint complains that the module is too big, but @@ -68,39 +75,50 @@ # Module-wide root directory for all the per-test subdirs we'll create on -# the fly. -MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' # env var name +# the fly (which are placed under wherever $CWD is when the test runs). +# Set by setupModule(). +module_tmp_root_dir = None TMP_REPO_DIR_NAME = 'tmp' # subdir under $CWD -# We clone these checked-in repositories on a per-test basis. The -# 'bare repo root' is the test/repos/ subdir that holds them all. -MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' # env var name -BARE_REPO_ROOT_NAME = 'repos' # subdir name - -# Subdirs under bare repo root, each holding a repository. -CONTAINER_REPO_NAME = 'container.git' -MIXED_REPO_NAME = 'mixed-cont-ext.git' -SIMPLE_REPO_NAME = 'simple-ext.git' -SIMPLE_FORK_NAME = 'simple-ext-fork.git' - - -# In each of the test repos, all externals live under this subdir. -EXTERNALS_NAME = 'externals' +# subdir under test/ that holds all of our checked-in repositories (which we +# will clone for these tests). +BARE_REPO_ROOT_NAME = 'repos' + +# Environment var referenced by checked-in externals file in mixed-cont-ext.git, +# which should be pointed to the fully-resolved BARE_REPO_ROOT_NAME directory. +# We explicitly clear this after every test, via tearDown(). +MIXED_CONT_EXT_ROOT_ENV_VAR = 'MANIC_TEST_BARE_REPO_ROOT' + +# Subdirs under bare repo root, each holding a repository. For more info +# on the contents of these repositories, see test/repos/README.md. In these +# tests the 'parent' repos are cloned as a starting point, whereas the 'child' +# repos are checked out when the tests run checkout_externals. +CONTAINER_REPO = 'container.git' # Parent repo +SIMPLE_REPO = 'simple-ext.git' # Child repo +SIMPLE_FORK_REPO = 'simple-ext-fork.git' # Child repo +MIXED_REPO = 'mixed-cont-ext.git' # Both parent and child + +# Standard (arbitrary) external names for test configs +TAG_SECTION = 'simp_tag' +BRANCH_SECTION = 'simp_branch' +HASH_SECTION = 'simp_hash' + +# All the configs we construct check out their externals into these local paths. +EXTERNALS_PATH = 'externals' SUB_EXTERNALS_PATH = 'src' # For mixed test repos, # For testing behavior with '.' instead of an explicit paths. SIMPLE_LOCAL_ONLY_NAME = '.' -# Filenames -CFG_NAME = 'externals.cfg' -CFG_SUB_NAME = 'sub-externals.cfg' -README_NAME = 'readme.txt' # Arbitrary file to check in. +# Externals files. +CFG_NAME = 'externals.cfg' # We construct this on a per-test basis. +CFG_SUB_NAME = 'sub-externals.cfg' # Already exists in mixed-cont-ext repo. -# Arbirary test branch name -REMOTE_BRANCH_FEATURE2 = 'feature2' +# Arbitrary text file in all the test repos. +README_NAME = 'readme.txt' -# Section names in container_nested_* test config. -NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney'] +# Branch that exists in both the simple and simple-fork repos. +REMOTE_BRANCH_FEATURE2 = 'feature2' SVN_TEST_REPO = 'https://github.com/escomp/cesm' @@ -123,152 +141,110 @@ def setUpModule(): # pylint: disable=C0103 pass # create clean dir for this run os.mkdir(repo_root) - # set into the environment so var will be expanded in externals - # files when executables are run - os.environ[MANIC_TEST_TMP_REPO_ROOT] = repo_root - - -class GenerateExternalsDescriptionCfgV1(object): - """Building blocks to create ExternalsDescriptionCfgV1 files. - - Basic usage: create_config() multiple create_*(), then write_config(). - Optionally after that: write_with_*(). - - Includes methods (like container_*()) to create predefined configs for the - checked-in repositories in test/repos/. - """ - - def __init__(self): - self._schema_version = '1.1.0' - self._config = None - - def container_full(self, dest_dir): - """Create the full container config file with simple and mixed use - externals - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) - - self.create_section(SIMPLE_REPO_NAME, 'simp_opt', - tag='tag1', required=False) - self.create_section(MIXED_REPO_NAME, 'mixed_req', - branch='master', externals=CFG_SUB_NAME) + # Make available to all tests in this file. + global module_tmp_root_dir + assert module_tmp_root_dir == None, module_tmp_root_dir + module_tmp_root_dir = repo_root - self.write_config(dest_dir) - - def container_simple_required(self, dest_dir): - """Create a container externals file with only simple externals. - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) - - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', - ref_hash='60b1cc1a38d63') - - self.write_config(dest_dir) - - def container_nested_required(self, dest_dir, order): - """Create a container externals file with only simple externals. - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', nested=True, - tag='tag1', path=NESTED_NAME[order[0]]) - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', nested=True, - branch=REMOTE_BRANCH_FEATURE2, path=NESTED_NAME[order[1]]) - - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', nested=True, - ref_hash='60b1cc1a38d63', path=NESTED_NAME[order[2]]) - - self.write_config(dest_dir) - - - def container_simple_optional(self, dest_dir): - """Create a container externals file with optional simple externals +class RepoUtils(object): + """Convenience methods for interacting with git repos.""" + @staticmethod + def create_branch(repo_base_dir, external_name, branch, with_commit=False): + """Create branch and optionally (with_commit) add a single commit. """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_req', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_opt', - tag='tag1', required=False) - - self.write_config(dest_dir) - - def container_simple_svn(self, dest_dir): - """Create a container externals file with only simple externals. + # pylint: disable=R0913 + cwd = os.getcwd() + repo_root = os.path.join(repo_base_dir, EXTERNALS_PATH, external_name) + os.chdir(repo_root) + cmd = ['git', 'checkout', '-b', branch, ] + execute_subprocess(cmd) + if with_commit: + msg = 'start work on {0}'.format(branch) + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) + @staticmethod + def create_commit(repo_base_dir, external_name): + """Make a commit to the given external. + + This is used to test sync state changes from local commits on + detached heads and tracking branches. """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', tag='tag1') - - self.create_svn_external('svn_branch', branch='trunk') - self.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') + cwd = os.getcwd() + repo_root = os.path.join(repo_base_dir, EXTERNALS_PATH, external_name) + os.chdir(repo_root) - self.write_config(dest_dir) + msg = 'work on great new feature!' + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) - def container_sparse(self, dest_dir): - """Create a container with a full external and a sparse external + @staticmethod + def clone_test_repo(bare_root, test_id, parent_repo_name, dest_dir_in): + """Clone repo at / into dest_dir_in or local per-test-subdir. + Returns output dir. """ - # Create a file for a sparse pattern match - sparse_filename = 'sparse_checkout' - with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile: - sfile.write('readme.txt') - - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag2') - - sparse_relpath = '../../{}'.format(sparse_filename) - self.create_section(SIMPLE_REPO_NAME, 'simp_sparse', - tag='tag2', sparse=sparse_relpath) + parent_repo_dir = os.path.join(bare_root, parent_repo_name) + if dest_dir_in is None: + # create unique subdir for this test + test_dir_name = test_id + print("Test repository name: {0}".format(test_dir_name)) + dest_dir = os.path.join(module_tmp_root_dir, test_dir_name) + else: + dest_dir = dest_dir_in - self.write_config(dest_dir) + # pylint: disable=W0212 + GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) + return dest_dir - def mixed_simple_base(self, dest_dir): - """Create a mixed-use base externals file with only simple externals. + @staticmethod + def add_file_to_repo(under_test_dir, filename, tracked): + """Add a file to the repository so we can put it into a dirty state """ - self.create_config() - self.create_section_ext_only('mixed_base') - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) + cwd = os.getcwd() + os.chdir(under_test_dir) + with open(filename, 'w') as tmp: + tmp.write('Hello, world!') - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', - ref_hash='60b1cc1a38d63') + if tracked: + # NOTE(bja, 2018-01) brittle hack to obtain repo dir and + # file name + path_data = filename.split('/') + repo_dir = os.path.join(path_data[0], path_data[1]) + os.chdir(repo_dir) + tracked_file = path_data[2] + cmd = ['git', 'add', tracked_file] + execute_subprocess(cmd) - self.write_config(dest_dir) + os.chdir(cwd) - def mixed_simple_sub(self, dest_dir): - """Create a mixed-use sub externals file with only simple externals. +class GenerateExternalsDescriptionCfgV1(object): + """Building blocks to create ExternalsDescriptionCfgV1 files. - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1', path=SUB_EXTERNALS_PATH) + Basic usage: create_config() multiple create_*(), then write_config(). + Optionally after that: write_with_*(). + """ - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - path=SUB_EXTERNALS_PATH) + def __init__(self, bare_root): + self._schema_version = '1.1.0' + self._config = None - self.write_config(dest_dir, filename=CFG_SUB_NAME) + # directory where we have test repositories (which we will clone for + # tests) + self._bare_root = bare_root def write_config(self, dest_dir, filename=CFG_NAME): """Write self._config to disk @@ -293,9 +269,15 @@ def create_metadata(self): self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, self._schema_version) + def url_for_repo_path(self, repo_path, repo_path_abs=None): + if repo_path_abs is not None: + return repo_path_abs + else: + return os.path.join(self._bare_root, repo_path) + def create_section(self, repo_path, name, tag='', branch='', - ref_hash='', required=True, path=EXTERNALS_NAME, - externals='', repo_path_abs=None, from_submodule=False, + ref_hash='', required=True, path=EXTERNALS_PATH, + sub_externals='', repo_path_abs=None, from_submodule=False, sparse='', nested=False): # pylint: disable=too-many-branches """Create a config ExternalsDescription section with the given name. @@ -303,7 +285,7 @@ def create_section(self, repo_path, name, tag='', branch='', Autofills some items and handles some optional items. repo_path_abs overrides repo_path (which is relative to the bare repo) - path is a subdir under repo_path. + path is a subdir under repo_path to check out to. """ # pylint: disable=R0913 self._config.add_section(name) @@ -328,10 +310,7 @@ def create_section(self, repo_path, name, tag='', branch='', ref_hash = '' branch = '' - if repo_path_abs is not None: - repo_url = repo_path_abs - else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) + repo_url = self.url_for_repo_path(repo_path, repo_path_abs) if not from_submodule: self._config.set(name, ExternalsDescription.REPO_URL, repo_url) @@ -347,8 +326,9 @@ def create_section(self, repo_path, name, tag='', branch='', if ref_hash: self._config.set(name, ExternalsDescription.HASH, ref_hash) - if externals: - self._config.set(name, ExternalsDescription.EXTERNALS, externals) + if sub_externals: + self._config.set(name, ExternalsDescription.EXTERNALS, + sub_externals) if sparse: self._config.set(name, ExternalsDescription.SPARSE, sparse) @@ -356,10 +336,8 @@ def create_section(self, repo_path, name, tag='', branch='', if from_submodule: self._config.set(name, ExternalsDescription.SUBMODULE, "True") - def create_section_ext_only(self, name, - required=True, externals=CFG_SUB_NAME): - """Create a config section with autofilling some items and handling - optional items. + def create_section_reference_to_subexternal(self, name): + """Just a reference to another externals file. """ # pylint: disable=R0913 @@ -372,10 +350,9 @@ def create_section_ext_only(self, name, self._config.set(name, ExternalsDescription.REPO_URL, LOCAL_PATH_INDICATOR) - self._config.set(name, ExternalsDescription.REQUIRED, str(required)) + self._config.set(name, ExternalsDescription.REQUIRED, str(True)) - if externals: - self._config.set(name, ExternalsDescription.EXTERNALS, externals) + self._config.set(name, ExternalsDescription.EXTERNALS, CFG_SUB_NAME) def create_svn_external(self, name, tag='', branch=''): """Create a config section for an svn repository. @@ -383,7 +360,7 @@ def create_svn_external(self, name, tag='', branch=''): """ self._config.add_section(name) self._config.set(name, ExternalsDescription.PATH, - os.path.join(EXTERNALS_NAME, name)) + os.path.join(EXTERNALS_PATH, name)) self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_SVN) @@ -398,54 +375,7 @@ def create_svn_external(self, name, tag='', branch=''): if branch: self._config.set(name, ExternalsDescription.BRANCH, branch) - @staticmethod - def create_branch(dest_dir, repo_name, branch, with_commit=False): - """Create branch and optionally (with_commit) add a single commit. - """ - # pylint: disable=R0913 - cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) - os.chdir(repo_root) - cmd = ['git', 'checkout', '-b', branch, ] - execute_subprocess(cmd) - if with_commit: - msg = 'start work on {0}'.format(branch) - with open(README_NAME, 'a') as handle: - handle.write(msg) - cmd = ['git', 'add', README_NAME, ] - execute_subprocess(cmd) - cmd = ['git', 'commit', '-m', msg, ] - execute_subprocess(cmd) - os.chdir(cwd) - - @staticmethod - def create_commit(dest_dir, repo_name, local_tracking_branch=None): - """Make a commit on whatever is currently checked out. - - This is used to test sync state changes from local commits on - detached heads and tracking branches. - - NOTE(jpalex, 2023-01) This has nothing to do with self._config, - should probably live elsewhere. - """ - cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) - os.chdir(repo_root) - if local_tracking_branch: - cmd = ['git', 'checkout', '-b', local_tracking_branch, ] - execute_subprocess(cmd) - - msg = 'work on great new feature!' - with open(README_NAME, 'a') as handle: - handle.write(msg) - cmd = ['git', 'add', README_NAME, ] - execute_subprocess(cmd) - cmd = ['git', 'commit', '-m', msg, ] - execute_subprocess(cmd) - os.chdir(cwd) - - def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, - filename=CFG_NAME): + def write_with_git_branch(self, dest_dir, name, branch, new_remote_repo_path=None): """Update fields in our config and write it to disk. name is the key of the ExternalsDescription in self._config to update. @@ -453,12 +383,11 @@ def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.BRANCH, branch) - if repo_path: - if repo_path == SIMPLE_LOCAL_ONLY_NAME: + if new_remote_repo_path: + if new_remote_repo_path == SIMPLE_LOCAL_ONLY_NAME: repo_url = SIMPLE_LOCAL_ONLY_NAME else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', - repo_path) + repo_url = os.path.join(self._bare_root, new_remote_repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -467,9 +396,9 @@ def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_with_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): + def write_with_svn_branch(self, dest_dir, name, branch): """Update a repository branch, and potentially the remote. """ # pylint: disable=R0913 @@ -481,11 +410,11 @@ def write_with_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_with_tag(self, dest_dir, name, tag, repo_path=None, - filename=CFG_NAME, remove_branch=True): - """Update a repository tag, and potentially the remote + def write_with_tag_and_remote_repo(self, dest_dir, name, tag, new_remote_repo_path, + remove_branch=True): + """Update a repository tag and the remote. NOTE(bja, 2017-11) remove_branch=False should result in an overspecified external with both a branch and tag. This is @@ -495,8 +424,8 @@ def write_with_tag(self, dest_dir, name, tag, repo_path=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.TAG, tag) - if repo_path: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) + if new_remote_repo_path: + repo_url = os.path.join(self._bare_root, new_remote_repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -506,10 +435,9 @@ def write_with_tag(self, dest_dir, name, tag, repo_path=None, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_without_branch_tag(self, dest_dir, name, - filename=CFG_NAME): + def write_without_branch_tag(self, dest_dir, name): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -525,10 +453,9 @@ def write_without_branch_tag(self, dest_dir, name, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_without_repo_url(self, dest_dir, name, - filename=CFG_NAME): + def write_without_repo_url(self, dest_dir, name): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -538,22 +465,59 @@ def write_without_repo_url(self, dest_dir, name, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_with_protocol(self, dest_dir, name, protocol, repo_path=None, - filename=CFG_NAME): + def write_with_protocol(self, dest_dir, name, protocol, repo_path=None): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.PROTOCOL, protocol) if repo_path: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) + repo_url = os.path.join(self._bare_root, repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) - self.write_config(dest_dir, filename) + self.write_config(dest_dir) + + +def _execute_checkout_in_dir(dirname, args, debug_env=''): + """Execute the checkout command in the appropriate repo dir with the + specified additional args. + args should be a list of strings. + debug_env shuld be a string of the form 'FOO=bar' or the empty string. + Note that we are calling the command line processing and main + routines and not using a subprocess call so that we get code + coverage results! Note this means that environment variables are passed + to checkout_externals via os.environ; debug_env is just used to aid + manual reproducibility of a given call. + + Returns (overall_status, tree_status) + where overall_status is 0 for success, nonzero otherwise. + and tree_status is set if --status was passed in, None otherwise. + + Note this command executes the checkout command, it doesn't + necessarily do any checking out (e.g. if --status is passed in). + """ + cwd = os.getcwd() + + # Construct a command line for reproducibility; this command is not + # actually executed in the test. + os.chdir(dirname) + cmdline = ['--externals', CFG_NAME, ] + cmdline += args + manual_cmd = ('Running equivalent of:\n' + 'pushd {dirname}; ' + '{debug_env} /path/to/checkout_externals {args}'.format( + dirname=dirname, debug_env=debug_env, + args=' '.join(cmdline))) + printlog(manual_cmd) + options = checkout.commandline_arguments(cmdline) + overall_status, tree_status = checkout.main(options) + os.chdir(cwd) + return overall_status, tree_status + class BaseTestSysCheckout(unittest.TestCase): """Base class of reusable systems level test setup for checkout_externals @@ -596,425 +560,57 @@ def setUp(self): # path to the executable self._checkout = os.path.join(root_dir, 'checkout_externals') - # directory where we have test repositories + # directory where we have test repositories (which we will clone for + # tests) self._bare_root = os.path.abspath( os.path.join(root_dir, 'test', BARE_REPO_ROOT_NAME)) - # set into the environment so var will be expanded in externals files - os.environ[MANIC_TEST_BARE_REPO_ROOT] = self._bare_root - # set the input file generator - self._generator = GenerateExternalsDescriptionCfgV1() + self._generator = GenerateExternalsDescriptionCfgV1(self._bare_root) # set the input file generator for secondary externals - self._sub_generator = GenerateExternalsDescriptionCfgV1() + self._sub_generator = GenerateExternalsDescriptionCfgV1(self._bare_root) def tearDown(self): """Tear down for individual tests """ - # remove the env var we added in setup - del os.environ[MANIC_TEST_BARE_REPO_ROOT] - # return to our common starting point os.chdir(self._return_dir) + + # (in case this was set) Don't pollute environment of other tests. + os.environ.pop(MIXED_CONT_EXT_ROOT_ENV_VAR, + None) # Don't care if key wasn't set. def clone_test_repo(self, parent_repo_name, dest_dir_in=None): - """Clone repo under bare_root into dest_dir_in or local per-test-subdir. - - Returns output dir. - """ - parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) - if dest_dir_in is None: - # unique repo for this test - test_dir_name = self._test_id - print("Test repository name: {0}".format(test_dir_name)) - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - test_dir_name) - else: - dest_dir = dest_dir_in - - # pylint: disable=W0212 - GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) - return dest_dir - - @staticmethod - def _add_file_to_repo(under_test_dir, filename, tracked): - """Add a file to the repository so we can put it into a dirty state - - """ - cwd = os.getcwd() - os.chdir(under_test_dir) - with open(filename, 'w') as tmp: - tmp.write('Hello, world!') - - if tracked: - # NOTE(bja, 2018-01) brittle hack to obtain repo dir and - # file name - path_data = filename.split('/') - repo_dir = os.path.join(path_data[0], path_data[1]) - os.chdir(repo_dir) - tracked_file = path_data[2] - cmd = ['git', 'add', tracked_file] - execute_subprocess(cmd) - - os.chdir(cwd) + """Clones repo under self._bare_root""" + return RepoUtils.clone_test_repo(self._bare_root, self._test_id, + parent_repo_name, dest_dir_in) + + def execute_checkout_in_dir(self, dirname, args, debug_env=''): + overall_status, tree_status = _execute_checkout_in_dir(dirname, args, + debug_env=debug_env) + self.assertEqual(overall_status, 0) + return tree_status + + def execute_checkout_with_status(self, dirname, args, debug_env=''): + """Calls checkout a second time to get status if needed.""" + tree_status = self.execute_checkout_in_dir( + dirname, args, debug_env=debug_env) + if tree_status is None: + tree_status = self.execute_checkout_in_dir(dirname, + self.status_args, + debug_env=debug_env) + self.assertNotEqual(tree_status, None) + return tree_status + + def _check_sync_clean(self, ext_status, expected_sync_state, + expected_clean_state): + self.assertEqual(ext_status.sync_state, expected_sync_state) + self.assertEqual(ext_status.clean_state, expected_clean_state) @staticmethod - def execute_checkout_in_dir(dirname, args): - """Extecute the checkout command in the appropriate repo dir with the - specified additional args - - Note that we are calling the command line processing and main - routines and not using a subprocess call so that we get code - coverage results! - - """ - cwd = os.getcwd() - checkout_path = os.path.abspath('{0}/../../checkout_externals') - os.chdir(dirname) - cmdline = ['--externals', CFG_NAME, ] - cmdline += args - repo_root = 'MANIC_TEST_BARE_REPO_ROOT={root}'.format( - root=os.environ[MANIC_TEST_BARE_REPO_ROOT]) - manual_cmd = ('Test cmd:\npushd {cwd}; {env} {checkout} {args}'.format( - cwd=dirname, env=repo_root, checkout=checkout_path, - args=' '.join(cmdline))) - printlog(manual_cmd) - options = checkout.commandline_arguments(cmdline) - overall_status, tree_status = checkout.main(options) - os.chdir(cwd) - return overall_status, tree_status - - # ---------------------------------------------------------------- - # - # Check results for generic perturbation of states - # - # 'tree' is a dict with ExternalStatus as values. - # 'name' is a key into that dict. - # ---------------------------------------------------------------- - def _check_generic_empty_default_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) - self.assertEqual(tree[name].clean_state, ExternalStatus.DEFAULT) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_ok_clean_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_ok_dirty_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.DIRTY) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_modified_ok_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.MODEL_MODIFIED) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_empty_default_optional(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) - self.assertEqual(tree[name].clean_state, ExternalStatus.DEFAULT) - self.assertEqual(tree[name].source_type, ExternalStatus.OPTIONAL) - - def _check_generic_ok_clean_optional(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.OPTIONAL) - - # ---------------------------------------------------------------- - # - # Check results for individual named externals - # - # 'tree' is a dict of string to ExternalStatus. - # ---------------------------------------------------------------- - def _check_simple_tag_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_nested_tag_empty(self, tree, name=EXTERNALS_NAME): - self._check_generic_empty_default_required(tree, name) - - def _check_simple_tag_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_nested_tag_ok(self, tree, name=EXTERNALS_NAME): - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_tag_dirty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_ok_dirty_required(tree, name) - - def _check_simple_tag_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_branch_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_nested_branch_empty(self, tree, name=EXTERNALS_NAME): - self._check_generic_empty_default_required(tree, name) - - def _check_simple_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_nested_branch_ok(self, tree, name=EXTERNALS_NAME): - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_branch_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_hash_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_nested_hash_empty(self, tree, name=EXTERNALS_NAME): - self._check_generic_empty_default_required(tree, name) - - def _check_simple_hash_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_nested_hash_ok(self, tree, name=EXTERNALS_NAME): - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_hash_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_req_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_req'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_req_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_req'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_opt_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_opt'.format(directory) - self._check_generic_empty_default_optional(tree, name) - - def _check_simple_opt_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_opt'.format(directory) - self._check_generic_ok_clean_optional(tree, name) - - def _check_mixed_ext_branch_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_mixed_ext_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_sparse'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_sparse'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - # ---------------------------------------------------------------- - # - # Check results for groups of externals under specific conditions - # - # ---------------------------------------------------------------- - def _check_container_simple_required_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_hash_empty(tree) - - def _check_container_nested_required_pre_checkout(self, overall, tree, order): - self.assertEqual(overall, 0) - self._check_nested_tag_empty(tree, name=NESTED_NAME[order[0]]) - self._check_nested_branch_empty(tree, name=NESTED_NAME[order[1]]) - self._check_nested_hash_empty(tree, name=NESTED_NAME[order[2]]) - - def _check_container_simple_required_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_hash_empty(tree) - - def _check_container_nested_required_checkout(self, overall, tree, order): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_nested_tag_empty(tree, name=NESTED_NAME[order[0]]) - self._check_nested_branch_empty(tree, name=NESTED_NAME[order[1]]) - self._check_nested_hash_empty(tree, name=NESTED_NAME[order[2]]) - - def _check_container_simple_required_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_hash_ok(tree) - - def _check_container_nested_required_post_checkout(self, overall, tree, order): - self.assertEqual(overall, 0) - self._check_nested_tag_ok(tree, name=NESTED_NAME[order[0]]) - self._check_nested_branch_ok(tree, name=NESTED_NAME[order[1]]) - self._check_nested_hash_ok(tree, name=NESTED_NAME[order[2]]) - - def _check_container_simple_required_out_of_sync(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_modified(tree) - self._check_simple_branch_modified(tree) - self._check_simple_hash_modified(tree) - - def _check_container_simple_optional_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_empty(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_empty(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_ok(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_post_optional(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_ok(tree) - self._check_simple_opt_ok(tree) - - def _check_container_simple_required_sb_modified(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_modified(tree) - self._check_simple_hash_ok(tree) - - def _check_container_simple_optional_st_dirty(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_dirty(tree) - self._check_simple_branch_ok(tree) - - def _check_container_full_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_pre_checkout(overall, tree) - - def _check_container_component_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_opt_ok(tree) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - - def _check_container_component_post_checkout2(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_ok(tree) - - def _check_container_component_post_checkout3(self, overall, tree): - self.assertEqual(overall, 0) - self.assertFalse("simp_opt" in tree) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - - def _check_container_full_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_post_checkout(overall, tree) - - def _check_container_full_pre_checkout_ext_change(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_pre_checkout_ext_change( - overall, tree) - - def _check_container_full_post_checkout_subext_modified( - self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_post_checkout_subext_modified( - overall, tree) - - def _check_mixed_ext_branch_required_pre_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_empty(tree, directory=EXTERNALS_NAME) - # NOTE: externals/mixed_req/src should not exist in the tree - # since this is the status before checkout of mixed_req. - - def _check_mixed_ext_branch_required_post_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_ok(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_ok(tree, directory=check_dir) - - def _check_mixed_ext_branch_required_pre_checkout_ext_change( - self, overall, tree): - # Note, this is the internal tree status just after change the - # externals description file, but before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_modified(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_ok(tree, directory=check_dir) - - def _check_mixed_ext_branch_required_post_checkout_subext_modified( - self, overall, tree): - # Note, this is the internal tree status just after change the - # externals description file, but before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_ok(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_modified(tree, directory=check_dir) - - def _check_mixed_cont_simple_required_pre_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=SUB_EXTERNALS_PATH) - - def _check_mixed_cont_simple_required_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=SUB_EXTERNALS_PATH) - - def _check_mixed_cont_simple_required_post_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH) - - def _check_container_sparse_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_sparse_empty(tree) - - def _check_container_sparse_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_sparse_ok(tree) - + def _external_path(section_name, base_path=EXTERNALS_PATH): + return './{0}/{1}'.format(base_path, section_name) + def _check_file_exists(self, repo_dir, pathname): "Check that exists in " self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname))) @@ -1023,9 +619,9 @@ def _check_file_absent(self, repo_dir, pathname): "Check that does not exist in " self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname))) + class TestSysCheckout(BaseTestSysCheckout): """Run systems level tests of checkout_externals - """ # NOTE(bja, 2017-11) pylint complains about long method names, but # it is hard to differentiate tests without making them more @@ -1037,246 +633,431 @@ class TestSysCheckout(BaseTestSysCheckout): # Run systems tests # # ---------------------------------------------------------------- - def test_container_simple_required(self): - """Verify that a container with simple subrepos - generates the correct initial status. - - """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_pre_checkout(overall, tree) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # status clean checked out - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) - + def test_required_bytag(self): + """Check out a required external pointing to a git tag.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(TAG_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the external is 'clean' aka at the correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired tag. + (tag_found, tag_name) = GitRepository._git_current_tag(local_path_abs) + self.assertEqual(tag_name, 'tag1') + + # Check existence of some simp_tag files + tag_path = os.path.join('externals', TAG_SECTION) + self._check_file_exists(cloned_repo_dir, + os.path.join(tag_path, README_NAME)) + # Subrepo should not exist (not referenced by configs). + self._check_file_absent(cloned_repo_dir, os.path.join(tag_path, + 'simple_subdir', + 'subdir_file.txt')) + + def test_required_bybranch(self): + """Check out a required external pointing to a git branch.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(BRANCH_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the external is 'clean' aka at the correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertTrue(os.path.exists(local_path_abs)) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired branch. + (branch_found, branch_name) = GitRepository._git_current_remote_branch( + local_path_abs) + self.assertEquals(branch_name, 'origin/' + REMOTE_BRANCH_FEATURE2) + + def test_required_byhash(self): + """Check out a required external pointing to a git hash.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, HASH_SECTION, + ref_hash='60b1cc1a38d63') + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(HASH_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the externals are 'clean' aka at their correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired hash. + (hash_found, hash_name) = GitRepository._git_current_hash( + local_path_abs) + self.assertTrue(hash_name.startswith('60b1cc1a38d63'), + msg=hash_name) + def test_container_nested_required(self): - """Verify that a container with nested subrepos - generates the correct initial status. + """Verify that a container with nested subrepos generates the correct initial status. Tests over all possible permutations """ + # Output subdirs for each of the externals, to test that one external can be + # checked out in a subdir of another. + NESTED_SUBDIR = ['./fred', './fred/wilma', './fred/wilma/barney'] + # Assert that each type of external (e.g. tag vs branch) can be at any parent level + # (e.g. child/parent/grandparent). orders = [[0, 1, 2], [1, 2, 0], [2, 0, 1], [0, 2, 1], [2, 1, 0], [1, 0, 2]] for n, order in enumerate(orders): - # create repo - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - self._test_id, "test"+str(n)) - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME, - dest_dir_in=dest_dir) - self._generator.container_nested_required(cloned_repo_dir, order) - - # status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_nested_required_pre_checkout(overall, tree, order) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_nested_required_checkout(overall, tree, order) - - # status clean checked out - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_nested_required_post_checkout(overall, tree, order) - + dest_dir = os.path.join(module_tmp_root_dir, self._test_id, + "test"+str(n)) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO, + dest_dir_in=dest_dir) + self._generator.create_config() + # We happen to check out each section via a different reference (tag/branch/hash) but + # those don't really matter, we just need to check out three repos into a nested set of + # directories. + self._generator.create_section( + SIMPLE_REPO, TAG_SECTION, nested=True, + tag='tag1', path=NESTED_SUBDIR[order[0]]) + self._generator.create_section( + SIMPLE_REPO, BRANCH_SECTION, nested=True, + branch=REMOTE_BRANCH_FEATURE2, path=NESTED_SUBDIR[order[1]]) + self._generator.create_section( + SIMPLE_REPO, HASH_SECTION, nested=True, + ref_hash='60b1cc1a38d63', path=NESTED_SUBDIR[order[2]]) + self._generator.write_config(cloned_repo_dir) + + # all externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + self._check_sync_clean(tree[NESTED_SUBDIR[order[0]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[NESTED_SUBDIR[order[1]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[NESTED_SUBDIR[order[2]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + # after checkout, all the repos are 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[NESTED_SUBDIR[order[0]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[NESTED_SUBDIR[order[1]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[NESTED_SUBDIR[order[2]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + def test_container_simple_optional(self): - """Verify that container with an optional simple subrepos - generates the correct initial status. + """Verify that container with an optional simple subrepos generates + the correct initial status. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_optional(cloned_repo_dir) - - # check status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_pre_checkout(overall, tree) - - # checkout required - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_optional_checkout(overall, tree) - - # status - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_post_checkout(overall, tree) + # create repo and externals config. + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, 'simp_req', + tag='tag1') - # checkout optional - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.optional_args) - self._check_container_simple_optional_post_checkout(overall, tree) + self._generator.create_section(SIMPLE_REPO, 'simp_opt', + tag='tag1', required=False) - # status - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_post_optional(overall, tree) + self._generator.write_config(cloned_repo_dir) + + # all externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) + + # after checkout, required external is clean, optional is still empty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) + + # after checking out optionals, the optional external is also clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.optional_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) def test_container_simple_verbose(self): - """Verify that container with simple subrepos runs with verbose status - output and generates the correct initial status. - + """Verify that verbose status matches non-verbose. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # check verbose status - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.verbose_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # after checkout, all externals should be 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # 'Verbose' status should tell the same story. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_simple_dirty(self): - """Verify that a container with simple subrepos - and a dirty status exits gracefully. - + """Verify that a container with a new tracked file is marked dirty. """ - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # add a file to the repo - tracked = True - self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', - tracked) - - # checkout: pre-checkout status should be dirty, did not - # modify working copy. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_optional_st_dirty(overall, tree) - - # verify status is still dirty - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_st_dirty(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # checkout, should start out clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # add a tracked file to the simp_tag external, should be dirty. + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/{0}/tmp.txt'.format(TAG_SECTION), + tracked=True) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.DIRTY) + + # Re-checkout; simp_tag should still be dirty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.DIRTY) def test_container_simple_untracked(self): """Verify that a container with simple subrepos and a untracked files is not considered 'dirty' and will attempt an update. """ - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # add a file to the repo - tracked = False - self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', - tracked) - - # checkout: pre-checkout status should be clean, ignoring the - # untracked file. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_post_checkout(overall, tree) - - # verify status is still clean - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # checkout, should start out clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # add an untracked file to the simp_tag external, should stay clean. + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/{0}/tmp.txt'.format(TAG_SECTION), + tracked=False) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # After checkout, the external should still be 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_simple_detached_sync(self): """Verify that a container with simple subrepos generates the correct out of sync status when making commits from a detached head - state. + state. + For more info about 'detached head' state: https://www.cloudbees.com/blog/git-detached-head """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_pre_checkout(overall, tree) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # make a commit on the detached head of the tag and hash externals - self._generator.create_commit(cloned_repo_dir, 'simp_tag') - self._generator.create_commit(cloned_repo_dir, 'simp_hash') - self._generator.create_commit(cloned_repo_dir, 'simp_branch') - - # status of repo, branch, tag and hash should all be out of sync! - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_out_of_sync(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + self._generator.create_section(SIMPLE_REPO, 'simp_hash', + ref_hash='60b1cc1a38d63') + + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - # same pre-checkout out of sync status - self._check_container_simple_required_out_of_sync(overall, tree) - - # now status should be in-sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) + + # Commit on top of the tag and hash (creating the detached head state in those two + # externals' repos) + # The branch commit does not create the detached head state, but here for completeness. + RepoUtils.create_commit(cloned_repo_dir, TAG_SECTION) + RepoUtils.create_commit(cloned_repo_dir, HASH_SECTION) + RepoUtils.create_commit(cloned_repo_dir, BRANCH_SECTION) + + # sync status of all three should be 'modified' (uncommitted changes) + # clean status is 'ok' (matches externals version) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # after checkout, all externals should be totally clean (no uncommitted changes, + # and matches externals version). + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_branch(self): """Verify that a container with remote branch change works """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # update the config file to point to a different remote with - # the same branch - self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - repo_path=SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) + + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) + + # update the branch external to point to a different remote with the same branch, + # then simp_branch should be out of sync + self._generator.write_with_git_branch(cloned_repo_dir, + name=BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2, + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, now simp_branch should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_tag_same_branch(self): """Verify that a container with remote tag change works. The new tag @@ -1285,276 +1066,323 @@ def test_container_remote_tag_same_branch(self): the branch. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the tag instead of branch. Tag MUST NOT be in the original - # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'forked-feature-v1', SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the new tag replacing the old branch. Tag MUST NOT be in the original + # repo! status of simp_branch should then be out of sync + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='forked-feature-v1', + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, then should be synced. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_tag_fetch_all(self): """Verify that a container with remote tag change works. The new tag should not be in the original repo, only the new remote - fork. It should also not be on a branch that will be fetch, + fork. It should also not be on a branch that will be fetched, and therefore not fetched by default with 'git fetch'. It will - only be retreived by 'git fetch --tags' - + only be retrieved by 'git fetch --tags' """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the tag instead of branch. Tag MUST NOT be in the original - # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'abandoned-feature', SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the new tag instead of the old branch. Tag MUST NOT be in the original + # repo! status of simp_branch should then be out of sync. + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='abandoned-feature', + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_preserve_dot(self): """Verify that after inital checkout, modifying an external git repo url to '.' and the current branch will leave it unchanged. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the same branch - self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - repo_path=SIMPLE_FORK_NAME) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the same branch. + self._generator.write_with_git_branch(cloned_repo_dir, name=BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2, + new_remote_repo_path=SIMPLE_FORK_REPO) + # after checkout, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) # update branch to point to a new branch that only exists in # the local fork - self._generator.create_branch(cloned_repo_dir, repo_name='simp_branch', - branch='private-feature', with_commit=True) - self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', - branch='private-feature', - repo_path=SIMPLE_LOCAL_ONLY_NAME) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) - - def test_container_full(self): - """Verify that 'full' container with simple and mixed subrepos - generates the correct initial status. + RepoUtils.create_branch(cloned_repo_dir, external_name=BRANCH_SECTION, + branch='private-feature', with_commit=True) + self._generator.write_with_git_branch(cloned_repo_dir, name=BRANCH_SECTION, + branch='private-feature', + new_remote_repo_path=SIMPLE_LOCAL_ONLY_NAME) + # after checkout, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + def test_container_mixed_subrepo(self): + """Verify container with mixed subrepo. The mixed subrepo has a sub-externals file with different sub-externals on different branches. """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - - # create the top level externals file - self._generator.container_full(cloned_repo_dir) - - # inital checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_full_pre_checkout(overall, tree) - - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_full_post_checkout(overall, tree) - - # Check existance of some files - subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(cloned_repo_dir, - os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, - 'simple_subdir', - 'subdir_file.txt')) - - # update the mixed-use repo to point to different branch - self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', - branch='new-feature', - repo_path=MIXED_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # check status out of sync for mixed_req, but sub-externals + self._generator.create_config() + self._generator.create_section(MIXED_REPO, 'mixed_req', + branch='master', sub_externals=CFG_SUB_NAME) + self._generator.write_config(cloned_repo_dir) + + # The subrepo has a repo_url that uses this environment variable. + # It'll be cleared in tearDown(). + os.environ[MIXED_CONT_EXT_ROOT_ENV_VAR] = self._bare_root + debug_env = MIXED_CONT_EXT_ROOT_ENV_VAR + '=' + self._bare_root + + # inital checkout: all requireds are clean, and optional is empty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args, + debug_env=debug_env) + mixed_req_path = self._external_path('mixed_req') + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + sub_ext_base_path = "{0}/{1}/{2}".format(EXTERNALS_PATH, 'mixed_req', SUB_EXTERNALS_PATH) + # The already-checked-in subexternals file has a 'simp_branch' section + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # update the mixed-use external to point to different branch + # status should become out of sync for mixed_req, but sub-externals # are still in sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_full_pre_checkout_ext_change(overall, tree) - - # run the checkout. Now the mixed use external and it's - # sub-exterals should be changed. Returned status is - # pre-checkout! - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_full_pre_checkout_ext_change(overall, tree) - - # check status out of sync for mixed_req, and sub-externals - # are in sync. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_full_post_checkout(overall, tree) - + self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', + branch='new-feature', + new_remote_repo_path=MIXED_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args, + debug_env=debug_env) + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # run the checkout. Now the mixed use external and its sub-externals should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args, + debug_env=debug_env) + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + def test_container_component(self): """Verify that optional component checkout works """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) # create the top level externals file - self._generator.container_full(cloned_repo_dir) - - # inital checkout, first try a nonexistant component argument noref + self._generator.create_config() + # Optional external, by tag. + self._generator.create_section(SIMPLE_REPO, 'simp_opt', + tag='tag1', required=False) + + # Required external, by branch. + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + # Required external, by hash. + self._generator.create_section(SIMPLE_REPO, HASH_SECTION, + ref_hash='60b1cc1a38d63') + self._generator.write_config(cloned_repo_dir) + + # inital checkout, first try a nonexistent component argument noref checkout_args = ['simp_opt', 'noref'] checkout_args.extend(self.checkout_args) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) + # Now explicitly check out one optional component.. + # Explicitly listed component (opt) should be present, the other two not. checkout_args = ['simp_opt'] checkout_args.extend(self.checkout_args) + tree = self.execute_checkout_with_status(cloned_repo_dir, + checkout_args) + self._check_sync_clean(tree[self._external_path('simp_opt')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + # Check out a second component, this one required. + # Explicitly listed component (branch) should be present, the still-unlisted one (tag) not. + checkout_args.append(BRANCH_SECTION) + tree = self.execute_checkout_with_status(cloned_repo_dir, + checkout_args) + self._check_sync_clean(tree[self._external_path('simp_opt')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - checkout_args) - - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_component_post_checkout(overall, tree) - checkout_args.append('simp_branch') - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - checkout_args) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_component_post_checkout2(overall, tree) def test_container_exclude_component(self): """Verify that exclude component checkout works """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - - # create the top level externals file - self._generator.container_full(cloned_repo_dir) - - # inital checkout, exclude simp_opt - checkout_args = ['--exclude', 'simp_opt'] + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + self._generator.create_section(SIMPLE_REPO, 'simp_hash', + ref_hash='60b1cc1a38d63') + + self._generator.write_config(cloned_repo_dir) + + # inital checkout should result in all externals being clean except excluded TAG_SECTION. + checkout_args = ['--exclude', TAG_SECTION] checkout_args.extend(self.checkout_args) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) - checkout_args.append("--status") - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) - self._check_container_component_post_checkout3(overall, tree) - - def test_mixed_simple(self): - """Verify that a mixed use repo can serve as a 'full' container, - pulling in a set of externals and a seperate set of sub-externals. + tree = self.execute_checkout_with_status(cloned_repo_dir, checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + def test_subexternal(self): + """Verify that an externals file can be brought in as a reference. """ - #import pdb; pdb.set_trace() - # create repository - cloned_repo_dir = self.clone_test_repo(MIXED_REPO_NAME) - # create top level externals file - self._generator.mixed_simple_base(cloned_repo_dir) - # NOTE: sub-externals file is already in the repo so we can - # switch branches during testing. Since this is a mixed-repo - # serving as the top level container repo, we can't switch - # during this test. + cloned_repo_dir = self.clone_test_repo(MIXED_REPO) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_mixed_cont_simple_required_checkout(overall, tree) - - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_mixed_cont_simple_required_post_checkout(overall, tree) + self._generator.create_config() + self._generator.create_section_reference_to_subexternal('mixed_base') + self._generator.write_config(cloned_repo_dir) + + # The subrepo has a repo_url that uses this environment variable. + # It'll be cleared in tearDown(). + os.environ[MIXED_CONT_EXT_ROOT_ENV_VAR] = self._bare_root + debug_env = MIXED_CONT_EXT_ROOT_ENV_VAR + '=' + self._bare_root + + # After checkout, confirm required's are clean and the referenced + # subexternal's contents are also clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args, + debug_env=debug_env) + + self._check_sync_clean( + tree[self._external_path(BRANCH_SECTION, base_path=SUB_EXTERNALS_PATH)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_sparse(self): """Verify that 'full' container with simple subrepo can run a sparse checkout and generate the correct initial status. """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # create the top level externals file - self._generator.container_sparse(cloned_repo_dir) - - # inital checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_sparse_pre_checkout(overall, tree) - - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_sparse_post_checkout(overall, tree) + # Create a file to list filenames to checkout. + sparse_filename = 'sparse_checkout' + with open(os.path.join(cloned_repo_dir, sparse_filename), 'w') as sfile: + sfile.write(README_NAME) - # Check existance of some files - subrepo_path = os.path.join('externals', 'simp_tag') + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag2') + + # Same tag as above, but with a sparse file too. + sparse_relpath = '../../' + sparse_filename + self._generator.create_section(SIMPLE_REPO, 'simp_sparse', + tag='tag2', sparse=sparse_relpath) + + self._generator.write_config(cloned_repo_dir) + + # inital checkout, confirm required's are clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_sparse')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Check existence of some files - full set in TAG_SECTION, and sparse set + # in 'simp_sparse'. + subrepo_path = os.path.join('externals', TAG_SECTION) self._check_file_exists(cloned_repo_dir, - os.path.join(subrepo_path, 'readme.txt')) + os.path.join(subrepo_path, README_NAME)) self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) subrepo_path = os.path.join('externals', 'simp_sparse') self._check_file_exists(cloned_repo_dir, - os.path.join(subrepo_path, 'readme.txt')) + os.path.join(subrepo_path, README_NAME)) self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) @@ -1591,42 +1419,27 @@ class TestSysCheckoutSVN(BaseTestSysCheckout): """ - def _check_svn_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_branch'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_svn_branch_dirty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_branch'.format(directory) - self._check_generic_ok_dirty_required(tree, name) - - def _check_svn_tag_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_tag'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_svn_tag_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_tag'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_container_simple_svn_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_branch_ok(tree) - self._check_svn_tag_ok(tree) - - def _check_container_simple_svn_sb_dirty_st_mod(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_tag_modified(tree) - self._check_svn_branch_dirty(tree) + @staticmethod + def _svn_branch_name(): + return './{0}/svn_branch'.format(EXTERNALS_PATH) - def _check_container_simple_svn_sb_clean_st_mod(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_tag_modified(tree) - self._check_svn_branch_ok(tree) + @staticmethod + def _svn_tag_name(): + return './{0}/svn_tag'.format(EXTERNALS_PATH) + + def _check_tag_branch_svn_tag_clean(self, tree): + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._svn_branch_name()], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._svn_tag_name()], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) @staticmethod - def have_svn_access(): + def _have_svn_access(): """Check if we have svn access so we can enable tests that use svn. """ @@ -1639,10 +1452,10 @@ def have_svn_access(): pass return have_svn - def skip_if_no_svn_access(self): + def _skip_if_no_svn_access(self): """Function decorator to disable svn tests when svn isn't available """ - have_svn = self.have_svn_access() + have_svn = self._have_svn_access() if not have_svn: raise unittest.SkipTest("No svn access") @@ -1650,60 +1463,55 @@ def test_container_simple_svn(self): """Verify that a container repo can pull in an svn branch and svn tag. """ - self.skip_if_no_svn_access() + self._skip_if_no_svn_access() # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_svn(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) + self._generator.create_config() + # Git repo. + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, tag='tag1') + + # Svn repos. + self._generator.create_svn_external('svn_branch', branch='trunk') + self._generator.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_svn_post_checkout(overall, tree) + self._generator.write_config(cloned_repo_dir) + + # checkout, make sure all sections are clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_tag_branch_svn_tag_clean(tree) # update description file to make the tag into a branch and # trigger a switch - self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', 'trunk') - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', + 'trunk') - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # checkout, again the results should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_tag_branch_svn_tag_clean(tree) # add an untracked file to the repo tracked = False - self._add_file_to_repo(cloned_repo_dir, - 'externals/svn_branch/tmp.txt', tracked) + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/svn_branch/tmp.txt', tracked) - # run a no-op checkout: pre-checkout status should be clean, - # ignoring the untracked file. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # run a no-op checkout. + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update description file to make the branch into a tag and # trigger a modified sync status self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', 'tags/cesm2.0.beta07') - # checkout: pre-checkout status should be clean and modified, - # will modify working copy. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_svn_sb_clean_st_mod(overall, tree) + self.execute_checkout_in_dir(cloned_repo_dir,self.checkout_args) # verify status is still clean and unmodified, last # checkout modified the working dir state. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.verbose_args) - self._check_container_simple_svn_post_checkout(overall, tree) + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) + self._check_tag_branch_svn_tag_clean(tree) class TestSubrepoCheckout(BaseTestSysCheckout): # Need to store information at setUp time for checking @@ -1730,20 +1538,19 @@ def setUp(self): self._bare_branch_name = 'subrepo_branch' self._config_branch_name = 'subrepo_config_branch' self._container_extern_name = 'externals_container.cfg' - self._my_test_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - self._test_id) + self._my_test_dir = os.path.join(module_tmp_root_dir, self._test_id) self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) self._checkout_dir = 'repo_with_submodules' - check_dir = self.clone_test_repo(CONTAINER_REPO_NAME, + check_dir = self.clone_test_repo(CONTAINER_REPO, dest_dir_in=self._repo_dir) self.assertTrue(self._repo_dir == check_dir) # Add the submodules cwd = os.getcwd() - fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_NAME) - simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO_NAME) - self._simple_ext_fork_name = os.path.splitext(SIMPLE_FORK_NAME)[0] + fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_REPO) + simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO) + self._simple_ext_fork_name = os.path.splitext(SIMPLE_FORK_REPO)[0] self._simple_ext_name = os.path.join('sourc', - os.path.splitext(SIMPLE_REPO_NAME)[0]) + os.path.splitext(SIMPLE_REPO)[0]) os.chdir(self._repo_dir) # Add a branch with a subrepo cmd = ['git', 'branch', self._bare_branch_name, 'master'] @@ -1764,7 +1571,7 @@ def setUp(self): execute_subprocess(cmd) cmd = ['git', 'checkout', self._config_branch_name] execute_subprocess(cmd) - cmd = ['git', 'submodule', 'add', '--name', SIMPLE_REPO_NAME, + cmd = ['git', 'submodule', 'add', '--name', SIMPLE_REPO, simple_repo_dir, self._simple_ext_name] execute_subprocess(cmd) # Checkout feature2 @@ -1776,8 +1583,8 @@ def setUp(self): # Save the fork repo hash for comparison self._simple_hash_check = self.get_git_hash() os.chdir(self._repo_dir) - self.create_externals_file(filename=self._container_extern_name, - dest_dir=self._repo_dir, from_submodule=True) + self.write_externals_config(filename=self._container_extern_name, + dest_dir=self._repo_dir, from_submodule=True) cmd = ['git', 'add', self._container_extern_name] execute_subprocess(cmd) cmd = ['git', 'commit', '-am', "'Added simple-ext as a submodule'"] @@ -1794,9 +1601,10 @@ def get_git_hash(revision="HEAD"): git_out = execute_subprocess(cmd, output_to_caller=True) return git_out.strip() - def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, - branch_name=None, sub_externals=None, - from_submodule=False): + def write_externals_config(self, name='', dest_dir=None, + filename=CFG_NAME, + branch_name=None, sub_externals=None, + from_submodule=False): # pylint: disable=too-many-arguments """Create a container externals file with only simple externals. @@ -1807,10 +1615,10 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, dest_dir = self._my_test_dir if from_submodule: - self._generator.create_section(SIMPLE_FORK_NAME, + self._generator.create_section(SIMPLE_FORK_REPO, self._simple_ext_fork_name, from_submodule=True) - self._generator.create_section(SIMPLE_REPO_NAME, + self._generator.create_section(SIMPLE_REPO, self._simple_ext_name, branch='feature3', path='', from_submodule=False) @@ -1821,7 +1629,7 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, self._generator.create_section(self._test_repo_name, self._checkout_dir, branch=branch_name, - path=name, externals=sub_externals, + path=name, sub_externals=sub_externals, repo_path_abs=self._repo_dir) self._generator.write_config(dest_dir, filename=filename) @@ -1831,12 +1639,10 @@ def idempotence_check(self, checkout_dir): checkout_externals --status does not cause errors""" cwd = os.getcwd() os.chdir(checkout_dir) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.status_args) - self.assertTrue(overall == 0) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) + self.execute_checkout_in_dir(self._my_test_dir, + self.status_args) os.chdir(cwd) def test_submodule_checkout_bare(self): @@ -1848,17 +1654,17 @@ def test_submodule_checkout_bare(self): """ simple_ext_fork_tag = "(tag1)" simple_ext_fork_status = " " - self.create_externals_file(branch_name=self._bare_branch_name) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._bare_branch_name) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, self._simple_ext_fork_name, "readme.txt") self.assertTrue(os.path.exists(fork_file)) - os.chdir(checkout_dir) + submods = git_submodule_status(checkout_dir) + print('checking status of', checkout_dir, ':', submods) self.assertEqual(len(submods.keys()), 1) self.assertTrue(self._simple_ext_fork_name in submods) submod = submods[self._simple_ext_fork_name] @@ -1868,7 +1674,6 @@ def test_submodule_checkout_bare(self): self.assertEqual(submod['status'], simple_ext_fork_status) self.assertTrue('tag' in submod) self.assertEqual(submod['tag'], simple_ext_fork_tag) - os.chdir(cwd) self.idempotence_check(checkout_dir) def test_submodule_checkout_none(self): @@ -1877,11 +1682,10 @@ def test_submodule_checkout_none(self): externals cfg file. Correct behavior is the submodle is not checked out. """ - self.create_externals_file(branch_name=self._bare_branch_name, - sub_externals="none") - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._bare_branch_name, + sub_externals="none") + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, @@ -1899,11 +1703,10 @@ def test_submodule_checkout_config(self): # pylint: disable=too-many-locals """ tag_check = None # Not checked out as submodule status_check = "-" # Not checked out as submodule - self.create_externals_file(branch_name=self._config_branch_name, - sub_externals=self._container_extern_name) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._config_branch_name, + sub_externals=self._container_extern_name) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, @@ -1965,14 +1768,17 @@ def test_error_unknown_protocol(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', - 'this-protocol-does-not-exist') + self._generator.write_with_protocol(cloned_repo_dir, BRANCH_SECTION, + 'this-protocol-does-not-exist') with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -1986,13 +1792,16 @@ def test_error_switch_protocol(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', 'svn') + self._generator.write_with_protocol(cloned_repo_dir, BRANCH_SECTION, 'svn') with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2002,14 +1811,18 @@ def test_error_unknown_tag(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME) + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='this-tag-does-not-exist', + new_remote_repo_path=SIMPLE_REPO) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2020,15 +1833,19 @@ def test_error_overspecify_tag_branch(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME, - remove_branch=False) + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='this-tag-does-not-exist', + new_remote_repo_path=SIMPLE_REPO, + remove_branch=False) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2039,14 +1856,16 @@ def test_error_underspecify_tag_branch(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_without_branch_tag(cloned_repo_dir, - 'simp_branch') + self._generator.write_without_branch_tag(cloned_repo_dir, BRANCH_SECTION) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2057,14 +1876,17 @@ def test_error_missing_url(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! self._generator.write_without_repo_url(cloned_repo_dir, - 'simp_branch') + BRANCH_SECTION) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) diff --git a/test/test_sys_repository_git.py b/test/test_sys_repository_git.py index 29d5433b95..7e5fb5020d 100644 --- a/test/test_sys_repository_git.py +++ b/test/test_sys_repository_git.py @@ -131,12 +131,12 @@ def tearDown(self): shutil.rmtree(self._tmpdir, ignore_errors=True) @staticmethod - def make_git_repo(): + def make_cwd_git_repo(): """Turn the current directory into an empty git repository""" execute_subprocess(['git', 'init']) @staticmethod - def add_git_commit(): + def add_cwd_git_commit(): """Add a git commit in the current directory""" with open('README', 'a') as myfile: myfile.write('more info') @@ -144,17 +144,17 @@ def add_git_commit(): execute_subprocess(['git', 'commit', '-m', 'my commit message']) @staticmethod - def checkout_git_branch(branchname): + def checkout_cwd_git_branch(branchname): """Checkout a new branch in the current directory""" execute_subprocess(['git', 'checkout', '-b', branchname]) @staticmethod - def make_git_tag(tagname): + def make_cwd_git_tag(tagname): """Make a lightweight tag at the current commit""" execute_subprocess(['git', 'tag', '-m', 'making a tag', tagname]) @staticmethod - def checkout_ref(refname): + def checkout_cwd_ref(refname): """Checkout the given refname in the current directory""" execute_subprocess(['git', 'checkout', refname]) @@ -164,72 +164,72 @@ def checkout_ref(refname): def test_currentHash_returnsHash(self): """Ensure that the _git_current_hash function returns a hash""" - self.make_git_repo() - self.add_git_commit() - hash_found, myhash = self._repo._git_current_hash() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + hash_found, myhash = self._repo._git_current_hash(os.getcwd()) self.assertTrue(hash_found) self.assertIsHash(myhash) def test_currentHash_outsideGitRepo(self): """Ensure that the _git_current_hash function returns False when outside a git repository""" - hash_found, myhash = self._repo._git_current_hash() + hash_found, myhash = self._repo._git_current_hash(os.getcwd()) self.assertFalse(hash_found) self.assertEqual('', myhash) def test_currentBranch_onBranch(self): """Ensure that the _git_current_branch function returns the name of the branch""" - self.make_git_repo() - self.add_git_commit() - self.checkout_git_branch('foo') - branch_found, mybranch = self._repo._git_current_branch() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.checkout_cwd_git_branch('foo') + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertTrue(branch_found) self.assertEqual('foo', mybranch) def test_currentBranch_notOnBranch(self): """Ensure that the _git_current_branch function returns False when not on a branch""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('mytag') - self.checkout_ref('mytag') - branch_found, mybranch = self._repo._git_current_branch() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('mytag') + self.checkout_cwd_ref('mytag') + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertFalse(branch_found) self.assertEqual('', mybranch) def test_currentBranch_outsideGitRepo(self): """Ensure that the _git_current_branch function returns False when outside a git repository""" - branch_found, mybranch = self._repo._git_current_branch() + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertFalse(branch_found) self.assertEqual('', mybranch) def test_currentTag_onTag(self): """Ensure that the _git_current_tag function returns the name of the tag""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('some_tag') - tag_found, mytag = self._repo._git_current_tag() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('some_tag') + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertTrue(tag_found) self.assertEqual('some_tag', mytag) def test_currentTag_notOnTag(self): """Ensure tha the _git_current_tag function returns False when not on a tag""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('some_tag') - self.add_git_commit() - tag_found, mytag = self._repo._git_current_tag() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('some_tag') + self.add_cwd_git_commit() + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertFalse(tag_found) self.assertEqual('', mytag) def test_currentTag_outsideGitRepo(self): """Ensure that the _git_current_tag function returns False when outside a git repository""" - tag_found, mytag = self._repo._git_current_tag() + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertFalse(tag_found) self.assertEqual('', mytag) diff --git a/test/test_unit_repository_git.py b/test/test_unit_repository_git.py index a6ad9f1003..1c01098acf 100644 --- a/test/test_unit_repository_git.py +++ b/test/test_unit_repository_git.py @@ -67,7 +67,7 @@ def setUp(self): def _git_current_branch(branch_found, branch_name): """Return a function that takes the place of repo._git_current_branch, which returns the given output.""" - def my_git_current_branch(): + def my_git_current_branch(dirname): """mock function that can take the place of repo._git_current_branch""" return branch_found, branch_name return my_git_current_branch @@ -76,7 +76,7 @@ def my_git_current_branch(): def _git_current_tag(tag_found, tag_name): """Return a function that takes the place of repo._git_current_tag, which returns the given output.""" - def my_git_current_tag(): + def my_git_current_tag(dirname): """mock function that can take the place of repo._git_current_tag""" return tag_found, tag_name return my_git_current_tag @@ -85,7 +85,7 @@ def my_git_current_tag(): def _git_current_hash(hash_found, hash_name): """Return a function that takes the place of repo._git_current_hash, which returns the given output.""" - def my_git_current_hash(): + def my_git_current_hash(dirname): """mock function that can take the place of repo._git_current_hash""" return hash_found, hash_name return my_git_current_hash @@ -102,7 +102,7 @@ def test_ref_branch(self): self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'foo_tag (branch feature3)' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_detached_tag(self): @@ -112,7 +112,7 @@ def test_ref_detached_tag(self): self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'foo_tag' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_detached_hash(self): @@ -123,7 +123,7 @@ def test_ref_detached_hash(self): self._repo._git_current_tag = self._git_current_tag(False, '') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'abc123' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_none(self): @@ -132,7 +132,7 @@ def test_ref_none(self): self._repo._git_current_branch = self._git_current_branch(False, '') self._repo._git_current_tag = self._git_current_tag(False, '') self._repo._git_current_hash = self._git_current_hash(False, '') - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, EMPTY_STR) @@ -206,11 +206,19 @@ def setUp(self): self._repo._current_ref = self._current_ref_empty self._create_tmp_git_dir() + # We have to override this class method rather than the self._repo + # instance method because it is called via + # GitRepository._remote_name_for_url, which is itself a @classmethod + # calls cls._git_remote_verbose(). + self._orignal_git_remote_verbose = GitRepository._git_remote_verbose + GitRepository._git_remote_verbose = self._git_remote_origin_upstream def tearDown(self): """Cleanup tmp stuff on the file system """ self._remove_tmp_git_dir() + GitRepository._git_remote_verbose = self._orignal_git_remote_verbose + def _create_tmp_git_dir(self): """Create a temporary fake git directory for testing purposes. """ @@ -227,29 +235,27 @@ def _remove_tmp_git_dir(self): # mock methods replacing git system calls # @staticmethod - def _current_ref_empty(): + def _current_ref_empty(dirname): """Return an empty string. + + Drop-in for GitRepository._current_ref """ return EMPTY_STR @staticmethod - def _git_remote_origin_upstream(): - """Return an info string that is a checkout hash - """ - return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM + def _git_remote_origin_upstream(dirname): + """Return an info string that is a checkout hash. - @staticmethod - def _git_remote_none(): - """Return an info string that is a checkout hash + Drop-in for GitRepository._git_remote_verbose. """ - return EMPTY_STR + return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM @staticmethod def _git_current_hash(myhash): """Return a function that takes the place of repo._git_current_hash, which returns the given hash """ - def my_git_current_hash(): + def my_git_current_hash(dirname): """mock function that can take the place of repo._git_current_hash""" return 0, myhash return my_git_current_hash @@ -263,7 +269,7 @@ def _git_revparse_commit(self, expected_ref, mystatus, myhash): status = 0 implies success, non-zero implies failure """ - def my_git_revparse_commit(ref): + def my_git_revparse_commit(ref, dirname): """mock function that can take the place of repo._git_revparse_commit""" self.assertEqual(expected_ref, ref) return mystatus, myhash @@ -291,9 +297,6 @@ def test_sync_dir_exist_no_git_info(self): """Test that a non-existent git repo returns an unknown status """ stat = ExternalStatus() - # Now we over-ride the _git_remote_verbose method on the repo to return - # a known value without requiring access to git. - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -313,7 +316,6 @@ def test_sync_invalid_reference(self): """Test that an invalid reference returns out-of-sync """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('abc123') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -333,7 +335,6 @@ def test_sync_tag_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('abc123') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -348,7 +349,6 @@ def test_sync_tag_on_different_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('def456') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -368,7 +368,6 @@ def test_sync_hash_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = '' self._repo._hash = 'abc' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -384,7 +383,6 @@ def test_sync_hash_on_different_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = '' self._repo._hash = 'abc' self._repo._git_current_hash = self._git_current_hash('def456') @@ -405,7 +403,6 @@ def test_sync_branch_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -421,7 +418,6 @@ def test_sync_branch_on_diff_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -433,11 +429,10 @@ def test_sync_branch_on_diff_hash(self): self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) def test_sync_branch_diff_remote(self): - """Test _determine_remote_name with a different remote + """Test _remote_name_for_url with a different remote """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/other/repo' @@ -449,11 +444,10 @@ def test_sync_branch_diff_remote(self): # expected argument def test_sync_branch_diff_remote2(self): - """Test _determine_remote_name with a different remote + """Test _remote_name_for_url with a different remote """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/local/repo2' @@ -469,7 +463,6 @@ def test_sync_branch_on_unknown_remote(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/unknown/repo' @@ -491,7 +484,6 @@ def test_sync_branch_on_untracked_local(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature3' self._repo._tag = '' self._repo._url = '.' @@ -611,24 +603,20 @@ def setUp(self): self._repo = GitRepository('test', repo) @staticmethod - def _shell_true(url, remote=None): - _ = url - _ = remote + def _shell_true(*args, **kwargs): return 0 @staticmethod - def _shell_false(url, remote=None): - _ = url - _ = remote + def _shell_false(*args, **kwargs): return 1 @staticmethod - def _mock_function_true(ref): + def _mock_revparse_commit(ref, dirname): _ = ref return (TestValidRef._shell_true, '97ebc0e0deadc0de') @staticmethod - def _mock_function_false(ref): + def _mock_revparse_commit_false(ref, dirname): _ = ref return (TestValidRef._shell_false, '97ebc0e0deadc0de') @@ -638,10 +626,11 @@ def test_tag_not_tag_branch_commit(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'something' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_not_tag(self): @@ -650,10 +639,11 @@ def test_tag_not_tag(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_true - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'tag1' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_indeterminant(self): @@ -662,10 +652,11 @@ def test_tag_indeterminant(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_true - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = 'something' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_is_unique(self): @@ -674,10 +665,11 @@ def test_tag_is_unique(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = 'tag1' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertTrue(received) def test_tag_is_not_hash(self): @@ -686,10 +678,11 @@ def test_tag_is_not_hash(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = '97ebc0e0' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_hash_is_commit(self): @@ -698,10 +691,11 @@ def test_hash_is_commit(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = '97ebc0e0' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) @@ -746,13 +740,14 @@ def _shell_false(url, remote=None): return 1 @staticmethod - def _mock_function_false(ref): + def _mock_revparse_commit_false(ref, dirname): _ = ref return (TestValidRef._shell_false, '') @staticmethod - def _mock_function_true(ref): + def _mock_revparse_commit_true(ref, dirname): _ = ref + _ = dirname return (TestValidRef._shell_true, '') def test_valid_ref_is_invalid(self): @@ -761,10 +756,12 @@ def test_valid_ref_is_invalid(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'invalid_ref' with self.assertRaises(RuntimeError): - self._repo._check_for_valid_ref(self._repo._tag) + self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) def test_valid_tag(self): """Verify a valid tag return true @@ -772,9 +769,11 @@ def test_valid_tag(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit_true self._repo._tag = 'tag1' - received = self._repo._check_for_valid_ref(self._repo._tag) + received = self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) def test_valid_branch(self): @@ -783,24 +782,28 @@ def test_valid_branch(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit_true self._repo._tag = 'tag1' - received = self._repo._check_for_valid_ref(self._repo._tag) + received = self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) def test_valid_hash(self): """Verify a valid hash return true """ - def _mock_revparse_commit(ref): + def _mock_revparse_commit_true(ref, dirname): _ = ref return (0, '56cc0b539426eb26810af9e') self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = _mock_revparse_commit + self._repo._git_revparse_commit = _mock_revparse_commit_true self._repo._hash = '56cc0b5394' - received = self._repo._check_for_valid_ref(self._repo._hash) + received = self._repo._check_for_valid_ref(self._repo._hash, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) From 66375364be49f083161900c383e96780ec133a22 Mon Sep 17 00:00:00 2001 From: ckoven Date: Mon, 6 Mar 2023 22:10:01 -0700 Subject: [PATCH 224/332] changed soil moisture initialization logic from if use_fates_hydro to if use_fates --- src/biogeophys/WaterStateType.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/biogeophys/WaterStateType.F90 b/src/biogeophys/WaterStateType.F90 index f61a7b943d..751f633875 100644 --- a/src/biogeophys/WaterStateType.F90 +++ b/src/biogeophys/WaterStateType.F90 @@ -14,7 +14,7 @@ module WaterStateType use decompMod , only : bounds_type use decompMod , only : subgrid_level_patch, subgrid_level_column, subgrid_level_gridcell use clm_varctl , only : use_bedrock, iulog - use clm_varctl , only : use_fates_planthydro + use clm_varctl , only : use_fates use clm_varpar , only : nlevgrnd, nlevsoi, nlevurb, nlevmaxurbgrnd, nlevsno use clm_varcon , only : spval use LandunitType , only : lun @@ -348,7 +348,7 @@ subroutine InitCold(this, bounds, & if (j > nbedrock) then this%h2osoi_vol_col(c,j) = 0.0_r8 else - if(use_fates_planthydro) then + if(use_fates) then this%h2osoi_vol_col(c,j) = 0.75_r8*watsat_col(c,j)*ratio else this%h2osoi_vol_col(c,j) = 0.15_r8*ratio From 397eb22678be387eceeed696cd34ea8feca1d21c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 7 Mar 2023 10:07:28 -0700 Subject: [PATCH 225/332] changed name of PRISM PRECIP stream --- cime_config/config_component.xml | 17 ++++++++++------- .../NEON/defaults/user_nl_datm_streams | 4 ++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index b5741e5b40..243c866bbd 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -295,13 +295,16 @@ ABBY,BLAN,CPER,DEJU,GRSM,HEAL,KONA,LENO,NIWO,ONAQ,PUUM,SERC,SRER,TALL,TREE,WOOD, BARR,BONA,DCFS,DELA,GUAN,JERC,KONZ,MLBS,NOGP,ORNL,RMNP,SJER,STEI,TEAK,UKFS,WREF, BART,CLBJ,DSNY,HARV,JORN,LAJA,MOAB,OAES,OSBS,SCBI,SOAP,STER,TOOL,UNDE,YELL, - ABBY.PRECIP,BLAN.PRECIP,CPER.PRECIP,DEJU.PRECIP,GRSM.PRECIP,HEAL.PRECIP,KONA.PRECIP, - LENO.PRECIP,NIWO.PRECIP,ONAQ.PRECIP,PUUM.PRECIP,SERC.PRECIP,SRER.PRECIP,TALL.PRECIP, - TREE.PRECIP,WOOD.PRECIP,BARR.PRECIP,BONA.PRECIP,DCFS.PRECIP,DELA.PRECIP,GUAN.PRECIP, - JERC.PRECIP,KONZ.PRECIP,MLBS.PRECIP,NOGP.PRECIP,ORNL.PRECIP,RMNP.PRECIP,SJER.PRECIP, - STEI.PRECIP,TEAK.PRECIP,UKFS.PRECIP,WREF.PRECIP,BART.PRECIP,CLBJ.PRECIP,DSNY.PRECIP, - HARV.PRECIP,JORN.PRECIP,LAJA.PRECIP,MOAB.PRECIP,OAES.PRECIP,OSBS.PRECIP,SCBI.PRECIP, - SOAP.PRECIP,STER.PRECIP,TOOL.PRECIP,UNDE.PRECIP,YELL.PRECIP, + PRISM_PRECIP.ABBY,PRISM_PRECIP.BLAN,PRISM_PRECIP.CPER,PRISM_PRECIP.DEJU,PRISM_PRECIP.GRSM, + PRISM_PRECIP.HEAL,PRISM_PRECIP.KONA,PRISM_PRECIP.LENO,PRISM_PRECIP.NIWO,PRISM_PRECIP.ONAQ, + PRISM_PRECIP.PUUM,PRISM_PRECIP.SERC,PRISM_PRECIP.SRER,PRISM_PRECIP.TALL,PRISM_PRECIP.TREE, + PRISM_PRECIP.WOOD,PRISM_PRECIP.BARR,PRISM_PRECIP.BONA,PRISM_PRECIP.DCFS,PRISM_PRECIP.DELA, + PRISM_PRECIP.GUAN,PRISM_PRECIP.JERC,PRISM_PRECIP.KONZ,PRISM_PRECIP.MLBS,PRISM_PRECIP.NOGP, + PRISM_PRECIP.ORNL,PRISM_PRECIP.RMNP,PRISM_PRECIP.SJER,PRISM_PRECIP.STEI,PRISM_PRECIP.TEAK, + PRISM_PRECIP.UKFS,PRISM_PRECIP.WREF,PRISM_PRECIP.BART,PRISM_PRECIP.CLBJ,PRISM_PRECIP.DSNY, + PRISM_PRECIP.HARV,PRISM_PRECIP.JORN,PRISM_PRECIP.LAJA,PRISM_PRECIP.MOAB,PRISM_PRECIP.OAES, + PRISM_PRECIP.OSBS,PRISM_PRECIP.SCBI,PRISM_PRECIP.SOAP,PRISM_PRECIP.STER,PRISM_PRECIP.TOOL, + PRISM_PRECIP.UNDE,PRISM_PRECIP.YELL, run_component_ctsm diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index 636ceb2ad8..78fff234d6 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -47,10 +47,10 @@ NEON.${NEONSITE}:datavars = \ PSRF Sa_pbot, \ FLDS Faxa_lwdn -NEON.${NEONSITE}.PRECIP:datavars = \ +NEON.PRISM_PRECIP.${NEONSITE}:datavars = \ PRECIP Faxa_precn -NEON.${NEONSITE}.PRECIP:datafiles = \ +NEON.PRISM_PRECIP${NEONSITE}:datafiles = \ /glade/u/home/tking/neon/prism_${NEONSITE}_2018.nc, \ /glade/u/home/tking/neon/prism_${NEONSITE}_2019.nc, \ /glade/u/home/tking/neon/prism_${NEONSITE}_2020.nc, \ From 4674b9069c23f2620f8ab3fc5ab822def9e5794b Mon Sep 17 00:00:00 2001 From: will wieder Date: Thu, 9 Mar 2023 10:23:46 -0700 Subject: [PATCH 226/332] Update README.md updates to SE list --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 298e5adc04..9de22e3663 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ Software engineering team: - [Bill Sacks](https://github.com/billsacks) - [Sam Levis](https://github.com/slevisconsulting) - [Adrianna Foster](https://github.com/adrifoster) +- [Sam Rabin](https://github.com/samsrabin) +- [Greg Lemieux](https://github.com/glemieux) +- [Ryan Knox](https://github.com/rgknox) Science team: - [Dave Lawrence](https://github.com/dlawrenncar) From c0bfe0105ef777d5df85c248201a67059b06afb4 Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Fri, 10 Mar 2023 13:31:32 -0700 Subject: [PATCH 227/332] mpasa files moved to ctsm5.1.dev052 --- bld/namelist_files/namelist_defaults_ctsm.xml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 338aebc95d..7e4cebb4d1 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1130,17 +1130,17 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_numaIA_hist_78pfts_CMIP6_si lnd/clm2/surfdata_map/ctsm1.0.dev094-2-g633be0eb/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c200521.nc -lnd/clm2/surfdata_map/surfdata_mpasa480_hist_78pfts_CMIP6_simyr2000_c211110.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa480_hist_78pfts_CMIP6_simyr2000_c211110.nc -lnd/clm2/surfdata_map/surfdata_mpasa240_hist_78pfts_CMIP6_simyr2000_c211115.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa240_hist_78pfts_CMIP6_simyr2000_c211115.nc -lnd/clm2/surfdata_map/surfdata_mpasa120_hist_78pfts_CMIP6_simyr2000_c211108.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa120_hist_78pfts_CMIP6_simyr2000_c211108.nc -lnd/clm2/surfdata_map/surfdata_mpasa60_hist_78pfts_CMIP6_simyr2000_c211110.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa60_hist_78pfts_CMIP6_simyr2000_c211110.nc -lnd/clm2/surfdata_map/surfdata_mpasa30_hist_78pfts_CMIP6_simyr2000_c211111.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa30_hist_78pfts_CMIP6_simyr2000_c211111.nc -lnd/clm2/surfdata_map/surfdata_mpasa15_hist_78pfts_CMIP6_simyr2000_c211111.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa15_hist_78pfts_CMIP6_simyr2000_c211111.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc @@ -1188,17 +1188,17 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_16pfts_Irrig_CMIP6_sim lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_1x1_brazil_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc -lnd/clm2/surfdata_map/surfdata_mpasa480_hist_78pfts_CMIP6_simyr1850_c211110.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa480_hist_78pfts_CMIP6_simyr1850_c211110.nc -lnd/clm2/surfdata_map/surfdata_mpasa240_hist_78pfts_CMIP6_simyr1850_c211115.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa240_hist_78pfts_CMIP6_simyr1850_c211115.nc -lnd/clm2/surfdata_map/surfdata_mpasa120_hist_78pfts_CMIP6_simyr1850_c211108.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa120_hist_78pfts_CMIP6_simyr1850_c211108.nc -lnd/clm2/surfdata_map/surfdata_mpasa60_hist_78pfts_CMIP6_simyr1850_c211110.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa60_hist_78pfts_CMIP6_simyr1850_c211110.nc -lnd/clm2/surfdata_map/surfdata_mpasa30_hist_78pfts_CMIP6_simyr1850_c211111.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa30_hist_78pfts_CMIP6_simyr1850_c211111.nc -lnd/clm2/surfdata_map/surfdata_mpasa15_hist_78pfts_CMIP6_simyr1850_c211111.nc +lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa15_hist_78pfts_CMIP6_simyr1850_c211111.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne30np4_hist_16pfts_Irrig_CMIP6_simyr1850_c190303.nc @@ -1302,17 +1302,17 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc lnd/clm2/surfdata_map/landuse.timeseries_mpasa480_hist_78pfts_CMIP6_simyr1850-2015_c211110.nc + use_crop=".true." >lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa480_hist_78pfts_CMIP6_simyr1850-2015_c211110.nc lnd/clm2/surfdata_map/landuse.timeseries_mpasa240_hist_78pfts_CMIP6_simyr1850-2015_c211115.nc + use_crop=".true." >lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa240_hist_78pfts_CMIP6_simyr1850-2015_c211115.nc lnd/clm2/surfdata_map/landuse.timeseries_mpasa120_hist_78pfts_CMIP6_simyr1850-2015_c211108.nc + use_crop=".true." >lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa120_hist_78pfts_CMIP6_simyr1850-2015_c211108.nc lnd/clm2/surfdata_map/landuse.timeseries_mpasa60_hist_78pfts_CMIP6_simyr1850-2015_c211110.nc +use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa60_hist_78pfts_CMIP6_simyr1850-2015_c211110.nc lnd/clm2/surfdata_map/landuse.timeseries_mpasa30_hist_78pfts_CMIP6_simyr1850-2015_c211111.nc +use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa30_hist_78pfts_CMIP6_simyr1850-2015_c211111.nc lnd/clm2/surfdata_map/landuse.timeseries_mpasa15_hist_78pfts_CMIP6_simyr1850-2015_c211111.nc +use_crop=".true.">lnd/clm2/surfdata_map/ctsm5.1.dev052/landuse.timeseries_mpasa15_hist_78pfts_CMIP6_simyr1850-2015_c211111.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc From fc3d4930ad97c1c7804052b537f30bb93f34341d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 14 Mar 2023 13:32:18 -0600 Subject: [PATCH 228/332] bug fix to get PRISM datm stream working --- cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index 78fff234d6..16b7c739a7 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -50,7 +50,7 @@ NEON.${NEONSITE}:datavars = \ NEON.PRISM_PRECIP.${NEONSITE}:datavars = \ PRECIP Faxa_precn -NEON.PRISM_PRECIP${NEONSITE}:datafiles = \ +NEON.PRISM_PRECIP.${NEONSITE}:datafiles = \ /glade/u/home/tking/neon/prism_${NEONSITE}_2018.nc, \ /glade/u/home/tking/neon/prism_${NEONSITE}_2019.nc, \ /glade/u/home/tking/neon/prism_${NEONSITE}_2020.nc, \ From ad8bc744534a4164237807f731cd8c43a7073bea Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 16 Mar 2023 08:47:50 -0600 Subject: [PATCH 229/332] Update externals to cesm2_3_alpha12c-ish For CMEPS, use the latest (fixing an issue with multi-instance cases relative to what was in alpha12c). --- Externals.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 7a4eeb4a6e..1a55310d19 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.38 +tag = ccs_config_cesm0.0.58 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -44,18 +44,18 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.45 +tag = cime6.0.94 required = True [cmeps] -tag = cmeps0.13.71 +tag = cmeps0.14.17 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.65 +tag = cdeps1.0.7 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -70,7 +70,7 @@ local_path = components/cpl7 required = True [share] -tag = share1.0.13 +tag = share1.0.16 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share @@ -84,7 +84,7 @@ local_path = libraries/mct required = True [parallelio] -tag = pio2_5_7 +tag = pio2_5_10 protocol = git repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio From deb0866e03dd9dcff47e65ab6dac68216219a846 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 16 Mar 2023 14:14:00 -0600 Subject: [PATCH 230/332] Update change file --- doc/ChangeLog | 24 +++++++++++++----------- doc/ChangeSum | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 6df1764ff4..a604f675f4 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,13 +1,16 @@ =============================================================== Tag name: ctsm5.1.dev119 Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310), ekluzek Erik Kluzek), lawrencepj1 (Peter Lawrence) -Date: Wed Feb 22 13:57:36 MST 2023 +Date: Thu Mar 16 14:13:37 MDT 2023 One-line Summary: Allow gross unrepresented land use transitions (PR #309) Purpose and description of changes ---------------------------------- - Get gross unrepresented land use transitions working in CLM5.1. + Get gross unrepresented land use transitions working in CLM5.1. This is additional optional + data added to the landuse.timeseries files for transient simulations. The current landuse.timseries + files have this data, but it's set to zero. This data will be part of the CTSM5.2 surface dataset + (that is upcoming) and be on by default for clm5_3 physics. Significant changes to scientifically-supported configurations -------------------------------------------------------------- @@ -30,9 +33,9 @@ hanges.] Bugs fixed or introduced ------------------------ -CTSM issues fixed (include CTSM Issue #): - I didn't find a corresponding issue. - + Known bugs introduced in this tag (include issue #): #1968 + #1968 -- Reduce the GULU tests down to one + Notes of particular relevance for users --------------------------------------- Changes made to namelist defaults (e.g., changed parameter values): @@ -42,11 +45,6 @@ Changes to the datasets (e.g., parameter, surface or initial files): Surface datasets may now contain non-zero gross unrepresented land use transitions. -Substantial timing or memory changes: -[e.g., check PFS test in the test suite and look at timings, if you -expect possible significant timing changes] - - Notes of particular relevance for developers: --------------------------------------------- Changes to tests or testing: @@ -77,7 +75,11 @@ Changes answers relative to baseline: NO (unless see next) Other details ------------- Pull Requests that document the changes (include PR ids): - https://github.com/ESCOMP/ctsm/pull/309 + https://github.com/ESCOMP/ctsm/pull/309 -- Added new files to allow Gross Unrepresented Land Use transition + https://github.com/ESCOMP/ctsm/pull/1965 -- update README + (NOT a PR) Update manage externals + + =============================================================== =============================================================== diff --git a/doc/ChangeSum b/doc/ChangeSum index 66e229d06c..e2bcff8e14 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev119 slevis 02/22/2023 Allow gross unrepresented land use transition (PR #309) + ctsm5.1.dev119 slevis 03/16/2023 Allow gross unrepresented land use transition (PR #309) ctsm5.1.dev118 slevis 02/05/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test ctsm5.1.dev117 afoster 02/02/2023 Updates to facilitate running FATES at NEON sites ctsm5.1.dev116 erik 01/26/2023 Small answer changes with bug fixes, zetamaxstable=2 for BHS, new single point fsurdat files From 1bd5e9ec54da627721da7e88f726da3e9036304a Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 17 Mar 2023 12:17:24 -0600 Subject: [PATCH 231/332] Update to latest CIME This is needed to fix an issue with CTSM-defined system tests --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 1a55310d19..c81dd77664 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -44,7 +44,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.94 +tag = cime6.0.98 required = True [cmeps] From fa421c8363e871073332bf6bbaa068dc3b0da836 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 20 Mar 2023 14:39:17 -0600 Subject: [PATCH 232/332] For the _Ln1 self-tests, ROF needs to couple every time step --- .../testdefs/testmods_dirs/clm/run_self_tests/shell_commands | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands old mode 100644 new mode 100755 index a66f52f6fd..d426269206 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/shell_commands @@ -1,3 +1,5 @@ #!/bin/bash ./xmlchange CLM_FORCE_COLDSTART="on" +# We use this testmod in a _Ln1 test; this requires forcing the ROF coupling frequency to every time step +./xmlchange ROF_NCPL=48 From f9e2a3bda960ede1b95e081063f6af5b66044c56 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 22 Mar 2023 12:19:24 -0600 Subject: [PATCH 233/332] temporary updates for nl and stream changes --- cime_config/config_component.xml | 4 +++- cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 243c866bbd..82c2d545b3 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -244,7 +244,9 @@ Dataset name for user-created datasets. This is used as the argument in Buildconf/clm.buildnml to build-namelist -clm_usr_name. An example of such a dataset would be 1x1pt_boulderCO_c090722. The default value is UNSET. - This is an advanced flag and should only be used by expert users. + This is an advanced flag and should only be used by expert users. For NEON + cases, this can be set to either NEON or NEON.PRISM, the latter of which would + PRISM precipitation instead of the default NEON precipitation. diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands index 7095e1def7..084e0027ba 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands @@ -1,5 +1,6 @@ #!/bin/bash -./xmlchange CLM_USRDAT_NAME=NEON +./xmlchange CLM_USRDAT_NAME=NEON.PRISM +# we should also ensure that this can be set to NEON.PRISM ./xmlchange CCSM_CO2_PPMV=408.83 # Set data forcing data to future scenario so will have data from 2018 to present-day ./xmlchange DATM_PRESAERO=SSP3-7.0 From c0574b4451d195119b83a6e5344c78a9b3e41d60 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 22 Mar 2023 12:29:20 -0600 Subject: [PATCH 234/332] temporary updates for nl and stream changes --- cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands | 2 +- cime_config/usermods_dirs/NEON/defaults/shell_commands | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands index 084e0027ba..aacfad8e15 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands @@ -1,5 +1,5 @@ #!/bin/bash -./xmlchange CLM_USRDAT_NAME=NEON.PRISM +./xmlchange CLM_USRDAT_NAME=NEON # we should also ensure that this can be set to NEON.PRISM ./xmlchange CCSM_CO2_PPMV=408.83 # Set data forcing data to future scenario so will have data from 2018 to present-day diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 7095e1def7..0cbba60f84 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -1,5 +1,6 @@ #!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON +# This is where we want to implement NEON.PRISM; in the current state, doing so causes SVN failures ./xmlchange CCSM_CO2_PPMV=408.83 # Set data forcing data to future scenario so will have data from 2018 to present-day ./xmlchange DATM_PRESAERO=SSP3-7.0 From a022f80e9dafa80824d3b4a316470113e23c3ce6 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 23 Mar 2023 07:20:37 -0600 Subject: [PATCH 235/332] Update COMP_ROOT_DIR_LND to mostly match the version in CIME The COMP_ROOT_DIR_LND is supposed to basically match the entry in CIME's config_files.xml except for the value for CLM. The motivation for this is to add an entry for SLIM: without this change, it seems like other config_files entries were using the CLM location of $SRCROOT for SLIM's COMP_ROOT_DIR_LND, which meant that we were getting duplicate entries of some files, which caused various problems (e.g., https://github.com/ESMCI/cime/issues/4373 - which is now fixed with this change instead of the originally-suggested CIME change - and seeing duplicate results from CIME's query_testlists and query_config scripts). But while making this change, I figured I might as well update the entire entry to be consistent with the latest version in CIME. --- .config_files.xml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.config_files.xml b/.config_files.xml index c6dcbbcdb3..d1a592e5db 100644 --- a/.config_files.xml +++ b/.config_files.xml @@ -5,31 +5,31 @@ char unset - $SRCROOT - $SRCROOT - $SRCROOT/components/cpl7/components/data_comps_mct/dlnd - $SRCROOT/components/cdeps/dlnd - $SRCROOT/components/cpl7/components/stub_comps_mct/slnd - $SRCROOT/components/cpl7/components/xcpl_comps_mct/xlnd - $CIMEROOT/src/components/stub_comps_nuopc/slnd - $CIMEROOT/src/components/xcpl_comps_nuopc/xlnd + $SRCROOT + $SRCROOT/components/slim/ + $SRCROOT/components/cpl7/components/data_comps_$COMP_INTERFACE/dlnd + $SRCROOT/components/cdeps/dlnd + $SRCROOT/components/cpl7/components/stub_comps_$COMP_INTERFACE/slnd + $CIMEROOT/CIME/non_py/src/components/stub_comps_$COMP_INTERFACE/slnd + $SRCROOT/components/cpl7/components/xcpl_comps_$COMP_INTERFACE/xlnd + $CIMEROOT/CIME/non_py/src/components/xcpl_comps_$COMP_INTERFACE/xlnd case_comps env_case.xml Root directory of the case land model component - $CIMEROOT/config/xml_schemas/config_compsets.xsd + $CIMEROOT/CIME/data/config/xml_schemas/config_compsets.xsd From 5b3e2460e31321827d4e5694c933e2e613f3b3d5 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 23 Mar 2023 11:20:54 -0600 Subject: [PATCH 236/332] Update cime --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index c81dd77664..57d472d8c0 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -44,7 +44,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.98 +tag = cime6.0.100 required = True [cmeps] From 2399486724cbe6b3b32808a3456b1102043fcdb7 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 23 Mar 2023 16:44:47 -0600 Subject: [PATCH 237/332] Reduce GU_LULCC tests down to a single test Resolves ESCOMP/CTSM#1968 --- cime_config/testdefs/ExpectedTestFails.xml | 2 +- cime_config/testdefs/testlist_clm.xml | 29 +++---------------- .../ciso_decStart_GU_LULCC/include_user_mods | 1 - .../clm/ciso_decStart_GU_LULCC/user_nl_clm | 1 - .../f09_dec1990Start_GU_LULCC/shell_commands | 2 +- .../include_user_mods | 2 -- 6 files changed, 6 insertions(+), 31 deletions(-) delete mode 100644 cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods delete mode 100644 cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm delete mode 100644 cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 6ddfea3437..d9a1c1b6fd 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -88,7 +88,7 @@ - + PEND #1045 diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 34e22e7272..34e8cf01b3 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -462,39 +462,18 @@ - - - - - - - - - - + - + - + - - - - - - - - - - - - @@ -2037,7 +2016,7 @@ - + diff --git a/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods deleted file mode 100644 index c0adfb5904..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/include_user_mods +++ /dev/null @@ -1 +0,0 @@ -../ciso_decStart diff --git a/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm deleted file mode 100644 index 8d3ca17778..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/ciso_decStart_GU_LULCC/user_nl_clm +++ /dev/null @@ -1 +0,0 @@ -do_grossunrep = .true. diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands index 972cd25f5c..3adf0390b1 100755 --- a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC/shell_commands @@ -1,3 +1,3 @@ -./xmlchange RUN_STARTDATE=1990-12-25 +./xmlchange RUN_STARTDATE=1990-12-30 # Ignore warnings because we are using crop, but starting from a different date than the initial conditions were for ./xmlchange CLM_BLDNML_OPTS=-ignore_warnings --append diff --git a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods deleted file mode 100644 index b10682f861..0000000000 --- a/cime_config/testdefs/testmods_dirs/clm/f09_dec1990Start_GU_LULCC_monthly/include_user_mods +++ /dev/null @@ -1,2 +0,0 @@ -../f09_dec1990Start_GU_LULCC -../monthly From af56c533e5f4a185b03ced15ae26d5285d6fc412 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 23 Mar 2023 17:16:23 -0600 Subject: [PATCH 238/332] Re-raise caught exception after printing error message Otherwise the test continues on after failure. Resolves ESCOMP/CTSM#1971 --- cime_config/SystemTests/fsurdatmodifyctsm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cime_config/SystemTests/fsurdatmodifyctsm.py b/cime_config/SystemTests/fsurdatmodifyctsm.py index 64d945760b..1427a7550f 100644 --- a/cime_config/SystemTests/fsurdatmodifyctsm.py +++ b/cime_config/SystemTests/fsurdatmodifyctsm.py @@ -89,6 +89,7 @@ def _run_modify_fsurdat(self): print("getting an error within the fsurdat_modifier tool itself. ") print("Default error message: ") print(error.output) + raise except: print("ERROR trying to run fsurdat_modifier tool.") raise From c9f8c28a894a1121627f34ca8a61dc5234f6fc47 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 23 Mar 2023 17:37:31 -0600 Subject: [PATCH 239/332] Increase time for C96 test With the recent externals update, this C96 test barely finished in the 10-minute allotted time once, and timed out a different time. Increasing the wall-clock time. (There has been investigation into the cause of recent slowness... the culprit seems to be mpt 2.25.) --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 34e8cf01b3..40ab7a48df 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -327,7 +327,7 @@ - + From 3cd1351964acd9e3fea7d8b3ced2f53de16bf13e Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sat, 25 Mar 2023 16:15:18 -0600 Subject: [PATCH 240/332] Add FUNITCTSM test to expected fails list --- cime_config/testdefs/ExpectedTestFails.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index d9a1c1b6fd..01d0b738be 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -51,6 +51,13 @@ + + + FAIL + #1972 + + + From 2ee9024597cd21c1b755bd24e55e4b1d856d256d Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sat, 25 Mar 2023 17:45:57 -0600 Subject: [PATCH 241/332] Add Fortran unit tests as a separate test to run This is needed until https://github.com/ESCOMP/CTSM/issues/1972 is resolved --- doc/.ChangeLog_template | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/.ChangeLog_template b/doc/.ChangeLog_template index 63f4628bad..ccf5752b7d 100644 --- a/doc/.ChangeLog_template +++ b/doc/.ChangeLog_template @@ -121,10 +121,15 @@ infrastructure should be run when appropriate, as described below. clm_pymods test suite on cheyenne - - regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + regular tests: + - aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing + - Fortran unit tests on cheyenne (until https://github.com/ESCOMP/CTSM/issues/1972 is resolved): from src, run: + ../cime/scripts/fortran_unit_testing/run_tests.py --build-dir `mktemp -d --tmpdir=. unit_tests.XXXXXXXX` + + aux_clm on cheyenne ------------ + aux_clm on izumi --------------- + Fortran unit tests on cheyenne - - cheyenne ---- - izumi ------- fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) cheyenne ---- From 97855df50fdbac07dbf759e0552acfa5f3526d5e Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sat, 25 Mar 2023 18:01:31 -0600 Subject: [PATCH 242/332] Update ChangeLog --- doc/ChangeLog | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 103 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index a604f675f4..0776ca96f0 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,106 @@ =============================================================== +Tag name: ctsm5.1.dev120 +Originator(s): sacks (Bill Sacks) +Date: Sat Mar 25 17:49:27 MDT 2023 +One-line Summary: Update externals and minor fixes + +Purpose and description of changes +---------------------------------- + +Main change is to update externals to cesm2_3_alpha12c-ish. + +Doing this exposed a few issues that are also fixed here. + +Also, reduce GU_LULCC tests down to a single test. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +- Resolves ESCOMP/CTSM#1968 (Reduce the GULU tests down to one) +- Resolves ESCOMP/CTSM#1971 (fsurdatmodifyctsm test should abort if it has trouble running the python script) + +Known bugs introduced in this tag (include issue #): +- ESCOMP/CTSM#1972 (FUNITCTSM test fails when run through run_sys_tests in upcoming ctsm5.1.dev120) + + +Notes of particular relevance for developers: +--------------------------------------------- + +Changes to tests or testing: +- Fortran unit tests now need to be run manually, since FUNITCTSM is + failing when run through run_sys_tests +- Reduced GU_LULCC tests down to a single test + + +Testing summary: +---------------- + + regular tests: + - aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing + - Fortran unit tests on cheyenne (until https://github.com/ESCOMP/CTSM/issues/1972 is resolved): from src, run: + ../cime/scripts/fortran_unit_testing/run_tests.py --build-dir `mktemp -d --tmpdir=. unit_tests.XXXXXXXX` + + aux_clm on cheyenne ------------ OK + aux_clm on izumi --------------- OK + Fortran unit tests on cheyenne - PASS + + For the two new tests (with BFAILs), ran them from dev119 with + comparison against this branch: + - ERP_D_Ld10_P36x2.f10_f10_mg37.IHistClm51BgcCrop.cheyenne_intel.clm-ciso_decStart + - SMS_Ld3_PS.f09_g17.IHistClm50BgcCrop.cheyenne_intel.clm-f09_dec1990Start_GU_LULCC + (with start date in the test mod changed to match the new version) + +Answer changes +-------------- + +Changes answers relative to baseline: YES, but just for certain compilers + + Summarize any changes to answers, i.e., + - what code configurations: all on certain compilers + - what platforms/compilers: + - nvhpc on cheyenne in non-debug cases (can be explained from + differences in compilation flags for non-debug cases, and also + some module differences) + - intel on izumi in debug cases (there were updates in ESMF + modules, though that's the same for other izumi compilers; I'm not + seeing other relevant diffs in ccs_config, so I'm not sure why we're + getting diffs here. I tried investigating, but ran into trouble trying + to get things to compile with the old ccs_config, so gave up on + tracking down the source of this difference) + + - nature of change (roundoff; larger than roundoff/same climate; new climate): + not investigated + +Other details +------------- +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): +- ccs_config: ccs_config_cesm0.0.38 -> ccs_config_cesm0.0.58 +- cime: cime6.0.45 -> cime6.0.100 +- cmeps: cmeps0.13.71 -> cmeps0.14.17 +- cdeps: cdeps0.12.65 -> cdeps1.0.7 +- share: share1.0.13 -> share1.0.16 +- pio: pio2_5_7 -> pio2_5_10 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev119 Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310), ekluzek Erik Kluzek), lawrencepj1 (Peter Lawrence) Date: Thu Mar 16 14:13:37 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index e2bcff8e14..e718109e03 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev120 sacks 03/25/2023 Update externals and minor fixes ctsm5.1.dev119 slevis 03/16/2023 Allow gross unrepresented land use transition (PR #309) ctsm5.1.dev118 slevis 02/05/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test ctsm5.1.dev117 afoster 02/02/2023 Updates to facilitate running FATES at NEON sites From 83d31e107a5d37400818ff59acc063e4762a9f60 Mon Sep 17 00:00:00 2001 From: adamrher Date: Sat, 25 Mar 2023 18:30:24 -0600 Subject: [PATCH 243/332] add new grids to namelist files --- bld/namelist_files/namelist_defaults_ctsm.xml | 18 ++++++++++++++++++ .../namelist_definition_ctsm.xml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 41103b5259..add537337a 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1092,8 +1092,14 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc + +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230221.nc + +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230221.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc + +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230221.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr2000_c200426.nc @@ -1167,6 +1173,12 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc + +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230221.nc + +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230221.nc + +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230221.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1233,6 +1245,12 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230221.nc +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230221.nc +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230221.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc + valid_values="512x1024,360x720cru,128x256,64x128,48x96,94x192,0.23x0.31,0.47x0.63,0.9x1.25,1.9x2.5,2.5x3.33,4x5,10x15,0.125nldas2,5x5_amazon,1x1_vancouverCAN,1x1_mexicocityMEX,1x1_asphaltjungleNJ,1x1_brazil,1x1_urbanc_alpha,1x1_numaIA,1x1_smallvilleIA,0.25x0.25,0.5x0.5,3x3min,5x5min,10x10min,0.33x0.33,0.125x0.125,ne3np4.pg3,ne4np4,ne5np4.pg3,ne16np4,ne16np4.pg3,ne30np4.pg2,ne30np4.pg3,ne30np4,ne60np4,ne120np4,ne120np4.pg2,ne120np4.pg3,ne0np4CONUS.ne30x8,ne0np4.ARCTIC.ne30x4,ne0np4.ARCTICGRIS.ne30x8,ne240np4,1km-merge-10min,C24,C48,C96,C192,C384"> Horizontal resolutions Note: 0.25x0.25, 0.5x0.5, 5x5min, 10x10min, 3x3min, 1km-merge-10min and 0.33x0.33 are only used for CLM toolsI From a1fb1a3087014fd5f67d5fbb4bab2496f8f6d563 Mon Sep 17 00:00:00 2001 From: adamrher Date: Mon, 27 Mar 2023 10:10:05 -0600 Subject: [PATCH 244/332] remade all surface datasets using dev120 tools --- bld/namelist_files/namelist_defaults_ctsm.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index add537337a..237b14f5e9 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1093,13 +1093,13 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc -/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230221.nc +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230326.nc -/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230221.nc +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230326.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc -/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230221.nc +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230326.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr2000_c200426.nc @@ -1174,11 +1174,11 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc -/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230221.nc +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230326.nc -/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230221.nc +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230326.nc -/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230221.nc +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230326.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1246,11 +1246,11 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc /glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230221.nc + >/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230326.nc /glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230221.nc + >/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230326.nc /glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230221.nc + >/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230326.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc Date: Mon, 27 Mar 2023 17:07:05 -0600 Subject: [PATCH 245/332] update parameter file pointer --- bld/namelist_files/namelist_defaults_ctsm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 41103b5259..ff9caff5c1 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -493,7 +493,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.25.0.0_12pft_c221128.nc +lnd/clm2/paramdata/fates_params_api.25.4.0_12pft_c230327.nc From 2c486781d2fc19b7ff678c393e803767feab8d15 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 28 Mar 2023 14:36:42 -0600 Subject: [PATCH 246/332] Increase wallclock time for fates f10 gnu test --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 40ab7a48df..38125d80bc 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1876,7 +1876,7 @@ - + From 02a4f9ccca597b32cda5adea59a632de38db72a7 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 28 Mar 2023 14:52:42 -0600 Subject: [PATCH 247/332] update fates tag --- Externals_CLM.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg index 219d1a9704..b1cffb373c 100644 --- a/Externals_CLM.cfg +++ b/Externals_CLM.cfg @@ -2,7 +2,7 @@ local_path = src/fates protocol = git repo_url = https://github.com/NGEET/fates -tag = sci.1.61.0_api.25.0.0 +tag = sci.1.65.2_api.25.4.0 required = True [externals_description] From d94f5d09a9b505e844920b2d4dff18d291af7c26 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 29 Mar 2023 15:11:07 -0600 Subject: [PATCH 248/332] include third datm stream for NEON --- cime_config/config_component.xml | 10 ++++++++++ cime_config/usermods_dirs/NEON/NIWO/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/defaults/shell_commands | 2 +- .../usermods_dirs/NEON/defaults/user_nl_datm_streams | 10 ---------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 82c2d545b3..7e11047b6b 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -297,6 +297,16 @@ ABBY,BLAN,CPER,DEJU,GRSM,HEAL,KONA,LENO,NIWO,ONAQ,PUUM,SERC,SRER,TALL,TREE,WOOD, BARR,BONA,DCFS,DELA,GUAN,JERC,KONZ,MLBS,NOGP,ORNL,RMNP,SJER,STEI,TEAK,UKFS,WREF, BART,CLBJ,DSNY,HARV,JORN,LAJA,MOAB,OAES,OSBS,SCBI,SOAP,STER,TOOL,UNDE,YELL, + NEON_PRECIP.ABBY,NEON_PRECIP.BLAN,NEON_PRECIP.CPER,NEON_PRECIP.DEJU,NEON_PRECIP.GRSM, + NEON_PRECIP.HEAL,NEON_PRECIP.KONA,NEON_PRECIP.LENO,NEON_PRECIP.NIWO,NEON_PRECIP.ONAQ, + NEON_PRECIP.PUUM,NEON_PRECIP.SERC,NEON_PRECIP.SRER,NEON_PRECIP.TALL,NEON_PRECIP.TREE, + NEON_PRECIP.WOOD,NEON_PRECIP.BARR,NEON_PRECIP.BONA,NEON_PRECIP.DCFS,NEON_PRECIP.DELA, + NEON_PRECIP.GUAN,NEON_PRECIP.JERC,NEON_PRECIP.KONZ,NEON_PRECIP.MLBS,NEON_PRECIP.NOGP, + NEON_PRECIP.ORNL,NEON_PRECIP.RMNP,NEON_PRECIP.SJER,NEON_PRECIP.STEI,NEON_PRECIP.TEAK, + NEON_PRECIP.UKFS,NEON_PRECIP.WREF,NEON_PRECIP.BART,NEON_PRECIP.CLBJ,NEON_PRECIP.DSNY, + NEON_PRECIP.HARV,NEON_PRECIP.JORN,NEON_PRECIP.LAJA,NEON_PRECIP.MOAB,NEON_PRECIP.OAES, + NEON_PRECIP.OSBS,NEON_PRECIP.SCBI,NEON_PRECIP.SOAP,NEON_PRECIP.STER,NEON_PRECIP.TOOL, + NEON_PRECIP.UNDE,NEON_PRECIP.YELL, PRISM_PRECIP.ABBY,PRISM_PRECIP.BLAN,PRISM_PRECIP.CPER,PRISM_PRECIP.DEJU,PRISM_PRECIP.GRSM, PRISM_PRECIP.HEAL,PRISM_PRECIP.KONA,PRISM_PRECIP.LENO,PRISM_PRECIP.NIWO,PRISM_PRECIP.ONAQ, PRISM_PRECIP.PUUM,PRISM_PRECIP.SERC,PRISM_PRECIP.SRER,PRISM_PRECIP.TALL,PRISM_PRECIP.TREE, diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index 65926f2188..746a47d8c0 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -1,3 +1,5 @@ ./xmlchange NEONSITE=NIWO ./xmlchange PTS_LON=254.41676 ./xmlchange PTS_LAT=40.05236 +./xmlchange DATM_YR_END=2018 +# remove DATM_YR_END xml change when running with PRISM precipitation diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index 0cbba60f84..ad23164b73 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -1,6 +1,6 @@ #!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON -# This is where we want to implement NEON.PRISM; in the current state, doing so causes SVN failures +# This can be set to either NEON or NEON.PRISM ./xmlchange CCSM_CO2_PPMV=408.83 # Set data forcing data to future scenario so will have data from 2018 to present-day ./xmlchange DATM_PRESAERO=SSP3-7.0 diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index 16b7c739a7..53c4e4e4a7 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -47,13 +47,3 @@ NEON.${NEONSITE}:datavars = \ PSRF Sa_pbot, \ FLDS Faxa_lwdn -NEON.PRISM_PRECIP.${NEONSITE}:datavars = \ - PRECIP Faxa_precn - -NEON.PRISM_PRECIP.${NEONSITE}:datafiles = \ - /glade/u/home/tking/neon/prism_${NEONSITE}_2018.nc, \ - /glade/u/home/tking/neon/prism_${NEONSITE}_2019.nc, \ - /glade/u/home/tking/neon/prism_${NEONSITE}_2020.nc, \ - /glade/u/home/tking/neon/prism_${NEONSITE}_2021.nc, \ - /glade/u/home/tking/neon/prism_${NEONSITE}_2022.nc - From e09b21c9e6a596567ac8a20b693df681943d0ab0 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 31 Mar 2023 12:26:49 -0600 Subject: [PATCH 249/332] implement shell command switch for datm_yr_* --- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 4 +++- cime_config/usermods_dirs/NEON/MOAB/shell_commands | 4 +++- cime_config/usermods_dirs/NEON/NIWO/shell_commands | 4 +++- cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 4 +++- cime_config/usermods_dirs/NEON/SJER/shell_commands | 5 ++++- cime_config/usermods_dirs/NEON/TEAK/shell_commands | 5 ++++- cime_config/usermods_dirs/NEON/WREF/shell_commands | 5 ++++- cime_config/usermods_dirs/NEON/YELL/shell_commands | 5 ++++- 8 files changed, 28 insertions(+), 8 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index d5de0f64eb..3070c7d594 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -2,7 +2,9 @@ ./xmlchange NEONSITE=MLBS ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 -./xmlchange DATM_YR_END=2019 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_END=2019 +fi # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange DATM_YR_END=2020 diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index 96d0bcbe68..c6e483d0a9 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -2,7 +2,9 @@ ./xmlchange NEONSITE=MOAB ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 -./xmlchange DATM_YR_END=2020 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_END=2020 +fi # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange DATM_YR_END=2021 diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index 746a47d8c0..3f40b8b514 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -1,5 +1,7 @@ ./xmlchange NEONSITE=NIWO ./xmlchange PTS_LON=254.41676 ./xmlchange PTS_LAT=40.05236 -./xmlchange DATM_YR_END=2018 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_END=2018 +fi # remove DATM_YR_END xml change when running with PRISM precipitation diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 511813398b..9b54cfb0a9 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -2,7 +2,9 @@ ./xmlchange NEONSITE=ONAQ ./xmlchange PTS_LON=247.54755 ./xmlchange PTS_LAT=40.17760 -./xmlchange DATM_YR_END=2019 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_END=2019 +fi # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange DATM_YR_END=2020 diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 6e05d23792..e521af98a8 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -2,7 +2,10 @@ ./xmlchange NEONSITE=SJER ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +./xmlchange DATM_YR_ALIGN=2019 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_START=2019 +fi # Different default start date and number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 9058eda022..c0930905a0 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -2,7 +2,10 @@ ./xmlchange NEONSITE=TEAK ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +./xmlchange DATM_YR_ALIGN=2019 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_START=2019 +fi # Different default start date and number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index 807055ae6e..e6fc742fe3 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -2,7 +2,10 @@ ./xmlchange NEONSITE=WREF ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +./xmlchange DATM_YR_ALIGN=2019 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_START=2019 +fi # Different default start date and number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index 800d7d01ad..d313af5dba 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -2,7 +2,10 @@ ./xmlchange NEONSITE=YELL ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 -./xmlchange DATM_YR_ALIGN=2019,DATM_YR_START=2019 +./xmlchange DATM_YR_ALIGN=2019 +if [[$CLM_USRDAT_NAME = 'NEON']]; then + ./xmlchange DATM_YR_START=2019 +fi # Different default start date and number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange RUN_STARTDATE=2019-01-01 From f11fe378abce6e68886994005967ea6cffa16914 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 31 Mar 2023 13:56:32 -0600 Subject: [PATCH 250/332] update shell command if nesting and non CONUS sites --- cime_config/config_component.xml | 20 +++++++++---------- .../usermods_dirs/NEON/MLBS/shell_commands | 2 +- .../usermods_dirs/NEON/MOAB/shell_commands | 13 ++++++------ .../usermods_dirs/NEON/NIWO/shell_commands | 1 - .../usermods_dirs/NEON/ONAQ/shell_commands | 13 ++++++------ .../usermods_dirs/NEON/SJER/shell_commands | 13 ++++++------ .../usermods_dirs/NEON/TEAK/shell_commands | 13 ++++++------ .../usermods_dirs/NEON/WREF/shell_commands | 13 ++++++------ .../usermods_dirs/NEON/YELL/shell_commands | 13 ++++++------ 9 files changed, 53 insertions(+), 48 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 7e11047b6b..786ce7cf42 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -307,16 +307,16 @@ NEON_PRECIP.HARV,NEON_PRECIP.JORN,NEON_PRECIP.LAJA,NEON_PRECIP.MOAB,NEON_PRECIP.OAES, NEON_PRECIP.OSBS,NEON_PRECIP.SCBI,NEON_PRECIP.SOAP,NEON_PRECIP.STER,NEON_PRECIP.TOOL, NEON_PRECIP.UNDE,NEON_PRECIP.YELL, - PRISM_PRECIP.ABBY,PRISM_PRECIP.BLAN,PRISM_PRECIP.CPER,PRISM_PRECIP.DEJU,PRISM_PRECIP.GRSM, - PRISM_PRECIP.HEAL,PRISM_PRECIP.KONA,PRISM_PRECIP.LENO,PRISM_PRECIP.NIWO,PRISM_PRECIP.ONAQ, - PRISM_PRECIP.PUUM,PRISM_PRECIP.SERC,PRISM_PRECIP.SRER,PRISM_PRECIP.TALL,PRISM_PRECIP.TREE, - PRISM_PRECIP.WOOD,PRISM_PRECIP.BARR,PRISM_PRECIP.BONA,PRISM_PRECIP.DCFS,PRISM_PRECIP.DELA, - PRISM_PRECIP.GUAN,PRISM_PRECIP.JERC,PRISM_PRECIP.KONZ,PRISM_PRECIP.MLBS,PRISM_PRECIP.NOGP, - PRISM_PRECIP.ORNL,PRISM_PRECIP.RMNP,PRISM_PRECIP.SJER,PRISM_PRECIP.STEI,PRISM_PRECIP.TEAK, - PRISM_PRECIP.UKFS,PRISM_PRECIP.WREF,PRISM_PRECIP.BART,PRISM_PRECIP.CLBJ,PRISM_PRECIP.DSNY, - PRISM_PRECIP.HARV,PRISM_PRECIP.JORN,PRISM_PRECIP.LAJA,PRISM_PRECIP.MOAB,PRISM_PRECIP.OAES, - PRISM_PRECIP.OSBS,PRISM_PRECIP.SCBI,PRISM_PRECIP.SOAP,PRISM_PRECIP.STER,PRISM_PRECIP.TOOL, - PRISM_PRECIP.UNDE,PRISM_PRECIP.YELL, + PRISM_PRECIP.ABBY,PRISM_PRECIP.BLAN,PRISM_PRECIP.CPER,PRISM_PRECIP.GRSM, + PRISM_PRECIP.KONA,PRISM_PRECIP.LENO,PRISM_PRECIP.NIWO,PRISM_PRECIP.ONAQ, + PRISM_PRECIP.SERC,PRISM_PRECIP.SRER,PRISM_PRECIP.TALL,PRISM_PRECIP.TREE, + PRISM_PRECIP.WOOD,PRISM_PRECIP.BONA,PRISM_PRECIP.DCFS,PRISM_PRECIP.DELA, + PRISM_PRECIP.JERC,PRISM_PRECIP.KONZ,PRISM_PRECIP.MLBS,PRISM_PRECIP.NOGP, + PRISM_PRECIP.ORNL,PRISM_PRECIP.RMNP,PRISM_PRECIP.SJER,PRISM_PRECIP.STEI, + PRISM_PRECIP.TEAK,PRISM_PRECIP.UKFS,PRISM_PRECIP.WREF,PRISM_PRECIP.BART, + PRISM_PRECIP.CLBJ,PRISM_PRECIP.DSNY,PRISM_PRECIP.HARV,PRISM_PRECIP.JORN, + PRISM_PRECIP.MOAB,PRISM_PRECIP.OAES,PRISM_PRECIP.OSBS,PRISM_PRECIP.SCBI, + PRISM_PRECIP.SOAP,PRISM_PRECIP.STER,PRISM_PRECIP.UNDE,PRISM_PRECIP.YELL, run_component_ctsm diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index 3070c7d594..376fc42add 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_END=2019 + ./xmlchange DATM_YR_END=2019 fi # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index c6e483d0a9..6a811c1e4a 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -3,10 +3,11 @@ ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_END=2020 -fi -# Different default number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then - ./xmlchange DATM_YR_END=2021 - ./xmlchange STOP_N=36 + # Different default number of months to run for transient case + if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2021 + ./xmlchange STOP_N=36 + else + ./xmlchange DATM_YR_END=2020 + fi fi diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index 3f40b8b514..483647d965 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -4,4 +4,3 @@ if [[$CLM_USRDAT_NAME = 'NEON']]; then ./xmlchange DATM_YR_END=2018 fi -# remove DATM_YR_END xml change when running with PRISM precipitation diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 9b54cfb0a9..f38cc944b6 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -3,10 +3,11 @@ ./xmlchange PTS_LON=247.54755 ./xmlchange PTS_LAT=40.17760 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_END=2019 -fi -# Different default number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then - ./xmlchange DATM_YR_END=2020 - ./xmlchange STOP_N=24 + # Different default number of months to run for transient case + if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_END=2020 + ./xmlchange STOP_N=24 + else + ./xmlchange DATM_YR_END=2019 + fi fi diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index e521af98a8..4b95814c7b 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -4,11 +4,12 @@ ./xmlchange PTS_LAT=37.107117 ./xmlchange DATM_YR_ALIGN=2019 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_START=2019 -fi -# Different default start date and number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then - ./xmlchange RUN_STARTDATE=2019-01-01 - ./xmlchange STOP_N=39 + # Different default start date and number of months to run for transient case + if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 + else + ./xmlchange DATM_YR_START=2019 + fi fi diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index c0930905a0..064354edb5 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -4,10 +4,11 @@ ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_START=2019 -fi -# Different default start date and number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then - ./xmlchange RUN_STARTDATE=2019-01-01 - ./xmlchange STOP_N=39 + # Different default start date and number of months to run for transient case + if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 + else + ./xmlchange DATM_YR_START=2019 + fi fi diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index e6fc742fe3..9be4d1719a 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -4,10 +4,11 @@ ./xmlchange PTS_LAT=45.81637 ./xmlchange DATM_YR_ALIGN=2019 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_START=2019 -fi -# Different default start date and number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then - ./xmlchange RUN_STARTDATE=2019-01-01 - ./xmlchange STOP_N=39 + # Different default start date and number of months to run for transient case + if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 + else + ./xmlchange DATM_YR_START=2019 + fi fi diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index d313af5dba..bb76a7cc34 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -4,10 +4,11 @@ ./xmlchange PTS_LAT=44.95597 ./xmlchange DATM_YR_ALIGN=2019 if [[$CLM_USRDAT_NAME = 'NEON']]; then - ./xmlchange DATM_YR_START=2019 -fi -# Different default start date and number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then - ./xmlchange RUN_STARTDATE=2019-01-01 - ./xmlchange STOP_N=39 + # Different default start date and number of months to run for transient case + if [[ $compset =~ ^HIST ]]; then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 + else + ./xmlchange DATM_YR_START=2019 + fi fi From 84d7f0df9acbd718002652e98c50d327e2b56b02 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 31 Mar 2023 14:36:24 -0600 Subject: [PATCH 251/332] remove BONA prism precip --- cime_config/config_component.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 786ce7cf42..b0f77cb1c3 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -310,13 +310,13 @@ PRISM_PRECIP.ABBY,PRISM_PRECIP.BLAN,PRISM_PRECIP.CPER,PRISM_PRECIP.GRSM, PRISM_PRECIP.KONA,PRISM_PRECIP.LENO,PRISM_PRECIP.NIWO,PRISM_PRECIP.ONAQ, PRISM_PRECIP.SERC,PRISM_PRECIP.SRER,PRISM_PRECIP.TALL,PRISM_PRECIP.TREE, - PRISM_PRECIP.WOOD,PRISM_PRECIP.BONA,PRISM_PRECIP.DCFS,PRISM_PRECIP.DELA, - PRISM_PRECIP.JERC,PRISM_PRECIP.KONZ,PRISM_PRECIP.MLBS,PRISM_PRECIP.NOGP, - PRISM_PRECIP.ORNL,PRISM_PRECIP.RMNP,PRISM_PRECIP.SJER,PRISM_PRECIP.STEI, - PRISM_PRECIP.TEAK,PRISM_PRECIP.UKFS,PRISM_PRECIP.WREF,PRISM_PRECIP.BART, - PRISM_PRECIP.CLBJ,PRISM_PRECIP.DSNY,PRISM_PRECIP.HARV,PRISM_PRECIP.JORN, - PRISM_PRECIP.MOAB,PRISM_PRECIP.OAES,PRISM_PRECIP.OSBS,PRISM_PRECIP.SCBI, - PRISM_PRECIP.SOAP,PRISM_PRECIP.STER,PRISM_PRECIP.UNDE,PRISM_PRECIP.YELL, + PRISM_PRECIP.WOOD,PRISM_PRECIP.DCFS,PRISM_PRECIP.DELA,PRISM_PRECIP.JERC, + PRISM_PRECIP.KONZ,PRISM_PRECIP.MLBS,PRISM_PRECIP.NOGP,PRISM_PRECIP.ORNL, + PRISM_PRECIP.RMNP,PRISM_PRECIP.SJER,PRISM_PRECIP.STEI,PRISM_PRECIP.TEAK, + PRISM_PRECIP.UKFS,PRISM_PRECIP.WREF,PRISM_PRECIP.BART,PRISM_PRECIP.CLBJ, + PRISM_PRECIP.DSNY,PRISM_PRECIP.HARV,PRISM_PRECIP.JORN,PRISM_PRECIP.MOAB, + PRISM_PRECIP.OAES,PRISM_PRECIP.OSBS,PRISM_PRECIP.SCBI,PRISM_PRECIP.SOAP, + PRISM_PRECIP.STER,PRISM_PRECIP.UNDE,PRISM_PRECIP.YELL, run_component_ctsm From 525d79ecc7d4db4af3a4222b94cdd7e854436845 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 31 Mar 2023 15:50:44 -0600 Subject: [PATCH 252/332] Revert some commits now that FUNITCTSM works again - Revert "Add Fortran unit tests as a separate test to run". This reverts commit 2ee9024597cd21c1b755bd24e55e4b1d856d256d. - Revert "Add FUNITCTSM test to expected fails list". This reverts commit 3cd1351964acd9e3fea7d8b3ced2f53de16bf13e. --- cime_config/testdefs/ExpectedTestFails.xml | 7 ------- doc/.ChangeLog_template | 11 +++-------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 01d0b738be..d9a1c1b6fd 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -51,13 +51,6 @@ - - - FAIL - #1972 - - - diff --git a/doc/.ChangeLog_template b/doc/.ChangeLog_template index ccf5752b7d..63f4628bad 100644 --- a/doc/.ChangeLog_template +++ b/doc/.ChangeLog_template @@ -121,15 +121,10 @@ infrastructure should be run when appropriate, as described below. clm_pymods test suite on cheyenne - - regular tests: - - aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing - - Fortran unit tests on cheyenne (until https://github.com/ESCOMP/CTSM/issues/1972 is resolved): from src, run: - ../cime/scripts/fortran_unit_testing/run_tests.py --build-dir `mktemp -d --tmpdir=. unit_tests.XXXXXXXX` - - aux_clm on cheyenne ------------ - aux_clm on izumi --------------- - Fortran unit tests on cheyenne - + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + cheyenne ---- + izumi ------- fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) cheyenne ---- From a0abc298330c2aceb51830c0013b4d3ae6bb53d5 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 3 Apr 2023 18:15:43 -0600 Subject: [PATCH 253/332] Update fates tag externals pointer This tag includes a fix to avoid line continuation issues seen on izumi. --- Externals_CLM.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg index b1cffb373c..f4a9eae9c6 100644 --- a/Externals_CLM.cfg +++ b/Externals_CLM.cfg @@ -2,7 +2,7 @@ local_path = src/fates protocol = git repo_url = https://github.com/NGEET/fates -tag = sci.1.65.2_api.25.4.0 +tag = sci.1.65.3_api.25.4.0 required = True [externals_description] From 55ce27f1242fc2c4b8f91cc3bc68ad5847a2bde2 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Tue, 4 Apr 2023 15:52:33 -0700 Subject: [PATCH 254/332] Update Changelog --- doc/ChangeLog | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 104 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 0776ca96f0..9311371add 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,107 @@ =============================================================== +Tag name: ctsm5.1.dev121 +Originator(s): glemieux (Gregory Lemieux,LBL/NGEET,510-486-5049) +Date: Tue Apr 4 04:34:20 PM MDT 2023 +One-line Summary: Changes soil moisture initialization logic for FATES + +Purpose and description of changes +---------------------------------- + +This PR changes the logic for soil moisture initialization to initialize +with wetter soils (75% of saturated water content, as opposed to 15% of +absolute water content) for all FATES configurations. The rationale for +this is that in FATES-nocomp simulations, Jessica Needham was finding very +high initial mortality rates in some seasonal tropical forest regions which +she traced it back to the initial soil moisture killing off plants before the +ecosystem could get established. + +This also updates the fates externals pointer to the latest tag which includes +a number of science updates since the last tag update and updates the the +default parameter file. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +Externals issues fixed (include issue #): + Partially addresses FATES#994 -- Bare ground establishment problem and increased soil moisture + +Notes of particular relevance for users +--------------------------------------- + +Changes made to namelist defaults (e.g., changed parameter values): + fates_paramfile updated to fates_params_api.25.4.0_12pft_c230327.nc + +Notes of particular relevance for developers: +--------------------------------------------- +NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + +Changes to tests or testing: + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- OK + izumi ------- OK + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + FATES tests run against fates-sci.1.65.3_api.25.4.0-ctsm5.1.dev120 baseline + + +Answer changes +-------------- + +Changes answers relative to baseline: + + Changes answers in fates suite for all non-hydro fates tests since the soil + moisture initialization matches that of fates hydro now. Changes answer + for all fates testmods in the aux_clm suite as the science tag has iterated + forward by 4 minor version updates. All diffs accounted for with prior fates + suite tests. + +Other details +------------- + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): + +- FATES: sci.1.61.0_api.25.0.0 -> sci.1.65.3_api.25.4.0 + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + + https://github.com/ESCOMP/CTSM/pull/1962 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev120 Originator(s): sacks (Bill Sacks) Date: Sat Mar 25 17:49:27 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index e718109e03..fb302ed794 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev121 glemieux 04/04/2023 Changes soil moisture initialization logic for FATES ctsm5.1.dev120 sacks 03/25/2023 Update externals and minor fixes ctsm5.1.dev119 slevis 03/16/2023 Allow gross unrepresented land use transition (PR #309) ctsm5.1.dev118 slevis 02/05/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test From 0c7b8c4f3ae372c6af70687fb3617a6d3b62f0b1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 5 Apr 2023 09:16:08 -0600 Subject: [PATCH 255/332] Changelog updates --- doc/ChangeLog | 16 +++++++++------- doc/ChangeSum | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 9311371add..53c04d01a4 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev121 Originator(s): glemieux (Gregory Lemieux,LBL/NGEET,510-486-5049) -Date: Tue Apr 4 04:34:20 PM MDT 2023 +Date: Wed Apr 5 09:15:55 MDT 2023 One-line Summary: Changes soil moisture initialization logic for FATES Purpose and description of changes @@ -42,6 +42,9 @@ Bugs fixed or introduced Externals issues fixed (include issue #): Partially addresses FATES#994 -- Bare ground establishment problem and increased soil moisture +Known bugs found since the previous tag (include issue #): + #1979 -- Need to loosen tolerance on near-zero truncation of h2osoi_ice in UpdateState_TopLayerFluxes + Notes of particular relevance for users --------------------------------------- @@ -50,13 +53,11 @@ Changes made to namelist defaults (e.g., changed parameter values): Notes of particular relevance for developers: --------------------------------------------- -NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide -[Remove any lines that don't apply. Remove entire section if nothing applies.] Caveats for developers (e.g., code that is duplicated that requires double maintenance): - -Changes to tests or testing: - + This changes the starting level for a COLD start of soil moisture for ALL FATES cases to a much + higher value than for non-FATES. In the long run we'd like to have these the same and/or + have the value changable on the namelist. Testing summary: ---------------- @@ -98,7 +99,8 @@ List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): Pull Requests that document the changes (include PR ids): (https://github.com/ESCOMP/ctsm/pull) - https://github.com/ESCOMP/CTSM/pull/1962 + https://github.com/ESCOMP/CTSM/pull/1962 -- Cold start moisture for FATES increased + https://github.com/ESCOMP/CTSM/pull/1978 -- revert some commits now that FUNITCTSM works again =============================================================== =============================================================== diff --git a/doc/ChangeSum b/doc/ChangeSum index fb302ed794..24a0b303c6 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev121 glemieux 04/04/2023 Changes soil moisture initialization logic for FATES + ctsm5.1.dev121 glemieux 04/05/2023 Changes soil moisture initialization logic for FATES ctsm5.1.dev120 sacks 03/25/2023 Update externals and minor fixes ctsm5.1.dev119 slevis 03/16/2023 Allow gross unrepresented land use transition (PR #309) ctsm5.1.dev118 slevis 02/05/2023 Use conda environment rather than ncar_pylib with the fsurdat_modifier system test From bbe21f418a700fec08dc90174e0431dfd93f0be0 Mon Sep 17 00:00:00 2001 From: adamrher Date: Wed, 5 Apr 2023 09:55:31 -0600 Subject: [PATCH 256/332] remade surface datasets with proper metadata --- bld/namelist_files/namelist_defaults_ctsm.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 237b14f5e9..659c2fd16b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1093,13 +1093,13 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc -/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230326.nc +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230405.nc -/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230326.nc +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc -/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230326.nc +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr2000_c200426.nc @@ -1174,11 +1174,11 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc -/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230326.nc +/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230405.nc -/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230326.nc +/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230405.nc -/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230326.nc +/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1246,11 +1246,11 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc /glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230326.nc + >/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc /glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230326.nc + >/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc /glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230326.nc + >/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc Date: Wed, 5 Apr 2023 13:34:23 -0600 Subject: [PATCH 257/332] Update date for changelog --- doc/ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 53c04d01a4..edb6223e2e 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev121 Originator(s): glemieux (Gregory Lemieux,LBL/NGEET,510-486-5049) -Date: Wed Apr 5 09:15:55 MDT 2023 +Date: Wed Apr 5 13:34:09 MDT 2023 One-line Summary: Changes soil moisture initialization logic for FATES Purpose and description of changes From cecbe6a65df970d218f56346fc4b5602ed98f891 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 6 Apr 2023 11:22:09 -0600 Subject: [PATCH 258/332] include tests --- cime_config/testdefs/testlist_clm.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 4d074eae11..98d07eb75f 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1552,6 +1552,16 @@ + + + + + + + + + + From 8d15cb3bbf4240a3e95a4fcf78366200b7152c3a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 7 Apr 2023 12:54:42 -0600 Subject: [PATCH 259/332] update shell commands --- .../usermods_dirs/NEON/MLBS/shell_commands | 6 ++++-- .../usermods_dirs/NEON/MOAB/shell_commands | 5 ++--- .../usermods_dirs/NEON/NIWO/shell_commands | 4 +++- .../usermods_dirs/NEON/ONAQ/shell_commands | 11 +++++++---- .../usermods_dirs/NEON/SJER/shell_commands | 9 +++++---- .../usermods_dirs/NEON/TEAK/shell_commands | 18 ++++++++++-------- .../usermods_dirs/NEON/WREF/shell_commands | 11 +++++++---- .../usermods_dirs/NEON/YELL/shell_commands | 9 +++++---- 8 files changed, 43 insertions(+), 30 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index 376fc42add..f21cbd680b 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -2,11 +2,13 @@ ./xmlchange NEONSITE=MLBS ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [ $CLM_USRDAT_NAME=='NEON'] +then ./xmlchange DATM_YR_END=2019 fi # Different default number of months to run for transient case -if [[ $compset =~ ^HIST ]]; then +if [ $compset =~ ^HIST ] +then ./xmlchange DATM_YR_END=2020 ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index 6a811c1e4a..c82e8879bf 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -2,12 +2,11 @@ ./xmlchange NEONSITE=MOAB ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [$CLM_USRDAT_NAME=='NEON']; then + ./xmlchange DATM_YR_END=2020 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]]; then ./xmlchange DATM_YR_END=2021 ./xmlchange STOP_N=36 - else - ./xmlchange DATM_YR_END=2020 fi fi diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index 483647d965..4848702826 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -1,6 +1,8 @@ +#!/bin/bash ./xmlchange NEONSITE=NIWO ./xmlchange PTS_LON=254.41676 ./xmlchange PTS_LAT=40.05236 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [ $CLM_USRDAT_NAME=="NEON" ] +then ./xmlchange DATM_YR_END=2018 fi diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index f38cc944b6..ae6f6d4a19 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -2,12 +2,15 @@ ./xmlchange NEONSITE=ONAQ ./xmlchange PTS_LON=247.54755 ./xmlchange PTS_LAT=40.17760 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [ $CLM_USRDAT_NAME=='NEON' ] +then + echo "IN CLM_USRDAT_NAME LOOP!" + ./xmlchange DATM_YR_END=2019 # Different default number of months to run for transient case - if [[ $compset =~ ^HIST ]]; then + if [[ $compset =~ ^HIST ]] + then + echo "IN HIST COMPSET LOOP!" ./xmlchange DATM_YR_END=2020 ./xmlchange STOP_N=24 - else - ./xmlchange DATM_YR_END=2019 fi fi diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 4b95814c7b..e76242e8b5 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -3,13 +3,14 @@ ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 ./xmlchange DATM_YR_ALIGN=2019 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [ $CLM_USRDAT_NAME=='NEON' ] +then + ./xmlchange DATM_YR_START=2019 # Different default start date and number of months to run for transient case - if [[ $compset =~ ^HIST ]]; then + if [[ $compset =~ ^HIST ]] + then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 - else - ./xmlchange DATM_YR_START=2019 fi fi diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 064354edb5..3775581719 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -3,12 +3,14 @@ ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019 -if [[$CLM_USRDAT_NAME = 'NEON']]; then - # Different default start date and number of months to run for transient case - if [[ $compset =~ ^HIST ]]; then - ./xmlchange RUN_STARTDATE=2019-01-01 - ./xmlchange STOP_N=39 - else - ./xmlchange DATM_YR_START=2019 - fi +./xmlchange DATM_YR_START=2019 +#if [ $CLM_USRDAT_NAME=='NEON'] +#then +# Different default start date and number of months to run for transient case +#./xmlchange DATM_YR_START=2019 +if [[ $compset =~ ^HIST ]] +then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 fi +#fi diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index 9be4d1719a..c8cb110cc4 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -3,12 +3,15 @@ ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 ./xmlchange DATM_YR_ALIGN=2019 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [ $CLM_USRDAT_NAME=='NEON' ] +then + echo "IN CLM_USRDAT_NAME LOOP!" # Different default start date and number of months to run for transient case - if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_START=2019 + if [[ $compset =~ ^HIST ]] + then + echo "IN HIST COMPSET LOOP!" ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 - else - ./xmlchange DATM_YR_START=2019 fi fi diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index bb76a7cc34..86eab8ca20 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -3,12 +3,13 @@ ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 ./xmlchange DATM_YR_ALIGN=2019 -if [[$CLM_USRDAT_NAME = 'NEON']]; then +if [ $CLM_USRDAT_NAME=='NEON' ] +then # Different default start date and number of months to run for transient case - if [[ $compset =~ ^HIST ]]; then + ./xmlchange DATM_YR_START=2019 + if [[ $compset =~ ^HIST ]] + then ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 - else - ./xmlchange DATM_YR_START=2019 fi fi From 44246738a18185d28e60ca807467e9dec30567f8 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 10 Apr 2023 11:58:29 -0600 Subject: [PATCH 260/332] update tests --- cime_config/testdefs/testlist_clm.xml | 2 +- cime_config/testdefs/testmods_dirs/clm/PRISM/include_user_mods | 1 + cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 cime_config/testdefs/testmods_dirs/clm/PRISM/include_user_mods create mode 100644 cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 98d07eb75f..5b7b6dfce4 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1552,7 +1552,7 @@ - + diff --git a/cime_config/testdefs/testmods_dirs/clm/PRISM/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/PRISM/include_user_mods new file mode 100644 index 0000000000..fe0e18cf88 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/PRISM/include_user_mods @@ -0,0 +1 @@ +../default diff --git a/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands b/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands new file mode 100644 index 0000000000..0b5ec8cff6 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange CLM_USRDAT_NAME=NEON From c8b80d84d732444a710c5f23fd1d331ce95dcba0 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sat, 8 Apr 2023 09:36:56 -0600 Subject: [PATCH 261/332] Rework handling of evaporation constraint in SoilFluxes Occasionally, h2osoi_ice was going significantly negative in UpdateState_TopLayerFluxes - see https://github.com/ESCOMP/CTSM/issues/1979. As noted in that issue, this seems to be due to h2osoi_ice having a very different magnitude from h2osoi_liq, leading to greater-than-roundoff-level differences from zero final state in a relative sense (i.e., relative to the magnitude of h2osoi_ice) - I think because of the appearance of the sum (h2osoi_ice + h2osoi_liq) in the equations that limit fluxes. To try to deal with this, I have reworked the handling of the evaporation constraint to directly limit both the liqevap and solidevap, so that both of them should result in the equivalent liq or ice states going to 0 within roundoff. To do that, I needed to move the partitioning of the total flux into liquid and solid to earlier in the subroutine and then recalculate those partitioning fluxes in conditions where we're applying an evaporation constraint. Note that the deleted lines in this commit have just been moved as-is to earlier in this subroutine. --- src/biogeophys/SoilFluxesMod.F90 | 159 +++++++++++++++++-------------- 1 file changed, 86 insertions(+), 73 deletions(-) diff --git a/src/biogeophys/SoilFluxesMod.F90 b/src/biogeophys/SoilFluxesMod.F90 index 5f4030c6e1..48820df41b 100644 --- a/src/biogeophys/SoilFluxesMod.F90 +++ b/src/biogeophys/SoilFluxesMod.F90 @@ -205,6 +205,74 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & endif end do + ! Partition evaporation into liquid and solid + do fp = 1, num_nolakep + p = filter_nolakep(fp) + c = patch%column(p) + l = patch%landunit(p) + j = col%snl(c)+1 + + qflx_liqevap_from_top_layer(p) = 0._r8 + qflx_solidevap_from_top_layer(p) = 0._r8 + qflx_soliddew_to_top_layer(p) = 0._r8 + qflx_liqdew_to_top_layer(p) = 0._r8 + + ! Partition the evaporation from snow/soil surface into liquid evaporation, + ! solid evaporation (sublimation), liquid dew, or solid dew. Note that the variables + ! affected here are all related to the snow subgrid patch only because of the use of qflx_ev_snow. + ! In the situations where there are snow layers or there is snow without an explicit snow layer, + ! the partitioned variables will represent the components of snow evaporation + ! (qflx_ev_snow = qflx_liqevap_from_top_layer + qflx_solidevap_from_top_layer + ! - qflx_liqdew_to_top_layer - qflx_soliddew_to_top_layer). + ! In the case of no snow, qflx_ev_snow has already been set equal to qflx_ev_soil (the evaporation + ! from the subgrid soil patch) and the partitioned variables will then represent evaporation from the + ! subgrid soil patch. + ! In the case of urban columns (and lake columns - see LakeHydrologyMod), there are no subgrid + ! patches and qflx_evap_soi is used. qflx_evap_soi = qflx_liqevap_from_top_layer + qflx_solidevap_from_top_layer + ! - qflx_liqdew_to_top_layer - qflx_soliddew_to_top_layer. + if (.not. lun%urbpoi(l)) then + if (qflx_ev_snow(p) >= 0._r8) then + ! for evaporation partitioning between liquid evap and ice sublimation, + ! use the ratio of liquid to (liquid+ice) in the top layer to determine split + if ((h2osoi_liq(c,j)+h2osoi_ice(c,j)) > 0._r8) then + qflx_liqevap_from_top_layer(p) = max(qflx_ev_snow(p)*(h2osoi_liq(c,j)/ & + (h2osoi_liq(c,j)+h2osoi_ice(c,j))), 0._r8) + else + qflx_liqevap_from_top_layer(p) = 0._r8 + end if + qflx_solidevap_from_top_layer(p) = qflx_ev_snow(p) - qflx_liqevap_from_top_layer(p) + else + if (t_grnd(c) < tfrz) then + qflx_soliddew_to_top_layer(p) = abs(qflx_ev_snow(p)) + else + qflx_liqdew_to_top_layer(p) = abs(qflx_ev_snow(p)) + end if + end if + + else ! Urban columns + + if (qflx_evap_soi(p) >= 0._r8) then + ! for evaporation partitioning between liquid evap and ice sublimation, + ! use the ratio of liquid to (liquid+ice) in the top layer to determine split + if ((h2osoi_liq(c,j)+h2osoi_ice(c,j)) > 0._r8) then + qflx_liqevap_from_top_layer(p) = max(qflx_evap_soi(p)*(h2osoi_liq(c,j)/ & + (h2osoi_liq(c,j)+h2osoi_ice(c,j))), 0._r8) + else + qflx_liqevap_from_top_layer(p) = 0._r8 + end if + qflx_solidevap_from_top_layer(p) = qflx_evap_soi(p) - qflx_liqevap_from_top_layer(p) + else + if (t_grnd(c) < tfrz) then + qflx_soliddew_to_top_layer(p) = abs(qflx_evap_soi(p)) + else + qflx_liqdew_to_top_layer(p) = abs(qflx_evap_soi(p)) + end if + end if + + end if + + end do + ! Constrain evaporation from snow to be <= available moisture do fp = 1,num_nolakep p = filter_nolakep(fp) @@ -221,6 +289,8 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & evaporation_demand = qflx_ev_snow(p) qflx_ev_snow(p) = evaporation_limit qflx_evap_soi(p) = qflx_evap_soi(p) - frac_sno_eff(c)*(evaporation_demand - evaporation_limit) + qflx_liqevap_from_top_layer(p) = h2osoi_liq(c,j)/(frac_sno_eff(c)*dtime) + qflx_solidevap_from_top_layer(p) = h2osoi_ice(c,j)/(frac_sno_eff(c)*dtime) ! conserve total energy flux eflx_sh_grnd(p) = eflx_sh_grnd(p) + frac_sno_eff(c)*(evaporation_demand - evaporation_limit)*htvp(c) endif @@ -235,11 +305,27 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & evaporation_demand = qflx_evap_soi(p) qflx_evap_soi(p) = evaporation_limit qflx_ev_snow(p) = qflx_evap_soi(p) + qflx_liqevap_from_top_layer(p) = h2osoi_liq(c,j)/dtime + qflx_solidevap_from_top_layer(p) = h2osoi_ice(c,j)/dtime ! conserve total energy flux eflx_sh_grnd(p) = eflx_sh_grnd(p) +(evaporation_demand -evaporation_limit)*htvp(c) endif endif + ! limit only solid evaporation (sublimation) from top soil layer + ! (liquid evaporation from soil should not be limited) + if (j==1 .and. frac_h2osfc(c) < 1._r8) then + evaporation_limit = h2osoi_ice(c,j)/(dtime*(1._r8 - frac_h2osfc(c))) + if (qflx_solidevap_from_top_layer(p) >= evaporation_limit) then + evaporation_demand = qflx_solidevap_from_top_layer(p) + qflx_solidevap_from_top_layer(p) & + = evaporation_limit + qflx_liqevap_from_top_layer(p) & + = qflx_liqevap_from_top_layer(p) & + + (evaporation_demand - evaporation_limit) + endif + endif + enddo call t_stopf('bgp2_loop_1') @@ -299,79 +385,6 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & eflx_sh_tot_u(p)= eflx_sh_tot(p) end if - qflx_liqevap_from_top_layer(p) = 0._r8 - qflx_solidevap_from_top_layer(p) = 0._r8 - qflx_soliddew_to_top_layer(p) = 0._r8 - qflx_liqdew_to_top_layer(p) = 0._r8 - - ! Partition the evaporation from snow/soil surface into liquid evaporation, - ! solid evaporation (sublimation), liquid dew, or solid dew. Note that the variables - ! affected here are all related to the snow subgrid patch only because of the use of qflx_ev_snow. - ! In the situations where there are snow layers or there is snow without an explicit snow layer, - ! the partitioned variables will represent the components of snow evaporation - ! (qflx_ev_snow = qflx_liqevap_from_top_layer + qflx_solidevap_from_top_layer - ! - qflx_liqdew_to_top_layer - qflx_soliddew_to_top_layer). - ! In the case of no snow, qflx_ev_snow has already been set equal to qflx_ev_soil (the evaporation - ! from the subgrid soil patch) and the partitioned variables will then represent evaporation from the - ! subgrid soil patch. - ! In the case of urban columns (and lake columns - see LakeHydrologyMod), there are no subgrid - ! patches and qflx_evap_soi is used. qflx_evap_soi = qflx_liqevap_from_top_layer + qflx_solidevap_from_top_layer - ! - qflx_liqdew_to_top_layer - qflx_soliddew_to_top_layer. - if (.not. lun%urbpoi(l)) then - if (qflx_ev_snow(p) >= 0._r8) then - ! for evaporation partitioning between liquid evap and ice sublimation, - ! use the ratio of liquid to (liquid+ice) in the top layer to determine split - if ((h2osoi_liq(c,j)+h2osoi_ice(c,j)) > 0._r8) then - qflx_liqevap_from_top_layer(p) = max(qflx_ev_snow(p)*(h2osoi_liq(c,j)/ & - (h2osoi_liq(c,j)+h2osoi_ice(c,j))), 0._r8) - else - qflx_liqevap_from_top_layer(p) = 0._r8 - end if - qflx_solidevap_from_top_layer(p) = qflx_ev_snow(p) - qflx_liqevap_from_top_layer(p) - else - if (t_grnd(c) < tfrz) then - qflx_soliddew_to_top_layer(p) = abs(qflx_ev_snow(p)) - else - qflx_liqdew_to_top_layer(p) = abs(qflx_ev_snow(p)) - end if - end if - - else ! Urban columns - - if (qflx_evap_soi(p) >= 0._r8) then - ! for evaporation partitioning between liquid evap and ice sublimation, - ! use the ratio of liquid to (liquid+ice) in the top layer to determine split - if ((h2osoi_liq(c,j)+h2osoi_ice(c,j)) > 0._r8) then - qflx_liqevap_from_top_layer(p) = max(qflx_evap_soi(p)*(h2osoi_liq(c,j)/ & - (h2osoi_liq(c,j)+h2osoi_ice(c,j))), 0._r8) - else - qflx_liqevap_from_top_layer(p) = 0._r8 - end if - qflx_solidevap_from_top_layer(p) = qflx_evap_soi(p) - qflx_liqevap_from_top_layer(p) - else - if (t_grnd(c) < tfrz) then - qflx_soliddew_to_top_layer(p) = abs(qflx_evap_soi(p)) - else - qflx_liqdew_to_top_layer(p) = abs(qflx_evap_soi(p)) - end if - end if - - end if - - ! limit only solid evaporation (sublimation) from top soil layer - ! (liquid evaporation from soil should not be limited) - if (j==1 .and. frac_h2osfc(c) < 1._r8) then - evaporation_limit = h2osoi_ice(c,j)/(dtime*(1._r8 - frac_h2osfc(c))) - if (qflx_solidevap_from_top_layer(p) >= evaporation_limit) then - evaporation_demand = qflx_solidevap_from_top_layer(p) - qflx_solidevap_from_top_layer(p) & - = evaporation_limit - qflx_liqevap_from_top_layer(p) & - = qflx_liqevap_from_top_layer(p) & - + (evaporation_demand - evaporation_limit) - endif - endif - ! Variables needed by history tape qflx_evap_can(p) = qflx_evap_veg(p) - qflx_tran_veg(p) From 3b35c9ca9343ba57abbfa0c8f088c75c9e1a721e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 11 Apr 2023 07:42:34 -0600 Subject: [PATCH 262/332] shell command updates/cleanup --- .../usermods_dirs/NEON/MLBS/shell_commands | 2 +- .../usermods_dirs/NEON/ONAQ/shell_commands | 2 -- .../usermods_dirs/NEON/TEAK/shell_commands | 18 +++++++++--------- .../usermods_dirs/NEON/WREF/shell_commands | 2 -- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index f21cbd680b..23e3a7014c 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -7,7 +7,7 @@ then ./xmlchange DATM_YR_END=2019 fi # Different default number of months to run for transient case -if [ $compset =~ ^HIST ] +if [[ $compset =~ ^HIST ]] then ./xmlchange DATM_YR_END=2020 ./xmlchange STOP_N=24 diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index ae6f6d4a19..0045025b15 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -4,12 +4,10 @@ ./xmlchange PTS_LAT=40.17760 if [ $CLM_USRDAT_NAME=='NEON' ] then - echo "IN CLM_USRDAT_NAME LOOP!" ./xmlchange DATM_YR_END=2019 # Different default number of months to run for transient case if [[ $compset =~ ^HIST ]] then - echo "IN HIST COMPSET LOOP!" ./xmlchange DATM_YR_END=2020 ./xmlchange STOP_N=24 fi diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 3775581719..0850dd2189 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -3,14 +3,14 @@ ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019 -./xmlchange DATM_YR_START=2019 -#if [ $CLM_USRDAT_NAME=='NEON'] -#then -# Different default start date and number of months to run for transient case -#./xmlchange DATM_YR_START=2019 -if [[ $compset =~ ^HIST ]] +if [ $CLM_USRDAT_NAME=='NEON'] then - ./xmlchange RUN_STARTDATE=2019-01-01 - ./xmlchange STOP_N=39 + ./xmlchange DATM_YR_START=2019 + # Different default start date and number of months to run for transient case + #./xmlchange DATM_YR_START=2019 + if [[ $compset =~ ^HIST ]] + then + ./xmlchange RUN_STARTDATE=2019-01-01 + ./xmlchange STOP_N=39 + fi fi -#fi diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index c8cb110cc4..497ffd73f3 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -5,12 +5,10 @@ ./xmlchange DATM_YR_ALIGN=2019 if [ $CLM_USRDAT_NAME=='NEON' ] then - echo "IN CLM_USRDAT_NAME LOOP!" # Different default start date and number of months to run for transient case ./xmlchange DATM_YR_START=2019 if [[ $compset =~ ^HIST ]] then - echo "IN HIST COMPSET LOOP!" ./xmlchange RUN_STARTDATE=2019-01-01 ./xmlchange STOP_N=39 fi From 00b8bf517e06f2559fa4e591c84e902ce41bb34e Mon Sep 17 00:00:00 2001 From: John Alex Date: Fri, 14 Apr 2023 13:11:29 +0000 Subject: [PATCH 263/332] Added cache for step size. --- src/utils/clm_time_manager.F90 | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/utils/clm_time_manager.F90 b/src/utils/clm_time_manager.F90 index 492dfb3a75..b8d9930b24 100644 --- a/src/utils/clm_time_manager.F90 +++ b/src/utils/clm_time_manager.F90 @@ -97,10 +97,11 @@ module clm_time_manager ref_ymd = uninit_int, &! reference date for time coordinate in yearmmdd format ref_tod = 0 ! reference time of day for time coordinate in seconds type(ESMF_Calendar), target, save :: tm_cal ! calendar - type(ESMF_Clock), save :: tm_clock ! model clock + type(ESMF_Clock), save :: tm_clock ! model clock + integer, save :: tm_clock_step_size_sec ! Cache of clock timestep. type(ESMF_Time), save :: tm_perp_date ! perpetual date - ! Data required to restart time manager: + ! Data required to restart time manager (only set if timemgr_restart_io is called): integer, save :: rst_step_sec = uninit_int ! timestep size seconds integer, save :: rst_start_ymd = uninit_int ! start date integer, save :: rst_start_tod = uninit_int ! start time of day @@ -309,6 +310,12 @@ subroutine init_clock( start_date, ref_date, curr_date ) call ESMF_ClockGet(tm_clock, currTime=current ) call chkrc(rc, sub//': error return from ESMF_ClockGet') end do + + + ! Cache step size, we query it a lot. + call ESMF_TimeIntervalGet(step_size, s=tm_clock_step_size_sec, rc=rc) + call chkrc(rc, sub//': error return from ESMF_ClockTimeIntervalGet') + end subroutine init_clock !========================================================================================= @@ -696,16 +703,10 @@ integer function get_step_size() ! Return the step size in seconds. character(len=*), parameter :: sub = 'clm::get_step_size' - type(ESMF_TimeInterval) :: step_size ! timestep size - integer :: rc if ( .not. check_timemgr_initialized(sub) ) return - call ESMF_ClockGet(tm_clock, timeStep=step_size, rc=rc) - call chkrc(rc, sub//': error return from ESMF_ClockGet') - - call ESMF_TimeIntervalGet(step_size, s=get_step_size, rc=rc) - call chkrc(rc, sub//': error return from ESMF_ClockTimeIntervalGet') + get_step_size = tm_clock_step_size_sec end function get_step_size From 1214de9fd5043571a76c228fc84949f102241403 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 17 Apr 2023 11:00:09 -0600 Subject: [PATCH 264/332] include run_neon updates --- Externals.cfg | 2 +- tools/site_and_regional/run_neon.py | 32 ++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 7a4eeb4a6e..da0f7b293b 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.38 +tag = ccs_config_cesm0.0.60 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 50f0640d03..4f4e6cb968 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -193,6 +193,18 @@ def get_parser(args, description, valid_neon_sites): default="transient", ) + parser.add_argument( + "--experiment", + help=""" + Appends the case name with string for model experiment + """, + action="store", + dest="experiment", + type=str, + required=False, + default=None, + ) + parser.add_argument( "--run-length", help=""" @@ -251,7 +263,7 @@ def get_parser(args, description, valid_neon_sites): dest="user_version", required = False, type = str, - choices= ['v1','v2'], + choices= ['v1','v2','v3'], ) @@ -294,6 +306,7 @@ def get_parser(args, description, valid_neon_sites): neon_sites, args.output_root, args.run_type, + args.experiment, args.overwrite, run_length, base_case_root, @@ -457,6 +470,7 @@ def run_case( setup_only=False, no_batch=False, rerun=False, + experiment=False, ): user_mods_dirs = [ os.path.join( @@ -475,9 +489,15 @@ def run_case( print ("using this version:", version) - case_root = os.path.abspath( - os.path.join(base_case_root, "..", self.name + "." + run_type) - ) + if experiment == None: + case_root = os.path.abspath( + os.path.join(base_case_root, "..", self.name + "." + run_type) + ) + else: + case_root = os.path.abspath( + os.path.join(base_case_root, "..", self.name + "." + experiment + "." + run_type) + ) + rundir = None if os.path.isdir(case_root): if overwrite: @@ -760,6 +780,7 @@ def main(description): site_list, output_root, run_type, + experiment, overwrite, run_length, base_case_root, @@ -767,7 +788,7 @@ def main(description): setup_only, no_batch, rerun, - user_version + user_version, ) = get_parser(sys.argv, description, valid_neon_sites) if output_root: @@ -809,6 +830,7 @@ def main(description): setup_only, no_batch, rerun, + experiment, ) From 1c099f3a752a2d4a07ea0c98ccae76a46ad96ccb Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 17 Apr 2023 15:14:33 -0600 Subject: [PATCH 265/332] Don't allow qflx_liqevap and qflx_solidevap fluxes to be negative These could sometimes be roundoff-level negative due to h2osoi_liq starting out roundoff-level negative. --- src/biogeophys/SoilFluxesMod.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/biogeophys/SoilFluxesMod.F90 b/src/biogeophys/SoilFluxesMod.F90 index 48820df41b..c316d30fe3 100644 --- a/src/biogeophys/SoilFluxesMod.F90 +++ b/src/biogeophys/SoilFluxesMod.F90 @@ -289,8 +289,8 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & evaporation_demand = qflx_ev_snow(p) qflx_ev_snow(p) = evaporation_limit qflx_evap_soi(p) = qflx_evap_soi(p) - frac_sno_eff(c)*(evaporation_demand - evaporation_limit) - qflx_liqevap_from_top_layer(p) = h2osoi_liq(c,j)/(frac_sno_eff(c)*dtime) - qflx_solidevap_from_top_layer(p) = h2osoi_ice(c,j)/(frac_sno_eff(c)*dtime) + qflx_liqevap_from_top_layer(p) = max(h2osoi_liq(c,j)/(frac_sno_eff(c)*dtime), 0._r8) + qflx_solidevap_from_top_layer(p) = max(h2osoi_ice(c,j)/(frac_sno_eff(c)*dtime), 0._r8) ! conserve total energy flux eflx_sh_grnd(p) = eflx_sh_grnd(p) + frac_sno_eff(c)*(evaporation_demand - evaporation_limit)*htvp(c) endif @@ -305,8 +305,8 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & evaporation_demand = qflx_evap_soi(p) qflx_evap_soi(p) = evaporation_limit qflx_ev_snow(p) = qflx_evap_soi(p) - qflx_liqevap_from_top_layer(p) = h2osoi_liq(c,j)/dtime - qflx_solidevap_from_top_layer(p) = h2osoi_ice(c,j)/dtime + qflx_liqevap_from_top_layer(p) = max(h2osoi_liq(c,j)/dtime, 0._r8) + qflx_solidevap_from_top_layer(p) = max(h2osoi_ice(c,j)/dtime, 0._r8) ! conserve total energy flux eflx_sh_grnd(p) = eflx_sh_grnd(p) +(evaporation_demand -evaporation_limit)*htvp(c) endif From 39de3a08b872958355756340517c42d2ba81ff40 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 18 Apr 2023 13:28:04 -0600 Subject: [PATCH 266/332] correction to get postAD and transient from postAD working --- tools/site_and_regional/run_neon.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 4f4e6cb968..19976285af 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -489,14 +489,11 @@ def run_case( print ("using this version:", version) - if experiment == None: - case_root = os.path.abspath( + if experiment != None: + self.name = self.name + "." + experiment + case_root = os.path.abspath( os.path.join(base_case_root, "..", self.name + "." + run_type) ) - else: - case_root = os.path.abspath( - os.path.join(base_case_root, "..", self.name + "." + experiment + "." + run_type) - ) rundir = None if os.path.isdir(case_root): From 81dda9d8a0fa52c13d1995d022bd36c12e0bdf99 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 18 Apr 2023 16:19:47 -0600 Subject: [PATCH 267/332] Updates needed for pFUnit4 A few specific notes: - Needed to increase tolerances a bit in the matrix test to make tests pass on my Mac (with gfortran). - Needed to remove some assertions in an expected-failure test for quadratic (it seems like the new pFUnit hits these assertions whereas the old did not). --- Externals.cfg | 16 ++++++++-------- src/CMakeLists.txt | 2 +- .../test/CNPhenology_test/CMakeLists.txt | 7 +++---- .../test/CNPhenology_test/test_CNPhenology.pf | 2 +- .../test/CNVegComputeSeed_test/CMakeLists.txt | 7 +++---- .../test_ComputeSeedAmounts.pf | 2 +- src/biogeochem/test/Species_test/CMakeLists.txt | 7 +++---- .../test/Species_test/test_SpeciesIsotope.pf | 2 +- .../test/Species_test/test_SpeciesNonIsotope.pf | 2 +- src/biogeophys/test/Balance_test/CMakeLists.txt | 7 +++---- src/biogeophys/test/Balance_test/test_Balance.pf | 2 +- .../test/Daylength_test/CMakeLists.txt | 7 +++---- .../Daylength_test/test_compute_max_daylength.pf | 2 +- .../test/Daylength_test/test_daylength.pf | 2 +- .../test/HumanStress_test/CMakeLists.txt | 7 +++---- .../test/HumanStress_test/test_humanstress.pf | 2 +- .../test/Irrigation_test/CMakeLists.txt | 12 +++--------- .../test/Irrigation_test/test_irrigation.pf | 2 +- .../test/Photosynthesis_test/CMakeLists.txt | 7 +++---- .../Photosynthesis_test/test_Photosynthesis.pf | 2 +- .../test/SnowHydrology_test/CMakeLists.txt | 7 +++---- .../test_SnowHydrology_SnowCappingExcess.pf | 2 +- .../test_SnowHydrology_initSnowLayers.pf | 2 +- .../test_SnowHydrology_newSnowBulkDensity.pf | 2 +- .../test/TotalWaterAndHeat_test/CMakeLists.txt | 12 +++--------- .../test_total_water_and_heat.pf | 2 +- .../WaterTracerContainerType_test/CMakeLists.txt | 12 +++--------- .../test_water_tracer_container.pf | 2 +- .../test/WaterTracerUtils_test/CMakeLists.txt | 12 +++--------- .../test_calc_tracer_from_bulk.pf | 2 +- .../test_calc_tracer_from_bulk_fixed_ratio.pf | 2 +- .../test_compare_bulk_to_tracer.pf | 2 +- .../test/WaterType_test/CMakeLists.txt | 12 +++--------- .../test/WaterType_test/test_water_type.pf | 2 +- .../test/Wateratm2lnd_test/CMakeLists.txt | 12 +++--------- .../test/Wateratm2lnd_test/test_set_tracers.pf | 2 +- .../dynColumnStateUpdater_test/CMakeLists.txt | 7 +++---- .../test_column_state_updater.pf | 2 +- .../test/dynColumnTemplate_test/CMakeLists.txt | 7 +++---- .../test_column_template.pf | 2 +- .../test/dynConsBiogeophys_test/CMakeLists.txt | 8 +++----- .../test_dyn_cons_biogeophys.pf | 2 +- .../test/dynInitColumns_test/CMakeLists.txt | 7 +++---- .../dynInitColumns_test/test_init_columns.pf | 2 +- .../test/dynLandunitArea_test/CMakeLists.txt | 8 +++----- .../test_update_landunit_weights.pf | 2 +- .../test_update_landunit_weights_one_gcell.pf | 2 +- .../dynPatchStateUpdater_test/CMakeLists.txt | 7 +++---- .../test_patch_state_updater.pf | 2 +- .../test/dynTimeInfo_test/CMakeLists.txt | 7 +++---- .../test/dynTimeInfo_test/test_dynTimeInfo.pf | 2 +- src/dyn_subgrid/test/dynVar_test/CMakeLists.txt | 8 ++++---- .../test/dynVar_test/test_dynVarTimeInterp.pf | 2 +- .../test/dynVar_test/test_dynVarTimeUninterp.pf | 2 +- .../test/initInterpMindist_test/CMakeLists.txt | 7 +++---- .../initInterpMindistTestUtils.pf | 2 +- .../initInterpMindist_test/test_set_mindist.pf | 2 +- .../test_set_single_match.pf | 2 +- .../initInterpMultilevel_test/CMakeLists.txt | 8 ++++---- .../initInterpMultilevelMock.pf | 2 +- .../test_init_interp_multilevel_interp.pf | 2 +- .../test_init_interp_multilevel_snow.pf | 2 +- .../test_init_interp_multilevel_split.pf | 2 +- .../test/initInterpUtils_test/CMakeLists.txt | 7 +++---- .../test_glc_elevclasses_are_same.pf | 2 +- src/main/test/accumul_test/CMakeLists.txt | 7 +++---- src/main/test/accumul_test/test_accumul.pf | 2 +- src/main/test/atm2lnd_test/CMakeLists.txt | 7 +++---- .../test/atm2lnd_test/test_downscale_forcings.pf | 2 +- .../test/atm2lnd_test/test_partition_precip.pf | 2 +- src/main/test/clm_glclnd_test/CMakeLists.txt | 7 +++---- src/main/test/clm_glclnd_test/test_clm_glclnd.pf | 2 +- src/main/test/filter_test/CMakeLists.txt | 7 +++---- src/main/test/filter_test/test_filter_col.pf | 2 +- src/main/test/glcBehavior_test/CMakeLists.txt | 7 +++---- .../test/glcBehavior_test/test_glcBehavior.pf | 2 +- src/main/test/initVertical_test/CMakeLists.txt | 7 +++---- .../test/initVertical_test/test_initVertical.pf | 2 +- src/main/test/ncdio_utils_test/CMakeLists.txt | 7 +++---- .../test/ncdio_utils_test/test_ncdio_utils.pf | 2 +- src/main/test/subgridWeights_test/CMakeLists.txt | 7 +++---- .../subgridWeights_test/test_subgridWeights.pf | 2 +- src/main/test/surfrdUtils_test/CMakeLists.txt | 7 +++---- .../test/surfrdUtils_test/test_surfrdUtils.pf | 2 +- src/main/test/topo_test/CMakeLists.txt | 7 +++---- src/main/test/topo_test/test_topo.pf | 2 +- .../test/assertions_test/CMakeLists.txt | 10 +++------- .../test/assertions_test/test_assertions.pf | 2 +- .../test/ACSpinup_test/CMakeLists.txt | 7 +++---- .../test/ACSpinup_test/test_acspinup.pf | 2 +- .../test/unittestArray_test/CMakeLists.txt | 7 +++---- .../unittestArray_test/test_unittestArray.pf | 2 +- .../unittestFilterBuilder_test/CMakeLists.txt | 7 +++---- .../test_filterBuilder.pf | 2 +- .../test/unittestSubgrid_test/CMakeLists.txt | 7 +++---- .../unittestSubgrid_test/test_unittestSubgrid.pf | 2 +- .../annual_flux_dribbler_test/CMakeLists.txt | 7 +++---- .../test_annual_flux_dribbler.pf | 2 +- src/utils/test/array_utils_test/CMakeLists.txt | 10 +++------- .../array_utils_test/test_convert_to_logical.pf | 2 +- .../array_utils_test/test_find_k_max_indices.pf | 2 +- .../test/clm_time_manager_test/CMakeLists.txt | 7 +++---- .../test_clm_time_manager.pf | 2 +- src/utils/test/matrix_test/CMakeLists.txt | 10 +++------- src/utils/test/matrix_test/test_matrix.pf | 4 ++-- src/utils/test/numerics_test/CMakeLists.txt | 10 +++------- .../numerics_test/test_truncate_small_values.pf | 2 +- src/utils/test/quadratic_test/CMakeLists.txt | 7 +++---- src/utils/test/quadratic_test/test_quadratic.pf | 4 +--- src/utils/test/sparse_matrix_test/CMakeLists.txt | 10 +++------- .../sparse_matrix_test/test_sparse_matrix.pf | 2 +- .../src/test/mkdomain_test/test_mkdomain.pf | 2 +- .../src/test/mkgridmap_test/test_mkgridmap.pf | 2 +- .../src/test/mkindexmap_test/test_mkindexmap.pf | 2 +- .../test/mkpctPftType_test/test_mkpctPftType.pf | 2 +- .../test_adjust_total_veg_area.pf | 2 +- .../mkpftUtils_test/test_convert_from_p2g.pf | 2 +- .../src/test/mkpftmod_test/test_pftInit.pf | 2 +- .../src/test/mkpftmod_test/test_pft_oride.pf | 2 +- .../src/test/mkpftmod_test/test_pftrun.pf | 2 +- .../mksoilUtils_test/test_dominant_soil_color.pf | 2 +- .../mksurfdata_map/src/unit_test_stubs/abort.F90 | 2 +- 122 files changed, 226 insertions(+), 320 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 57d472d8c0..bc9b22f258 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,17 +34,17 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.58 +hash = bd942aae8fbc2848420d2d3249212ef38611724f protocol = git -repo_url = https://github.com/ESMCI/ccs_config_cesm.git +repo_url = https://github.com/billsacks/ccs_config_cesm.git local_path = ccs_config required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/ESMCI/cime -tag = cime6.0.100 +repo_url = https://github.com/billsacks/cime +hash = a42f55a0c31706c377c77ed72b6ebbe8f02f8e92 required = True [cmeps] @@ -63,16 +63,16 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl7.0.14 +hash = 4b3e95623cdd37cb7e72b55b1deb5442bc302faa protocol = git -repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps +repo_url = https://github.com/billsacks/CESM_CPL7andDataComps local_path = components/cpl7 required = True [share] -tag = share1.0.16 +hash = 988c9ec7a94840be80b1851769ce93af859e3005 protocol = git -repo_url = https://github.com/ESCOMP/CESM_share +repo_url = https://github.com/billsacks/CESM_share local_path = share required = True diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 395c2e4868..27d85d464f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -27,7 +27,7 @@ set (drv_sources_needed_base ) extract_sources("${drv_sources_needed_base}" "${drv_sources}" drv_sources_needed) -# Add CLM source directories (these add their own test directories) +# Add CLM source directories add_subdirectory(${CLM_ROOT}/src/utils clm_utils) add_subdirectory(${CLM_ROOT}/src/biogeochem clm_biogeochem) add_subdirectory(${CLM_ROOT}/src/soilbiogeochem clm_soilbiogeochem) diff --git a/src/biogeochem/test/CNPhenology_test/CMakeLists.txt b/src/biogeochem/test/CNPhenology_test/CMakeLists.txt index f265e12ab0..2367c86612 100644 --- a/src/biogeochem/test/CNPhenology_test/CMakeLists.txt +++ b/src/biogeochem/test/CNPhenology_test/CMakeLists.txt @@ -1,7 +1,6 @@ set (pfunit_sources test_CNPhenology.pf) -create_pFUnit_test(CNPhenology test_CNPhenology_exe - "${pfunit_sources}" "") - -target_link_libraries(test_CNPhenology_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(CNPhenology + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeochem/test/CNPhenology_test/test_CNPhenology.pf b/src/biogeochem/test/CNPhenology_test/test_CNPhenology.pf index c93f15cb8f..135364e474 100644 --- a/src/biogeochem/test/CNPhenology_test/test_CNPhenology.pf +++ b/src/biogeochem/test/CNPhenology_test/test_CNPhenology.pf @@ -2,7 +2,7 @@ module test_CNPhenology ! Tests of CNPhenologyMod - use pfunit_mod + use funit use unittestSubgridMod use CNPhenologyMod use unittestTimeManagerMod, only : unittest_timemgr_setup, unittest_timemgr_teardown diff --git a/src/biogeochem/test/CNVegComputeSeed_test/CMakeLists.txt b/src/biogeochem/test/CNVegComputeSeed_test/CMakeLists.txt index 35173a6dc3..dd6bdba723 100644 --- a/src/biogeochem/test/CNVegComputeSeed_test/CMakeLists.txt +++ b/src/biogeochem/test/CNVegComputeSeed_test/CMakeLists.txt @@ -1,7 +1,6 @@ set (pfunit_sources test_ComputeSeedAmounts.pf) -create_pFUnit_test(CNVegComputeSeed test_CNVegComputeSeed_exe - "${pfunit_sources}" "") - -target_link_libraries(test_CNVegComputeSeed_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(CNVegComputeSeed + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeochem/test/CNVegComputeSeed_test/test_ComputeSeedAmounts.pf b/src/biogeochem/test/CNVegComputeSeed_test/test_ComputeSeedAmounts.pf index f5e8aeff9b..be87b4c989 100644 --- a/src/biogeochem/test/CNVegComputeSeed_test/test_ComputeSeedAmounts.pf +++ b/src/biogeochem/test/CNVegComputeSeed_test/test_ComputeSeedAmounts.pf @@ -2,7 +2,7 @@ module test_ComputeSeedAmounts ! Tests of CNVegComputeSeedMod: ComputeSeedAmounts - use pfunit_mod + use funit use CNVegComputeSeedMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/biogeochem/test/Species_test/CMakeLists.txt b/src/biogeochem/test/Species_test/CMakeLists.txt index 3a0bc4b50c..14d091d1ef 100644 --- a/src/biogeochem/test/Species_test/CMakeLists.txt +++ b/src/biogeochem/test/Species_test/CMakeLists.txt @@ -2,7 +2,6 @@ set (pfunit_sources test_SpeciesNonIsotope.pf test_SpeciesIsotope.pf) -create_pFUnit_test(Species test_Species_exe - "${pfunit_sources}" "") - -target_link_libraries(test_Species_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(Species + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/biogeochem/test/Species_test/test_SpeciesIsotope.pf b/src/biogeochem/test/Species_test/test_SpeciesIsotope.pf index 8199b1c92a..6017d9a866 100644 --- a/src/biogeochem/test/Species_test/test_SpeciesIsotope.pf +++ b/src/biogeochem/test/Species_test/test_SpeciesIsotope.pf @@ -2,7 +2,7 @@ module test_SpeciesIsotope ! Tests of SpeciesIsotopeType - use pfunit_mod + use funit use SpeciesIsotopeType use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/biogeochem/test/Species_test/test_SpeciesNonIsotope.pf b/src/biogeochem/test/Species_test/test_SpeciesNonIsotope.pf index e9e4b9fac5..fdcc204db1 100644 --- a/src/biogeochem/test/Species_test/test_SpeciesNonIsotope.pf +++ b/src/biogeochem/test/Species_test/test_SpeciesNonIsotope.pf @@ -2,7 +2,7 @@ module test_SpeciesNonIsotope ! Tests of SpeciesNonIsotopeType - use pfunit_mod + use funit use SpeciesNonIsotopeType use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/biogeophys/test/Balance_test/CMakeLists.txt b/src/biogeophys/test/Balance_test/CMakeLists.txt index 2d0cd75c00..541d4fb266 100644 --- a/src/biogeophys/test/Balance_test/CMakeLists.txt +++ b/src/biogeophys/test/Balance_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(balance test_balance_exe - "test_Balance.pf" "") - -target_link_libraries(test_balance_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(balance + TEST_SOURCES "test_Balance.pf" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/Balance_test/test_Balance.pf b/src/biogeophys/test/Balance_test/test_Balance.pf index 824eebb78c..af5fbbbccb 100644 --- a/src/biogeophys/test/Balance_test/test_Balance.pf +++ b/src/biogeophys/test/Balance_test/test_Balance.pf @@ -2,7 +2,7 @@ module test_balance ! Some tests of the balance check system - use pfunit_mod + use funit use shr_kind_mod , only : r8 => shr_kind_r8 use unittestTimeManagerMod, only : unittest_timemgr_setup, unittest_timemgr_teardown diff --git a/src/biogeophys/test/Daylength_test/CMakeLists.txt b/src/biogeophys/test/Daylength_test/CMakeLists.txt index 6182551580..2e5cb58cf9 100644 --- a/src/biogeophys/test/Daylength_test/CMakeLists.txt +++ b/src/biogeophys/test/Daylength_test/CMakeLists.txt @@ -2,7 +2,6 @@ set (pfunit_sources test_daylength.pf test_compute_max_daylength.pf) -create_pFUnit_test(Daylength test_Daylength_exe - "${pfunit_sources}" "") - -target_link_libraries(test_Daylength_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(Daylength + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/Daylength_test/test_compute_max_daylength.pf b/src/biogeophys/test/Daylength_test/test_compute_max_daylength.pf index 3c8d080128..ac06b8f0c7 100644 --- a/src/biogeophys/test/Daylength_test/test_compute_max_daylength.pf +++ b/src/biogeophys/test/Daylength_test/test_compute_max_daylength.pf @@ -2,7 +2,7 @@ module test_compute_max_daylength ! Tests of DaylengthMod: ComputeMaxDaylength - use pfunit_mod + use funit use DaylengthMod, only: ComputeMaxDaylength, daylength use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod, only : unittest_subgrid_teardown, bounds diff --git a/src/biogeophys/test/Daylength_test/test_daylength.pf b/src/biogeophys/test/Daylength_test/test_daylength.pf index b326a1190a..0445832b40 100644 --- a/src/biogeophys/test/Daylength_test/test_daylength.pf +++ b/src/biogeophys/test/Daylength_test/test_daylength.pf @@ -2,7 +2,7 @@ module test_daylength ! Tests of the daylength function in DaylengthMod - use pfunit_mod + use funit use shr_kind_mod , only : r8 => shr_kind_r8 use shr_const_mod, only : SHR_CONST_PI diff --git a/src/biogeophys/test/HumanStress_test/CMakeLists.txt b/src/biogeophys/test/HumanStress_test/CMakeLists.txt index d2583a3a47..d23fcb8f71 100644 --- a/src/biogeophys/test/HumanStress_test/CMakeLists.txt +++ b/src/biogeophys/test/HumanStress_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(humanstress test_humanstress_exe - "test_humanstress.pf" "") - -target_link_libraries(test_humanstress_exe clm csm_share) +add_pfunit_ctest(humanstress + TEST_SOURCES "test_humanstress.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/biogeophys/test/HumanStress_test/test_humanstress.pf b/src/biogeophys/test/HumanStress_test/test_humanstress.pf index a6cf85bbdd..5a51cb4689 100644 --- a/src/biogeophys/test/HumanStress_test/test_humanstress.pf +++ b/src/biogeophys/test/HumanStress_test/test_humanstress.pf @@ -2,7 +2,7 @@ module test_humanstress ! Tests of the humanstress functions in HumanIndexMod - use pfunit_mod + use funit use shr_kind_mod , only : r8 => shr_kind_r8 use HumanIndexMod diff --git a/src/biogeophys/test/Irrigation_test/CMakeLists.txt b/src/biogeophys/test/Irrigation_test/CMakeLists.txt index 7a1b7cc5ee..8cb531ba1a 100644 --- a/src/biogeophys/test/Irrigation_test/CMakeLists.txt +++ b/src/biogeophys/test/Irrigation_test/CMakeLists.txt @@ -1,12 +1,6 @@ set (pfunit_sources test_irrigation.pf) -# extra sources used for this test, which are not .pf files -# (currently none) -set (extra_sources - ) - -create_pFUnit_test(irrigation test_irrigation_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_irrigation_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(irrigation + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/Irrigation_test/test_irrigation.pf b/src/biogeophys/test/Irrigation_test/test_irrigation.pf index e810d39ddb..d83fc94329 100644 --- a/src/biogeophys/test/Irrigation_test/test_irrigation.pf +++ b/src/biogeophys/test/Irrigation_test/test_irrigation.pf @@ -2,7 +2,7 @@ module test_irrigation ! Tests of IrrigationMod - use pfunit_mod + use funit use unittestSubgridMod use unittestTimeManagerMod, only : unittest_timemgr_setup, unittest_timemgr_teardown use unittestTimeManagerMod, only : unittest_timemgr_set_curr_date diff --git a/src/biogeophys/test/Photosynthesis_test/CMakeLists.txt b/src/biogeophys/test/Photosynthesis_test/CMakeLists.txt index 149ec47c1d..29810cbd9d 100644 --- a/src/biogeophys/test/Photosynthesis_test/CMakeLists.txt +++ b/src/biogeophys/test/Photosynthesis_test/CMakeLists.txt @@ -1,7 +1,6 @@ set (pfunit_sources test_Photosynthesis.pf) -create_pFUnit_test(Photosynthesis test_Photosynthesis_exe - "${pfunit_sources}" "") - -target_link_libraries(test_Photosynthesis_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(Photosynthesis + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/Photosynthesis_test/test_Photosynthesis.pf b/src/biogeophys/test/Photosynthesis_test/test_Photosynthesis.pf index 74ca9c9c4c..faa506a99f 100644 --- a/src/biogeophys/test/Photosynthesis_test/test_Photosynthesis.pf +++ b/src/biogeophys/test/Photosynthesis_test/test_Photosynthesis.pf @@ -2,7 +2,7 @@ module test_Photosynthesis ! Tests of PhotosynthesisMod.F90 - use pfunit_mod + use funit use PhotosynthesisMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod, only : unittest_subgrid_teardown, bounds diff --git a/src/biogeophys/test/SnowHydrology_test/CMakeLists.txt b/src/biogeophys/test/SnowHydrology_test/CMakeLists.txt index cfd1e3a4bb..600356b2ff 100644 --- a/src/biogeophys/test/SnowHydrology_test/CMakeLists.txt +++ b/src/biogeophys/test/SnowHydrology_test/CMakeLists.txt @@ -3,7 +3,6 @@ set (pfunit_sources test_SnowHydrology_newSnowBulkDensity.pf test_SnowHydrology_SnowCappingExcess.pf) -create_pFUnit_test(SnowHydrology test_SnowHydrology_exe - "${pfunit_sources}" "") - -target_link_libraries(test_SnowHydrology_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(SnowHydrology + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_SnowCappingExcess.pf b/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_SnowCappingExcess.pf index 5f42d37226..65448a9207 100644 --- a/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_SnowCappingExcess.pf +++ b/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_SnowCappingExcess.pf @@ -2,7 +2,7 @@ module test_SnowHydrology_SnowCappingExcess ! Tests of SnowHydrologyMod: SnowCappingExcess - use pfunit_mod + use funit use SnowHydrologyMod use TopoMod, only : topo_type use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_initSnowLayers.pf b/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_initSnowLayers.pf index ab1ab810fb..dc5afcab78 100644 --- a/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_initSnowLayers.pf +++ b/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_initSnowLayers.pf @@ -2,7 +2,7 @@ module test_SnowHydrology_initSnowLayers ! Tests of SnowHydrologyMod: initSnowLayers - use pfunit_mod + use funit use SnowHydrologyMod use shr_kind_mod, only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_newSnowBulkDensity.pf b/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_newSnowBulkDensity.pf index db8c7ba980..e3df17b9a0 100644 --- a/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_newSnowBulkDensity.pf +++ b/src/biogeophys/test/SnowHydrology_test/test_SnowHydrology_newSnowBulkDensity.pf @@ -2,7 +2,7 @@ module test_SnowHydrology_newSnowBulkDensity ! Tests of SnowHydrologyMod: newSnowBulkDensity - use pfunit_mod + use funit use unittestSubgridMod use SnowHydrologyMod use atm2lndType , only : atm2lnd_type diff --git a/src/biogeophys/test/TotalWaterAndHeat_test/CMakeLists.txt b/src/biogeophys/test/TotalWaterAndHeat_test/CMakeLists.txt index 67f2a4812e..a82532f69b 100644 --- a/src/biogeophys/test/TotalWaterAndHeat_test/CMakeLists.txt +++ b/src/biogeophys/test/TotalWaterAndHeat_test/CMakeLists.txt @@ -1,12 +1,6 @@ set (pfunit_sources test_total_water_and_heat.pf) -# extra sources used for this test, which are not .pf files -# (currently none) -set (extra_sources - ) - -create_pFUnit_test(total_water_and_heat test_total_water_and_heat_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_total_water_and_heat_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(total_water_and_heat + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/TotalWaterAndHeat_test/test_total_water_and_heat.pf b/src/biogeophys/test/TotalWaterAndHeat_test/test_total_water_and_heat.pf index 05cb5eed57..88f9dae5b2 100644 --- a/src/biogeophys/test/TotalWaterAndHeat_test/test_total_water_and_heat.pf +++ b/src/biogeophys/test/TotalWaterAndHeat_test/test_total_water_and_heat.pf @@ -2,7 +2,7 @@ module test_total_water_and_heat ! Tests of TotalWaterAndHeatMod - use pfunit_mod + use funit use TotalWaterAndHeatMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/biogeophys/test/WaterTracerContainerType_test/CMakeLists.txt b/src/biogeophys/test/WaterTracerContainerType_test/CMakeLists.txt index 10bb931346..283906a442 100644 --- a/src/biogeophys/test/WaterTracerContainerType_test/CMakeLists.txt +++ b/src/biogeophys/test/WaterTracerContainerType_test/CMakeLists.txt @@ -1,12 +1,6 @@ set (pfunit_sources test_water_tracer_container.pf) -# extra sources used for this test, which are not .pf files -# (currently none) -set (extra_sources - ) - -create_pFUnit_test(water_tracer_container test_water_tracer_container_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_water_tracer_container_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(water_tracer_container + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/WaterTracerContainerType_test/test_water_tracer_container.pf b/src/biogeophys/test/WaterTracerContainerType_test/test_water_tracer_container.pf index 58bee7792b..ce37c896a1 100644 --- a/src/biogeophys/test/WaterTracerContainerType_test/test_water_tracer_container.pf +++ b/src/biogeophys/test/WaterTracerContainerType_test/test_water_tracer_container.pf @@ -2,7 +2,7 @@ module test_water_tracer_container ! Tests of WaterTracerContainerType - use pfunit_mod + use funit use WaterTracerContainerType use shr_kind_mod , only : r8 => shr_kind_r8 use decompMod, only : subgrid_level_gridcell diff --git a/src/biogeophys/test/WaterTracerUtils_test/CMakeLists.txt b/src/biogeophys/test/WaterTracerUtils_test/CMakeLists.txt index ed16d6200f..321e06883a 100644 --- a/src/biogeophys/test/WaterTracerUtils_test/CMakeLists.txt +++ b/src/biogeophys/test/WaterTracerUtils_test/CMakeLists.txt @@ -3,12 +3,6 @@ set (pfunit_sources test_calc_tracer_from_bulk.pf test_compare_bulk_to_tracer.pf) -# extra sources used for this test, which are not .pf files -# (currently none) -set (extra_sources - ) - -create_pFUnit_test(water_tracer_utils test_water_tracer_utils_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_water_tracer_utils_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(water_tracer_utils + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk.pf b/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk.pf index df3afd8d0f..e6e7aac1f1 100644 --- a/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk.pf +++ b/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk.pf @@ -2,7 +2,7 @@ module test_calc_tracer_from_bulk ! Tests of WaterTracerUtils: CalcTracerFromBulk - use pfunit_mod + use funit use WaterTracerUtils use shr_kind_mod , only : r8 => shr_kind_r8 use decompMod, only : subgrid_level_unspecified diff --git a/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk_fixed_ratio.pf b/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk_fixed_ratio.pf index 5bde146ac3..1871fc281d 100644 --- a/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk_fixed_ratio.pf +++ b/src/biogeophys/test/WaterTracerUtils_test/test_calc_tracer_from_bulk_fixed_ratio.pf @@ -2,7 +2,7 @@ module test_calc_tracer_from_bulk_fixed_ratio ! Tests of WaterTracerUtils: CalcTracerFromBulkFixedRatio - use pfunit_mod + use funit use WaterTracerUtils, only : CalcTracerFromBulkFixedRatio use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/biogeophys/test/WaterTracerUtils_test/test_compare_bulk_to_tracer.pf b/src/biogeophys/test/WaterTracerUtils_test/test_compare_bulk_to_tracer.pf index 01292402ff..3e185b9e85 100644 --- a/src/biogeophys/test/WaterTracerUtils_test/test_compare_bulk_to_tracer.pf +++ b/src/biogeophys/test/WaterTracerUtils_test/test_compare_bulk_to_tracer.pf @@ -2,7 +2,7 @@ module test_compare_bulk_to_tracer ! Tests of WaterTracerUtils: CompareBulkToTracer - use pfunit_mod + use funit use WaterTracerUtils, only : CompareBulkToTracer use shr_kind_mod , only : r8 => shr_kind_r8 use shr_infnan_mod , only : nan => shr_infnan_nan, assignment(=) diff --git a/src/biogeophys/test/WaterType_test/CMakeLists.txt b/src/biogeophys/test/WaterType_test/CMakeLists.txt index 5ca5d23d72..506179aabd 100644 --- a/src/biogeophys/test/WaterType_test/CMakeLists.txt +++ b/src/biogeophys/test/WaterType_test/CMakeLists.txt @@ -1,12 +1,6 @@ set (pfunit_sources test_water_type.pf) -# extra sources used for this test, which are not .pf files -# (currently none) -set (extra_sources - ) - -create_pFUnit_test(water_type test_water_type_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_water_type_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(water_type + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/WaterType_test/test_water_type.pf b/src/biogeophys/test/WaterType_test/test_water_type.pf index c709f6e991..f3edc731a3 100644 --- a/src/biogeophys/test/WaterType_test/test_water_type.pf +++ b/src/biogeophys/test/WaterType_test/test_water_type.pf @@ -2,7 +2,7 @@ module test_water_type ! Tests of WaterType - use pfunit_mod + use funit use WaterType use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod, only : bounds, unittest_subgrid_teardown diff --git a/src/biogeophys/test/Wateratm2lnd_test/CMakeLists.txt b/src/biogeophys/test/Wateratm2lnd_test/CMakeLists.txt index 35e12aea3e..c2157952e0 100644 --- a/src/biogeophys/test/Wateratm2lnd_test/CMakeLists.txt +++ b/src/biogeophys/test/Wateratm2lnd_test/CMakeLists.txt @@ -1,12 +1,6 @@ set (pfunit_sources test_set_tracers.pf) -# extra sources used for this test, which are not .pf files -# (currently none) -set (extra_sources - ) - -create_pFUnit_test(water_atm2lnd test_water_atm2lnd_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_water_atm2lnd_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(water_atm2lnd + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/biogeophys/test/Wateratm2lnd_test/test_set_tracers.pf b/src/biogeophys/test/Wateratm2lnd_test/test_set_tracers.pf index e16eeea73a..13928b566b 100644 --- a/src/biogeophys/test/Wateratm2lnd_test/test_set_tracers.pf +++ b/src/biogeophys/test/Wateratm2lnd_test/test_set_tracers.pf @@ -2,7 +2,7 @@ module test_set_tracers ! Tests of Wateratm2lndType: routines that set tracers - use pfunit_mod + use funit use Wateratm2lndType use WaterInfoBulkType, only : water_info_bulk_type use WaterInfoTracerType, only : water_info_tracer_type diff --git a/src/dyn_subgrid/test/dynColumnStateUpdater_test/CMakeLists.txt b/src/dyn_subgrid/test/dynColumnStateUpdater_test/CMakeLists.txt index 8384506601..5329042a59 100644 --- a/src/dyn_subgrid/test/dynColumnStateUpdater_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynColumnStateUpdater_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(ColumnStateUpdater test_ColumnStateUpdater_exe - "test_column_state_updater.pf" "") - -target_link_libraries(test_ColumnStateUpdater_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(ColumnStateUpdater + TEST_SOURCES "test_column_state_updater.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/dyn_subgrid/test/dynColumnStateUpdater_test/test_column_state_updater.pf b/src/dyn_subgrid/test/dynColumnStateUpdater_test/test_column_state_updater.pf index a9c456ab04..0758b84ed2 100644 --- a/src/dyn_subgrid/test/dynColumnStateUpdater_test/test_column_state_updater.pf +++ b/src/dyn_subgrid/test/dynColumnStateUpdater_test/test_column_state_updater.pf @@ -2,7 +2,7 @@ module test_column_state_updater ! Tests of dynColumnStateUpdaterMod - use pfunit_mod + use funit use dynColumnStateUpdaterMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/dyn_subgrid/test/dynColumnTemplate_test/CMakeLists.txt b/src/dyn_subgrid/test/dynColumnTemplate_test/CMakeLists.txt index 0f6ad1c753..e809bed311 100644 --- a/src/dyn_subgrid/test/dynColumnTemplate_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynColumnTemplate_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(ColumnTemplate test_ColumnTemplate_exe - "test_column_template.pf" "") - -target_link_libraries(test_ColumnTemplate_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(ColumnTemplate + TEST_SOURCES "test_column_template.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/dyn_subgrid/test/dynColumnTemplate_test/test_column_template.pf b/src/dyn_subgrid/test/dynColumnTemplate_test/test_column_template.pf index e5ab68fe29..ba4e692601 100644 --- a/src/dyn_subgrid/test/dynColumnTemplate_test/test_column_template.pf +++ b/src/dyn_subgrid/test/dynColumnTemplate_test/test_column_template.pf @@ -2,7 +2,7 @@ module test_column_template ! Tests of dynColumnTemplateMod - use pfunit_mod + use funit use dynColumnTemplateMod use unittestSubgridMod use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/dyn_subgrid/test/dynConsBiogeophys_test/CMakeLists.txt b/src/dyn_subgrid/test/dynConsBiogeophys_test/CMakeLists.txt index 2ef1ab38b9..5e981270a4 100644 --- a/src/dyn_subgrid/test/dynConsBiogeophys_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynConsBiogeophys_test/CMakeLists.txt @@ -1,8 +1,6 @@ set(pfunit_sources test_dyn_cons_biogeophys.pf) -create_pFUnit_test(dynConsBiogeophys test_dynConsBiogeophys_exe - "${pfunit_sources}" "") - -target_link_libraries(test_dynConsBiogeophys_exe clm csm_share esmf_wrf_timemgr) - +add_pfunit_ctest(dynConsBiogeophys + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/dyn_subgrid/test/dynConsBiogeophys_test/test_dyn_cons_biogeophys.pf b/src/dyn_subgrid/test/dynConsBiogeophys_test/test_dyn_cons_biogeophys.pf index 7a847c24c4..d991f0cc54 100644 --- a/src/dyn_subgrid/test/dynConsBiogeophys_test/test_dyn_cons_biogeophys.pf +++ b/src/dyn_subgrid/test/dynConsBiogeophys_test/test_dyn_cons_biogeophys.pf @@ -2,7 +2,7 @@ module test_dyn_cons_biogeophys ! Tests of dynConsBiogeophysMod - use pfunit_mod + use funit use dynConsBiogeophysMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/dyn_subgrid/test/dynInitColumns_test/CMakeLists.txt b/src/dyn_subgrid/test/dynInitColumns_test/CMakeLists.txt index 15647e0517..7952f66756 100644 --- a/src/dyn_subgrid/test/dynInitColumns_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynInitColumns_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(dynInitColumns test_dynInitColumns_exe - "test_init_columns.pf" "") - -target_link_libraries(test_dynInitColumns_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(dynInitColumns + TEST_SOURCES "test_init_columns.pf" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/dyn_subgrid/test/dynInitColumns_test/test_init_columns.pf b/src/dyn_subgrid/test/dynInitColumns_test/test_init_columns.pf index 829d9e8260..ade2e6d955 100644 --- a/src/dyn_subgrid/test/dynInitColumns_test/test_init_columns.pf +++ b/src/dyn_subgrid/test/dynInitColumns_test/test_init_columns.pf @@ -2,7 +2,7 @@ module test_init_columns ! Tests of the dynInitColumns module - use pfunit_mod + use funit use unittestSubgridMod use dynInitColumnsMod use ColumnType , only : col diff --git a/src/dyn_subgrid/test/dynLandunitArea_test/CMakeLists.txt b/src/dyn_subgrid/test/dynLandunitArea_test/CMakeLists.txt index 2547dc4a17..cd0100d72a 100644 --- a/src/dyn_subgrid/test/dynLandunitArea_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynLandunitArea_test/CMakeLists.txt @@ -2,8 +2,6 @@ set(pfunit_sources test_update_landunit_weights_one_gcell.pf test_update_landunit_weights.pf) -create_pFUnit_test(dynLandunitArea test_dynLandunitArea_exe - "${pfunit_sources}" "") - -target_link_libraries(test_dynLandunitArea_exe clm csm_share) - +add_pfunit_ctest(dynLandunitArea + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights.pf b/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights.pf index 7bfc76429b..d4e7c8c7ed 100644 --- a/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights.pf +++ b/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights.pf @@ -2,7 +2,7 @@ module test_update_landunit_weights ! Tests of the update_landunit_weights routine in the dynLandunitArea module - use pfunit_mod + use funit use unittestSubgridMod use dynLandunitAreaMod use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights_one_gcell.pf b/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights_one_gcell.pf index 03ee916a06..1f255dfd9b 100644 --- a/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights_one_gcell.pf +++ b/src/dyn_subgrid/test/dynLandunitArea_test/test_update_landunit_weights_one_gcell.pf @@ -2,7 +2,7 @@ module test_update_landunit_weights_one_gcell ! Tests of the update_landunit_weights_one_gcell routine in the dynLandunitArea module - use pfunit_mod + use funit use dynLandunitAreaMod use landunit_varcon, only : istsoil, istcrop, isturb_md, istice, istdlak, max_lunit use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/dyn_subgrid/test/dynPatchStateUpdater_test/CMakeLists.txt b/src/dyn_subgrid/test/dynPatchStateUpdater_test/CMakeLists.txt index 932fa98ea2..10c05b8c14 100644 --- a/src/dyn_subgrid/test/dynPatchStateUpdater_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynPatchStateUpdater_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(PatchStateUpdater test_PatchStateUpdater_exe - "test_patch_state_updater.pf" "") - -target_link_libraries(test_PatchStateUpdater_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(PatchStateUpdater + TEST_SOURCES "test_patch_state_updater.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/dyn_subgrid/test/dynPatchStateUpdater_test/test_patch_state_updater.pf b/src/dyn_subgrid/test/dynPatchStateUpdater_test/test_patch_state_updater.pf index 6e0cd9fc10..af0d1992d5 100644 --- a/src/dyn_subgrid/test/dynPatchStateUpdater_test/test_patch_state_updater.pf +++ b/src/dyn_subgrid/test/dynPatchStateUpdater_test/test_patch_state_updater.pf @@ -2,7 +2,7 @@ module test_patch_state_updater ! Tests of dynPatchStateUpdaterMod - use pfunit_mod + use funit use dynPatchStateUpdaterMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/dyn_subgrid/test/dynTimeInfo_test/CMakeLists.txt b/src/dyn_subgrid/test/dynTimeInfo_test/CMakeLists.txt index 3e2e20e756..66f2027c36 100644 --- a/src/dyn_subgrid/test/dynTimeInfo_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynTimeInfo_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(dynTimeInfo test_dynTimeInfo_exe - "test_dynTimeInfo.pf" "") - -target_link_libraries(test_dynTimeInfo_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(dynTimeInfo + TEST_SOURCES "test_dynTimeInfo.pf" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/dyn_subgrid/test/dynTimeInfo_test/test_dynTimeInfo.pf b/src/dyn_subgrid/test/dynTimeInfo_test/test_dynTimeInfo.pf index cdb1e394ec..3c6092381e 100644 --- a/src/dyn_subgrid/test/dynTimeInfo_test/test_dynTimeInfo.pf +++ b/src/dyn_subgrid/test/dynTimeInfo_test/test_dynTimeInfo.pf @@ -2,7 +2,7 @@ module test_dynTimeInfo ! Tests of the dynTimeInfo class - use pfunit_mod + use funit use dynTimeInfoMod use shr_kind_mod, only: r8 => shr_kind_r8 use unittestTimeManagerMod, only : unittest_timemgr_setup, unittest_timemgr_teardown diff --git a/src/dyn_subgrid/test/dynVar_test/CMakeLists.txt b/src/dyn_subgrid/test/dynVar_test/CMakeLists.txt index 2fadf5d844..fc4cf07b30 100644 --- a/src/dyn_subgrid/test/dynVar_test/CMakeLists.txt +++ b/src/dyn_subgrid/test/dynVar_test/CMakeLists.txt @@ -6,7 +6,7 @@ set (pfunit_sources set (extra_sources test_dynVarShared.F90) -create_pfUnit_test(dynVar test_dynVar_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_dynVar_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(dynVar + TEST_SOURCES "${pfunit_sources}" + OTHER_SOURCES "${extra_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeInterp.pf b/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeInterp.pf index e50acf09a4..da2168f2ce 100644 --- a/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeInterp.pf +++ b/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeInterp.pf @@ -2,7 +2,7 @@ module test_dynVarTimeInterp ! Tests of dyn_var_time_interp - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use dynVarTimeInterpMod, only : dyn_var_time_interp_type use test_dynVarShared diff --git a/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeUninterp.pf b/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeUninterp.pf index 9f41a8c1ac..1e2fe31836 100644 --- a/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeUninterp.pf +++ b/src/dyn_subgrid/test/dynVar_test/test_dynVarTimeUninterp.pf @@ -2,7 +2,7 @@ module test_dynVarTimeUninterp ! Tests of dyn_var_time_uninterp - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use dynVarTimeUninterpMod, only : dyn_var_time_uninterp_type use test_dynVarShared diff --git a/src/init_interp/test/initInterpMindist_test/CMakeLists.txt b/src/init_interp/test/initInterpMindist_test/CMakeLists.txt index 03d669ef34..f349f0e021 100644 --- a/src/init_interp/test/initInterpMindist_test/CMakeLists.txt +++ b/src/init_interp/test/initInterpMindist_test/CMakeLists.txt @@ -3,7 +3,6 @@ set (pfunit_sources test_set_single_match.pf initInterpMindistTestUtils.pf) -create_pFUnit_test(initInterpMindist test_initInterpMindist_exe - "${pfunit_sources}" "") - -target_link_libraries(test_initInterpMindist_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(initInterpMindist + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf b/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf index 83f524c5d8..04f09cb55d 100644 --- a/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf +++ b/src/init_interp/test/initInterpMindist_test/initInterpMindistTestUtils.pf @@ -2,7 +2,7 @@ module initInterpMindistTestUtils ! Utilities to aid the testing of initInterpMindist - use pfunit_mod + use funit use shr_kind_mod , only : r8 => shr_kind_r8 use initInterpMindist, only : subgrid_type, subgrid_special_indices_type use glcBehaviorMod, only: glc_behavior_type diff --git a/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf b/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf index 67509c35af..06ce20d7de 100644 --- a/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf +++ b/src/init_interp/test/initInterpMindist_test/test_set_mindist.pf @@ -2,7 +2,7 @@ module test_set_mindist ! Tests of initInterpMindist: set_mindist - use pfunit_mod + use funit use initInterpMindist use initInterpMindistTestUtils, only : create_subgrid_info, create_glc_behavior use initInterpMindistTestUtils, only : subgrid_special_indices, ilun_special diff --git a/src/init_interp/test/initInterpMindist_test/test_set_single_match.pf b/src/init_interp/test/initInterpMindist_test/test_set_single_match.pf index 64e66a8d3c..e8ac6bf672 100644 --- a/src/init_interp/test/initInterpMindist_test/test_set_single_match.pf +++ b/src/init_interp/test/initInterpMindist_test/test_set_single_match.pf @@ -2,7 +2,7 @@ module test_set_single_match ! Tests of initInterpMindist: set_single_match - use pfunit_mod + use funit use initInterpMindist use initInterpMindistTestUtils, only : create_subgrid_info, create_glc_behavior use initInterpMindistTestUtils, only : subgrid_special_indices, ilun_special diff --git a/src/init_interp/test/initInterpMultilevel_test/CMakeLists.txt b/src/init_interp/test/initInterpMultilevel_test/CMakeLists.txt index d87d37529c..92ab54e54b 100644 --- a/src/init_interp/test/initInterpMultilevel_test/CMakeLists.txt +++ b/src/init_interp/test/initInterpMultilevel_test/CMakeLists.txt @@ -8,7 +8,7 @@ set (pfunit_sources set (extra_sources multilevel_interp_factory.F90) -create_pFUnit_test(initInterpMultilevel test_initInterpMultilevel_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_initInterpMultilevel_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(initInterpMultilevel + TEST_SOURCES "${pfunit_sources}" + OTHER_SOURCES "${extra_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/init_interp/test/initInterpMultilevel_test/initInterpMultilevelMock.pf b/src/init_interp/test/initInterpMultilevel_test/initInterpMultilevelMock.pf index 04911c0278..988f1e5e19 100644 --- a/src/init_interp/test/initInterpMultilevel_test/initInterpMultilevelMock.pf +++ b/src/init_interp/test/initInterpMultilevel_test/initInterpMultilevelMock.pf @@ -6,7 +6,7 @@ module initInterpMultilevelMock ! interp_multilevel routine is called correctly. ! ! !USES: - use pfunit_mod + use funit use shr_kind_mod , only : r8 => shr_kind_r8 use initInterpMultilevelBase , only : interp_multilevel_type diff --git a/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_interp.pf b/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_interp.pf index cf5a2279a8..1635ab2528 100644 --- a/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_interp.pf +++ b/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_interp.pf @@ -2,7 +2,7 @@ module test_init_interp_multilevel_interp ! Tests of initInterpMultilevelInterp - use pfunit_mod + use funit use initInterpMultilevelInterp use multilevel_interp_factory use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_snow.pf b/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_snow.pf index 0e31df0c68..4a4d3be0d1 100644 --- a/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_snow.pf +++ b/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_snow.pf @@ -2,7 +2,7 @@ module test_init_interp_multilevel_snow ! Tests of initInterpMultilevelSnow - use pfunit_mod + use funit use initInterpMultilevelSnow use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_split.pf b/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_split.pf index de9256aa45..c86696ba5e 100644 --- a/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_split.pf +++ b/src/init_interp/test/initInterpMultilevel_test/test_init_interp_multilevel_split.pf @@ -2,7 +2,7 @@ module test_init_interp_multilevel_split ! Tests of initInterpMultilevelSplit - use pfunit_mod + use funit use initInterpMultilevelSplit use initInterpMultilevelInterp, only : interp_multilevel_interp_type use initInterpMultilevelCopy, only : interp_multilevel_copy_type diff --git a/src/init_interp/test/initInterpUtils_test/CMakeLists.txt b/src/init_interp/test/initInterpUtils_test/CMakeLists.txt index 099120472c..874900e21a 100644 --- a/src/init_interp/test/initInterpUtils_test/CMakeLists.txt +++ b/src/init_interp/test/initInterpUtils_test/CMakeLists.txt @@ -1,7 +1,6 @@ set (pfunit_sources test_glc_elevclasses_are_same.pf) -create_pFUnit_test(initInterpUtils test_initInterpUtils_exe - "${pfunit_sources}" "") - -target_link_libraries(test_initInterpUtils_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(initInterpUtils + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/init_interp/test/initInterpUtils_test/test_glc_elevclasses_are_same.pf b/src/init_interp/test/initInterpUtils_test/test_glc_elevclasses_are_same.pf index 987429cf5d..0835dd6534 100644 --- a/src/init_interp/test/initInterpUtils_test/test_glc_elevclasses_are_same.pf +++ b/src/init_interp/test/initInterpUtils_test/test_glc_elevclasses_are_same.pf @@ -2,7 +2,7 @@ module test_glc_elevclasses_are_same ! Tests of initInterpUtils: glc_elevclasses_are_same - use pfunit_mod + use funit use initInterpUtils use shr_kind_mod , only : r8 => shr_kind_r8 use ncdio_pio, only : file_desc_t, ncd_set_dim, ncd_set_var diff --git a/src/main/test/accumul_test/CMakeLists.txt b/src/main/test/accumul_test/CMakeLists.txt index 105ef0dac1..0b06d9e87e 100644 --- a/src/main/test/accumul_test/CMakeLists.txt +++ b/src/main/test/accumul_test/CMakeLists.txt @@ -1,7 +1,6 @@ set(pfunit_sources test_accumul.pf) -create_pFUnit_test(accumul test_accumul_exe - "${pfunit_sources}" "") - -target_link_libraries(test_accumul_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(accumul + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/main/test/accumul_test/test_accumul.pf b/src/main/test/accumul_test/test_accumul.pf index 429aeac594..15d2e7403e 100644 --- a/src/main/test/accumul_test/test_accumul.pf +++ b/src/main/test/accumul_test/test_accumul.pf @@ -2,7 +2,7 @@ module test_accumul ! Tests of accumulMod - use pfunit_mod + use funit use accumulMod use unittestSubgridMod use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch diff --git a/src/main/test/atm2lnd_test/CMakeLists.txt b/src/main/test/atm2lnd_test/CMakeLists.txt index e42192b45b..51c4732205 100644 --- a/src/main/test/atm2lnd_test/CMakeLists.txt +++ b/src/main/test/atm2lnd_test/CMakeLists.txt @@ -2,7 +2,6 @@ set(pfunit_sources test_downscale_forcings.pf test_partition_precip.pf) -create_pFUnit_test(atm2lnd test_atm2lnd_exe - "${pfunit_sources}" "") - -target_link_libraries(test_atm2lnd_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(atm2lnd + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/main/test/atm2lnd_test/test_downscale_forcings.pf b/src/main/test/atm2lnd_test/test_downscale_forcings.pf index d7705f2d56..d688ad809d 100644 --- a/src/main/test/atm2lnd_test/test_downscale_forcings.pf +++ b/src/main/test/atm2lnd_test/test_downscale_forcings.pf @@ -2,7 +2,7 @@ module test_downscale_forcings ! Tests of atm2lndMod: downscale_forcings - use pfunit_mod + use funit use atm2lndMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/main/test/atm2lnd_test/test_partition_precip.pf b/src/main/test/atm2lnd_test/test_partition_precip.pf index c0d9065007..48c12c3f3c 100644 --- a/src/main/test/atm2lnd_test/test_partition_precip.pf +++ b/src/main/test/atm2lnd_test/test_partition_precip.pf @@ -2,7 +2,7 @@ module test_partition_precip ! Tests of atm2lndMod: partition_precip - use pfunit_mod + use funit use atm2lndMod use atm2lndType use shr_kind_mod, only : r8 => shr_kind_r8 diff --git a/src/main/test/clm_glclnd_test/CMakeLists.txt b/src/main/test/clm_glclnd_test/CMakeLists.txt index f7ac27caf5..12caa0d851 100644 --- a/src/main/test/clm_glclnd_test/CMakeLists.txt +++ b/src/main/test/clm_glclnd_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(clm_glclnd test_clm_glclnd_exe - "test_clm_glclnd.pf" "") - -target_link_libraries(test_clm_glclnd_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(clm_glclnd + TEST_SOURCES "test_clm_glclnd.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/clm_glclnd_test/test_clm_glclnd.pf b/src/main/test/clm_glclnd_test/test_clm_glclnd.pf index e02acc3990..7d233e1f73 100644 --- a/src/main/test/clm_glclnd_test/test_clm_glclnd.pf +++ b/src/main/test/clm_glclnd_test/test_clm_glclnd.pf @@ -2,7 +2,7 @@ module test_clm_glclnd ! Tests of clm_glclnd - use pfunit_mod + use funit use unittestSubgridMod use shr_kind_mod, only : r8 => shr_kind_r8 use lnd2glcMod diff --git a/src/main/test/filter_test/CMakeLists.txt b/src/main/test/filter_test/CMakeLists.txt index c5cd2b3eb4..291da0367b 100644 --- a/src/main/test/filter_test/CMakeLists.txt +++ b/src/main/test/filter_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(filter test_filter_exe - "test_filter_col.pf" "") - -target_link_libraries(test_filter_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(filter + TEST_SOURCES "test_filter_col.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/filter_test/test_filter_col.pf b/src/main/test/filter_test/test_filter_col.pf index 55c11c4fb6..d1d087df82 100644 --- a/src/main/test/filter_test/test_filter_col.pf +++ b/src/main/test/filter_test/test_filter_col.pf @@ -2,7 +2,7 @@ module test_filter_col ! Tests of filterColMod - use pfunit_mod + use funit use filterColMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/main/test/glcBehavior_test/CMakeLists.txt b/src/main/test/glcBehavior_test/CMakeLists.txt index caf52f4439..8a83d43b74 100644 --- a/src/main/test/glcBehavior_test/CMakeLists.txt +++ b/src/main/test/glcBehavior_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(glcBehavior test_glcBehavior_exe - "test_glcBehavior.pf" "") - -target_link_libraries(test_glcBehavior_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(glcBehavior + TEST_SOURCES "test_glcBehavior.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/glcBehavior_test/test_glcBehavior.pf b/src/main/test/glcBehavior_test/test_glcBehavior.pf index d57da25610..ff104458b1 100644 --- a/src/main/test/glcBehavior_test/test_glcBehavior.pf +++ b/src/main/test/glcBehavior_test/test_glcBehavior.pf @@ -2,7 +2,7 @@ module test_glcBehavior ! Tests of glcBehaviorMod - use pfunit_mod + use funit use glcBehaviorMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/main/test/initVertical_test/CMakeLists.txt b/src/main/test/initVertical_test/CMakeLists.txt index c6e3938fb0..8fe9648a50 100644 --- a/src/main/test/initVertical_test/CMakeLists.txt +++ b/src/main/test/initVertical_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(initVertical test_initVertical_exe - "test_initVertical.pf" "") - -target_link_libraries(test_initVertical_exe clm csm_share esmf_wrf_timemgr) \ No newline at end of file +add_pfunit_ctest(initVertical + TEST_SOURCES "test_initVertical.pf" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/main/test/initVertical_test/test_initVertical.pf b/src/main/test/initVertical_test/test_initVertical.pf index 5a52671246..0726f962d1 100644 --- a/src/main/test/initVertical_test/test_initVertical.pf +++ b/src/main/test/initVertical_test/test_initVertical.pf @@ -2,7 +2,7 @@ module test_initVertical ! Tests of initVerticalMod - use pfunit_mod + use funit use initVerticalMod use shr_kind_mod , only : r8 => shr_kind_r8 use clm_varpar, only : nlevlak, nlevgrnd, nlevdecomp_full, nlayer diff --git a/src/main/test/ncdio_utils_test/CMakeLists.txt b/src/main/test/ncdio_utils_test/CMakeLists.txt index 95ff84ac1c..27f37d5abb 100644 --- a/src/main/test/ncdio_utils_test/CMakeLists.txt +++ b/src/main/test/ncdio_utils_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(ncdio_utils test_ncdio_utils_exe - "test_ncdio_utils.pf" "") - -target_link_libraries(test_ncdio_utils_exe clm csm_share) +add_pfunit_ctest(ncdio_utils + TEST_SOURCES "test_ncdio_utils.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/ncdio_utils_test/test_ncdio_utils.pf b/src/main/test/ncdio_utils_test/test_ncdio_utils.pf index c87a89e4f7..9ee1d9ba8e 100644 --- a/src/main/test/ncdio_utils_test/test_ncdio_utils.pf +++ b/src/main/test/ncdio_utils_test/test_ncdio_utils.pf @@ -2,7 +2,7 @@ module test_ncdio_utils ! Tests of ncdio_utils - use pfunit_mod + use funit use ncdio_utils use ncdio_pio ! use the fake version of this module use shr_kind_mod, only : r8 => shr_kind_r8 diff --git a/src/main/test/subgridWeights_test/CMakeLists.txt b/src/main/test/subgridWeights_test/CMakeLists.txt index 45b4d53b01..190223929c 100644 --- a/src/main/test/subgridWeights_test/CMakeLists.txt +++ b/src/main/test/subgridWeights_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(subgridWeights test_subgridWeights_exe - "test_subgridWeights.pf" "") - -target_link_libraries(test_subgridWeights_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(subgridWeights + TEST_SOURCES "test_subgridWeights.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/subgridWeights_test/test_subgridWeights.pf b/src/main/test/subgridWeights_test/test_subgridWeights.pf index 7d29ef0510..34d5f0f660 100644 --- a/src/main/test/subgridWeights_test/test_subgridWeights.pf +++ b/src/main/test/subgridWeights_test/test_subgridWeights.pf @@ -2,7 +2,7 @@ module test_subgridWeights ! Tests of subgridWeightsMod - use pfunit_mod + use funit use unittestSubgridMod use subgridWeightsMod use shr_kind_mod, only : r8 => shr_kind_r8 diff --git a/src/main/test/surfrdUtils_test/CMakeLists.txt b/src/main/test/surfrdUtils_test/CMakeLists.txt index 4139bf6faa..b41357d646 100644 --- a/src/main/test/surfrdUtils_test/CMakeLists.txt +++ b/src/main/test/surfrdUtils_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(surfrdUtils test_surfrdUtils_exe - "test_surfrdUtils.pf" "") - -target_link_libraries(test_surfrdUtils_exe clm csm_share) +add_pfunit_ctest(surfrdUtils + TEST_SOURCES "test_surfrdUtils.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/surfrdUtils_test/test_surfrdUtils.pf b/src/main/test/surfrdUtils_test/test_surfrdUtils.pf index 065dbc9b39..98191fbe99 100644 --- a/src/main/test/surfrdUtils_test/test_surfrdUtils.pf +++ b/src/main/test/surfrdUtils_test/test_surfrdUtils.pf @@ -2,7 +2,7 @@ module test_surfrdUtils ! Tests of surfrdUtilsMod - use pfunit_mod + use funit use surfrdUtilsMod use shr_kind_mod, only : r8 => shr_kind_r8 diff --git a/src/main/test/topo_test/CMakeLists.txt b/src/main/test/topo_test/CMakeLists.txt index c01625994a..7329ab8bd7 100644 --- a/src/main/test/topo_test/CMakeLists.txt +++ b/src/main/test/topo_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(topo test_topo_exe - "test_topo.pf" "") - -target_link_libraries(test_topo_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(topo + TEST_SOURCES "test_topo.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/main/test/topo_test/test_topo.pf b/src/main/test/topo_test/test_topo.pf index 28a46b474d..196ce34763 100644 --- a/src/main/test/topo_test/test_topo.pf +++ b/src/main/test/topo_test/test_topo.pf @@ -2,7 +2,7 @@ module test_topo ! Tests of TopoMod - use pfunit_mod + use funit use TopoMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestGlcMec diff --git a/src/self_tests/test/assertions_test/CMakeLists.txt b/src/self_tests/test/assertions_test/CMakeLists.txt index de7a0febef..d36d8c7675 100644 --- a/src/self_tests/test/assertions_test/CMakeLists.txt +++ b/src/self_tests/test/assertions_test/CMakeLists.txt @@ -1,10 +1,6 @@ set (pfunit_sources test_assertions.pf) -set (extra_sources - ) - -create_pFUnit_test(assertions test_assertions_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_assertions_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(assertions + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/self_tests/test/assertions_test/test_assertions.pf b/src/self_tests/test/assertions_test/test_assertions.pf index 10eaaafce6..6350048302 100644 --- a/src/self_tests/test/assertions_test/test_assertions.pf +++ b/src/self_tests/test/assertions_test/test_assertions.pf @@ -2,7 +2,7 @@ module test_assertions ! Tests of Assertions - use pfunit_mod + use funit use Assertions use shr_kind_mod , only : r8 => shr_kind_r8 use unittestUtils, only : endrun_msg diff --git a/src/soilbiogeochem/test/ACSpinup_test/CMakeLists.txt b/src/soilbiogeochem/test/ACSpinup_test/CMakeLists.txt index 17f01d6d22..c74163bd52 100644 --- a/src/soilbiogeochem/test/ACSpinup_test/CMakeLists.txt +++ b/src/soilbiogeochem/test/ACSpinup_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(acspinup test_acspinup_exe - "test_acspinup.pf" "") - -target_link_libraries(test_acspinup_exe clm csm_share) +add_pfunit_ctest(acspinup + TEST_SOURCES "test_acspinup.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/soilbiogeochem/test/ACSpinup_test/test_acspinup.pf b/src/soilbiogeochem/test/ACSpinup_test/test_acspinup.pf index df9c4f80fb..b3de19f4ad 100644 --- a/src/soilbiogeochem/test/ACSpinup_test/test_acspinup.pf +++ b/src/soilbiogeochem/test/ACSpinup_test/test_acspinup.pf @@ -2,7 +2,7 @@ module test_acspinup ! Tests of the acspinup functions in SoilBiogeochemStateType - use pfunit_mod + use funit use shr_kind_mod , only : r8 => shr_kind_r8 use SoilBiogeochemStateType, only : get_spinup_latitude_term diff --git a/src/unit_test_shr/test/unittestArray_test/CMakeLists.txt b/src/unit_test_shr/test/unittestArray_test/CMakeLists.txt index 9a18380545..9889d96e0a 100644 --- a/src/unit_test_shr/test/unittestArray_test/CMakeLists.txt +++ b/src/unit_test_shr/test/unittestArray_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(unittestArray test_unittestArray_exe - "test_unittestArray.pf" "") - -target_link_libraries(test_unittestArray_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(unittestArray + TEST_SOURCES "test_unittestArray.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/unit_test_shr/test/unittestArray_test/test_unittestArray.pf b/src/unit_test_shr/test/unittestArray_test/test_unittestArray.pf index b11d62e612..f48fd7ad52 100644 --- a/src/unit_test_shr/test/unittestArray_test/test_unittestArray.pf +++ b/src/unit_test_shr/test/unittestArray_test/test_unittestArray.pf @@ -2,7 +2,7 @@ module test_unittestArray ! Tests of unittestArrayMod - use pfunit_mod + use funit use unittestArrayMod use unittestSubgridMod use unittestSimpleSubgridSetupsMod diff --git a/src/unit_test_shr/test/unittestFilterBuilder_test/CMakeLists.txt b/src/unit_test_shr/test/unittestFilterBuilder_test/CMakeLists.txt index c767479aee..698cd525b6 100644 --- a/src/unit_test_shr/test/unittestFilterBuilder_test/CMakeLists.txt +++ b/src/unit_test_shr/test/unittestFilterBuilder_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(unittestFilterBuilder test_filterBuilder_exe - "test_filterBuilder.pf" "") - -target_link_libraries(test_filterBuilder_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(unittestFilterBuilder + TEST_SOURCES "test_filterBuilder.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/unit_test_shr/test/unittestFilterBuilder_test/test_filterBuilder.pf b/src/unit_test_shr/test/unittestFilterBuilder_test/test_filterBuilder.pf index 58674110a9..5f017c10b8 100644 --- a/src/unit_test_shr/test/unittestFilterBuilder_test/test_filterBuilder.pf +++ b/src/unit_test_shr/test/unittestFilterBuilder_test/test_filterBuilder.pf @@ -2,7 +2,7 @@ module test_filterBuilder ! Tests of unittestFilterBuilder - use pfunit_mod + use funit use unittestFilterBuilderMod implicit none diff --git a/src/unit_test_shr/test/unittestSubgrid_test/CMakeLists.txt b/src/unit_test_shr/test/unittestSubgrid_test/CMakeLists.txt index 2e0bc4f03e..e6bf0e58b4 100644 --- a/src/unit_test_shr/test/unittestSubgrid_test/CMakeLists.txt +++ b/src/unit_test_shr/test/unittestSubgrid_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(unittestSubgrid test_unittestSubgrid_exe - "test_unittestSubgrid.pf" "") - -target_link_libraries(test_unittestSubgrid_exe clm csm_share) \ No newline at end of file +add_pfunit_ctest(unittestSubgrid + TEST_SOURCES "test_unittestSubgrid.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/unit_test_shr/test/unittestSubgrid_test/test_unittestSubgrid.pf b/src/unit_test_shr/test/unittestSubgrid_test/test_unittestSubgrid.pf index a0bea34dc4..c8ae194249 100644 --- a/src/unit_test_shr/test/unittestSubgrid_test/test_unittestSubgrid.pf +++ b/src/unit_test_shr/test/unittestSubgrid_test/test_unittestSubgrid.pf @@ -2,7 +2,7 @@ module test_unittestSubgrid ! Tests of unittestSubgridMod - use pfunit_mod + use funit use unittestSubgridMod use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/src/utils/test/annual_flux_dribbler_test/CMakeLists.txt b/src/utils/test/annual_flux_dribbler_test/CMakeLists.txt index 74923ce9a7..a0a60e7431 100644 --- a/src/utils/test/annual_flux_dribbler_test/CMakeLists.txt +++ b/src/utils/test/annual_flux_dribbler_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(annual_flux_dribbler test_annual_flux_dribbler_exe - "test_annual_flux_dribbler.pf" "") - -target_link_libraries(test_annual_flux_dribbler_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(annual_flux_dribbler + TEST_SOURCES "test_annual_flux_dribbler.pf" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/utils/test/annual_flux_dribbler_test/test_annual_flux_dribbler.pf b/src/utils/test/annual_flux_dribbler_test/test_annual_flux_dribbler.pf index bd9f8e489a..79b9718547 100644 --- a/src/utils/test/annual_flux_dribbler_test/test_annual_flux_dribbler.pf +++ b/src/utils/test/annual_flux_dribbler_test/test_annual_flux_dribbler.pf @@ -2,7 +2,7 @@ module test_annual_flux_dribbler ! Tests of AnnualFluxDribbler - use pfunit_mod + use funit use AnnualFluxDribbler use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSubgridMod diff --git a/src/utils/test/array_utils_test/CMakeLists.txt b/src/utils/test/array_utils_test/CMakeLists.txt index 11cbd47e5a..21c7d91d81 100644 --- a/src/utils/test/array_utils_test/CMakeLists.txt +++ b/src/utils/test/array_utils_test/CMakeLists.txt @@ -3,10 +3,6 @@ set (pfunit_sources test_convert_to_logical.pf ) -set (extra_sources - ) - -create_pFUnit_test(array_utils test_array_utils_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_array_utils_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(array_utils + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/utils/test/array_utils_test/test_convert_to_logical.pf b/src/utils/test/array_utils_test/test_convert_to_logical.pf index 6e726d443a..3a7ba7b4de 100644 --- a/src/utils/test/array_utils_test/test_convert_to_logical.pf +++ b/src/utils/test/array_utils_test/test_convert_to_logical.pf @@ -2,7 +2,7 @@ module test_convert_to_logical ! Tests of array_utils: convert_to_logical - use pfunit_mod + use funit use array_utils use shr_kind_mod , only : r8 => shr_kind_r8 use unittestUtils, only : endrun_msg diff --git a/src/utils/test/array_utils_test/test_find_k_max_indices.pf b/src/utils/test/array_utils_test/test_find_k_max_indices.pf index 084f5fc415..e125491085 100644 --- a/src/utils/test/array_utils_test/test_find_k_max_indices.pf +++ b/src/utils/test/array_utils_test/test_find_k_max_indices.pf @@ -2,7 +2,7 @@ module test_find_k_max_indices ! Tests of array_utils: find_k_max_indices - use pfunit_mod + use funit use array_utils use shr_kind_mod , only : r8 => shr_kind_r8 use unittestUtils, only : endrun_msg diff --git a/src/utils/test/clm_time_manager_test/CMakeLists.txt b/src/utils/test/clm_time_manager_test/CMakeLists.txt index 3651eaf984..f34e77dfc9 100644 --- a/src/utils/test/clm_time_manager_test/CMakeLists.txt +++ b/src/utils/test/clm_time_manager_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(clm_time_manager test_clm_time_manager_exe - "test_clm_time_manager.pf" "") - -target_link_libraries(test_clm_time_manager_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(clm_time_manager + TEST_SOURCES "test_clm_time_manager.pf" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/utils/test/clm_time_manager_test/test_clm_time_manager.pf b/src/utils/test/clm_time_manager_test/test_clm_time_manager.pf index 9e49bad266..435d795e50 100644 --- a/src/utils/test/clm_time_manager_test/test_clm_time_manager.pf +++ b/src/utils/test/clm_time_manager_test/test_clm_time_manager.pf @@ -2,7 +2,7 @@ module test_clm_time_manager ! Tests of clm_time_manager - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use clm_time_manager use unittestTimeManagerMod, only : unittest_timemgr_setup, unittest_timemgr_teardown diff --git a/src/utils/test/matrix_test/CMakeLists.txt b/src/utils/test/matrix_test/CMakeLists.txt index a9c8456eea..50ed6673d8 100644 --- a/src/utils/test/matrix_test/CMakeLists.txt +++ b/src/utils/test/matrix_test/CMakeLists.txt @@ -1,10 +1,6 @@ set (pfunit_sources test_matrix.pf) -set (extra_sources - ) - -create_pFUnit_test(matrix test_matrix_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_matrix_exe clm csm_share) +add_pfunit_ctest(matrix + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/utils/test/matrix_test/test_matrix.pf b/src/utils/test/matrix_test/test_matrix.pf index 5ffa8726b3..67fc6ded20 100644 --- a/src/utils/test/matrix_test/test_matrix.pf +++ b/src/utils/test/matrix_test/test_matrix.pf @@ -2,7 +2,7 @@ module test_matrix ! Tests of Matrix: inverse - use pfunit_mod + use funit use MatrixMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestUtils, only : endrun_msg @@ -20,7 +20,7 @@ module test_matrix integer, parameter :: ndims = 20 - real(r8), parameter :: tol = 1.e-15_r8 + real(r8), parameter :: tol = 1.e-14_r8 contains diff --git a/src/utils/test/numerics_test/CMakeLists.txt b/src/utils/test/numerics_test/CMakeLists.txt index 83bf0cc1d0..19d2c67451 100644 --- a/src/utils/test/numerics_test/CMakeLists.txt +++ b/src/utils/test/numerics_test/CMakeLists.txt @@ -1,10 +1,6 @@ set (pfunit_sources test_truncate_small_values.pf) -set (extra_sources - ) - -create_pFUnit_test(numerics test_numerics_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_numerics_exe clm csm_share esmf_wrf_timemgr) +add_pfunit_ctest(numerics + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf_wrf_timemgr) diff --git a/src/utils/test/numerics_test/test_truncate_small_values.pf b/src/utils/test/numerics_test/test_truncate_small_values.pf index e4673d5388..c1faf8e622 100644 --- a/src/utils/test/numerics_test/test_truncate_small_values.pf +++ b/src/utils/test/numerics_test/test_truncate_small_values.pf @@ -2,7 +2,7 @@ module test_truncate_small_values ! Tests of NumericsMod: truncate_small_values - use pfunit_mod + use funit use NumericsMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestSimpleSubgridSetupsMod diff --git a/src/utils/test/quadratic_test/CMakeLists.txt b/src/utils/test/quadratic_test/CMakeLists.txt index 50d6c92f1a..daea7647cc 100644 --- a/src/utils/test/quadratic_test/CMakeLists.txt +++ b/src/utils/test/quadratic_test/CMakeLists.txt @@ -1,4 +1,3 @@ -create_pFUnit_test(quadratic test_quadratic_exe - "test_quadratic.pf" "") - -target_link_libraries(test_quadratic_exe clm csm_share) +add_pfunit_ctest(quadratic + TEST_SOURCES "test_quadratic.pf" + LINK_LIBRARIES clm csm_share) diff --git a/src/utils/test/quadratic_test/test_quadratic.pf b/src/utils/test/quadratic_test/test_quadratic.pf index 34ecc12eb7..7acfbae6e9 100644 --- a/src/utils/test/quadratic_test/test_quadratic.pf +++ b/src/utils/test/quadratic_test/test_quadratic.pf @@ -2,7 +2,7 @@ module test_quadratic ! Tests of quadratic - use pfunit_mod + use funit use quadraticMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestUtils, only : endrun_msg @@ -176,8 +176,6 @@ contains b = 4.0_r8 c = 4.0_r8 + 100.0_r8*epsilon(b) call quadratic (a, b, c, r1, r2) - call check_root(a, b, c, r1) - call check_root(a, b, c, r2) expected_msg = endrun_msg( & 'quadratic ERROR: Quadratic solution error: b^2 - 4ac is negative') @assertExceptionRaised(expected_msg) diff --git a/src/utils/test/sparse_matrix_test/CMakeLists.txt b/src/utils/test/sparse_matrix_test/CMakeLists.txt index 7ba943b272..ba8a7966c8 100644 --- a/src/utils/test/sparse_matrix_test/CMakeLists.txt +++ b/src/utils/test/sparse_matrix_test/CMakeLists.txt @@ -1,10 +1,6 @@ set (pfunit_sources test_sparse_matrix.pf) -set (extra_sources - ) - -create_pFUnit_test(sparse_matrix test_sparse_matrix_exe - "${pfunit_sources}" "${extra_sources}") - -target_link_libraries(test_sparse_matrix_exe clm csm_share) +add_pfunit_ctest(sparse_matrix + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share) diff --git a/src/utils/test/sparse_matrix_test/test_sparse_matrix.pf b/src/utils/test/sparse_matrix_test/test_sparse_matrix.pf index 9e69fdace5..2f8bc1ca7b 100644 --- a/src/utils/test/sparse_matrix_test/test_sparse_matrix.pf +++ b/src/utils/test/sparse_matrix_test/test_sparse_matrix.pf @@ -2,7 +2,7 @@ module test_sparse_matrix ! Tests of Sparse Matrix Multiply module - use pfunit_mod + use funit use SparseMatrixMultiplyMod use shr_kind_mod , only : r8 => shr_kind_r8 use unittestUtils, only : endrun_msg diff --git a/tools/mksurfdata_map/src/test/mkdomain_test/test_mkdomain.pf b/tools/mksurfdata_map/src/test/mkdomain_test/test_mkdomain.pf index c54e299356..fd6a2e1e7b 100644 --- a/tools/mksurfdata_map/src/test/mkdomain_test/test_mkdomain.pf +++ b/tools/mksurfdata_map/src/test/mkdomain_test/test_mkdomain.pf @@ -2,7 +2,7 @@ module test_mkdomain ! Tests of mkdomainMod - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkgridmapMod, only : gridmap_type, for_test_create_gridmap diff --git a/tools/mksurfdata_map/src/test/mkgridmap_test/test_mkgridmap.pf b/tools/mksurfdata_map/src/test/mkgridmap_test/test_mkgridmap.pf index c8eda9d007..a15944384f 100644 --- a/tools/mksurfdata_map/src/test/mkgridmap_test/test_mkgridmap.pf +++ b/tools/mksurfdata_map/src/test/mkgridmap_test/test_mkgridmap.pf @@ -2,7 +2,7 @@ module test_mkgridmap ! Tests of mkgridmapMod - use pfunit_mod + use funit use mkgridmapMod use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/tools/mksurfdata_map/src/test/mkindexmap_test/test_mkindexmap.pf b/tools/mksurfdata_map/src/test/mkindexmap_test/test_mkindexmap.pf index 98e9590478..7f6f1d6745 100644 --- a/tools/mksurfdata_map/src/test/mkindexmap_test/test_mkindexmap.pf +++ b/tools/mksurfdata_map/src/test/mkindexmap_test/test_mkindexmap.pf @@ -2,7 +2,7 @@ module test_mkindexmap ! Tests of mkindexmapMod - use pfunit_mod + use funit use mkindexmapMod use mkgridmapMod, only : gridmap_type, for_test_create_gridmap, gridmap_clean use shr_kind_mod , only : r8 => shr_kind_r8 diff --git a/tools/mksurfdata_map/src/test/mkpctPftType_test/test_mkpctPftType.pf b/tools/mksurfdata_map/src/test/mkpctPftType_test/test_mkpctPftType.pf index 47e7e90f48..1652a742a2 100644 --- a/tools/mksurfdata_map/src/test/mkpctPftType_test/test_mkpctPftType.pf +++ b/tools/mksurfdata_map/src/test/mkpctPftType_test/test_mkpctPftType.pf @@ -2,7 +2,7 @@ module test_mkpctPftType ! Tests of pct_pft_type - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkpctPftTypeMod diff --git a/tools/mksurfdata_map/src/test/mkpftUtils_test/test_adjust_total_veg_area.pf b/tools/mksurfdata_map/src/test/mkpftUtils_test/test_adjust_total_veg_area.pf index 345c1a7370..1223e5bc68 100644 --- a/tools/mksurfdata_map/src/test/mkpftUtils_test/test_adjust_total_veg_area.pf +++ b/tools/mksurfdata_map/src/test/mkpftUtils_test/test_adjust_total_veg_area.pf @@ -2,7 +2,7 @@ module test_adjust_total_veg_area ! Tests of mkpftUtilsMod: adjust_total_veg_area - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkpctPftTypeMod, only : pct_pft_type diff --git a/tools/mksurfdata_map/src/test/mkpftUtils_test/test_convert_from_p2g.pf b/tools/mksurfdata_map/src/test/mkpftUtils_test/test_convert_from_p2g.pf index 53548e4e6c..3227031726 100644 --- a/tools/mksurfdata_map/src/test/mkpftUtils_test/test_convert_from_p2g.pf +++ b/tools/mksurfdata_map/src/test/mkpftUtils_test/test_convert_from_p2g.pf @@ -2,7 +2,7 @@ module test_convert_from_p2g ! Tests of mkpftUtilsMod: convert_from_p2g - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkpctPftTypeMod, only : pct_pft_type diff --git a/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftInit.pf b/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftInit.pf index 1ddb143961..a555e3e8ca 100644 --- a/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftInit.pf +++ b/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftInit.pf @@ -2,7 +2,7 @@ module test_pftInit ! Tests of mkpftMod: pft_override functions - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkpftMod diff --git a/tools/mksurfdata_map/src/test/mkpftmod_test/test_pft_oride.pf b/tools/mksurfdata_map/src/test/mkpftmod_test/test_pft_oride.pf index 97cfc66d1e..bc9cda88de 100644 --- a/tools/mksurfdata_map/src/test/mkpftmod_test/test_pft_oride.pf +++ b/tools/mksurfdata_map/src/test/mkpftmod_test/test_pft_oride.pf @@ -2,7 +2,7 @@ module test_pft_oride ! Tests of mkpftMod: pft_override functions - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkpftMod diff --git a/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftrun.pf b/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftrun.pf index 389748764b..77c8f0e623 100644 --- a/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftrun.pf +++ b/tools/mksurfdata_map/src/test/mkpftmod_test/test_pftrun.pf @@ -2,7 +2,7 @@ module test_pftrun ! Tests of mkpftMod: pft_override functions - use pfunit_mod + use funit use shr_kind_mod, only : r8 => shr_kind_r8 use mkpftMod diff --git a/tools/mksurfdata_map/src/test/mksoilUtils_test/test_dominant_soil_color.pf b/tools/mksurfdata_map/src/test/mksoilUtils_test/test_dominant_soil_color.pf index b506549d87..011f20d70c 100644 --- a/tools/mksurfdata_map/src/test/mksoilUtils_test/test_dominant_soil_color.pf +++ b/tools/mksurfdata_map/src/test/mksoilUtils_test/test_dominant_soil_color.pf @@ -2,7 +2,7 @@ module test_dominant_soil_color ! Tests of mksoilUtilsMod: dominant_soil_color - use pfunit_mod + use funit use mksoilUtilsMod use shr_kind_mod , only : r8 => shr_kind_r8 use mkgridmapMod, only : gridmap_type, gridmap_clean, for_test_create_gridmap diff --git a/tools/mksurfdata_map/src/unit_test_stubs/abort.F90 b/tools/mksurfdata_map/src/unit_test_stubs/abort.F90 index aa1d8b76c2..8f56fc82fc 100644 --- a/tools/mksurfdata_map/src/unit_test_stubs/abort.F90 +++ b/tools/mksurfdata_map/src/unit_test_stubs/abort.F90 @@ -18,7 +18,7 @@ subroutine abort() ! call assertExceptionRaised ! ! then this will result in the given pFUnit test failing. - use pfunit_mod, only : throw + use funit, only : throw implicit none call throw("ABORTED:") From 9784e9773be50f40f45c191152da5aca7e619298 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 18 Apr 2023 16:35:13 -0600 Subject: [PATCH 268/332] Update ccs_config external --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index bc9b22f258..78b9e8ec05 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -hash = bd942aae8fbc2848420d2d3249212ef38611724f +hash = bf7e85f30c9d434e047bd2494779db3b4ef5e3cb protocol = git repo_url = https://github.com/billsacks/ccs_config_cesm.git local_path = ccs_config From e0c9b9fa5350ed632dd7b9b97a0829326d0fd2ea Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 18 Apr 2023 16:38:20 -0600 Subject: [PATCH 269/332] Another update to ccs_config --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 78b9e8ec05..1083fd4965 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -hash = bf7e85f30c9d434e047bd2494779db3b4ef5e3cb +hash = 8a7b4c61c6ace2a4905bf6aff973a28509e63a93 protocol = git repo_url = https://github.com/billsacks/ccs_config_cesm.git local_path = ccs_config From 2eb757822f4dd7a2ffbb7b99fe7468d422b737ff Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 18 Apr 2023 16:59:52 -0600 Subject: [PATCH 270/332] Update ccs_config --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 1083fd4965..5d51de8ae1 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -hash = 8a7b4c61c6ace2a4905bf6aff973a28509e63a93 +hash = dc66fd06256d6eb5e6fa493042123b4b54302e81 protocol = git repo_url = https://github.com/billsacks/ccs_config_cesm.git local_path = ccs_config From 9ca2e3f8164a426a274511407989db8a74b629ce Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Tue, 18 Apr 2023 17:04:54 -0600 Subject: [PATCH 271/332] Update CIME external --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 5d51de8ae1..0649acf642 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -44,7 +44,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/billsacks/cime -hash = a42f55a0c31706c377c77ed72b6ebbe8f02f8e92 +hash = 8316e3b4e11bb0f51b57aac42481fdacd69e60df required = True [cmeps] From acc32ee89613c96b1a6163e2bd0f4c6baef74ad4 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 20 Apr 2023 16:58:09 -0600 Subject: [PATCH 272/332] documentation update --- cime_config/usermods_dirs/NEON/defaults/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/shell_commands b/cime_config/usermods_dirs/NEON/defaults/shell_commands index ad23164b73..d7173100be 100644 --- a/cime_config/usermods_dirs/NEON/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/defaults/shell_commands @@ -1,6 +1,6 @@ #!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON -# This can be set to either NEON or NEON.PRISM +# CLM_USRDAT_NAME can be set to either NEON or NEON.PRISM ./xmlchange CCSM_CO2_PPMV=408.83 # Set data forcing data to future scenario so will have data from 2018 to present-day ./xmlchange DATM_PRESAERO=SSP3-7.0 From 686c64deeef48f947a149604b84003ab7759adad Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 20 Apr 2023 19:35:04 -0600 Subject: [PATCH 273/332] Add CDEPS branch to externals --- Externals.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index dfc4191642..9d7adf3a7c 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,9 +55,11 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.7 +#tag = cdeps1.0.7 +branch = main protocol = git -repo_url = https://github.com/ESCOMP/CDEPS.git +#repo_url = https://github.com/ESCOMP/CDEPS.git +repo_url = https://github.com/TeaganKing/CDEPS.git local_path = components/cdeps externals = Externals_CDEPS.cfg required = True From f92d389e518446f417328ea7e03aab1c812db3b6 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 21 Apr 2023 23:25:04 -0600 Subject: [PATCH 274/332] Set the testmod to PRISM --- cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands b/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands index 0b5ec8cff6..39812a8706 100644 --- a/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/PRISM/shell_commands @@ -1,2 +1,2 @@ #!/bin/bash -./xmlchange CLM_USRDAT_NAME=NEON +./xmlchange CLM_USRDAT_NAME=NEON.PRISM From e3ee3140ee2339af15e8b68eab8c1c1d6a5cba34 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 22 Apr 2023 13:40:47 -0600 Subject: [PATCH 275/332] This change is not needed anymore --- .../usermods_dirs/NEON/defaults/user_nl_datm_streams | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams index 53c4e4e4a7..bae77db6b5 100644 --- a/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams +++ b/cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams @@ -38,12 +38,3 @@ preso3.SSP3-7.0:year_last=2022 preso3.SSP3-7.0:year_align=2018 preso3.SSP3-7.0:dtlimit=30 -NEON.${NEONSITE}:datavars = \ - FSDS Faxa_swdn, \ - ZBOT Sa_z, \ - TBOT Sa_tbot, \ - WIND Sa_wind, \ - RH Sa_rh, \ - PSRF Sa_pbot, \ - FLDS Faxa_lwdn - From 2e5913a57a17c8f70cb07b50ec0efb3196276c67 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 22 Apr 2023 14:46:55 -0600 Subject: [PATCH 276/332] Update the description of CLM_USRDAT_NAME --- cime_config/config_component.xml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index b0f77cb1c3..6f1933c858 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -241,12 +241,13 @@ UNSET run_component_ctsm env_run.xml - Dataset name for user-created datasets. This is used as the argument - in Buildconf/clm.buildnml to build-namelist -clm_usr_name. An example of - such a dataset would be 1x1pt_boulderCO_c090722. The default value is UNSET. - This is an advanced flag and should only be used by expert users. For NEON - cases, this can be set to either NEON or NEON.PRISM, the latter of which would - PRISM precipitation instead of the default NEON precipitation. + Resolution name for user-created resolutions. This is especially used + for single point and regional resolutions created via subset_data from + the global datasets. This should be set when you use CLM_USRDAT as the grid + to create_newcase. The default value is UNSET. + For NEON cases, this can be set to either NEON or NEON.PRISM, the latter of which would + use PRISM precipitation instead of the default NEON precipitation. NEON cases then also + use the variable NEONSITE to specify the exact site. From 8fd2ee136cd63c4b9c882fc9fb842714ef9f654c Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 23 Apr 2023 06:42:46 -0600 Subject: [PATCH 277/332] Update externals to tagged versions on the main branches --- Externals.cfg | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 0649acf642..6702c569ee 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,17 +34,17 @@ hash = 34723c2 required = True [ccs_config] -hash = dc66fd06256d6eb5e6fa493042123b4b54302e81 +tag = ccs_config_cesm0.0.63 protocol = git -repo_url = https://github.com/billsacks/ccs_config_cesm.git +repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/billsacks/cime -hash = 8316e3b4e11bb0f51b57aac42481fdacd69e60df +repo_url = https://github.com/ESMCI/cime +tag = cime6.0.108 required = True [cmeps] @@ -63,16 +63,16 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -hash = 4b3e95623cdd37cb7e72b55b1deb5442bc302faa +tag = cpl77.0.5 protocol = git -repo_url = https://github.com/billsacks/CESM_CPL7andDataComps +repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 required = True [share] -hash = 988c9ec7a94840be80b1851769ce93af859e3005 +tag = share1.0.17 protocol = git -repo_url = https://github.com/billsacks/CESM_share +repo_url = https://github.com/ESCOMP/CESM_share local_path = share required = True From a72ef45746c0621c2578bb9674e60a9bc226ad5e Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 23 Apr 2023 07:14:24 -0600 Subject: [PATCH 278/332] Update some other externals to more recent versions Use versions of externals from latest CESM tag --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 6702c569ee..72a914fbba 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -48,14 +48,14 @@ tag = cime6.0.108 required = True [cmeps] -tag = cmeps0.14.17 +tag = cmeps0.14.21 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.7 +tag = cdeps1.0.9 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps From 4e58ce37dd8cce4a0fe10ed96594c606635682be Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Sun, 23 Apr 2023 19:48:52 -0600 Subject: [PATCH 279/332] Update ChangeLog --- doc/ChangeLog | 166 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 167 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index edb6223e2e..f25b413a38 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,170 @@ =============================================================== +Tag name: ctsm5.1.dev122 +Originator(s): sacks (Bill Sacks) +Date: Sun Apr 23 19:36:37 MDT 2023 +One-line Summary: Rework handling of evaporation constraint in SoilFluxes + +Purpose and description of changes +---------------------------------- + +Occasionally, h2osoi_ice was going significantly negative in +UpdateState_TopLayerFluxes - see +https://github.com/ESCOMP/CTSM/issues/1979. As noted in that issue, this +seems to be due to h2osoi_ice having a very different magnitude from +h2osoi_liq, leading to greater-than-roundoff-level differences from zero +final state in a relative sense (i.e., relative to the magnitude of +h2osoi_ice) - I think because of the appearance of the sum (h2osoi_ice + +h2osoi_liq) in the equations that limit fluxes. + +To try to deal with this, I have reworked the handling of the +evaporation constraint to directly limit both the liqevap and solidevap, +so that both of them should result in the equivalent liq or ice states +going to 0 within roundoff. + +To do that, I needed to move the partitioning of the total flux into +liquid and solid to earlier in the subroutine and then recalculate those +partitioning fluxes in conditions where we're applying an evaporation +constraint. + +Note that I applied a max of 0 to the new fluxes because many initial +conditions files have roundoff-level negative H2OSOI_LIQ, so without +this limit, we were getting roundoff-level negative fluxes. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +- Resolves ESCOMP/CTSM#1979 (Need some changes to avoid negative h2osoi_ice in UpdateState_TopLayerFluxes) + + +Testing summary: +---------------- + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + Tests passed, some baseline differences as expected. + +Answer changes +-------------- + +Changes answers relative to baseline: YES + + Summarize any changes to answers, i.e., + - what code configurations: potentially all + - what platforms/compilers: potentially all + - nature of change (roundoff; larger than roundoff/same climate; new climate): + roundoff + + Differences were only observed in a few tests: + - ERP_P36x2_Ld30.f45_f45_mg37.I2000Clm51FatesSpCruRsGs.cheyenne_intel.clm-FatesColdSatPhen + - ERI_D_Ld9_P48x1.T31_g37.I2000Clm50Sp.izumi_nag.clm-reduceOutput + - SMS_D_Ln9_P36x3.f19_g17.IHistClm50Sp.cheyenne_intel.clm-waccmx_offline + - SMS_D_Ln9_P36x3_Vmct.f19_g17.IHistClm50Sp.cheyenne_intel.clm-waccmx_offline + + If bitwise differences were observed, how did you show they were no worse + than roundoff? + + Only two tests had greater-than-roundoff-level differences in the + cprnc output: + SMS_D_Ln9_P36x3.f19_g17.IHistClm50Sp.cheyenne_intel.clm-waccmx_offline + and the mct equivalent, + SMS_D_Ln9_P36x3_Vmct.f19_g17.IHistClm50Sp.cheyenne_intel.clm-waccmx_offline. + To verify that differences were fundamentally no greater than + roundoff-level, I introduced temporary code; this minimal diff + ended up being enough to give just roundoff-level differences from baseline: + + diff --git a/src/biogeophys/SoilFluxesMod.F90 b/src/biogeophys/SoilFluxesMod.F90 + index c316d30fe..6a958c0ee 100644 + --- a/src/biogeophys/SoilFluxesMod.F90 + +++ b/src/biogeophys/SoilFluxesMod.F90 + @@ -45,7 +45,7 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & + ! Update surface fluxes based on the new ground temperature + ! + ! !USES: + - use clm_time_manager , only : get_step_size_real + + use clm_time_manager , only : get_step_size_real, get_nstep + use clm_varcon , only : hvap, cpair, grav, vkc, tfrz, sb + use landunit_varcon , only : istsoil, istcrop + use column_varcon , only : icol_roof, icol_sunwall, icol_shadewall, icol_road_perv + @@ -79,7 +79,9 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & + real(r8) :: t_grnd0(bounds%begc:bounds%endc) ! t_grnd of previous time step + real(r8) :: lw_grnd + real(r8) :: evaporation_limit ! top layer moisture available for evaporation + - real(r8) :: evaporation_demand ! evaporative demand + + real(r8) :: evaporation_demand ! evaporative demand + + real(r8) :: qflx_liqevap_orig + + real(r8) :: qflx_solidevap_orig + !----------------------------------------------------------------------- + + associate( & + @@ -291,6 +293,7 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & + qflx_evap_soi(p) = qflx_evap_soi(p) - frac_sno_eff(c)*(evaporation_demand - evaporation_limit) + qflx_liqevap_from_top_layer(p) = max(h2osoi_liq(c,j)/(frac_sno_eff(c)*dtime), 0._r8) + qflx_solidevap_from_top_layer(p) = max(h2osoi_ice(c,j)/(frac_sno_eff(c)*dtime), 0._r8) + + + ! conserve total energy flux + eflx_sh_grnd(p) = eflx_sh_grnd(p) + frac_sno_eff(c)*(evaporation_demand - evaporation_limit)*htvp(c) + endif + @@ -307,6 +310,24 @@ subroutine SoilFluxes (bounds, num_urbanl, filter_urbanl, & + qflx_ev_snow(p) = qflx_evap_soi(p) + qflx_liqevap_from_top_layer(p) = max(h2osoi_liq(c,j)/dtime, 0._r8) + qflx_solidevap_from_top_layer(p) = max(h2osoi_ice(c,j)/dtime, 0._r8) + + + + if (h2osoi_liq(c,j) + h2osoi_ice(c,j) > 0._r8) then + + qflx_liqevap_orig = max(qflx_evap_soi(p)*(h2osoi_liq(c,j)/ & + + (h2osoi_liq(c,j)+h2osoi_ice(c,j))), 0._r8) + + else + + qflx_liqevap_orig = 0._r8 + + end if + + qflx_solidevap_orig = qflx_evap_soi(p) - qflx_liqevap_orig + + if (qflx_solidevap_from_top_layer(p) == 0._r8 .and. & + + qflx_solidevap_orig < 0._r8 .and. & + + qflx_solidevap_orig > -1.e-16_r8) then + + write(iulog,'(a, i0, 1x, i0, 1x, 5e24.17)') & + + 'WJS adj urb: solid orig le 0, new 0: nstep, p, orig, new, qflx_evap_soi, h2osoi_liq, h2osoi_ice = ', & + + get_nstep(), p, qflx_solidevap_orig, qflx_solidevap_from_top_layer(p), & + + qflx_evap_soi(p), h2osoi_liq(c,j), h2osoi_ice(c,j) + + qflx_solidevap_from_top_layer(p) = qflx_solidevap_orig + + end if + + + ! conserve total energy flux + eflx_sh_grnd(p) = eflx_sh_grnd(p) +(evaporation_demand -evaporation_limit)*htvp(c) + endif + + (Note that the diffs in + ERP_P36x2_Ld30.f45_f45_mg37.I2000Clm51FatesSpCruRsGs.cheyenne_intel.clm-FatesColdSatPhen + were ambiguous as to whether they were roundoff-level due to the + single-precision output in that test; I reran with double precision + for the baseline and the branch and was able to verify that the + diffs were only double-precision roundoff-level.) + +Other details +------------- + +Pull Requests that document the changes (include PR ids): +https://github.com/ESCOMP/CTSM/pull/1987 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev121 Originator(s): glemieux (Gregory Lemieux,LBL/NGEET,510-486-5049) Date: Wed Apr 5 13:34:09 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 24a0b303c6..faadd8cee6 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev122 sacks 04/23/2023 Rework handling of evaporation constraint in SoilFluxes ctsm5.1.dev121 glemieux 04/05/2023 Changes soil moisture initialization logic for FATES ctsm5.1.dev120 sacks 03/25/2023 Update externals and minor fixes ctsm5.1.dev119 slevis 03/16/2023 Allow gross unrepresented land use transition (PR #309) From 90fcb22221658eb2c9571d618083f316a7fdaeff Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Apr 2023 11:19:32 -0600 Subject: [PATCH 280/332] Change the PRISM test to one of the sites where the dates were different for PRISM, and add a FATES PRISM test for another such site --- cime_config/testdefs/testlist_clm.xml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 89cf33cfe1..049d505ef6 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1561,7 +1561,7 @@ - + @@ -1582,6 +1582,17 @@ + + + + + + + + + + + From 0a89caeec35f2058497b256f245195b42bf81100 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Apr 2023 13:22:30 -0600 Subject: [PATCH 281/332] Add a comment about why the year is earlier for some sites --- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/MOAB/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/NIWO/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/SJER/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/TEAK/shell_commands | 3 ++- cime_config/usermods_dirs/NEON/WREF/shell_commands | 2 ++ cime_config/usermods_dirs/NEON/YELL/shell_commands | 2 ++ 8 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index 23e3a7014c..c4c215772a 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -2,6 +2,8 @@ ./xmlchange NEONSITE=MLBS ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON'] then ./xmlchange DATM_YR_END=2019 diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index c82e8879bf..782863e9d0 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -2,6 +2,8 @@ ./xmlchange NEONSITE=MOAB ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [$CLM_USRDAT_NAME=='NEON']; then ./xmlchange DATM_YR_END=2020 # Different default number of months to run for transient case diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index 4848702826..f5e9d1bab3 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -2,6 +2,8 @@ ./xmlchange NEONSITE=NIWO ./xmlchange PTS_LON=254.41676 ./xmlchange PTS_LAT=40.05236 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=="NEON" ] then ./xmlchange DATM_YR_END=2018 diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 0045025b15..07b9a43ccc 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -2,6 +2,8 @@ ./xmlchange NEONSITE=ONAQ ./xmlchange PTS_LON=247.54755 ./xmlchange PTS_LAT=40.17760 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then ./xmlchange DATM_YR_END=2019 diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index e76242e8b5..9eb170ecea 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -3,6 +3,8 @@ ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 ./xmlchange DATM_YR_ALIGN=2019 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then ./xmlchange DATM_YR_START=2019 diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 0850dd2189..5b067f56b1 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -3,11 +3,12 @@ ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON'] then ./xmlchange DATM_YR_START=2019 # Different default start date and number of months to run for transient case - #./xmlchange DATM_YR_START=2019 if [[ $compset =~ ^HIST ]] then ./xmlchange RUN_STARTDATE=2019-01-01 diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index 497ffd73f3..34e3bdc881 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -3,6 +3,8 @@ ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 ./xmlchange DATM_YR_ALIGN=2019 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then # Different default start date and number of months to run for transient case diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index 86eab8ca20..836f0ce5f1 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -3,6 +3,8 @@ ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 ./xmlchange DATM_YR_ALIGN=2019 +# NEON precipitation data for this siste is bad so end early +# If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then # Different default start date and number of months to run for transient case From d2e9296ca41415cc34fd91254ab6fd2dd782aa50 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Apr 2023 13:25:26 -0600 Subject: [PATCH 282/332] Nest the transient year change inside the NEON if so won't apply for NEON.PRISM --- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index c4c215772a..8967701e92 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -7,10 +7,10 @@ if [ $CLM_USRDAT_NAME=='NEON'] then ./xmlchange DATM_YR_END=2019 -fi -# Different default number of months to run for transient case -if [[ $compset =~ ^HIST ]] -then - ./xmlchange DATM_YR_END=2020 - ./xmlchange STOP_N=24 + # Different default number of months to run for transient case + if [[ $compset =~ ^HIST ]] + then + ./xmlchange DATM_YR_END=2020 + ./xmlchange STOP_N=24 + fi fi From 0f262d41314143538f310d55b67e56573c0dc9d3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Apr 2023 13:34:18 -0600 Subject: [PATCH 283/332] Add a FATES PRISM testmod --- .../testdefs/testmods_dirs/clm/FatesPRISM/include_user_mods | 1 + .../testdefs/testmods_dirs/clm/FatesPRISM/shell_commands | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/clm/FatesPRISM/include_user_mods create mode 100644 cime_config/testdefs/testmods_dirs/clm/FatesPRISM/shell_commands diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesPRISM/include_user_mods b/cime_config/testdefs/testmods_dirs/clm/FatesPRISM/include_user_mods new file mode 100644 index 0000000000..4c7aa0f2b4 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesPRISM/include_user_mods @@ -0,0 +1 @@ +../Fates diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesPRISM/shell_commands b/cime_config/testdefs/testmods_dirs/clm/FatesPRISM/shell_commands new file mode 100644 index 0000000000..39812a8706 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/FatesPRISM/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange CLM_USRDAT_NAME=NEON.PRISM From 64485caa4462a0683ea1df1a853599a185a1c2e1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 25 Apr 2023 22:22:54 -0600 Subject: [PATCH 284/332] Add --prism option for run_neon --- tools/site_and_regional/run_neon.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/site_and_regional/run_neon.py b/tools/site_and_regional/run_neon.py index 19976285af..7ec5d7577d 100755 --- a/tools/site_and_regional/run_neon.py +++ b/tools/site_and_regional/run_neon.py @@ -193,6 +193,19 @@ def get_parser(args, description, valid_neon_sites): default="transient", ) + parser.add_argument( + "--prism", + help=""" + Uses the PRISM reanaylsis precipitation data for the site instead of the NEON data + (only available over Continental US) + """, + action="store_true", + dest="prism", + required=False, + default=False, + ) + + parser.add_argument( "--experiment", help=""" @@ -307,6 +320,7 @@ def get_parser(args, description, valid_neon_sites): args.output_root, args.run_type, args.experiment, + args.prism, args.overwrite, run_length, base_case_root, @@ -464,6 +478,7 @@ def run_case( self, base_case_root, run_type, + prism, run_length, user_version, overwrite=False, @@ -555,6 +570,9 @@ def run_case( case.set_value("REST_OPTION", "end") case.set_value("CONTINUE_RUN", False) case.set_value("NEONVERSION", version) + if prism: + case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") + if run_type == "ad": case.set_value("CLM_FORCE_COLDSTART", "on") case.set_value("CLM_ACCELERATED_SPINUP", "on") @@ -778,6 +796,7 @@ def main(description): output_root, run_type, experiment, + prism, overwrite, run_length, base_case_root, @@ -821,6 +840,7 @@ def main(description): neon_site.run_case( base_case_root, run_type, + prism, run_length, user_version, overwrite, From d95ada651e9e7c4517cb99a65c1a2ecbe734bda5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 26 Apr 2023 00:37:15 -0600 Subject: [PATCH 285/332] Add option for v3 NEONVERSION --- cime_config/config_component.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 6f1933c858..743e9229b9 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -327,7 +327,7 @@ char - v1,v2,latest + v1,v2,v3,latest latest run_component_ctsm env_run.xml From 830af35ae1cfebc05c0e088a2560bd28143b9857 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 26 Apr 2023 10:37:53 -0600 Subject: [PATCH 286/332] Add a test for run_neon with PRISM --- test/tools/README.testnames | 1 + test/tools/input_tests_master | 6 ++++-- test/tools/nl_files/run_neon_YELL_PRISM | 1 + test/tools/tests_pretag_nompi_neon | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 test/tools/nl_files/run_neon_YELL_PRISM diff --git a/test/tools/README.testnames b/test/tools/README.testnames index 11d9e23d4c..f42864facc 100644 --- a/test/tools/README.testnames +++ b/test/tools/README.testnames @@ -43,6 +43,7 @@ m is the resolution 9 -- 4x5 a -- NEON YELL b -- NEON KONA +c -- NEON OSBS d -- region1 c -- single point from the 0.9x1.25 grid g -- unused diff --git a/test/tools/input_tests_master b/test/tools/input_tests_master index 63e4e5173f..784df20ea5 100644 --- a/test/tools/input_tests_master +++ b/test/tools/input_tests_master @@ -24,8 +24,10 @@ bli58 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_10x15_crp_1850-200 smi64 TSMscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_5x5_amazon_hirespft_2005^tools__ds bli64 TBLscript_tools.sh mksurfdata_map mksurfdata.pl mksrfdt_5x5_amazon_hirespft_2005^tools__ds -sm0a1 TSMscript_tools.sh site_and_regional run_neon.py run_neon_OSBS -bl0a1 TBLscript_tools.sh site_and_regional run_neon.py run_neon_OSBS +sm0c1 TSMscript_tools.sh site_and_regional run_neon.py run_neon_OSBS +bl0c1 TBLscript_tools.sh site_and_regional run_neon.py run_neon_OSBS +sm0a1 TSMscript_tools.sh site_and_regional run_neon.py run_neon_YELL_PRISM +bl0a1 TBLscript_tools.sh site_and_regional run_neon.py run_neon_YELL_PRISM smba1 TSMscript_tools.sh site_and_regional subset_data subset_data_YELL blba1 TBLscript_tools.sh site_and_regional subset_data subset_data_YELL diff --git a/test/tools/nl_files/run_neon_YELL_PRISM b/test/tools/nl_files/run_neon_YELL_PRISM new file mode 100644 index 0000000000..f5ebdf9fdf --- /dev/null +++ b/test/tools/nl_files/run_neon_YELL_PRISM @@ -0,0 +1 @@ +--verbose --run-type transient --setup-only --neon-site YELL --prism --neon-version v2 --experiment toolstest diff --git a/test/tools/tests_pretag_nompi_neon b/test/tools/tests_pretag_nompi_neon index 43167e71c0..e5fa27e6c4 100644 --- a/test/tools/tests_pretag_nompi_neon +++ b/test/tools/tests_pretag_nompi_neon @@ -1,4 +1,5 @@ sm0a1 bl0a1 +sm0c1 bl0c1 smaa2 blaa2 smba1 blba1 smbb1 blbb1 From b3d782c0e8552b91e8c5aac23dfca0578fd74cba Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 26 Apr 2023 17:05:44 -0600 Subject: [PATCH 287/332] Add a warning if this is NEON and transient and the NEONSITE.transient casename is not being used (or if it's PRISM) for the user make sure the finidat file is appropriately spunup --- cime_config/buildnml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cime_config/buildnml b/cime_config/buildnml index 4e04951474..25724def82 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -3,7 +3,7 @@ """ CTSM namelist creator """ -import sys, os, shutil +import sys, os, shutil, re _CIMEROOT = os.environ.get("CIMEROOT") if _CIMEROOT is None: @@ -41,6 +41,7 @@ def buildnml(case, caseroot, compname): configuration = case.get_value("CLM_CONFIGURATION") structure = case.get_value("CLM_STRUCTURE") ccsm_co2_ppmv = case.get_value("CCSM_CO2_PPMV") + casename = case.get_value("CASE") clm_co2_type = case.get_value("CLM_CO2_TYPE") clm_namelist_opts = case.get_value("CLM_NAMELIST_OPTS") clm_bldnml_opts = case.get_value("CLM_BLDNML_OPTS") @@ -133,6 +134,13 @@ def buildnml(case, caseroot, compname): clm_usrdat_name = case.get_value("CLM_USRDAT_NAME") lnd_grid = clm_usrdat_name clmusr = " -clm_usr_name %s "%clm_usrdat_name + # Write warning about initial condition data + if "NEON" in clm_usrdat_name and clm_force_coldstart == "off": + if ("_transient" in clm_nml_use_case) \ + and (re.fullmatch(r"\w\w\w\w\.transient", casename) is None \ + or clm_usrdat_name is "NEON.PRISM"): + logger.warning( "WARNING: Do you have approriprate initial conditions for this simulation?" \ + + " Check that the finidat file used in the lnd_in namelist is apprporiately spunup for your case" ) if comp_atm != "datm": nomeg = "-no-megan" From 1acbca96e702fdb900b5001c1b56b46c80db45c0 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 27 Apr 2023 11:22:11 -0600 Subject: [PATCH 288/332] Update ccs_config version This just updates the nag compiler setup to remove the recently-added NaN flag, which is causing test failures. --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 72a914fbba..888b7e1079 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.63 +tag = ccs_config_cesm0.0.64 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config From 7f0c833bf12ba3bb9da4f0711e6b90a7b43b29f1 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:50:36 -0600 Subject: [PATCH 289/332] Update shell_commands --- cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands index aacfad8e15..a41a15018e 100644 --- a/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands +++ b/cime_config/usermods_dirs/NEON/FATES/defaults/shell_commands @@ -1,6 +1,6 @@ #!/bin/bash ./xmlchange CLM_USRDAT_NAME=NEON -# we should also ensure that this can be set to NEON.PRISM +# CLM_USRDAT_NAME can also be set to NEON.PRISM ./xmlchange CCSM_CO2_PPMV=408.83 # Set data forcing data to future scenario so will have data from 2018 to present-day ./xmlchange DATM_PRESAERO=SSP3-7.0 From 8303e3fa92b47bbbc17a260890c6cd03561e7ded Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:51:47 -0600 Subject: [PATCH 290/332] Update shell_commands --- cime_config/usermods_dirs/NEON/YELL/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/YELL/shell_commands b/cime_config/usermods_dirs/NEON/YELL/shell_commands index 836f0ce5f1..a490e044b9 100644 --- a/cime_config/usermods_dirs/NEON/YELL/shell_commands +++ b/cime_config/usermods_dirs/NEON/YELL/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=249.45803999999998 ./xmlchange PTS_LAT=44.95597 ./xmlchange DATM_YR_ALIGN=2019 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then From 5b772a609a3f663d55cc2f7b034a906150775b38 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:52:30 -0600 Subject: [PATCH 291/332] Update shell_commands --- cime_config/usermods_dirs/NEON/WREF/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/WREF/shell_commands b/cime_config/usermods_dirs/NEON/WREF/shell_commands index 34e3bdc881..ace585a894 100644 --- a/cime_config/usermods_dirs/NEON/WREF/shell_commands +++ b/cime_config/usermods_dirs/NEON/WREF/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=238.04162 ./xmlchange PTS_LAT=45.81637 ./xmlchange DATM_YR_ALIGN=2019 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then From 20ba10632ba05612c32f7c2c0f0e752ae9e4b155 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:52:46 -0600 Subject: [PATCH 292/332] Update shell_commands --- cime_config/usermods_dirs/NEON/MLBS/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/MLBS/shell_commands b/cime_config/usermods_dirs/NEON/MLBS/shell_commands index 8967701e92..54d854e1b0 100644 --- a/cime_config/usermods_dirs/NEON/MLBS/shell_commands +++ b/cime_config/usermods_dirs/NEON/MLBS/shell_commands @@ -2,7 +2,7 @@ ./xmlchange NEONSITE=MLBS ./xmlchange PTS_LON=279.47575 ./xmlchange PTS_LAT=37.37783 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON'] then From 36028793c9f5ec66f535910556683dc027703d24 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:53:01 -0600 Subject: [PATCH 293/332] Update shell_commands --- cime_config/usermods_dirs/NEON/MOAB/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/MOAB/shell_commands b/cime_config/usermods_dirs/NEON/MOAB/shell_commands index 782863e9d0..93ca5e2870 100644 --- a/cime_config/usermods_dirs/NEON/MOAB/shell_commands +++ b/cime_config/usermods_dirs/NEON/MOAB/shell_commands @@ -2,7 +2,7 @@ ./xmlchange NEONSITE=MOAB ./xmlchange PTS_LON=250.61118 ./xmlchange PTS_LAT=38.25136 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [$CLM_USRDAT_NAME=='NEON']; then ./xmlchange DATM_YR_END=2020 From 5b71760a34e737a20e2e0e6c2070574481be059f Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:53:19 -0600 Subject: [PATCH 294/332] Update shell_commands --- cime_config/usermods_dirs/NEON/NIWO/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/NIWO/shell_commands b/cime_config/usermods_dirs/NEON/NIWO/shell_commands index f5e9d1bab3..a33719a0b4 100644 --- a/cime_config/usermods_dirs/NEON/NIWO/shell_commands +++ b/cime_config/usermods_dirs/NEON/NIWO/shell_commands @@ -2,7 +2,7 @@ ./xmlchange NEONSITE=NIWO ./xmlchange PTS_LON=254.41676 ./xmlchange PTS_LAT=40.05236 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=="NEON" ] then From 42868ba4fdab31d4e01127036a6a30939a80e442 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:53:36 -0600 Subject: [PATCH 295/332] Update shell_commands --- cime_config/usermods_dirs/NEON/ONAQ/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands index 07b9a43ccc..6185167008 100644 --- a/cime_config/usermods_dirs/NEON/ONAQ/shell_commands +++ b/cime_config/usermods_dirs/NEON/ONAQ/shell_commands @@ -2,7 +2,7 @@ ./xmlchange NEONSITE=ONAQ ./xmlchange PTS_LON=247.54755 ./xmlchange PTS_LAT=40.17760 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then From 46d107af4a77441f9678dacdb606c1a3add6f40f Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:53:50 -0600 Subject: [PATCH 296/332] Update shell_commands --- cime_config/usermods_dirs/NEON/SJER/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/SJER/shell_commands b/cime_config/usermods_dirs/NEON/SJER/shell_commands index 9eb170ecea..3525405e09 100644 --- a/cime_config/usermods_dirs/NEON/SJER/shell_commands +++ b/cime_config/usermods_dirs/NEON/SJER/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=240.267 ./xmlchange PTS_LAT=37.107117 ./xmlchange DATM_YR_ALIGN=2019 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON' ] then From 95c5c87f546f75db85dada70be48659c9cbbf4c0 Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Thu, 27 Apr 2023 13:54:04 -0600 Subject: [PATCH 297/332] Update shell_commands --- cime_config/usermods_dirs/NEON/TEAK/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/usermods_dirs/NEON/TEAK/shell_commands b/cime_config/usermods_dirs/NEON/TEAK/shell_commands index 5b067f56b1..f584056ba6 100644 --- a/cime_config/usermods_dirs/NEON/TEAK/shell_commands +++ b/cime_config/usermods_dirs/NEON/TEAK/shell_commands @@ -3,7 +3,7 @@ ./xmlchange PTS_LON=240.99424199999999 ./xmlchange PTS_LAT=37.006472 ./xmlchange DATM_YR_ALIGN=2019 -# NEON precipitation data for this siste is bad so end early +# NEON precipitation data for this site is bad so end early # If CLM_USRDAT_NAME is NEON.PRISM you can run to the end of the data if [ $CLM_USRDAT_NAME=='NEON'] then From 1e57258df4776e7a68a5c4a24f4f692bd168ce8c Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 1 May 2023 11:37:31 -0600 Subject: [PATCH 298/332] Update ChangeLog --- doc/ChangeLog | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 59 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index f25b413a38..61ea818302 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,62 @@ =============================================================== +Tag name: ctsm5.1.dev123 +Originator(s): sacks (Bill Sacks) +Date: Thu Apr 27 15:00:44 MDT 2023 +One-line Summary: Updates needed for pFUnit 4 and other externals updates + +Purpose and description of changes +---------------------------------- + +(1) Lots of small changes needed for the update to pFUnit4. Note that + this is a backwards-incompatible update, so we will require pFUnit 4 + moving forward. + +(2) Externals updates: some of these are needed for the update to pFUnit + 4; others are included to update externals to those in a recent CESM + alpha tag. + +Notes of particular relevance for developers: +--------------------------------------------- +Caveats for developers (e.g., code that is duplicated that requires double maintenance): +- Running the Fortran unit tests now requires pFUnit 4 +- I didn't run the mksurfdata_map unit tests... these will be removed + soon anyway with the replacement of mksurfdata_map with mksurfdata_esmf + +Testing summary: +---------------- + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- PASS + izumi ------- PASS + + Note: most testing run on 46968da7b; reran just izumi-nag testing on + the latest version (the only difference was in the version of the + ccs_config external, and the only diff there was for nag). + + any other testing (give details below): + - Fortran unit tests (under src) on izumi and my Mac + +Answer changes +-------------- + +Changes answers relative to baseline: NO + +Other details +------------- +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): +- ccs_config: ccs_config_cesm0.0.58 -> ccs_config_cesm0.0.64 +- cime: cime6.0.100 -> cime6.0.108 +- cmeps: cmeps0.14.17 -> cmeps0.14.21 +- cdeps: cdeps1.0.7 -> cdeps1.0.9 +- cpl7: cpl7.0.14 -> cpl77.0.5 +- share: share1.0.16 -> share1.0.17 + +Pull Requests that document the changes (include PR ids): +https://github.com/ESCOMP/CTSM/pull/1989 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev122 Originator(s): sacks (Bill Sacks) Date: Sun Apr 23 19:36:37 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index faadd8cee6..06204a828d 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev123 sacks 04/27/2023 Updates needed for pFUnit 4 and other externals updates ctsm5.1.dev122 sacks 04/23/2023 Rework handling of evaporation constraint in SoilFluxes ctsm5.1.dev121 glemieux 04/05/2023 Changes soil moisture initialization logic for FATES ctsm5.1.dev120 sacks 03/25/2023 Update externals and minor fixes From d1092e70135803a7927930fd9703ba68120a05a5 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 1 May 2023 11:38:01 -0600 Subject: [PATCH 299/332] Update ChangeLog date --- doc/ChangeLog | 2 +- doc/ChangeSum | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 61ea818302..aa4f7055c4 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev123 Originator(s): sacks (Bill Sacks) -Date: Thu Apr 27 15:00:44 MDT 2023 +Date: Mon May 1 11:37:51 MDT 2023 One-line Summary: Updates needed for pFUnit 4 and other externals updates Purpose and description of changes diff --git a/doc/ChangeSum b/doc/ChangeSum index 06204a828d..4aea4f599a 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev123 sacks 04/27/2023 Updates needed for pFUnit 4 and other externals updates + ctsm5.1.dev123 sacks 05/01/2023 Updates needed for pFUnit 4 and other externals updates ctsm5.1.dev122 sacks 04/23/2023 Rework handling of evaporation constraint in SoilFluxes ctsm5.1.dev121 glemieux 04/05/2023 Changes soil moisture initialization logic for FATES ctsm5.1.dev120 sacks 03/25/2023 Update externals and minor fixes From b369985bb10ad75f9ce1c37f6d9840d56ceb0160 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 3 May 2023 10:49:06 -0600 Subject: [PATCH 300/332] Update CDEPS to version with NEON.PRISM changes in it --- Externals.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 85696ec980..9f6b4596fb 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,8 +55,6 @@ local_path = components/cmeps required = True [cdeps] -#tag = cdeps1.0.7 -branch = main tag = cdeps1.0.11 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git From 7f9e9ab92d7719be779d4aad2722dfd6f4265a4e Mon Sep 17 00:00:00 2001 From: John Alex Date: Wed, 3 May 2023 12:08:50 -0600 Subject: [PATCH 301/332] Trivial refactorings in 3 for loops to make the code clearer, per https://github.com/ESCOMP/CTSM/issues/1514 --- src/cpl/share_esmf/UrbanTimeVarType.F90 | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/cpl/share_esmf/UrbanTimeVarType.F90 b/src/cpl/share_esmf/UrbanTimeVarType.F90 index 6a6f982438..b638d31b7a 100644 --- a/src/cpl/share_esmf/UrbanTimeVarType.F90 +++ b/src/cpl/share_esmf/UrbanTimeVarType.F90 @@ -252,16 +252,13 @@ subroutine urbantv_interp(this, bounds) ! Determine this%tbuilding_max for all landunits do l = bounds%begl,bounds%endl if (lun%urbpoi(l)) then - ig = 0 - do g = bounds%begg,bounds%endg - ig = ig+1 - if (g == lun%gridcell(l)) exit - end do - do n = isturb_MIN,isturb_MAX - if (stream_varnames(lun%itype(l)) == stream_varnames(n)) then - this%t_building_max(l) = dataptr2d(ig,n) - end if - end do + ! Note that since l is within [begl, endl] bounds, we can assume + ! lun%gricell(l) is within [begg, endg] + ig = lun%gridcell(l) - bounds%begg + 1 + + ! Since we are within an urban land unit, we know that + ! lun%itype is within [pisturb_MIN, isturb_MAX] + this%t_building_max(l) = dataptr2d(ig, lun%itype(l)) else this%t_building_max(l) = spval end if @@ -272,9 +269,7 @@ subroutine urbantv_interp(this, bounds) found = .false. do l = bounds%begl,bounds%endl if (lun%urbpoi(l)) then - ig = 0 do g = bounds%begg,bounds%endg - ig = ig+1 if (g == lun%gridcell(l)) exit end do if ( .not. urban_valid(g) .or. (this%t_building_max(l) <= 0._r8)) then From 13f599720d917d8d15fbf436a8bb44ea05ed3d18 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 10:31:15 -0600 Subject: [PATCH 302/332] Add test for ne3pg3_ne3pg3_mg37 --- cime_config/testdefs/testlist_clm.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 38125d80bc..67e1f10710 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1405,6 +1405,15 @@ + + + + + + + + + From 3a875dc0e6f60f49280ad3b94b69bd57bcfb3b72 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 10:38:01 -0600 Subject: [PATCH 303/332] Increse number of tests because of increased resolutions --- bld/unit_testers/build-namelist_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 79add93d88..972fc04411 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,7 +163,7 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1850; +my $ntests = 1886; if ( defined($opts{'compare'}) ) { $ntests += 1254; } From b53ff4596f0cb3d04492ad2e89c2bc0a2ad05a61 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 10:58:12 -0600 Subject: [PATCH 304/332] Increase compare tests because of increased resolutions --- bld/unit_testers/build-namelist_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 972fc04411..9e6ec62826 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -165,7 +165,7 @@ sub cat_and_create_namelistinfile { # my $ntests = 1886; if ( defined($opts{'compare'}) ) { - $ntests += 1254; + $ntests += 1281; } plan( tests=>$ntests ); From 0b463641cc48f35b6ff76c5f827fe78bb29be6a4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 12:58:35 -0600 Subject: [PATCH 305/332] Move the new files to inputdata --- bld/namelist_files/namelist_defaults_ctsm.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2502aa2c5d..118888f1df 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1093,13 +1093,13 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc -/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120//surfdata_ne3np4.pg3_hist_78pfts_CMIP6_simyr2000_c230405.nc -/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne5np4.pg3_hist_78pfts_CMIP6_simyr2000_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc -/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr2000_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne16np4.pg3_hist_78pfts_CMIP6_simyr2000_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr2000_c200426.nc @@ -1174,11 +1174,11 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc -/glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/surfdata_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne3np4.pg3_hist_78pfts_CMIP6_simyr1850_c230405.nc -/glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/surfdata_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne5np4.pg3_hist_78pfts_CMIP6_simyr1850_c230405.nc -/glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/surfdata_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne16np4.pg3_hist_78pfts_CMIP6_simyr1850_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc @@ -1246,11 +1246,11 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts use_crop=".true." >lnd/clm2/surfdata_map/landuse.timeseries_1x1_numaIA_hist_78pfts_CMIP6_simyr1850-2015_c170917.nc /glade/work/aherring/grids/uniform-res/ne3np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc + >lnd/clm2/surfdata_map/ctsm5.1.dev120/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc /glade/work/aherring/grids/uniform-res/ne5np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc + >lnd/clm2/surfdata_map/ctsm5.1.dev120/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc /glade/work/aherring/grids/uniform-res/ne16np4.pg3/clm_surfdata_5_0/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc + >lnd/clm2/surfdata_map/ctsm5.1.dev120/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc Date: Fri, 5 May 2023 13:48:05 -0600 Subject: [PATCH 306/332] Remove the use_crop attribute as unneeded --- bld/namelist_files/namelist_defaults_ctsm.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 118888f1df..cc1cd118a8 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1092,13 +1092,13 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_smallvilleIA_hist_78pfts_CMIP6_simyr2000_c230123.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev120//surfdata_ne3np4.pg3_hist_78pfts_CMIP6_simyr2000_c230405.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne5np4.pg3_hist_78pfts_CMIP6_simyr2000_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_ne16np4_hist_78pfts_CMIP6_simyr2000_c190214.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne16np4.pg3_hist_78pfts_CMIP6_simyr2000_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr2000_c200426.nc @@ -1173,17 +1173,17 @@ lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_numaIA_hist_78pfts_CMIP6_simyr lnd/clm2/surfdata_map/ctsm5.1.dev116/surfdata_1x1_brazil_hist_78pfts_CMIP6_simyr1850_c230123.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne3np4.pg3_hist_78pfts_CMIP6_simyr1850_c230405.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne5np4.pg3_hist_78pfts_CMIP6_simyr1850_c230405.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev120/surfdata_ne16np4.pg3_hist_78pfts_CMIP6_simyr1850_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4_hist_78pfts_CMIP6_simyr1850_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg2_hist_78pfts_CMIP6_simyr1850_c200426.nc - + lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne30np4.pg3_hist_78pfts_CMIP6_simyr1850_c200426.nc lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne120np4_hist_78pfts_CMIP6_simyr1850_c200427.nc From d1e9b43be287f19091935b0ac6908830ce75e93c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 14:01:10 -0600 Subject: [PATCH 307/332] Allow the landuse.timeseries files for the new fields to be used for SSP5-8.5 --- bld/namelist_files/namelist_defaults_ctsm.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index cc1cd118a8..2e6714b366 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1289,6 +1289,12 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_C96_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200317.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/landuse.timeseries_ne3np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/landuse.timeseries_ne5np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc +lnd/clm2/surfdata_map/ctsm5.1.dev120/landuse.timeseries_ne16np4.pg3_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c230405.nc lnd/clm2/surfdata_map/release-clm5.0.30/landuse.timeseries_ne30np4_SSP5-8.5_78pfts_CMIP6_simyr1850-2100_c200426.nc Date: Fri, 5 May 2023 18:33:42 -0600 Subject: [PATCH 308/332] Remove hardcoding of lexist for memory testing --- src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index f7b356e0aa..61809443b1 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -466,8 +466,6 @@ subroutine lnd_set_lndmask_from_maskmesh(mesh_lnd, mesh_mask, vm, gsize, lndmask ! Determine if lndfrac/lndmask file exists inquire(file=trim(flandfrac), exist=lexist) - lexist = .false. ! TODO: fix this - if (lexist) then ! If file exists - read in lndmask and lndfrac From 96c58a969524cda6cc966da0cc9916665c428ca5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 18:36:31 -0600 Subject: [PATCH 309/332] Remove print statements that should no longer be needed --- src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index 61809443b1..02cad3c32e 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -208,10 +208,8 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes if (trim(meshfile_mask) /= trim(meshfile_lnd)) then field_lnd = ESMF_FieldCreate(mesh_lndinput, ESMF_TYPEKIND_R8, meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - print *,__FILE__,__LINE__ field_ctsm = ESMF_FieldCreate(mesh_ctsm, ESMF_TYPEKIND_R8, meshloc=ESMF_MESHLOC_ELEMENT, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - print *,__FILE__,__LINE__ call ESMF_FieldRedistStore(field_lnd, field_ctsm, routehandle=rhandle_lnd2ctsm, & ignoreUnmatchedIndices=.true., rc=rc) if (chkerr(rc,__LINE__,u_FILE_u)) return From 052c670970b13f561b33df7aeed523e1ec7713fc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 5 May 2023 18:39:29 -0600 Subject: [PATCH 310/332] Add comment about the ESMF_VMLogMemInfo calls --- src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 index 02cad3c32e..592d5f441c 100644 --- a/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 +++ b/src/cpl/share_esmf/lnd_set_decomp_and_domain.F90 @@ -113,11 +113,13 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes ! Read in mask meshfile if needed if (trim(meshfile_mask) /= trim(meshfile_lnd)) then #ifdef DEBUG + ! This will get added to the ESMF PET files if DEBUG=TRUE and CREATE_ESMF_PET_FILES=TRUE call ESMF_VMLogMemInfo("clm: Before lnd mesh create in ") #endif mesh_maskinput = ESMF_MeshCreate(filename=trim(meshfile_mask), fileformat=ESMF_FILEFORMAT_ESMFMESH, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return #ifdef DEBUG + ! This will get added to the ESMF PET files if DEBUG=TRUE and CREATE_ESMF_PET_FILES=TRUE call ESMF_VMLogMemInfo("clm: After lnd mesh create in ") #endif ! Determine lndmask_glob and lndfrac_glob @@ -127,6 +129,7 @@ subroutine lnd_set_decomp_and_domain_from_readmesh(driver, vm, meshfile_lnd, mes lndfrac_loc_input, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return #ifdef DEBUG + ! This will get added to the ESMF PET files if DEBUG=TRUE and CREATE_ESMF_PET_FILES=TRUE call ESMF_VMLogMemInfo("clm: After lnd_set_lndmask_from_maskmesh ") #endif else From 3044ca9981be2385a47a73fea7e2b7f6c9104b45 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 6 May 2023 15:29:46 -0600 Subject: [PATCH 311/332] Update test numbers, because of increased resolutions --- bld/unit_testers/build-namelist_test.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 79add93d88..a87a0ccac3 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,9 +163,9 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1850; +my $ntests = 1922; if ( defined($opts{'compare'}) ) { - $ntests += 1254; + $ntests += 1308; } plan( tests=>$ntests ); From 45a902c494a21a956a40470d55f94fb9c43e68f1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 15:37:28 -0600 Subject: [PATCH 312/332] Add two mpasa tests to help ensure MPASA resolutions will be functional in CTSM tags and can be used in CAM --- cime_config/testdefs/testlist_clm.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 38125d80bc..475bafd9d6 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -53,6 +53,15 @@ + + + + + + + + + @@ -1405,6 +1414,15 @@ + + + + + + + + + From 1f8cae445b6c5be8999db910defa5989a9830405 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 16:11:38 -0600 Subject: [PATCH 313/332] Add mpasa480 and mpasa120 to the standard resolution list as a reminder to include mpasa resolutions as a standard for surface dataset creation --- tools/mksurfdata_map/Makefile.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index c1df608436..09857f5d46 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -81,7 +81,7 @@ SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha --o # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore # The variable resolution grids for ARCTIC, ARCTICGRIS and CONUS are also standard STANDARD_RES_NO_CROP = 0.9x1.25,1.9x2.5,10x15 -STANDARD_RES = 0.9x1.25,1.9x2.5,10x15,4x5,ne30np4,C96,ne30pg2,ne30pg3,ne30pg4,ne120np4pg3,ne0np4ARCTICGRISne30x8,ne0np4ARCTICne30x4,ne0np4CONUSne30x8 +STANDARD_RES = 0.9x1.25,1.9x2.5,10x15,4x5,ne30np4,C96,ne30pg2,ne30pg3,ne30pg4,ne120np4pg3,ne0np4ARCTICGRISne30x8,ne0np4ARCTICne30x4,ne0np4CONUSne30x8,mpasa480,mpasa120 # For future CMIP6 scenarios: SSP-RCP's FUTURE_RES = 0.9x1.25,1.9x2.5,10x15 From b7f64b7ce190c5f6518193b8afc25c6739b72164 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 16:35:32 -0600 Subject: [PATCH 314/332] Remove attributes no longer needed for init_interp_attributes, and add one used in the others --- bld/namelist_files/namelist_defaults_ctsm.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index c99dea7118..7a2a89adf2 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -796,32 +796,32 @@ attributes from the config_cache.xml file (with keys converted to upper-case). hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 +>hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 +>hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 +>hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 +>hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 +>hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_nitrif_denitrif=.true. use_vertsoilc=.true. use_crop=.true. irrigate=.true. glc_nec=10 +>hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. From 95934832d6c926f3b87301779f895ab527a60e9e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 16:48:48 -0600 Subject: [PATCH 315/332] Use_crop attribute not needed for mpasa 1850 surface datasets as crop datasets are used for both SP, BGC, and crop cases --- bld/namelist_files/namelist_defaults_ctsm.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 7a2a89adf2..2d028e1ee0 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -1177,17 +1177,17 @@ lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_10x15_hist_16pfts_Irrig_CMIP6_s lnd/clm2/surfdata_map/release-clm5.0.18/surfdata_4x5_hist_16pfts_Irrig_CMIP6_simyr1850_c190214.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa480_hist_78pfts_CMIP6_simyr1850_c211110.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa240_hist_78pfts_CMIP6_simyr1850_c211115.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa120_hist_78pfts_CMIP6_simyr1850_c211108.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa60_hist_78pfts_CMIP6_simyr1850_c211110.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa30_hist_78pfts_CMIP6_simyr1850_c211111.nc - + lnd/clm2/surfdata_map/ctsm5.1.dev052/surfdata_mpasa15_hist_78pfts_CMIP6_simyr1850_c211111.nc From 7001b8f6ca296d1d55aca97cb453602a1477d122 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 17:35:48 -0600 Subject: [PATCH 316/332] Move directory for dynGrossUnrepMod resolving #1969 --- src/{biogeochem => dyn_subgrid}/dynGrossUnrepMod.F90 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{biogeochem => dyn_subgrid}/dynGrossUnrepMod.F90 (100%) diff --git a/src/biogeochem/dynGrossUnrepMod.F90 b/src/dyn_subgrid/dynGrossUnrepMod.F90 similarity index 100% rename from src/biogeochem/dynGrossUnrepMod.F90 rename to src/dyn_subgrid/dynGrossUnrepMod.F90 From 514a3b979c2b843aa0221e74c8d37677fa89701f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 19:14:54 -0600 Subject: [PATCH 317/332] Add reminder to make new surface datasets for the new resolutions here --- tools/mksurfdata_map/Makefile.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mksurfdata_map/Makefile.data b/tools/mksurfdata_map/Makefile.data index c1df608436..904aec8d7d 100644 --- a/tools/mksurfdata_map/Makefile.data +++ b/tools/mksurfdata_map/Makefile.data @@ -81,7 +81,7 @@ SUBSETDATA_1X1_URBALPHA := --lat -37.7308 --lon 0 --site 1x1_urbanc_alpha --o # The ne30np4 series (including pg2, pg3, pg4) are standard for SE dycore # The variable resolution grids for ARCTIC, ARCTICGRIS and CONUS are also standard STANDARD_RES_NO_CROP = 0.9x1.25,1.9x2.5,10x15 -STANDARD_RES = 0.9x1.25,1.9x2.5,10x15,4x5,ne30np4,C96,ne30pg2,ne30pg3,ne30pg4,ne120np4pg3,ne0np4ARCTICGRISne30x8,ne0np4ARCTICne30x4,ne0np4CONUSne30x8 +STANDARD_RES = 0.9x1.25,1.9x2.5,10x15,4x5,ne30np4,C96,ne30pg2,ne30pg3,ne30pg4,ne120np4pg3,ne0np4ARCTICGRISne30x8,ne0np4ARCTICne30x4,ne0np4CONUSne30x8,ne3np4.pg3,ne5np4.pg3,ne16np4.pg3 # For future CMIP6 scenarios: SSP-RCP's FUTURE_RES = 0.9x1.25,1.9x2.5,10x15 From f6881492ad476e27c112ac5b68cd809e3a0c76c1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sun, 7 May 2023 20:43:08 -0600 Subject: [PATCH 318/332] Use the f09 finidat file rather than ne120 for these new low resolution grids --- bld/namelist_files/namelist_defaults_ctsm.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml index 2e6714b366..7ff5750e5b 100644 --- a/bld/namelist_files/namelist_defaults_ctsm.xml +++ b/bld/namelist_files/namelist_defaults_ctsm.xml @@ -758,11 +758,26 @@ attributes from the config_cache.xml file (with keys converted to upper-case). >hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + + +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + + hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. +hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. + + hgrid=0.9x1.25 maxpft=79 mask=gx1v7 use_cn=.true. use_crop=.true. irrigate=.true. glc_nec=10 do_transient_pfts=.false. From cae251a0e21eb8521e473bb53b222bd87b7112e4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Mon, 8 May 2023 12:14:24 -0600 Subject: [PATCH 319/332] Need to have CLM version match the lnd_tuning version --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 2d1aafb728..b3c43df8f6 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1414,7 +1414,7 @@ - + From 18ac79080a0ee315c8e960f594fa16c771d1e90b Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Mon, 8 May 2023 17:35:40 -0600 Subject: [PATCH 320/332] Fix nuopc cplhist test Make some changes to the cplhist testmod that should fix the cplhist test, based on testing done by Keith Oleson: - Point to new cplhist forcing data generated and used by Adam Herrington and Keith Oleson - Use DATM_PRESNDEP=none until https://github.com/escomp/ctsm/issues/1844 is resolved Also, remove mct cplhist test --- cime_config/testdefs/ExpectedTestFails.xml | 7 ------- cime_config/testdefs/testlist_clm.xml | 11 +--------- .../testmods_dirs/clm/cplhist/shell_commands | 20 ++++++++----------- 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index d9a1c1b6fd..24f7062798 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -44,13 +44,6 @@ - - - FAIL - #1844 - - - diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 38125d80bc..475d5db97d 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1306,7 +1306,7 @@ - + @@ -1315,15 +1315,6 @@ - - - - - - - - - diff --git a/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands b/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands index ac079d5334..a284aefcb9 100755 --- a/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/cplhist/shell_commands @@ -1,12 +1,8 @@ -driver=`./xmlquery --value COMP_INTERFACE` -if [ "$driver" = "nuopc" ]; then - ./xmlchange DATM_YR_ALIGN=1 - ./xmlchange DATM_YR_END=84 - ./xmlchange DATM_YR_START=82 -else - ./xmlchange DATM_CPLHIST_YR_ALIGN=1 - ./xmlchange DATM_CPLHIST_YR_END=84 - ./xmlchange DATM_CPLHIST_YR_START=82 -fi -./xmlchange DATM_CPLHIST_CASE=b.e20.B1850.f09_g17.pi_control.all.221.cplhist -./xmlchange DATM_CPLHIST_DIR=/glade/p/cesm/bgcwg_dev/forcing/b.e20.B1850.f09_g17.pi_control.all.221.cplhist/cpl/hist.mon +./xmlchange DATM_CPLHIST_CASE=cam6ctsm51_cesm23a12c_ne30pg3g17_CPLHIST_1850 +./xmlchange DATM_CPLHIST_DIR=/glade/p/cgd/tss/CTSM_datm_forcing_data/atm_forcing.cdeps_datm.CPLHIST_cam6ctsm51_cesm23a12c_ne30pg3g17_1850 +./xmlchange DATM_YR_START=1 +./xmlchange DATM_YR_END=1 +./xmlchange DATM_YR_ALIGN=1 + +# Needed until we have cplhist forcing with ndep - see https://github.com/escomp/ctsm/issues/1844 +./xmlchange DATM_PRESNDEP=none From 53ce361f0e6da787185984dd23d859c5f6b4c574 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 9 May 2023 10:36:21 -0600 Subject: [PATCH 321/332] Update cdeps version to one working --- Externals.cfg | 2 +- doc/ChangeLog | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 3 files changed, 173 insertions(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 9f6b4596fb..3b2e66164b 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,7 +55,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.11 +tag = cdeps1.0.12 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps diff --git a/doc/ChangeLog b/doc/ChangeLog index aa4f7055c4..a70837d7f8 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,175 @@ =============================================================== +Tag name: ctsm5.1.dev124 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Tue May 9 01:47:18 MDT 2023 +One-line Summary: Initialization memory update, new surface datasets for new grids, add option for running NEON with PRISM data + +Purpose and description of changes +---------------------------------- + +[Fill this in with details] + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +CTSM issues fixed (include CTSM Issue #): + +Externals issues fixed (include issue #): + +Known bugs introduced in this tag (include issue #): + +Known bugs found since the previous tag (include issue #): + + +Notes of particular relevance for users +--------------------------------------- +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +Caveats for users (e.g., need to interpolate initial conditions): + +Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): + +Changes made to namelist defaults (e.g., changed parameter values): + +Changes to the datasets (e.g., parameter, surface or initial files): + +Substantial timing or memory changes: +[e.g., check PFS test in the test suite and look at timings, if you +expect possible significant timing changes] + + +Notes of particular relevance for developers: +--------------------------------------------- +NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + +Changes to tests or testing: + + +Testing summary: +---------------- +[... Remove before making master tag. + +Nearly all CTSM tags should undergo 'regular' (aux_clm) testing. +However, it occasionally makes sense to do more or less system testing; +here is guidance on different available levels of system testing: + a) no system testing (for use when the only changes are ones that + have absolutely no impact on system runs; this + includes documentation-only tags, tags that + just change the tools or some python code that + does not impact system runs, etc.) + b) minimal (for use in rare cases where only a small change with + known behavior is added ... eg. a minor bug fix. This + might be to just run the "short" test list, or to run + a single test. Whatever makes sense for the particular case.) + c) python only (for use where the only changes are in the python directory: + run the python testing listed below) + d) regular (regular tests on normal machines if CTSM source is modified) + e) release (regular tests plus the fates, ctsm_sci, mosart and rtm test lists + and normally all of the ancillary tests (build-namelist, python, ptclm, etc.) + would be run as well) + +In addition, various other tests of the tools, python and perl +infrastructure should be run when appropriate, as described below. + +...] + +[Remove any lines that don't apply.] + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + cheyenne - + + tools-tests (test/tools) (if tools have been changed): + + cheyenne - + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + (any machine) - + + [If python code has changed and you are NOT running aux_clm (e.g., because the only changes are in python + code) then also run the clm_pymods test suite; this is a small subset of aux_clm that runs the system + tests impacted by python changes. The best way to do this, if you expect no changes from the last tag in + either model output or namelists, is: create sym links pointing to the last tag's baseline directory, + named with the upcoming tag; then run the clm_pymods test suite comparing against these baselines but NOT + doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to + separately run the clm_pymods test suite, and you can remove the following line.] + + clm_pymods test suite on cheyenne - + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- + izumi ------- + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- + izumi ------- + + any other testing (give details below): + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + +Answer changes +-------------- + +Changes answers relative to baseline: + + [ If a tag changes answers relative to baseline comparison the + following should be filled in (otherwise remove this section). + And always remove these three lines and parts that don't apply. ] + + Summarize any changes to answers, i.e., + - what code configurations: + - what platforms/compilers: + - nature of change (roundoff; larger than roundoff/same climate; new climate): + + If bitwise differences were observed, how did you show they were no worse + than roundoff? + + If this tag changes climate describe the run(s) done to evaluate the new + climate (put details of the simulations in the experiment database) + - casename: + + URL for LMWG diagnostics output used to validate new climate: + + +Other details +------------- +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev123 Originator(s): sacks (Bill Sacks) Date: Mon May 1 11:37:51 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 4aea4f599a..c73ffb1b64 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev124 erik 05/09/2023 Initialization memory update, new surface datasets for new grids, add option for running NEON with PRISM data ctsm5.1.dev123 sacks 05/01/2023 Updates needed for pFUnit 4 and other externals updates ctsm5.1.dev122 sacks 04/23/2023 Rework handling of evaporation constraint in SoilFluxes ctsm5.1.dev121 glemieux 04/05/2023 Changes soil moisture initialization logic for FATES From 08094df364b18059dd5e346a3660076d47cf9be7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 9 May 2023 11:16:58 -0600 Subject: [PATCH 322/332] Izumi tests need to be with DEBUG on --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index b3c43df8f6..59e7be0d92 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1609,7 +1609,7 @@ - + From 5d3314b9dfc3837325639d6bb558494962f17078 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 9 May 2023 16:52:19 -0600 Subject: [PATCH 323/332] Update changelog --- doc/ChangeLog | 164 ++++++++++++++++++++++---------------------------- 1 file changed, 72 insertions(+), 92 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index a70837d7f8..e82ff156bb 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,13 +1,51 @@ =============================================================== Tag name: ctsm5.1.dev124 -Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) -Date: Tue May 9 01:47:18 MDT 2023 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) / mvertens / adamrher / MiCurry / jtrusdal / TeaganKing +Date: Tue May 9 16:52:05 MDT 2023 One-line Summary: Initialization memory update, new surface datasets for new grids, add option for running NEON with PRISM data Purpose and description of changes ---------------------------------- -[Fill this in with details] +Lower memory usage at initialization: + + Some work by Mariana Vertenstein to lower the memory usage at initiatlization. + +New Low Resolutions for SE grids: + + ne3np4.pg3, ne5np4.pg3, ne16np4.pg3 + +New MPASA Resolutions: + + mpasa480 --------- Course resolution + mpasa120 --------- Near 1-degree + mpasa60, mpasa30 - High resolution + mpasa15 ---------- Very high resolution + + This merge adds the surface data and landuse.timeseries files for the CAM-MPAS dycore. At present, these grids are: + + mpasa480 - 480 km quasi-uniform, global with 2,462 horizontal grid columns + mpasa120 - 120 km quasi-uniform, global with 40,962 horizontal grid columns + mpasa60 - 60 km quasi-uniform, global with 163,842 horizontal grid columns + mpasa30 - 30 km quasi-uniform, global with 655,362 horizontal grid columns + mpasa15 - 15 km quasi-uniform, global with 2,621,442 horizontal grid columns + + +Option to Run NEON with PRISM preciption data: + + Some NEON sites have bad or incomplete precitation data. By setting CLM_USRDAT_NAME="NEON.PRISM" + the PRISM 4km CONUS ReAnayslis data is used in place of the NEON precipitation data. This + is helpful for several NEON sites: + MLBS, MOAB, ONAQ, SJER, NIWO, TEAK, WREF, YELL + + Allow PRISM precipitation to be used as a new datm stream. + + Updates to cime_config/config_component.xml include additional valid values for PRECIP data stream names. Changes in + cime_config/usermods_dirs/NEON/defaults/user_nl_datm_streams specify which input variables are gathered from which streams and + specifies file location for PRISM data. + + Using PRISM precipitation instead of NEON precipitation does have a substantial impact on CTSM output (eg. latent heat flux + biases). Significant changes to scientifically-supported configurations @@ -29,106 +67,60 @@ Does this tag change answers significantly for any of the following physics conf Bugs fixed or introduced ------------------------ -[Remove any lines that don't apply. Remove entire section if nothing applies.] CTSM issues fixed (include CTSM Issue #): - -Externals issues fixed (include issue #): - -Known bugs introduced in this tag (include issue #): - -Known bugs found since the previous tag (include issue #): - + Fixes #1969 -- Move dynGrossUnrepMod.F90 from biogeochem to dyn_subgrid subdirectory + Fixes #1904 -- Other precipitation streams for NEON + Fixes #1927 -- Course SE resolutions support + Fixes #1313 -- MPASA resolution support Notes of particular relevance for users --------------------------------------- -[Remove any lines that don't apply. Remove entire section if nothing applies.] Caveats for users (e.g., need to interpolate initial conditions): + New resolutions do NOT have specific initial conditions for them, they use the general ones Changes to CTSM's user interface (e.g., new/renamed XML or namelist variables): + NEON cases will give a warning about spinup data for non default transient cases + if you set a NEON case with CLM_USRDAT_NAME=NEON.PRISM, PRISM data will be used for precip + + Two new options to run_neon.py "--prism" and "--experiment" Changes made to namelist defaults (e.g., changed parameter values): + "v3" data option for NEONVERSION Changes to the datasets (e.g., parameter, surface or initial files): - -Substantial timing or memory changes: -[e.g., check PFS test in the test suite and look at timings, if you -expect possible significant timing changes] - + New surface and landuse.timeseries datasets for: + ne3np4.pg3, ne5np4.pg3, ne16np4.pg3 + mpasa480, mpasa120, mpasa60, mpasa30, mpasa15 Notes of particular relevance for developers: --------------------------------------------- -NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide -[Remove any lines that don't apply. Remove entire section if nothing applies.] - -Caveats for developers (e.g., code that is duplicated that requires double maintenance): - -Changes to tests or testing: +Changes to tests or testing: Add new tests for new resolutions -Testing summary: +Testing summary: regular, tools ---------------- -[... Remove before making master tag. - -Nearly all CTSM tags should undergo 'regular' (aux_clm) testing. -However, it occasionally makes sense to do more or less system testing; -here is guidance on different available levels of system testing: - a) no system testing (for use when the only changes are ones that - have absolutely no impact on system runs; this - includes documentation-only tags, tags that - just change the tools or some python code that - does not impact system runs, etc.) - b) minimal (for use in rare cases where only a small change with - known behavior is added ... eg. a minor bug fix. This - might be to just run the "short" test list, or to run - a single test. Whatever makes sense for the particular case.) - c) python only (for use where the only changes are in the python directory: - run the python testing listed below) - d) regular (regular tests on normal machines if CTSM source is modified) - e) release (regular tests plus the fates, ctsm_sci, mosart and rtm test lists - and normally all of the ancillary tests (build-namelist, python, ptclm, etc.) - would be run as well) - -In addition, various other tests of the tools, python and perl -infrastructure should be run when appropriate, as described below. - -...] - -[Remove any lines that don't apply.] [PASS means all tests PASS; OK means tests PASS other than expected fails.] build-namelist tests (if CLMBuildNamelist.pm has changed): - cheyenne - + cheyenne - PASS (81 new tests) tools-tests (test/tools) (if tools have been changed): - cheyenne - + cheyenne - OK + cheyenne - PASS (tests_pretag_nompi_neon) python testing (if python code has changed; see instructions in python/README.md; document testing done): - (any machine) - - - [If python code has changed and you are NOT running aux_clm (e.g., because the only changes are in python - code) then also run the clm_pymods test suite; this is a small subset of aux_clm that runs the system - tests impacted by python changes. The best way to do this, if you expect no changes from the last tag in - either model output or namelists, is: create sym links pointing to the last tag's baseline directory, - named with the upcoming tag; then run the clm_pymods test suite comparing against these baselines but NOT - doing their own baseline generation. If you are already running the full aux_clm then you do NOT need to - separately run the clm_pymods test suite, and you can remove the following line.] - - clm_pymods test suite on cheyenne - + cheyenne - PASS regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - cheyenne ---- - izumi ------- - - fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) - cheyenne ---- - izumi ------- + cheyenne ---- OK + izumi ------- OK any other testing (give details below): @@ -138,36 +130,24 @@ If the tag used for baseline comparisons was NOT the previous tag, note that her Answer changes -------------- -Changes answers relative to baseline: - - [ If a tag changes answers relative to baseline comparison the - following should be filled in (otherwise remove this section). - And always remove these three lines and parts that don't apply. ] - - Summarize any changes to answers, i.e., - - what code configurations: - - what platforms/compilers: - - nature of change (roundoff; larger than roundoff/same climate; new climate): - - If bitwise differences were observed, how did you show they were no worse - than roundoff? - - If this tag changes climate describe the run(s) done to evaluate the new - climate (put details of the simulations in the experiment database) - - casename: - - URL for LMWG diagnostics output used to validate new climate: - +Changes answers relative to baseline: No bit-for-bit Other details ------------- -[Remove any lines that don't apply. Remove entire section if nothing applies.] -List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): cdeps + Update CDEPS to cdeps1.0.12 Pull Requests that document the changes (include PR ids): (https://github.com/ESCOMP/ctsm/pull) + #1998 -- combination of below PR's... + + #1954 -- PRISM + #1973 -- SE + #1501 -- MPASA + #1899 -- memory scaling + =============================================================== =============================================================== Tag name: ctsm5.1.dev123 From 4db1d204279a2a9ce6f3ac4c753fa4a754892491 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 12 May 2023 12:55:17 -0600 Subject: [PATCH 324/332] Point to latest cdeps (needed for this branch) --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 888b7e1079..cd6855c9dc 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -55,7 +55,7 @@ local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.9 +tag = cdeps1.0.13 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps From 67009619235fe4292a0bce42302e3d11ff0b685b Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 17 May 2023 08:59:21 -0600 Subject: [PATCH 325/332] Update ChangeLog --- doc/ChangeLog | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 55 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index e82ff156bb..3ec1e8385c 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,58 @@ =============================================================== +Tag name: ctsm5.1.dev125 +Originator(s): jpalex (John Paul Alex) +Date: Wed May 17 08:56:41 MDT 2023 +One-line Summary: Added cache for clock step_size in clm_time_manager.F90 + +Purpose and description of changes +---------------------------------- + +Added cache for clock step_size in clm_time_manager.F90 to improve +performance. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +- Resolves ESCOMP/CTSM#207 (Improve performance of get_step_size) + + +Testing summary: +---------------- + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- PASS + izumi ------- PASS + +Answer changes +-------------- + +Changes answers relative to baseline: NO + +Other details +------------- +Pull Requests that document the changes (include PR ids): +https://github.com/ESCOMP/CTSM/pull/2004 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev124 Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) / mvertens / adamrher / MiCurry / jtrusdal / TeaganKing Date: Tue May 9 16:52:05 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index c73ffb1b64..a3e68c6581 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev125 jpalex 05/17/2023 Added cache for clock step_size in clm_time_manager.F90 ctsm5.1.dev124 erik 05/09/2023 Initialization memory update, new surface datasets for new grids, add option for running NEON with PRISM data ctsm5.1.dev123 sacks 05/01/2023 Updates needed for pFUnit 4 and other externals updates ctsm5.1.dev122 sacks 04/23/2023 Rework handling of evaporation constraint in SoilFluxes From 002f95dbbedbe1f20c8ea7246bcab4735dd3215d Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 17 May 2023 09:07:55 -0600 Subject: [PATCH 326/332] Minor edit to ChangeLog --- doc/ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 3ec1e8385c..c9bc6eedbd 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,6 +1,6 @@ =============================================================== Tag name: ctsm5.1.dev125 -Originator(s): jpalex (John Paul Alex) +Originator(s): jpalex (John Alex) Date: Wed May 17 08:56:41 MDT 2023 One-line Summary: Added cache for clock step_size in clm_time_manager.F90 From b93a1f7bf2459fd264f86996668e349c543cc412 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 17 May 2023 09:19:24 -0600 Subject: [PATCH 327/332] Clean up whitespace --- src/cpl/share_esmf/UrbanTimeVarType.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpl/share_esmf/UrbanTimeVarType.F90 b/src/cpl/share_esmf/UrbanTimeVarType.F90 index b638d31b7a..088ec9eeae 100644 --- a/src/cpl/share_esmf/UrbanTimeVarType.F90 +++ b/src/cpl/share_esmf/UrbanTimeVarType.F90 @@ -256,7 +256,7 @@ subroutine urbantv_interp(this, bounds) ! lun%gricell(l) is within [begg, endg] ig = lun%gridcell(l) - bounds%begg + 1 - ! Since we are within an urban land unit, we know that + ! Since we are within an urban land unit, we know that ! lun%itype is within [pisturb_MIN, isturb_MAX] this%t_building_max(l) = dataptr2d(ig, lun%itype(l)) else From ff7eb0513f8cb7d00cc0fb3df139a2dcb3bc513c Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 18 May 2023 17:24:02 -0600 Subject: [PATCH 328/332] Update ChangeLog --- doc/ChangeLog | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 54 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index c9bc6eedbd..219621c9b2 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,57 @@ =============================================================== +Tag name: ctsm5.1.dev126 +Originator(s): jpalex (John Alex) +Date: Thu May 18 17:21:59 MDT 2023 +One-line Summary: Clean up some loops in UrbanTimeVarType + +Purpose and description of changes +---------------------------------- + +Refactor some inefficient and confusing looping structures in +UrbanTimeVarType.F90 + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +- Resolves ESCOMP/CTSM#1514 (Inefficient and confusing looping structures in UrbanTimeVarType.F90) + +Testing summary: +---------------- + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- PASS + izumi ------- PASS + +Answer changes +-------------- + +Changes answers relative to baseline: NO + +Other details +------------- +Pull Requests that document the changes (include PR ids): +https://github.com/ESCOMP/CTSM/pull/2005 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev125 Originator(s): jpalex (John Alex) Date: Wed May 17 08:56:41 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index a3e68c6581..72c9987387 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev126 jpalex 05/18/2023 Clean up some loops in UrbanTimeVarType ctsm5.1.dev125 jpalex 05/17/2023 Added cache for clock step_size in clm_time_manager.F90 ctsm5.1.dev124 erik 05/09/2023 Initialization memory update, new surface datasets for new grids, add option for running NEON with PRISM data ctsm5.1.dev123 sacks 05/01/2023 Updates needed for pFUnit 4 and other externals updates From 4ab7026ea9ba5ce4e37bb16964c574647317a016 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 19 May 2023 04:53:05 -0600 Subject: [PATCH 329/332] Update ChangeLog --- doc/ChangeLog | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 74 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 219621c9b2..a620f38209 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,77 @@ =============================================================== +Tag name: ctsm5.1.dev127 +Originator(s): sacks (Bill Sacks) +Date: Fri May 19 04:48:30 MDT 2023 +One-line Summary: Fix nuopc cplhist test + +Purpose and description of changes +---------------------------------- + +Make some changes to the cplhist testmod that fix the cplhist test, +based on testing done by Keith Oleson: +- Point to new cplhist forcing data generated and used by Adam + Herrington and Keith Oleson +- Use DATM_PRESNDEP=none until + https://github.com/escomp/ctsm/issues/1844 is resolved + +Also, remove mct cplhist test. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ +CTSM issues fixed (include CTSM Issue #): +- Takes steps towards addressing ESCOMP/CTSM#1844 (Create new auxiliary + history file for cplhist test with ndep data) + +Notes of particular relevance for developers: +--------------------------------------------- +Changes to tests or testing: +- Changes cplhist test; new test is + SMS_D_Ld1.ne30pg3_t061.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist +- Removes mct cplhist test + +Testing summary: +---------------- + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- PASS + izumi ------- PASS + + Note that there were no baselines for the new test + (SMS_D_Ld1.ne30pg3_t061.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist) + +Answer changes +-------------- + +Changes answers relative to baseline: NO + +Other details +------------- +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): +- cdeps: cdeps1.0.12 -> cdeps1.0.13 + +Pull Requests that document the changes (include PR ids): +https://github.com/ESCOMP/CTSM/pull/1999 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev126 Originator(s): jpalex (John Alex) Date: Thu May 18 17:21:59 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index 72c9987387..ca203e6f21 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev127 sacks 05/19/2023 Fix nuopc cplhist test ctsm5.1.dev126 jpalex 05/18/2023 Clean up some loops in UrbanTimeVarType ctsm5.1.dev125 jpalex 05/17/2023 Added cache for clock step_size in clm_time_manager.F90 ctsm5.1.dev124 erik 05/09/2023 Initialization memory update, new surface datasets for new grids, add option for running NEON with PRISM data From 1474f2a4a3b75d41febf7c848e8a756d8a262443 Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Mon, 22 May 2023 11:01:25 -0700 Subject: [PATCH 330/332] update fates history tests to use double precision --- cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm | 1 + .../testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm | 1 + cime_config/usermods_dirs/fates_sp/user_nl_clm | 1 + 3 files changed, 3 insertions(+) diff --git a/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm index 2a915f2fd7..c27e69b531 100644 --- a/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/Fates/user_nl_clm @@ -2,6 +2,7 @@ hist_mfilt = 365 hist_nhtfrq = -24 hist_empty_htapes = .true. +hist_ndens = 1 fates_spitfire_mode = 1 hist_fincl1 = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS', 'FATES_AREA_TREES', 'FATES_COLD_STATUS', 'FATES_DROUGHT_STATUS', 'FATES_GDD', diff --git a/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm index 0a0726dcce..85f54fa214 100644 --- a/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/FatesColdAllVars/user_nl_clm @@ -3,6 +3,7 @@ hist_mfilt = 365 hist_nhtfrq = -24 hist_empty_htapes = .false. fates_spitfire_mode = 1 +hist_ndens = 1 hist_fincl1 = 'FATES_CROWNAREA_PF', 'FATES_CANOPYCROWNAREA_PF', 'FATES_NCL_AP', 'FATES_NPATCH_AP', 'FATES_VEGC_AP', 'FATES_SECONDARY_FOREST_FRACTION', 'FATES_WOOD_PRODUCT', diff --git a/cime_config/usermods_dirs/fates_sp/user_nl_clm b/cime_config/usermods_dirs/fates_sp/user_nl_clm index f2f68eb74c..4987eae969 100644 --- a/cime_config/usermods_dirs/fates_sp/user_nl_clm +++ b/cime_config/usermods_dirs/fates_sp/user_nl_clm @@ -6,6 +6,7 @@ use_lch4 = .false. fates_spitfire_mode = 0 use_fates_fixed_biogeog = .true. use_fates_nocomp = .true. +hist_ndens = 1 ! Turn off a list of fields that are not needed for FATES-SP mode hist_fexcl1 = 'FATES_TRIMMING', 'FATES_COLD_STATUS', 'FATES_DROUGHT_STATUS', 'FATES_GDD', 'FATES_NCHILLDAYS', 'FATES_NCOLDDAYS', 'FATES_DAYSINCE_COLDLEAFOFF', 'FATES_DAYSINCE_COLDLEAFON', 'FATES_DAYSINCE_DROUGHTLEAFOFF', From ebf9601621f4b2a162bb2b2bef75c59b06a736ee Mon Sep 17 00:00:00 2001 From: Gregory Lemieux Date: Fri, 26 May 2023 15:27:07 -0700 Subject: [PATCH 331/332] update changelog --- doc/ChangeLog | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 69 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index a620f38209..937e686014 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,72 @@ =============================================================== +Tag name: ctsm5.1.dev128 +Originator(s): glemieux (Gregory Lemieux,LBL/NGEET,510-486-5049) +Date: Fri May 26 15:45:00 MDT 2023 +One-line Summary: Update FATES tests to double precision + +Purpose and description of changes +---------------------------------- + +This pull request updates the fates tests to set the output +precision to double precision. The usermod fates_sp is similarly +updated. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed or introduced +------------------------ + +CTSM issues fixed (include CTSM Issue #): +- Resolves https://github.com/ESCOMP/CTSM/issues/1986 + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + cheyenne ---- OK + izumi ------- OK + + fates tests: (give name of baseline if different from CTSM tagname, normally fates baselines are fates--) + cheyenne ---- OK + izumi ------- OK + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + FATES tests run against fates-sci.1.65.6_api.25.4.0-ctsm5.1.dev127 baseline + +Answer changes +-------------- + +Changes answers relative to baseline: Yes, but only for fates tests and compsets + + Summarize any changes to answers, i.e., + - Differences are due to changing hist_ndens to 1 (double precision) + +Other details +------------- + +Pull Requests that document the changes (include PR ids): +https://github.com/ESCOMP/CTSM/pull/2010 + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev127 Originator(s): sacks (Bill Sacks) Date: Fri May 19 04:48:30 MDT 2023 diff --git a/doc/ChangeSum b/doc/ChangeSum index ca203e6f21..69357c6d66 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev128 glemieux 05/26/2023 Update FATES tests to double precision ctsm5.1.dev127 sacks 05/19/2023 Fix nuopc cplhist test ctsm5.1.dev126 jpalex 05/18/2023 Clean up some loops in UrbanTimeVarType ctsm5.1.dev125 jpalex 05/17/2023 Added cache for clock step_size in clm_time_manager.F90 From cc333285d4c37a0aa0e33220fe2cfa355348092a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 1 Jun 2023 15:32:53 -0600 Subject: [PATCH 332/332] Update the time stamps --- doc/ChangeLog | 2 +- doc/ChangeSum | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 937e686014..ced4fc388e 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,7 +1,7 @@ =============================================================== Tag name: ctsm5.1.dev128 Originator(s): glemieux (Gregory Lemieux,LBL/NGEET,510-486-5049) -Date: Fri May 26 15:45:00 MDT 2023 +Date: Thu Jun 1 15:31:52 MDT 2023 One-line Summary: Update FATES tests to double precision Purpose and description of changes diff --git a/doc/ChangeSum b/doc/ChangeSum index 69357c6d66..aca77a5238 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,6 +1,6 @@ Tag Who Date Summary ============================================================================================================================ - ctsm5.1.dev128 glemieux 05/26/2023 Update FATES tests to double precision + ctsm5.1.dev128 glemieux 06/01/2023 Update FATES tests to double precision ctsm5.1.dev127 sacks 05/19/2023 Fix nuopc cplhist test ctsm5.1.dev126 jpalex 05/18/2023 Clean up some loops in UrbanTimeVarType ctsm5.1.dev125 jpalex 05/17/2023 Added cache for clock step_size in clm_time_manager.F90