diff --git a/Makefile b/Makefile index bc0e047da9..6d16433a7c 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,10 @@ test-coupled: ## Test if the coupling with MAgPIE works. Takes significantly $(info Coupling tests take around 75 minutes to run, please be patient) @R_PROFILE_USER= TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_file("tests/testthat/test_20-coupled.R")' +test-coupled-slurm: ## test-coupled, but on slurm + $(info Coupling tests take around 75 minutes to run. Sent to slurm, find log in test-coupled.log) + @sbatch --qos=priority --wrap="make test-coupled" --job-name=test-coupled --mail-type=END --output=test-coupled.log + test-full: ## Run all tests, including coupling tests and a default ## REMIND scenario. Takes significantly longer than 10 minutes to run. $(info Full tests take more than an hour to run, please be patient) diff --git a/main.gms b/main.gms index 30bd747a62..a59888c284 100755 --- a/main.gms +++ b/main.gms @@ -147,6 +147,21 @@ *' * Put the equation sign (`=g=`, `=l=`, `=e=`) in a single line without anything else *' *' +*' #### Switches: +*' A switch must be defined in main.gms +*' Follow this mode of definition for parameters, including the indentation: +*' -------- +*' parameter +*' param_name "explanation what it means" +*' ; +*' param_name = 0; !! def = 0 !! regexp = 0|1 +*' -------- +*' * The value behind 'def' contains the default value and is added only for the user to remember if changed manually +*' * The value behind regexp is read by scripts/start/checkFixCfg.R to check the validity of the input. +*' In this case, it checks whether the value fits this regular expression: ^(0|1)$ +*' Use 'value1|value2' for specific values, use '[1-7]' for a row of integers and use 'is.numeric' for numeric values. +*' +*' *' #### Other general rules: *' * Decompose large model equations into several small equations to enhance readability and model diagnostics *' * Don't use hard-coded numbers in the equations part of the model @@ -273,7 +288,7 @@ $setglobal capitalMarket debt_limit !! def = debt_limit *'---------------------- 24_trade --------------------------------------- *' *' * (standard): macro-economic commodities and primary energy commodities trading -*' * (se_trade): macro-economic commodities, primary energy commodities and secondary energy hydrogen and electricitiy trading +*' * (se_trade): macro-economic commodities, primary energy commodities and secondary energy hydrogen and electricity trading *' * (capacity): capacity-based trade implementation $setglobal trade standard !! def = standard *'---------------------- 26_agCosts ---------------------------------------- @@ -289,11 +304,11 @@ $setglobal agCosts costs !! def = costs $setglobal CES_parameters load !! def = load *'--------------------- 30_biomass ---------------------------------------- *' -*' * (magpie_40): using supplycurved derived from MAgpIE 4.0 +*' * (magpie_40): using supply curves derived from MAgpIE 4.0 $setglobal biomass magpie_40 !! def = magpie_hightccost *'--------------------- 31_fossil ---------------------------------------- *' -*' * (timeDepGrades): time-dependent grade stucture of fossil resources (oil & gas only) +*' * (timeDepGrades): time-dependent grade structure of fossil resources (oil & gas only) *' * (grades2poly) : simplified version of the time-dependent grade realization (using polynomial functions) *' * (exogenous) : exogenous fossil extraction and costs $setglobal fossil grades2poly !! def = grades2poly @@ -341,7 +356,7 @@ $setglobal CCU on !! def = on *' * (coalPhaseoutRegional): [works only with Negishi] global phase-out of new freely-emitting coal conversion, caps all coal routes with the exception of coaltr: coal solids can still expand *' * (CombLowCandCoalPO): [works only with Negishi] combination of lowCarbonPush and coalPhaseout *' * (NDC): Technology targets for 2030 for spv,wind,tnrs. -*' * (NPi): Reference technology targets, mostly already enacted (N)ational (P]ol(i)cies, mostly for 2020 +*' * (NPi): Reference technology targets, mostly already enacted (N)ational (P)ol(i)cies, mostly for 2020 *' * (EVmandates): mandate for electric vehicles - used for UBA project $setglobal techpol none !! def = none *'--------------------- 41_emicapregi ---------------------------------------- @@ -407,7 +422,7 @@ $setGlobal damages off !! def = off *' * (DiceLikeItr): Internalize DICE-like damages (calculate the SCC) adjust cm_damages_SccHorizon. Requires cm_emiscen set to 9 for now. *' * (BurkeLikeItr): Internalize Burke-like damages (calculate the SCC) adjust cm_damages_SccHorizo. Requires cm_emiscen set to 9 for now. *' * (KWlikeItr): Internalize damage function based on Kalkuhl & Wenz (2020). Requires cm_emiscen set to 9 for now. -*' * (KWlikeItrCPnash): Internalize damage function based on Kalkuhl & Wenz (2020), but with Nash SCC, i.e. each region only internalizes its own damages. Requires cm_emiscen set to9 for now. +*' * (KWlikeItrCPnash): Internalize damage function based on Kalkuhl & Wenz (2020), but with Nash SCC, i.e. each region only internalizes its own damages. Requires cm_emiscen set to 9 for now. *' * (KWlikeItrCPreg): Internalize damage function based on Kalkuhl & Wenz (2020), but with regional SCC instead of a global uniform price. Requires cm_emiscen set to 9 for now. *' * (KW_SEitr): Internalize damage function based on Kalkuhl & Wenz (2020) for upper limit based on standard error. Requires cm_emiscen set to 9 for now. *' * (KWTCintItr): Internalize combined damages from Kalkuhl & Wenz (2020) and from tropical cyclones. Requires cm_emiscen set to 9 for now. @@ -455,7 +470,7 @@ parameter parameter c_keep_iteration_gdxes "save intermediate iteration gdxes" ; - c_keep_iteration_gdxes = 0; !! def = 0 + c_keep_iteration_gdxes = 0; !! def = 0 !! regexp = 0|1 *' in default we do not save gdx files from each iteration but this might be helpful for debugging *' *' * (0) gdx files from each iteration are NOT saved @@ -463,12 +478,12 @@ parameter parameter cm_keep_presolve_gdxes "save gdxes for all regions/solver tries/nash iterations for debugging" ; - cm_keep_presolve_gdxes = 0; !! def = 0 + cm_keep_presolve_gdxes = 0; !! def = 0 !! regexp = 0|1 *' parameter cm_nash_autoconverge "choice of nash convergence mode" ; - cm_nash_autoconverge = 1; !! def = 1 + cm_nash_autoconverge = 1; !! def = 1 !! regexp = [0-3] *' * (0): manually set number of iterations by adjusting cm_iteration_max *' * (1): run until solution is sufficiently converged - coarse tolerances, quick solution. ! do not use in production runs ! *' * (2): run until solution is sufficiently converged - fine tolerances, for production runs. @@ -477,37 +492,38 @@ parameter parameter cm_emiscen "policy scenario choice" ; - cm_emiscen = 1; !! def = 1 + cm_emiscen = 1; !! def = 1 !! regexp = 0|1|4|6|9|10 *' * (0): no global budget. Policy may still be prescribed by 41_emicaprei module. *' * (1): BAU *' * (4): emission time path *' * (6): budget *' * (9): tax scenario (requires running module 21_tax "on"), tax level controlled by module 45_carbonprice and cm_co2_tax_2020, other ghg etc. controlled by cm_rcp_scen +*' * (10): used for cost-benefit analysis *' *JeS* WARNING: data for cm_emiscen 4 only exists for multigas_scen 2 bau scenarios and for multigas_scen 1 *' parameter cm_co2_tax_2020 "level of co2 tax in year 2020 in $ per t CO2eq, makes sense only for emiscen eq 9 and 45_carbonprice exponential" ; - cm_co2_tax_2020 = -1; !! def = -1 + cm_co2_tax_2020 = -1; !! def = -1 !! regexp = is.numeric *' * (-1): default setting equivalent to no carbon tax *' * (any number >= 0): tax level in 2020, with 5% exponential increase over time *' parameter cm_co2_tax_growth "growth rate of carbon tax" ; - cm_co2_tax_growth = 1.05; !! def = 1.05 + cm_co2_tax_growth = 1.05; !! def = 1.05 !! regexp = is.numeric *' (any number >= 0): rate of exponential increase over time parameter c_macscen "use of mac" ; - c_macscen = 1; !! def = 1 + c_macscen = 1; !! def = 1 !! regexp = 1|2 *' * (1): on *' * (2): off *' parameter cm_nucscen "nuclear option choice" ; - cm_nucscen = 2; !! def = 2 + cm_nucscen = 2; !! def = 2 !! regexp = 2|5|6 *' * (2): no fnrs, tnrs with restricted new builds until 2030 (based on current data on plants under construction, planned or proposed) *' * (5): no new nuclear investments after 2020 *' * (6): +33% investment costs (tnrs) & uranium resources increased by a factor of 10 @@ -515,7 +531,7 @@ parameter parameter cm_ccapturescen "carbon capture option choice, no carbon capture only if CCS and CCU are switched off!" ; - cm_ccapturescen = 1; !! def = 1 + cm_ccapturescen = 1; !! def = 1 !! regexp = [1-4] *' * (1): yes *' * (2): no carbon capture (only if CCS and CCU are switched off!) *' * (3): no bio carbon capture @@ -524,14 +540,14 @@ parameter parameter c_bioliqscen "2nd generation bioenergy liquids technology choice" ; - c_bioliqscen = 1; !! def = 1 + c_bioliqscen = 1; !! def = 1 !! regexp = 0|1 *' * (0): no technologies *' * (1): all technologies *' parameter c_bioh2scen "bioenergy hydrogen technology choice" ; - c_bioh2scen = 1; !! def = 1 + c_bioh2scen = 1; !! def = 1 !! regexp = 0|1 *' * (0): no technologies *' * (1): all technologies *' @@ -583,7 +599,7 @@ parameter parameter c_solscen "solar option choice" ; - c_solscen = 1; !! def = 1 + c_solscen = 1; !! def = 1 !! regexp = [1-3] *' * (1): yes *' * (2): no solar *' * (3): fix at bau level @@ -633,14 +649,14 @@ parameter parameter cm_1stgen_phaseout "choose if 1st generation biofuels should phase out after 2030 (vm_deltaCap equals 0)" ; - cm_1stgen_phaseout = 0; !! def = 0 + cm_1stgen_phaseout = 0; !! def = 0 !! regexp = 0|1 *' * (0): 1st generation biofuels after 2020 are fixed at upper limit of resource potential (maxprod) *' * (1): no new capacities for 1st generation biofuel technologies may be built after 2030 -> phaseout until ~2060 *' parameter cm_phaseoutBiolc "Switch that allows for a full phaseout of all bioenergy technologies globally" ; - cm_phaseoutBiolc = 0; !! def = 0 + cm_phaseoutBiolc = 0; !! def = 0 !! regexp = 0|1 *** Only working with magpie_40 realization of 30_biomass module. *** (0): (default) No phaseout *** (1): Phaseout capacities of all bioenergy technologies using pebiolc, as far @@ -653,8 +669,8 @@ parameter parameter cm_startyear "first optimized modelling time step [year]" ; - cm_startyear = 2005; !! def = 2005 for a baseline -*' * (2005): standard for basline to check if model is well calibrated + cm_startyear = 2005; !! def = 2005 for a baseline !! regexp = 20[0-9](0|5) +*' * (2005): standard for baseline to check if model is well calibrated *' * (2015): standard for all policy runs (eq. to fix2010), NDC, NPi and production baselines, especially if various baselines with varying parameters are explored *' * (....): later start for delay policy runs, eg. 2025 for what used to be delay2020 *' @@ -666,14 +682,14 @@ parameter parameter cm_prtpScen "pure rate of time preference standard values" ; - cm_prtpScen = 3; !! def = 3 + cm_prtpScen = 3; !! def = 3 !! regexp = 1|3 *' * (1): 1 % *' * (3): 3 % *' parameter cm_fetaxscen "choice of final energy tax path, subsidy path and inconvenience cost path, values other than zero enable final energy tax" ; - cm_fetaxscen = 3; !! def = 3 + cm_fetaxscen = 3; !! def = 3 !! regexp = [0-4] *' even if set to 0, the PE inconvenience cost per SO2-cost for coal are always on if module 21_tax is on *' * (0): no tax, sub, inconv *' * (1): constant t,s,i (used in SSP 5 and ADVANCE WP3.1 HighOilSub) @@ -684,14 +700,14 @@ parameter parameter cm_distrBeta "elasticity of tax revenue redistribution" ; - cm_distrBeta = 1; !! def = 1 + cm_distrBeta = 1; !! def = 1 !! regexp = 0|1 *' (0): equal per capita redistribution *' (1): proportional redistribution *' parameter cm_multigasscen "scenario on GHG portfolio to be included in permit trading scheme" ; - cm_multigasscen = 2; !! def = 2 + cm_multigasscen = 2; !! def = 2 !! regexp = [1-3] *' * (1): CO2 only *' * (2): all GHG *' * (3): all GHG excl CO2 emissions from LULUCF @@ -699,7 +715,7 @@ parameter parameter cm_permittradescen "scenario on permit trade" ; - cm_permittradescen = 1; !! def = 1 + cm_permittradescen = 1; !! def = 1 !! regexp = [1-3] *' * (1): full permit trade (no restrictions) *' * (2): no permit trade (only domestic mitigation) *' * (3): limited trade (certain percentage of GDP) @@ -752,14 +768,14 @@ parameter parameter c_cint_scen "additional GHG emissions from mining fossil fuels" ; - c_cint_scen = 1; !! def = 1 + c_cint_scen = 1; !! def = 1 !! regexp = 0|1 *' * (0): switch is off (emissions are not accounted) *' * (1): switch is on (emissions are accounted) *' parameter cm_so2tax_scen "level of SO2 tax" ; - cm_so2tax_scen = 1; !! def = + cm_so2tax_scen = 1; !! def = 1 !! regexp = [0-4] *' * (0): so2 tax is set to zero *' * (1): so2 tax is low *' * (2): so2 tax is standard @@ -769,7 +785,7 @@ parameter parameter cm_solwindenergyscen "scenario for fluctuating renewables, 1 is reference, 2 is pessimistic with limits to fluctuating SE el share" ; - cm_solwindenergyscen = 1; !! def = 1 + cm_solwindenergyscen = 1; !! def = 1 !! regexp = [0-4] *' * (0) advanced - cheap investment costs, higher learning rates for pv, csp and wind *' * (1) reference - normal investment costs & learning rates for pv, csp and wind EMF27-3nd round Adv scenario *' * (2) pessimistic EMF27 and AWP 2 - share of PV, wind&CSP limited to 20%. Learning rates reduced by 20%, floor costs increased by 40%. EMF27-3nd round Cons scenario @@ -779,7 +795,7 @@ parameter parameter c_techAssumptScen "scenario for assumptions of energy technologies based on SSP scenarios, 1: SSP2 (default), 2: SSP1, 3: SSP5" ; - c_techAssumptScen = 1; !! def = 1 + c_techAssumptScen = 1; !! def = 1 !! regexp = [1-3] *' This flag defines an energy technology scenario *' * (1) SSP2: reference scenario - default investment costs & learning rates for pv, csp and wind *' * (2) SSP1: advanced renewable energy techno., pessimistic for nuclear and CCS @@ -788,7 +804,7 @@ parameter parameter c_ccsinjecratescen "CCS injection rate factor, 0.5% by default yielding a 60 Mt per year IR" ; - c_ccsinjecratescen = 1; !! def = 1 + c_ccsinjecratescen = 1; !! def = 1 !! regexp = [0-5] *' This flag determines the upper bound of the CCS injection rate *' CCS refers to carbon sequestration, carbon capture is modelled separately *' * (0) no "CCS" as in no carbon sequestration at all @@ -801,7 +817,7 @@ parameter parameter c_ccscapratescen "CCS capture rate" ; - c_ccscapratescen = 1; !! def = 1 + c_ccscapratescen = 1; !! def = 1 !! regexp = 1|2 *' This flag determines the CO2 capture rate of CCS technologies *' * (1) reference *' * (2) increased capture rate @@ -809,12 +825,12 @@ parameter parameter c_export_tax_scen "choose which oil export tax is used in the model. 0 = none, 1 = fix" ; - c_export_tax_scen = 0; !! def = 0 + c_export_tax_scen = 0; !! def = 0 !! regexp = 0|1 *' parameter cm_iterative_target_adj "whether or not a tax or a budget target should be iteratively adjusted depending on actual emission or forcing level" ; - cm_iterative_target_adj = 0; !! def = 0 + cm_iterative_target_adj = 0; !! def = 0 !! regexp = 0|2|3|4|5|6|7|9 *' * (0): no iterative adjustment *' * (2): iterative adjustment II based on magicc calculated forcing (for both budget and tax runs), see modules/ 0 /magicc/postsolve.gms for direct algorithms of adjustment *' * (3): [requires 45_carbonprice = "NDC" and emiscen = 9] iterative adjustment III for tax based on 2025 or 2030 regionally differentiated emissions, see module/45_carbonprice//postsolve.gms for algorithm of adjustment @@ -827,7 +843,7 @@ parameter parameter cm_NDC_divergentScenario "choose scenario about convergence of CO2eq prices in NDC realization of module 45_carbonprice" ; - cm_NDC_divergentScenario = 0; !! def = 0 + cm_NDC_divergentScenario = 0; !! def = 0 !! regexp = [0-2] *' * (0) 70 years after 2030 *' * (1) 120 years after 2030 *' * (2) until year 3000 ("never") @@ -835,14 +851,14 @@ parameter parameter cm_gdximport_target "whether or not the starting value for iteratively adjusted budgets, tax scenarios, or forcing targets (emiscen 5,6,8,9) should be read in from the input.gdx" ; - cm_gdximport_target = 0; !! def = 0 + cm_gdximport_target = 0; !! def = 0 !! regexp = 0|1 *' * (0): no import, the default starting value as specified in modules/ 0 /on/input/tax_CO2.inc, core/input/data_emibudget.inc, modules/15_climate/box/datainput.gms is used *' * (1): the values from the gdx are read in (works only if the gdx has a parameter value) ATTENTION: make sure that the values from the gdx have the right structure (e.g. regionally differentiated or not) *' parameter cm_gs_ew "grain size (for enhanced weathering, CDR module) [micrometre]" ; - cm_gs_ew = 20; !! def = 20 + cm_gs_ew = 20; !! def = 20 !! regexp = is.numeric *' parameter cm_LimRock "limit amount of rock spread each year [Gt]" @@ -928,7 +944,7 @@ parameter parameter cm_DiscRateScen "Scenario for the implicit discount rate applied to the energy efficiency capital" ; - cm_DiscRateScen = 0;!! def = 0 + cm_DiscRateScen = 0; !! def = 0 !! regexp = [0-4] *' * (0) Baseline without higher discount rate: No additional discount rate *' * (1) Baseline with higher discount rate: Increase the discount rate by 10%pts from 2005 until the end *' * (2) Energy Efficiency policy: 10%pts higher discount rate until cm_start_year and 0 afterwards. @@ -968,7 +984,7 @@ parameter parameter cm_TaxConvCheck "switch for enabling tax convergence check in nash mode" ; - cm_TaxConvCheck = 0; !! def 0, which means tax convergence check is off + cm_TaxConvCheck = 0; !! def 0, which means tax convergence check is off !! regexp = 0|1 *' switches tax convergence check in nash mode on and off (check that tax revenue in all regions, periods be smaller than 0.01% of GDP) *' * 0 (off) *' * 1 (on), default @@ -1011,7 +1027,7 @@ parameter parameter cm_VRE_supply_assumptions "default (0), optimistic (1), sombre (2), or bleak (3) assumptions on VRE supply" ; - cm_VRE_supply_assumptions = 0; !! 0 - default, 1 - optimistic, 2 - sombre, 3 - bleak + cm_VRE_supply_assumptions = 0; !! 0 - default, 1 - optimistic, 2 - sombre, 3 - bleak !! regexp = [0-3] *' * for 1 - optimistic, modify *' - inco0, incolearn, and learn parameters for spv and storspv *' - ease capacity constraints on storage @@ -1133,7 +1149,7 @@ $setglobal cm_MAgPIE_coupling off !! def = "off" *' * (rcp45): RCP4.5 *' * (rcp60): RCP6.0 *' * (rcp85): RCP8.5 -$setglobal cm_rcp_scen none !! def = "none" +$setglobal cm_rcp_scen none !! def = "none" !! regexp = none|rcp20|rcp26|rcp37|rcp45|rcp60|rcp85 *' cm_NDC_version "choose version year of NDC targets as well as conditional vs. unconditional targets" *' * (2023_cond): all NDCs conditional to international financial support published until February 24, 2023 *' * (2023_uncond): all NDCs independent of international financial support published until February 24, 2023 @@ -1143,12 +1159,12 @@ $setglobal cm_rcp_scen none !! def = "none" *' * (2021_uncond): all NDCs independent of international financial support published until December 31, 2021 *' * (2018_cond): all NDCs conditional to international financial support published until December 31, 2018 *' * (2018_uncond): all NDCs independent of international financial support published until December 31, 2018 -$setglobal cm_NDC_version 2023_cond !! def = "2023_cond", "2023_uncond", "2022_cond", "2022_uncond", "2021_cond", "2021_uncond", "2018_cond", "2018_uncond" +$setglobal cm_NDC_version 2023_cond !! def = "2023_cond" !! regexp = 20[1-9][0-9]_(un)?cond *** cm_netZeroScen "choose scenario of net zero targets of netZero realization of module 46_carbonpriceRegi" *** (NGFS_v4): settings used for NGFS v4, 2023 *** (NGFS_v4_20pc): settings used for NGFS v4, 2023, with still 20% of 2020 emissions in netZero year *** (ENGAGE4p5_GlP): settings used for ENGAGE 4.5 Glasgow+ scenario -$setglobal cm_netZeroScen NGFS_v4 !! def = "NGFS_v4" +$setglobal cm_netZeroScen NGFS_v4 !! def = "NGFS_v4" !! regexp = NGFS_v4|NGFS_v4_20pc|ENGAGE4p5_GlP *** c_regi_earlyreti_rate "maximum percentage of capital stock that can be retired early (before reaching their expected lifetimes) in one year in specified regions, if they are not economically viable. It is applied to all techs unless otherwise specified in c_tech_earlyreti_rate." *** GLO 0.09, EUR_regi 0.15: default value. (0.09 means full retirement after 11 years, 10% standing after 10 years) $setglobal c_regi_earlyreti_rate GLO 0.09, EUR_regi 0.15 !! def = GLO 0.09, EUR_regi 0.15 diff --git a/modules/46_carbonpriceRegi/NDC/datainput.gms b/modules/46_carbonpriceRegi/NDC/datainput.gms index 8d6c728d24..c6cb4e26b2 100644 --- a/modules/46_carbonpriceRegi/NDC/datainput.gms +++ b/modules/46_carbonpriceRegi/NDC/datainput.gms @@ -21,7 +21,7 @@ $offdelim $onlisting ; -Parameter p46_factorTargetyear(t,all_regi) "Multiplier for target year emissions vs 2005 emissions, as weighted average for all countries with quantifyable emissions under NDC in particular region"; +Parameter p46_factorTargetyear(ttot,all_regi) "Multiplier for target year emissions vs 2005 emissions, as weighted average for all countries with quantifyable emissions under NDC in particular region"; p46_factorTargetyear(t,all_regi) = f46_factorTargetyear(t,all_regi,"%cm_NDC_version%","%cm_GDPscen%"); display p46_factorTargetyear; @@ -34,7 +34,7 @@ $offdelim $onlisting ; -Parameter p46_2005shareTarget(t,all_regi) "2005 GHG emission share of countries with quantifyable emissions under NDC in particular region, time dimension specifies alternative future target years"; +Parameter p46_2005shareTarget(ttot,all_regi) "2005 GHG emission share of countries with quantifyable emissions under NDC in particular region, time dimension specifies alternative future target years"; p46_2005shareTarget(t,all_regi) = f46_2005shareTarget(t,all_regi,"%cm_NDC_version%","%cm_GDPscen%"); display p46_2005shareTarget; @@ -52,7 +52,7 @@ p46_histShare(tall,all_regi) = f46_histShare(tall,all_regi,"%cm_NDC_version%"); display p46_histShare; -Parameter p46_BAU_reg_emi_wo_LU_bunkers(t,all_regi) "regional GHG emissions (without LU and bunkers) in BAU scenario" +Parameter p46_BAU_reg_emi_wo_LU_bunkers(ttot,all_regi) "regional GHG emissions (without LU and bunkers) in BAU scenario" / $ondelim $ifthen exist "./modules/46_carbonpriceRegi/NDC/input/pm_BAU_reg_emi_wo_LU_bunkers.cs4r" @@ -86,9 +86,9 @@ Scalar p46_ignoreNDCafter "NDC targets after this years are ignored, Scalar p46_minRatioOfCoverageToMax "only targets whose coverage is this times p46_bestNDCcoverage are considered. Use 1 for only best." /0.2/; Scalar p46_useSingleYearCloseTo "if 0: use all. If > 0: use only one single NDC target per country closest to this year (use 2030.4 to prefer 2030 over 2035 over 2025)" /0/; -Set p46_NDCyearSet(t,all_regi) "YES for years whose NDC targets is used"; +Set p46_NDCyearSet(ttot,all_regi) "YES for years whose NDC targets is used"; Parameter p46_bestNDCcoverage(all_regi) "highest coverage of NDC targets within region"; -Parameter p46_distanceToOptyear(t,all_regi) "distance to p46_useSingleYearCloseTo to favor years in case of multiple equally good targets"; +Parameter p46_distanceToOptyear(ttot,all_regi) "distance to p46_useSingleYearCloseTo to favor years in case of multiple equally good targets"; Parameter p46_minDistanceToOptyear(all_regi) "minimal distance to p46_useSingleYearCloseTo per region"; p46_bestNDCcoverage(regi) = smax(t$(t.val <= p46_ignoreNDCafter AND t.val >= p46_ignoreNDCbefore), p46_2005shareTarget(t,regi)); diff --git a/scripts/start/checkFixCfg.R b/scripts/start/checkFixCfg.R new file mode 100644 index 0000000000..be2cbe8785 --- /dev/null +++ b/scripts/start/checkFixCfg.R @@ -0,0 +1,50 @@ +checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { + refcfg <- gms::readDefaultConfig(remindPath) + gms::check_config(cfg, reference_file = refcfg, modulepath = file.path(remindPath, "modules"), + settings_config = file.path(remindPath, "config", "settings_config.csv"), + extras = c("backup", "remind_folder", "pathToMagpieReport", "cm_nash_autoconverge_lastrun", + "gms$c_expname", "restart_subsequent_runs", "gms$c_GDPpcScen", + "gms$cm_CES_configuration", "gms$c_description", "model")) + + errorsfound <- 0 + + code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE) + grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)" + + for (n in names(cfg$gms)) { + errormsg <- NULL + paramdef <- paste0("^([ ]*", n, "[ ]*=|\\$setglobal[ ]+", n, " )") + filtered <- grep(paste0(paramdef, ".*regexp[ ]*=[ ]*"), code, value = TRUE) + if (length(filtered) == 1) { + # search for string '!! regexp = whatever', potentially followed by '!! otherstuff' and extract 'whatever' + regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$") + useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE) + if (! grepl(useregexp, cfg$gms[[n]])) { + errormsg <- paste0("Parameter cfg$gms$", n, "=", cfg$gms[[n]], " does not fit this regular expression: ", regexp) + } + } else if (length(filtered) > 1) { + errormsg <- paste0("More than one regexp found for ", n, ". These are the code lines:\n", paste(filtered, collapse = "\n")) + } + if (! is.null(errormsg)) { + errorsfound <- errorsfound + 1 + if (testmode) warning(errormsg) else message(errormsg) + } + } + + # Check for compatibility with subsidizeLearning + if ((cfg$gms$optimization != "nash") && (cfg$gms$subsidizeLearning == "globallyOptimal") ) { + message("Only optimization='nash' is compatible with subsidizeLearning='globallyOptimal'. Switching subsidizeLearning to 'off' now.\n") + cfg$gms$subsidizeLearning <- "off" + } + + # reportCEScalib only works with the calibrate module + if (! isTRUE(cfg$gms$CES_parameters == "calibrate")) { + cfg$output <- setdiff(cfg$output, "reportCEScalib") + } + if (errorsfound > 0) { + if (testmode) warning(errorsfound, " errors found.") + else stop(errorsfound, " errors found, see above. Either adapt the parameter choice or the regexp in main.gms") + } + + return(cfg) +} diff --git a/scripts/start/prepare.R b/scripts/start/prepare.R index ffc47290c6..4e4add82b1 100644 --- a/scripts/start/prepare.R +++ b/scripts/start/prepare.R @@ -68,28 +68,13 @@ prepare <- function() { # change to REMIND main folder setwd(cfg$remind_folder) - # Check configuration for consistency -# cfg <- check_config(cfg, reference_file="config/default.cfg", -# settings_config = "config/settings_config.csv", -# extras = c("backup", "remind_folder", "pathToMagpieReport", "cm_nash_autoconverge_lastrun", -# "gms$c_expname", "restart_subsequent_runs", "gms$c_GDPpcScen", -# "gms$cm_CES_configuration", "gms$c_description")) - - # Check for compatibility with subsidizeLearning - if ( (cfg$gms$optimization != 'nash') & (cfg$gms$subsidizeLearning == 'globallyOptimal') ) { - cat("Only optimization='nash' is compatible with subsudizeLearning='globallyOptimal'. Switching subsidizeLearning to 'off' now. \n") - cfg$gms$subsidizeLearning = 'off' - } - - # reportCEScalib only works with the calibrate module - if ( cfg$gms$CES_parameters != "calibrate" ) cfg$output <- setdiff(cfg$output,"reportCEScalib") + cfg <- checkFixCfg(cfg, remindPath = cfg$remind_folder) #AJS quit if title is too long - GAMS can't handle that if( nchar(cfg$title) > 75 | grepl("\\.",cfg$title) ) { stop("This title is too long or the name contains dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ") } - # adjust GDPpcScen based on GDPscen cfg$gms$c_GDPpcScen <- gsub("gdp_","",cfg$gms$cm_GDPscen) diff --git a/scripts/start/runGamsCompile.R b/scripts/start/runGamsCompile.R index de2f3ae43f..885d112b6f 100644 --- a/scripts/start/runGamsCompile.R +++ b/scripts/start/runGamsCompile.R @@ -19,12 +19,12 @@ runGamsCompile <- function(modelFile, cfg, interactive = TRUE) { gcdir <- file.path(dirname(modelFile), "output", "gamscompile") dir.create(gcdir, recursive = TRUE, showWarnings = FALSE) tmpModelFile <- file.path(gcdir, paste0("main_", cfg$title, ".gms")) + tmpModelLst <- gsub("gms$", "lst", tmpModelFile) file.copy(modelFile, tmpModelFile, overwrite = TRUE) lucode2::manipulateConfig(tmpModelFile, cfg$gms) exitcode <- system2( command = cfg$gamsv, - args = paste(tmpModelFile, "-o", gsub("gms$", "lst", tmpModelFile), - "-action=c -errmsg=1 -pw=132 -ps=0 -logoption=0")) + args = paste(tmpModelFile, "-o", tmpModelLst, "-action=c -errmsg=1 -pw=132 -ps=0 -logoption=0")) if (0 < exitcode) { message(red, "FAIL ", NC, gsub("gms$", "lst", tmpModelFile)) if (interactive) { @@ -39,6 +39,9 @@ runGamsCompile <- function(modelFile, cfg, interactive = TRUE) { return(FALSE) } else { message(green, " OK ", NC, gsub("gms$", "lst", tmpModelFile)) + if (isTRUE(grepl("TESTTHAT_scenario_config", cfg$title))) { # for test_04-gamscompile + unlink(c(tmpModelFile, tmpModelLst)) + } return(TRUE) } } diff --git a/scripts/start/selectScenarios.R b/scripts/start/selectScenarios.R index 070a2b9e00..926d48b442 100644 --- a/scripts/start/selectScenarios.R +++ b/scripts/start/selectScenarios.R @@ -13,6 +13,7 @@ #' @return dataframe with scenarios from settings which should be started selectScenarios <- function (settings, interactive, startgroup) { + if (isTRUE(startgroup == "*")) return(settings) scenariosInGroup <- grepl(paste0("(^|,)", startgroup, "($|,)"), as.character(settings$start), perl = TRUE) if (interactive | ! any(scenariosInGroup)) { scenariosInGroup <- gms::chooseFromList(setNames(rownames(settings), settings$start), type = "runs", returnBoolean = TRUE) diff --git a/start.R b/start.R index 44e72ddf33..605a051763 100755 --- a/start.R +++ b/start.R @@ -39,7 +39,7 @@ helpText <- " #' --testOneRegi, -1: starting the REMIND run(s) in testOneRegi mode #' startgroup=MYGROUP when reading a scenario config .csv file, don't start #' everything specified by \"start = 1\", instead start everything -#' specified by \"start = MYGROUP\" +#' specified by \"start = MYGROUP\". Use startgroup=* to start all. #' titletag=MYTAG append \"-MYTAG\" to all titles of all runs that are started #' slurmConfig=CONFIG use the provided CONFIG as slurmConfig: a string, or an integer <= 16 #' to select one of the options shown when running './start.R -t'. @@ -328,6 +328,8 @@ if (any(c("--reprepare", "--restart") %in% flags)) { # abort on too long paths ---- cfg$gms$cm_CES_configuration <- calculate_CES_configuration(cfg, check = TRUE) + cfg <- checkFixCfg(cfg) + # save the cfg object for the later automatic start of subsequent runs (after preceding run finished) if (! "--gamscompile" %in% flags) { filename <- paste0(cfg$title,".RData") diff --git a/start_bundle_coupled.R b/start_bundle_coupled.R index e567ef35e9..c74ee0534f 100755 --- a/start_bundle_coupled.R +++ b/start_bundle_coupled.R @@ -142,7 +142,7 @@ if (length(argv) > 0) { if (sum(file_exists) > 1) stop("Enter only a scenario_config_coupled* file via command line or set all files manually in start_bundle_coupled.R") if (!all(file_exists)) stop("Unknown parameter provided: ", paste(argv[!file_exists], collapse = ", ")) # set config file to not known parameter where the file actually exists - path_settings_coupled <- file.path(path_remind, argv[[1]]) + path_settings_coupled <- normalizePath(argv[[1]]) if (! isTRUE(grepl("scenario_config_coupled", path_settings_coupled))) stop("Enter only a scenario_config_coupled* file via command line or set all files manually in start_bundle_coupled.R.\n", "Your command line arguments were: ", paste0(argv, collapse = " ")) @@ -549,6 +549,8 @@ for(scen in common){ numberOfTasks <- 1 } + cfg_rem <- checkFixCfg(cfg_rem, remindPath = path_remind) + Rdatafile <- paste0(fullrunname, ".RData") message("Save settings to ", Rdatafile) save(path_remind, path_magpie, cfg_rem, cfg_mag, runname, fullrunname, max_iterations, start_iter, diff --git a/tests/testthat/test_01-checkFixCfg.R b/tests/testthat/test_01-checkFixCfg.R new file mode 100644 index 0000000000..2bba25d801 --- /dev/null +++ b/tests/testthat/test_01-checkFixCfg.R @@ -0,0 +1,34 @@ +# | (C) 2006-2023 Potsdam Institute for Climate Impact Research (PIK) +# | authors, and contributors see CITATION.cff file. This file is part +# | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of +# | AGPL-3.0, you are granted additional permissions described in the +# | REMIND License Exception, version 1.0 (see LICENSE file). +# | Contact: remind@pik-potsdam.de +test_that("checkFixCfg works", { + remind_folder <- "../.." + savecfg <- cfg <- gms::readDefaultConfig(remind_folder) + # regexp = NA means: no warning + expect_warning(checkFixCfg(cfg, remind_folder, testmode = TRUE), regexp = NA) + + wrongsetting <- c( + "cm_NDC_version" = "2004_cond", + "cm_emiscen" = "123", + "cm_nash_autoconverge" = "NA", + "cm_co2_tax_2020" = "2.2.2", + "cm_co2_tax_growth" = "333++", + "c_macscen" = "-1", + "cm_keep_presolve_gdxes" = "1.1", + "cm_startyear" = "1985", + "cm_netZeroScen" = "NöööGFS_v4", + "cm_rcp_scen" = "apocalypse", + NULL) + + cfg <- savecfg + cfg$gms[names(wrongsetting)] <- wrongsetting + w <- capture_warnings(checkFixCfg(cfg, remind_folder, testmode = TRUE)) + for (n in names(wrongsetting)) { + expect_match(w, paste0(n, "=", wrongsetting[[n]]), all = FALSE, fixed = TRUE) + } + expect_match(w, paste0(length(wrongsetting), " errors found"), all = FALSE, fixed = TRUE) + expect_equal(length(w), length(wrongsetting) + 1) +}) diff --git a/tests/testthat/test_01-readCheckScenarioConfig.R b/tests/testthat/test_01-readCheckScenarioConfig.R index e014de3dec..40ba090d62 100644 --- a/tests/testthat/test_01-readCheckScenarioConfig.R +++ b/tests/testthat/test_01-readCheckScenarioConfig.R @@ -10,7 +10,7 @@ if (length(csvfiles) == 0) { file.path("../../config", "*", "scenario_config*.csv"))) } for (csvfile in csvfiles) { - test_that(paste("perform readCheckScenarioConfig with", gsub("../../config/", "", csvfile, fixed = TRUE)), { + test_that(paste("perform readCheckScenarioConfig with", basename(csvfile)), { # regexp = NA means: expect no warning expect_warning(readCheckScenarioConfig(csvfile, remindPath = "../../", testmode = TRUE), regexp = NA) }) @@ -24,15 +24,15 @@ test_that("readCheckScenarioConfig fails on error-loaden config", { "PBScopy;0;PBS;;;mustbedifferenttoPBS"), con = csvfile, sep = "\n") w <- capture_warnings(scenConf <- readCheckScenarioConfig(csvfile, remindPath = "../../", testmode = TRUE)) - expect_match(w, "11 errors found", all = FALSE) - expect_match(w, "These titles are too long", all = FALSE) - expect_match(w, "These titles may be confused with regions", all = FALSE) - expect_match(w, "These titles contain illegal characters", all = FALSE) + expect_match(w, "11 errors found", all = FALSE, fixed = TRUE) + expect_match(w, "These titles are too long", all = FALSE, fixed = TRUE) + expect_match(w, "These titles may be confused with regions", all = FALSE, fixed = TRUE) + expect_match(w, "These titles contain illegal characters", all = FALSE, fixed = TRUE) expect_match(w, "\\.@&", all = FALSE) - expect_match(w, "Outdated column names found that must not be used", all = FALSE) - expect_match(w, "contain whitespaces", all = FALSE) - expect_match(w, "scenario names indicated in copyConfigFrom column were not found", all = FALSE) - expect_match(w, "specify in copyConfigFrom column a scenario name defined below in the file", all = FALSE) + expect_match(w, "Outdated column names found that must not be used", all = FALSE, fixed = TRUE) + expect_match(w, "contain whitespaces", all = FALSE, fixed = TRUE) + expect_match(w, "scenario names indicated in copyConfigFrom column were not found", all = FALSE, fixed = TRUE) + expect_match(w, "specify in copyConfigFrom column a scenario name defined below in the file", all = FALSE, fixed = TRUE) copiedFromPBS <- c("c_budgetCO2", "path_gdx", "path_gdx_ref") expect_identical(unlist(scenConf["PBS", copiedFromPBS]), unlist(scenConf["PBScopy", copiedFromPBS])) diff --git a/tests/testthat/test_01-start.R b/tests/testthat/test_01-start.R index dabc7da1f6..3be8cbed54 100644 --- a/tests/testthat/test_01-start.R +++ b/tests/testthat/test_01-start.R @@ -14,3 +14,45 @@ test_that("start.R fails on missing path_gdx* files", { expect_true(any(grepl("2 errors were identified", output))) expectFailStatus(output) }) + +test_that("start.R --test startgroup=AMT titletag=AMT config/scenario_config.csv works", { + with_mocked_bindings({ + skipIfPreviousFailed() + output <- localSystem2("Rscript", + c("start.R", "--test", "slurmConfig=16", "startgroup=AMT", "titletag=TESTTHAT", "config/scenario_config.csv")) + printIfFailed(output) + expectSuccessStatus(output) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) + +test_that("start.R --test succeeds on all configs", { + # skipIfFast() + skipIfPreviousFailed() + csvfiles <- system("git ls-files ../../config/scenario_config*.csv ../../config/*/scenario_config*.csv", intern = TRUE) + if (length(csvfiles) == 0) { + csvfiles <- Sys.glob(c(file.path("../../config/scenario_config*.csv"), + file.path("../../config", "*", "scenario_config*.csv"))) + } + csvfiles <- normalizePath(grep("scenario_config_coupled", csvfiles, invert = TRUE, value = TRUE)) + skipfiles <- c("scenario_config_21_EU11_ECEMF", "scenario_config_DeepEl", + "scenario_config_EDGE-T_NDC_NPi_pkbudget", "scenario_config_NAVIGATE_300") + csvfiles <- grep(paste(skipfiles, collapse = "|"), csvfiles, invert = TRUE, value = TRUE) + expect_true(length(csvfiles) > 0) + with_mocked_bindings( + for (csvfile in csvfiles) { + test_that(paste("perform start.R --test with", basename(csvfile)), { + output <- localSystem2("Rscript", + c("start.R", "--test", "slurmConfig=16", "startgroup=*", "titletag=TESTTHAT", csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) diff --git a/tests/testthat/test_04-gamscompile.R b/tests/testthat/test_04-gamscompile.R index 3724a854f6..817278fa3f 100644 --- a/tests/testthat/test_04-gamscompile.R +++ b/tests/testthat/test_04-gamscompile.R @@ -4,10 +4,48 @@ # | AGPL-3.0, you are granted additional permissions described in the # | REMIND License Exception, version 1.0 (see LICENSE file). # | Contact: remind@pik-potsdam.de -test_that("start.R --gamscompile startgroup=AMT titletag=AMT config/scenario_config.csv works", { +test_that("start.R --gamscompile startgroup=AMT config/scenario_config.csv works", { + unlink(paste0("../../output/gamscompile/*TESTTHAT", c(".gms", ".lst"))) + csvfile <- "config/scenario_config.csv" + titletag <- paste0("titletag=TESTTHAT-", gsub(".csv$", "", basename(csvfile))) + with_mocked_bindings({ + skipIfPreviousFailed() + output <- localSystem2("Rscript", + c("start.R", "--gamscompile", "startgroup=AMT", titletag, csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) + +test_that("start.R --gamscompile works on all configs and scenarios", { + skipIfFast() skipIfPreviousFailed() - output <- localSystem2("Rscript", - c("start.R", "--gamscompile", "startgroup=AMT", "titletag=AMT", "config/scenario_config.csv")) - printIfFailed(output) - expectSuccessStatus(output) + csvfiles <- system("git ls-files ../../config/scenario_config*.csv ../../config/*/scenario_config*.csv", intern = TRUE) + if (length(csvfiles) == 0) { + csvfiles <- Sys.glob(c(file.path("../../config/scenario_config*.csv"), + file.path("../../config", "*", "scenario_config*.csv"))) + } + csvfiles <- normalizePath(grep("^scenario_config_coupled.*", csvfiles, invert = TRUE, value = TRUE)) + skipfiles <- c("scenario_config_21_EU11_ECEMF", "scenario_config_EDGE-T_NDC_NPi_pkbudget", + "scenario_config_NAVIGATE_300", "21_regions_EU11", "scenario_config_tradeCap_standalone") + csvfiles <- grep(paste(skipfiles, collapse = "|"), csvfiles, invert = TRUE, value = TRUE) + expect_true(length(csvfiles) > 0) + with_mocked_bindings( + for (csvfile in csvfiles) { + test_that(paste("perform start.R --gamscompile with", basename(csvfile)), { + titletag <- paste0("titletag=TESTTHAT-", gsub(".csv$", "", basename(csvfile))) + output <- localSystem2("Rscript", + c("start.R", "--gamscompile", "startgroup=*", titletag, csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") }) diff --git a/tests/testthat/test_06-output.R b/tests/testthat/test_06-output.R index 4cd51cecb7..3f75c90534 100644 --- a/tests/testthat/test_06-output.R +++ b/tests/testthat/test_06-output.R @@ -4,9 +4,10 @@ # | AGPL-3.0, you are granted additional permissions described in the # | REMIND License Exception, version 1.0 (see LICENSE file). # | Contact: remind@pik-potsdam.de +skipIfFast() +skipIfPreviousFailed() + test_that("output.R -> single -> reporting works", { - skipIfFast() - skipIfPreviousFailed() output <- localSystem2("Rscript", c("output.R", "comp=single", "output=reporting", "outputdir=output/testOneRegi", "slurmConfig='--qos=priority --mem=8000 --wait'")) printIfFailed(output) @@ -15,8 +16,6 @@ test_that("output.R -> single -> reporting works", { }) test_that("output.R -> export -> xlsx_IIASA works", { - skipIfFast() - skipIfPreviousFailed() exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*")) unlink(exportfiles) output <- localSystem2("Rscript", c("output.R", "project=TESTTHAT", "filename_prefix=TESTTHAT", @@ -31,8 +30,6 @@ test_that("output.R -> export -> xlsx_IIASA works", { }) test_that("cleanup output.R", { - skipIfFast() - skipIfPreviousFailed() exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*")) unlink(exportfiles) }) diff --git a/tests/testthat/test_20-coupled.R b/tests/testthat/test_20-coupled.R index f90a7314c9..ec23420766 100644 --- a/tests/testthat/test_20-coupled.R +++ b/tests/testthat/test_20-coupled.R @@ -239,6 +239,32 @@ test_that("Check path_mif_ghgprice_land with file", { } }) +test_that("start_bundle_coupled.R --test succeeds on all configs", { + skipIfPreviousFailed() + csvfiles <- system("git ls-files ../../config/scenario_config_coupled*.csv ../../config/*/scenario_config_coupled*.csv", intern = TRUE) + if (length(csvfiles) == 0) { + csvfiles <- Sys.glob(c(file.path("../../config/scenario_config_coupled*.csv"), + file.path("../../config", "*", "scenario_config_coupled*.csv"))) + } + skipfiles <- c("scenario_config_coupled_shortCascade", # fails on missing mif file which is present while running other tests + "scenario_config_coupled_GCS") # GCFS setting not in standard MAgPIE release + csvfiles <- normalizePath(grep(paste(skipfiles, collapse = "|"), csvfiles, invert = TRUE, value = TRUE)) + expect_true(length(csvfiles) > 0) + with_mocked_bindings( + for (csvfile in csvfiles) { + test_that(paste("perform start_bundle_coupled.R --test with", basename(csvfile)), { + output <- localSystem2("Rscript", + c("start_bundle_coupled.R", "--test", "startgroup=*", csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) + test_that("delete files to leave clean state", { # leave clean state skipIfPreviousFailed()