From 53e56e607c386751850b1145fe88c3a64df14ee7 Mon Sep 17 00:00:00 2001 From: orichters Date: Tue, 11 Jul 2023 12:01:48 +0200 Subject: [PATCH 1/4] =?UTF-8?q?reactive=20check=5Fconfig,=20add=20regexp?= =?UTF-8?q?=20check=20for=20cfg$=E2=80=A6=20values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 7 +- Makefile | 4 + main.gms | 188 ++++++++++-------- scripts/start/checkFixCfg.R | 50 +++++ scripts/start/prepare.R | 17 +- scripts/start/runGamsCompile.R | 7 +- scripts/start/selectScenarios.R | 1 + start.R | 4 +- start_bundle_coupled.R | 4 +- tests/testthat/test_01-checkFixCfg.R | 34 ++++ .../test_01-readCheckScenarioConfig.R | 18 +- tests/testthat/test_01-start.R | 42 ++++ tests/testthat/test_04-gamscompile.R | 48 ++++- tests/testthat/test_06-output.R | 9 +- tests/testthat/test_20-coupled.R | 26 +++ 15 files changed, 333 insertions(+), 126 deletions(-) create mode 100644 scripts/start/checkFixCfg.R create mode 100644 tests/testthat/test_01-checkFixCfg.R diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ffc63914..c5e02896c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### added +- **config** regex tests for many parameters +- **testthat** test and compile all config files ## [3.2.1] - 2023-07-13 (incomplete) ### changed - **documentation** MAgPIE coupling, DIETER coupling, input changes -- **config** NGFS, SHAPE +- **config** NGFS_v4, SHAPE - **scripts** re-enable summation checks for IIASA submission ### added -- **45_carbonprice** added realization for National Policies Implemented +- **45_carbonprice** added realization `NPi` (National Policies Implemented) - **47_carbonpriceRegi** now supports BECCS quantity targets - **MAgPIE coupling** added `qos=auto` mode - **MAgPIE coupling** added renv support mode diff --git a/Makefile b/Makefile index bc0e047da..6d16433a7 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,10 @@ test-coupled: ## Test if the coupling with MAgPIE works. Takes significantly $(info Coupling tests take around 75 minutes to run, please be patient) @R_PROFILE_USER= TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_file("tests/testthat/test_20-coupled.R")' +test-coupled-slurm: ## test-coupled, but on slurm + $(info Coupling tests take around 75 minutes to run. Sent to slurm, find log in test-coupled.log) + @sbatch --qos=priority --wrap="make test-coupled" --job-name=test-coupled --mail-type=END --output=test-coupled.log + test-full: ## Run all tests, including coupling tests and a default ## REMIND scenario. Takes significantly longer than 10 minutes to run. $(info Full tests take more than an hour to run, please be patient) diff --git a/main.gms b/main.gms index 30bd747a6..5290031eb 100755 --- a/main.gms +++ b/main.gms @@ -147,6 +147,21 @@ *' * Put the equation sign (`=g=`, `=l=`, `=e=`) in a single line without anything else *' *' +*' #### Switches: +*' A switch must be defined in main.gms +*' Follow this mode of definition for parameters, including the indentation: +*' -------- +*' parameter +*' param_name "explanation what it means" +*' ; +*' param_name = 0; !! def = 0 !! regexp = 0|1 +*' -------- +*' * The value behind 'def' contains the default value and is added only for the user to remember if changed manually +*' * The value behind regexp is read by scripts/start/checkFixCfg.R to check the validity of the input. +*' In this case, it checks whether the value fits this regular expression: ^(0|1)$ +*' Use 'value1|value2' for specific values, use '[1-7]' for a row of integers and use 'is.numeric' for numeric values. +*' +*' *' #### Other general rules: *' * Decompose large model equations into several small equations to enhance readability and model diagnostics *' * Don't use hard-coded numbers in the equations part of the model @@ -273,7 +288,7 @@ $setglobal capitalMarket debt_limit !! def = debt_limit *'---------------------- 24_trade --------------------------------------- *' *' * (standard): macro-economic commodities and primary energy commodities trading -*' * (se_trade): macro-economic commodities, primary energy commodities and secondary energy hydrogen and electricitiy trading +*' * (se_trade): macro-economic commodities, primary energy commodities and secondary energy hydrogen and electricity trading *' * (capacity): capacity-based trade implementation $setglobal trade standard !! def = standard *'---------------------- 26_agCosts ---------------------------------------- @@ -289,11 +304,11 @@ $setglobal agCosts costs !! def = costs $setglobal CES_parameters load !! def = load *'--------------------- 30_biomass ---------------------------------------- *' -*' * (magpie_40): using supplycurved derived from MAgpIE 4.0 +*' * (magpie_40): using supply curves derived from MAgpIE 4.0 $setglobal biomass magpie_40 !! def = magpie_hightccost *'--------------------- 31_fossil ---------------------------------------- *' -*' * (timeDepGrades): time-dependent grade stucture of fossil resources (oil & gas only) +*' * (timeDepGrades): time-dependent grade structure of fossil resources (oil & gas only) *' * (grades2poly) : simplified version of the time-dependent grade realization (using polynomial functions) *' * (exogenous) : exogenous fossil extraction and costs $setglobal fossil grades2poly !! def = grades2poly @@ -341,7 +356,7 @@ $setglobal CCU on !! def = on *' * (coalPhaseoutRegional): [works only with Negishi] global phase-out of new freely-emitting coal conversion, caps all coal routes with the exception of coaltr: coal solids can still expand *' * (CombLowCandCoalPO): [works only with Negishi] combination of lowCarbonPush and coalPhaseout *' * (NDC): Technology targets for 2030 for spv,wind,tnrs. -*' * (NPi): Reference technology targets, mostly already enacted (N)ational (P]ol(i)cies, mostly for 2020 +*' * (NPi): Reference technology targets, mostly already enacted (N)ational (P)olicies (i)mplemented, mostly for 2020 *' * (EVmandates): mandate for electric vehicles - used for UBA project $setglobal techpol none !! def = none *'--------------------- 41_emicapregi ---------------------------------------- @@ -374,6 +389,7 @@ $setglobal banking off !! def = off *' * (diffCurvPhaseIn2Lin): [REMIND 2.1 default for validation peakBudget runs, in combination with "iterative_target_adj" = 9] curved convergence of CO2 prices between regions until cm_CO2priceRegConvEndYr; developed countries have linear path from 0 in 2010 through cm_co2_tax_2020 in 2020; *' * (diffPhaseIn2Constant): !experimental! linearly phase in global constant price, with starting values differentiated by GDP/cap *' * (NDC): implements a carbon price trajectory consistent with the NDC targets (up to 2030) and a trajectory of comparable ambition post 2030 (1.25%/yr price increase and regional convergence of carbon price). Choose version using cm_NDC_version "2023_cond", "2023_uncond", or replace 2023 by 2022, 2021 or 2018 to get all NDC published until end of these years. +*' * (NPi): National Policies Implemented, extrapolation of historical (until 2020) carbon prices $setglobal carbonprice none !! def = none *'--------------------- 46_carbonpriceRegi --------------------------------- *' @@ -407,7 +423,7 @@ $setGlobal damages off !! def = off *' * (DiceLikeItr): Internalize DICE-like damages (calculate the SCC) adjust cm_damages_SccHorizon. Requires cm_emiscen set to 9 for now. *' * (BurkeLikeItr): Internalize Burke-like damages (calculate the SCC) adjust cm_damages_SccHorizo. Requires cm_emiscen set to 9 for now. *' * (KWlikeItr): Internalize damage function based on Kalkuhl & Wenz (2020). Requires cm_emiscen set to 9 for now. -*' * (KWlikeItrCPnash): Internalize damage function based on Kalkuhl & Wenz (2020), but with Nash SCC, i.e. each region only internalizes its own damages. Requires cm_emiscen set to9 for now. +*' * (KWlikeItrCPnash): Internalize damage function based on Kalkuhl & Wenz (2020), but with Nash SCC, i.e. each region only internalizes its own damages. Requires cm_emiscen set to 9 for now. *' * (KWlikeItrCPreg): Internalize damage function based on Kalkuhl & Wenz (2020), but with regional SCC instead of a global uniform price. Requires cm_emiscen set to 9 for now. *' * (KW_SEitr): Internalize damage function based on Kalkuhl & Wenz (2020) for upper limit based on standard error. Requires cm_emiscen set to 9 for now. *' * (KWTCintItr): Internalize combined damages from Kalkuhl & Wenz (2020) and from tropical cyclones. Requires cm_emiscen set to 9 for now. @@ -455,7 +471,7 @@ parameter parameter c_keep_iteration_gdxes "save intermediate iteration gdxes" ; - c_keep_iteration_gdxes = 0; !! def = 0 + c_keep_iteration_gdxes = 0; !! def = 0 !! regexp = 0|1 *' in default we do not save gdx files from each iteration but this might be helpful for debugging *' *' * (0) gdx files from each iteration are NOT saved @@ -463,12 +479,12 @@ parameter parameter cm_keep_presolve_gdxes "save gdxes for all regions/solver tries/nash iterations for debugging" ; - cm_keep_presolve_gdxes = 0; !! def = 0 + cm_keep_presolve_gdxes = 0; !! def = 0 !! regexp = 0|1 *' parameter cm_nash_autoconverge "choice of nash convergence mode" ; - cm_nash_autoconverge = 1; !! def = 1 + cm_nash_autoconverge = 1; !! def = 1 !! regexp = [0-3] *' * (0): manually set number of iterations by adjusting cm_iteration_max *' * (1): run until solution is sufficiently converged - coarse tolerances, quick solution. ! do not use in production runs ! *' * (2): run until solution is sufficiently converged - fine tolerances, for production runs. @@ -477,37 +493,38 @@ parameter parameter cm_emiscen "policy scenario choice" ; - cm_emiscen = 1; !! def = 1 + cm_emiscen = 1; !! def = 1 !! regexp = 0|1|4|6|9|10 *' * (0): no global budget. Policy may still be prescribed by 41_emicaprei module. *' * (1): BAU *' * (4): emission time path *' * (6): budget *' * (9): tax scenario (requires running module 21_tax "on"), tax level controlled by module 45_carbonprice and cm_co2_tax_2020, other ghg etc. controlled by cm_rcp_scen +*' * (10): used for cost-benefit analysis *' *JeS* WARNING: data for cm_emiscen 4 only exists for multigas_scen 2 bau scenarios and for multigas_scen 1 *' parameter cm_co2_tax_2020 "level of co2 tax in year 2020 in $ per t CO2eq, makes sense only for emiscen eq 9 and 45_carbonprice exponential" ; - cm_co2_tax_2020 = -1; !! def = -1 + cm_co2_tax_2020 = -1; !! def = -1 !! regexp = is.numeric *' * (-1): default setting equivalent to no carbon tax *' * (any number >= 0): tax level in 2020, with 5% exponential increase over time *' parameter cm_co2_tax_growth "growth rate of carbon tax" ; - cm_co2_tax_growth = 1.05; !! def = 1.05 + cm_co2_tax_growth = 1.05; !! def = 1.05 !! regexp = is.numeric *' (any number >= 0): rate of exponential increase over time parameter c_macscen "use of mac" ; - c_macscen = 1; !! def = 1 + c_macscen = 1; !! def = 1 !! regexp = 1|2 *' * (1): on *' * (2): off *' parameter cm_nucscen "nuclear option choice" ; - cm_nucscen = 2; !! def = 2 + cm_nucscen = 2; !! def = 2 !! regexp = 2|5|6 *' * (2): no fnrs, tnrs with restricted new builds until 2030 (based on current data on plants under construction, planned or proposed) *' * (5): no new nuclear investments after 2020 *' * (6): +33% investment costs (tnrs) & uranium resources increased by a factor of 10 @@ -515,7 +532,7 @@ parameter parameter cm_ccapturescen "carbon capture option choice, no carbon capture only if CCS and CCU are switched off!" ; - cm_ccapturescen = 1; !! def = 1 + cm_ccapturescen = 1; !! def = 1 !! regexp = [1-4] *' * (1): yes *' * (2): no carbon capture (only if CCS and CCU are switched off!) *' * (3): no bio carbon capture @@ -524,14 +541,14 @@ parameter parameter c_bioliqscen "2nd generation bioenergy liquids technology choice" ; - c_bioliqscen = 1; !! def = 1 + c_bioliqscen = 1; !! def = 1 !! regexp = 0|1 *' * (0): no technologies *' * (1): all technologies *' parameter c_bioh2scen "bioenergy hydrogen technology choice" ; - c_bioh2scen = 1; !! def = 1 + c_bioh2scen = 1; !! def = 1 !! regexp = 0|1 *' * (0): no technologies *' * (1): all technologies *' @@ -583,7 +600,7 @@ parameter parameter c_solscen "solar option choice" ; - c_solscen = 1; !! def = 1 + c_solscen = 1; !! def = 1 !! regexp = [1-3] *' * (1): yes *' * (2): no solar *' * (3): fix at bau level @@ -633,14 +650,14 @@ parameter parameter cm_1stgen_phaseout "choose if 1st generation biofuels should phase out after 2030 (vm_deltaCap equals 0)" ; - cm_1stgen_phaseout = 0; !! def = 0 + cm_1stgen_phaseout = 0; !! def = 0 !! regexp = 0|1 *' * (0): 1st generation biofuels after 2020 are fixed at upper limit of resource potential (maxprod) *' * (1): no new capacities for 1st generation biofuel technologies may be built after 2030 -> phaseout until ~2060 *' parameter cm_phaseoutBiolc "Switch that allows for a full phaseout of all bioenergy technologies globally" ; - cm_phaseoutBiolc = 0; !! def = 0 + cm_phaseoutBiolc = 0; !! def = 0 !! regexp = 0|1 *** Only working with magpie_40 realization of 30_biomass module. *** (0): (default) No phaseout *** (1): Phaseout capacities of all bioenergy technologies using pebiolc, as far @@ -653,8 +670,8 @@ parameter parameter cm_startyear "first optimized modelling time step [year]" ; - cm_startyear = 2005; !! def = 2005 for a baseline -*' * (2005): standard for basline to check if model is well calibrated + cm_startyear = 2005; !! def = 2005 for a baseline !! regexp = 20[0-9](0|5) +*' * (2005): standard for baseline to check if model is well calibrated *' * (2015): standard for all policy runs (eq. to fix2010), NDC, NPi and production baselines, especially if various baselines with varying parameters are explored *' * (....): later start for delay policy runs, eg. 2025 for what used to be delay2020 *' @@ -666,14 +683,14 @@ parameter parameter cm_prtpScen "pure rate of time preference standard values" ; - cm_prtpScen = 3; !! def = 3 + cm_prtpScen = 3; !! def = 3 !! regexp = 1|3 *' * (1): 1 % *' * (3): 3 % *' parameter cm_fetaxscen "choice of final energy tax path, subsidy path and inconvenience cost path, values other than zero enable final energy tax" ; - cm_fetaxscen = 3; !! def = 3 + cm_fetaxscen = 3; !! def = 3 !! regexp = [0-4] *' even if set to 0, the PE inconvenience cost per SO2-cost for coal are always on if module 21_tax is on *' * (0): no tax, sub, inconv *' * (1): constant t,s,i (used in SSP 5 and ADVANCE WP3.1 HighOilSub) @@ -684,14 +701,14 @@ parameter parameter cm_distrBeta "elasticity of tax revenue redistribution" ; - cm_distrBeta = 1; !! def = 1 + cm_distrBeta = 1; !! def = 1 !! regexp = 0|1 *' (0): equal per capita redistribution *' (1): proportional redistribution *' parameter cm_multigasscen "scenario on GHG portfolio to be included in permit trading scheme" ; - cm_multigasscen = 2; !! def = 2 + cm_multigasscen = 2; !! def = 2 !! regexp = [1-3] *' * (1): CO2 only *' * (2): all GHG *' * (3): all GHG excl CO2 emissions from LULUCF @@ -699,7 +716,7 @@ parameter parameter cm_permittradescen "scenario on permit trade" ; - cm_permittradescen = 1; !! def = 1 + cm_permittradescen = 1; !! def = 1 !! regexp = [1-3] *' * (1): full permit trade (no restrictions) *' * (2): no permit trade (only domestic mitigation) *' * (3): limited trade (certain percentage of GDP) @@ -707,59 +724,59 @@ parameter parameter cm_rentdiscoil "[grades2poly] discount factor for the oil rent" ; - cm_rentdiscoil = 0.2; !! def 0.2 + cm_rentdiscoil = 0.2; !! def = 0.2 *' parameter cm_rentdiscoil2 "[grades2poly] discount factor for the oil rent achieved in 2100" ; - cm_rentdiscoil2 = 0.9; !! def 0.9 + cm_rentdiscoil2 = 0.9; !! def = 0.9 *' parameter cm_rentconvoil "[grades2poly] number of years required to converge to the 2100 oil rent" ; - cm_rentconvoil = 50; !! def 50 + cm_rentconvoil = 50; !! def = 50 *' parameter cm_rentdiscgas "[grades2poly] discount factor for the gas rent" ; - cm_rentdiscgas = 0.6; !! def 0.6 + cm_rentdiscgas = 0.6; !! def = 0.6 *' parameter cm_rentdiscgas2 "[grades2poly] discount factor for the gas rent achieved in 2100" ; - cm_rentdiscgas2 = 0.8; !! def 0.8 + cm_rentdiscgas2 = 0.8; !! def = 0.8 *' parameter cm_rentconvgas "[grades2poly] number of years required to converge to the 2100 gas rent" ; - cm_rentconvgas = 50; !! def 50 + cm_rentconvgas = 50; !! def = 50 *' parameter cm_rentdisccoal "[grades2poly] discount factor for the coal rent" ; - cm_rentdisccoal = 0.4; !! def 0.4 + cm_rentdisccoal = 0.4; !! def = 0.4 *' parameter cm_rentdisccoal2 "[grades2poly] discount factor for the coal rent achieved in 2100" ; - cm_rentdisccoal2 = 0.6; !! def 0.6 + cm_rentdisccoal2 = 0.6; !! def = 0.6 *' parameter cm_rentconvcoal "[grades2poly] number of years required to converge to the 2100 coal rent" ; - cm_rentconvcoal = 50; !! def 50 + cm_rentconvcoal = 50; !! def = 50 *' parameter c_cint_scen "additional GHG emissions from mining fossil fuels" ; - c_cint_scen = 1; !! def = 1 + c_cint_scen = 1; !! def = 1 !! regexp = 0|1 *' * (0): switch is off (emissions are not accounted) *' * (1): switch is on (emissions are accounted) *' parameter cm_so2tax_scen "level of SO2 tax" ; - cm_so2tax_scen = 1; !! def = + cm_so2tax_scen = 1; !! def = 1 !! regexp = [0-4] *' * (0): so2 tax is set to zero *' * (1): so2 tax is low *' * (2): so2 tax is standard @@ -769,7 +786,7 @@ parameter parameter cm_solwindenergyscen "scenario for fluctuating renewables, 1 is reference, 2 is pessimistic with limits to fluctuating SE el share" ; - cm_solwindenergyscen = 1; !! def = 1 + cm_solwindenergyscen = 1; !! def = 1 !! regexp = [0-4] *' * (0) advanced - cheap investment costs, higher learning rates for pv, csp and wind *' * (1) reference - normal investment costs & learning rates for pv, csp and wind EMF27-3nd round Adv scenario *' * (2) pessimistic EMF27 and AWP 2 - share of PV, wind&CSP limited to 20%. Learning rates reduced by 20%, floor costs increased by 40%. EMF27-3nd round Cons scenario @@ -779,7 +796,7 @@ parameter parameter c_techAssumptScen "scenario for assumptions of energy technologies based on SSP scenarios, 1: SSP2 (default), 2: SSP1, 3: SSP5" ; - c_techAssumptScen = 1; !! def = 1 + c_techAssumptScen = 1; !! def = 1 !! regexp = [1-3] *' This flag defines an energy technology scenario *' * (1) SSP2: reference scenario - default investment costs & learning rates for pv, csp and wind *' * (2) SSP1: advanced renewable energy techno., pessimistic for nuclear and CCS @@ -788,7 +805,7 @@ parameter parameter c_ccsinjecratescen "CCS injection rate factor, 0.5% by default yielding a 60 Mt per year IR" ; - c_ccsinjecratescen = 1; !! def = 1 + c_ccsinjecratescen = 1; !! def = 1 !! regexp = [0-5] *' This flag determines the upper bound of the CCS injection rate *' CCS refers to carbon sequestration, carbon capture is modelled separately *' * (0) no "CCS" as in no carbon sequestration at all @@ -801,7 +818,7 @@ parameter parameter c_ccscapratescen "CCS capture rate" ; - c_ccscapratescen = 1; !! def = 1 + c_ccscapratescen = 1; !! def = 1 !! regexp = 1|2 *' This flag determines the CO2 capture rate of CCS technologies *' * (1) reference *' * (2) increased capture rate @@ -809,12 +826,12 @@ parameter parameter c_export_tax_scen "choose which oil export tax is used in the model. 0 = none, 1 = fix" ; - c_export_tax_scen = 0; !! def = 0 + c_export_tax_scen = 0; !! def = 0 !! regexp = 0|1 *' parameter cm_iterative_target_adj "whether or not a tax or a budget target should be iteratively adjusted depending on actual emission or forcing level" ; - cm_iterative_target_adj = 0; !! def = 0 + cm_iterative_target_adj = 0; !! def = 0 !! regexp = 0|2|3|4|5|6|7|9 *' * (0): no iterative adjustment *' * (2): iterative adjustment II based on magicc calculated forcing (for both budget and tax runs), see modules/ 0 /magicc/postsolve.gms for direct algorithms of adjustment *' * (3): [requires 45_carbonprice = "NDC" and emiscen = 9] iterative adjustment III for tax based on 2025 or 2030 regionally differentiated emissions, see module/45_carbonprice//postsolve.gms for algorithm of adjustment @@ -827,7 +844,7 @@ parameter parameter cm_NDC_divergentScenario "choose scenario about convergence of CO2eq prices in NDC realization of module 45_carbonprice" ; - cm_NDC_divergentScenario = 0; !! def = 0 + cm_NDC_divergentScenario = 0; !! def = 0 !! regexp = [0-2] *' * (0) 70 years after 2030 *' * (1) 120 years after 2030 *' * (2) until year 3000 ("never") @@ -835,14 +852,14 @@ parameter parameter cm_gdximport_target "whether or not the starting value for iteratively adjusted budgets, tax scenarios, or forcing targets (emiscen 5,6,8,9) should be read in from the input.gdx" ; - cm_gdximport_target = 0; !! def = 0 + cm_gdximport_target = 0; !! def = 0 !! regexp = 0|1 *' * (0): no import, the default starting value as specified in modules/ 0 /on/input/tax_CO2.inc, core/input/data_emibudget.inc, modules/15_climate/box/datainput.gms is used *' * (1): the values from the gdx are read in (works only if the gdx has a parameter value) ATTENTION: make sure that the values from the gdx have the right structure (e.g. regionally differentiated or not) *' parameter cm_gs_ew "grain size (for enhanced weathering, CDR module) [micrometre]" ; - cm_gs_ew = 20; !! def = 20 + cm_gs_ew = 20; !! def = 20 !! regexp = is.numeric *' parameter cm_LimRock "limit amount of rock spread each year [Gt]" @@ -928,7 +945,7 @@ parameter parameter cm_DiscRateScen "Scenario for the implicit discount rate applied to the energy efficiency capital" ; - cm_DiscRateScen = 0;!! def = 0 + cm_DiscRateScen = 0; !! def = 0 !! regexp = [0-4] *' * (0) Baseline without higher discount rate: No additional discount rate *' * (1) Baseline with higher discount rate: Increase the discount rate by 10%pts from 2005 until the end *' * (2) Energy Efficiency policy: 10%pts higher discount rate until cm_start_year and 0 afterwards. @@ -968,7 +985,7 @@ parameter parameter cm_TaxConvCheck "switch for enabling tax convergence check in nash mode" ; - cm_TaxConvCheck = 0; !! def 0, which means tax convergence check is off + cm_TaxConvCheck = 0; !! def = 0, which means tax convergence check is off !! regexp = 0|1 *' switches tax convergence check in nash mode on and off (check that tax revenue in all regions, periods be smaller than 0.01% of GDP) *' * 0 (off) *' * 1 (on), default @@ -976,12 +993,12 @@ parameter parameter cm_maxFadeOutPriceAnticip "switch to determine maximum allowed fadeout price anticipation to consider that the model converged." ; - cm_maxFadeOutPriceAnticip = 1e-4; !! def 1e-4, the fadeout price anticipation term needs to be lower than 1e-4 to consider that the model converged. + cm_maxFadeOutPriceAnticip = 1e-4; !! def = 1e-4, the fadeout price anticipation term needs to be lower than 1e-4 to consider that the model converged. *' parameter cm_flex_tax "switch for enabling flexibility tax" ; - cm_flex_tax = 1; !! def 1 + cm_flex_tax = 1; !! def = 1 !! regexp = 0|1 *' cm_flex_tax "switch for flexibility tax/subsidy, switching it on activates a tax on a number of technologies with flexible or inflexible electricity input." *' technologies with flexible eletricity input get a subsidy corresponding to the lower-than-average electricity prices that they see, while *' inflexible technologies get a tax corresponding to the higher-than-average electricity prices that they see @@ -996,14 +1013,14 @@ parameter parameter cm_PriceDurSlope_elh2 "slope of price duration curve of electrolysis" ; - cm_PriceDurSlope_elh2 = 15; !! def 15 + cm_PriceDurSlope_elh2 = 15; !! def = 15 *' cm_PriceDurSlope_elh2, slope of price duration curve for electrolysis (increase means more flexibility subsidy for electrolysis H2) *' This switch only has an effect if the flexibility tax is on by cm_flex_tax set to 1 *' parameter cm_FlexTaxFeedback "switch deciding whether flexibility tax feedback on buildlings and industry electricity prices is on" ; - cm_FlexTaxFeedback = 0; !! def 0 + cm_FlexTaxFeedback = 0; !! def = 0 !! regexp = 0|1 *' cm_FlexTaxFeedback, switches on feedback of flexibility tax on buildings and industry. *' That is, electricity price decrease for electrolysis has to be matched by electrictiy price increase in buildings and industry. *' This switch only has an effect if the flexibility tax is on by cm_flex_tax set to 1. @@ -1011,7 +1028,7 @@ parameter parameter cm_VRE_supply_assumptions "default (0), optimistic (1), sombre (2), or bleak (3) assumptions on VRE supply" ; - cm_VRE_supply_assumptions = 0; !! 0 - default, 1 - optimistic, 2 - sombre, 3 - bleak + cm_VRE_supply_assumptions = 0; !! 0 - default, 1 - optimistic, 2 - sombre, 3 - bleak !! regexp = [0-3] *' * for 1 - optimistic, modify *' - inco0, incolearn, and learn parameters for spv and storspv *' - ease capacity constraints on storage @@ -1024,32 +1041,32 @@ parameter parameter cm_build_H2costAddH2Inv "additional h2 distribution costs for low diffusion levels (default value: 6.5$/kg = 0.2 $/Kwh)" ; - cm_build_H2costAddH2Inv = 0.2; !! def 6.5$/kg = 0.2 $/Kwh + cm_build_H2costAddH2Inv = 0.2; !! def = 6.5$/kg = 0.2 $/Kwh *' parameter cm_build_H2costDecayStart "simplified logistic function end of full value (ex. 5% -> between 0 and 5% the function will have the value 1). [%]" ; - cm_build_H2costDecayStart = 0.05; !! def 5% + cm_build_H2costDecayStart = 0.05; !! def = 0.05 *' parameter cm_build_H2costDecayEnd "simplified logistic function start of null value (ex. 10% -> after 10% the function will have the value 0). [%]" ; - cm_build_H2costDecayEnd = 0.1; !! def 10% + cm_build_H2costDecayEnd = 0.1; !! def = 0.1 *' parameter cm_build_AdjCostActive "Activate adjustment cost to penalise inter-temporal variation of area-specific weatherisation demand and space cooling efficiency slope (only in putty)" ; - cm_build_AdjCostActive = 0; !! def 0 = Adjustment cost deactivated (set to 1 to activate) + cm_build_AdjCostActive = 0; !! def = 0: Adjustment cost deactivated (set to 1 to activate) !! regexp = 0|1 *' parameter cm_indst_H2costAddH2Inv "additional h2 distribution costs for low diffusion levels (default value: 3.25$kg = 0.1 $/kWh)" ; - cm_indst_H2costAddH2Inv = 0.1; !! def 3.25$/kg = 0.1 $/Kwh + cm_indst_H2costAddH2Inv = 0.1; !! def = 3.25$/kg = 0.1 $/Kwh *' parameter cm_indst_costDecayStart "simplified logistic function end of full value (ex. 5% -> between 0 and 5% the simplified logistic function will have the value 1). [%]" ; - cm_indst_costDecayStart = 0.05; !! def 5% + cm_indst_costDecayStart = 0.05; !! def = 5% *' parameter cm_indst_H2costDecayEnd "simplified logistic function start of null value (ex. 10% -> between 10% and 100% the simplified logistic function will have the value 0). [%]" @@ -1067,7 +1084,7 @@ parameter parameter cm_noPeFosCCDeu "switch to suppress Pe2Se Fossil Carbon Capture in Germany" ; - cm_noPeFosCCDeu = 0; !! def 0 + cm_noPeFosCCDeu = 0; !! def = 0 !! regexp = 0|1 *' CCS limitations for Germany *' def 0, no suppression of Pe2Se Fossil Carbon Capture in Germany, if 1 then no pe2se fossil CO2 capture in Germany *' fossil CCS limitations in Germany+ @@ -1078,17 +1095,17 @@ parameter parameter cm_logitCal_markup_conv_b "value to which logit calibration markup of standard fe2ue technologies in detailed buildings module converges to" ; - cm_logitCal_markup_conv_b = 0.8; !! def 0.8 + cm_logitCal_markup_conv_b = 0.8; !! def = 0.8 *' long-term convergence value of detailed buildings fe2ue conventional techs price markup parameter cm_logitCal_markup_newtech_conv_b "value to which logit calibration markup of new fe2ue technologies in detailed buildings module converges to" ; - cm_logitCal_markup_newtech_conv_b = 0.3; !! def 0.3 + cm_logitCal_markup_newtech_conv_b = 0.3; !! def = 0.3 *' long-term convergence value of detailed buildings fe2ue new techs price markup parameter cm_startIter_EDGET "starting iteration of EDGE-T" ; - cm_startIter_EDGET = 14; !! def 14, by default EDGE-T is run first in iteration 14 + cm_startIter_EDGET = 14; !! def = 14, by default EDGE-T is run first in iteration 14 !! regexp = [0-9]+ *' EDGE-T transport starting iteration of coupling *' def 14, EDGE-T coupling starts at 14, if you want to test whether infeasibilities after EDGE-T -> set it to 1 to check after first iteration *' @@ -1100,7 +1117,7 @@ parameter parameter cm_EnSecScen_limit "switch for limiting the gas demand from 2025 onward, currently only applied to Germany" ; - cm_EnSecScen_limit = 0; !! def = 0 + cm_EnSecScen_limit = 0; !! def = 0 *' This switch is used to represent a limited gas supply in a energy security scenario. [EJ per yr] *' *' * (0) default, equals "off", no limit imposed @@ -1109,12 +1126,12 @@ parameter parameter c_SlackMultiplier "Muliplicative factor to up/downscale the slack size for v_changeProdStartyearSlack" ; - c_SlackMultiplier = 1; !! def 1 + c_SlackMultiplier = 1; !! def = 1 *' parameter c_changeProdCost "Muliplicative factor to up/downscale the costs for vm_changeProdStartyearCost" ; - c_changeProdCost = 5; !! def 5 + c_changeProdCost = 5; !! def = 5 *' ***----------------------------------------------------------------------------- *' #### FLAGS @@ -1123,7 +1140,7 @@ parameter *' *' * (off): off = REMIND expects to be run standalone (emission factors are used, shiftfactors are set to zero) *' * (on): on = REMIND expects to be run based on a MAgPIE reporting file (emission factors are set to zero because emissions are retrieved from the MAgPIE reporting, shift factors for supply curves are calculated) -$setglobal cm_MAgPIE_coupling off !! def = "off" +$setglobal cm_MAgPIE_coupling off !! def = "off" !! regexp = on|off *' cm_rcp_scen "chooses RCP scenario" *' *' * (none): no RCP scenario, standard setting @@ -1133,7 +1150,7 @@ $setglobal cm_MAgPIE_coupling off !! def = "off" *' * (rcp45): RCP4.5 *' * (rcp60): RCP6.0 *' * (rcp85): RCP8.5 -$setglobal cm_rcp_scen none !! def = "none" +$setglobal cm_rcp_scen none !! def = "none" !! regexp = none|rcp20|rcp26|rcp37|rcp45|rcp60|rcp85 *' cm_NDC_version "choose version year of NDC targets as well as conditional vs. unconditional targets" *' * (2023_cond): all NDCs conditional to international financial support published until February 24, 2023 *' * (2023_uncond): all NDCs independent of international financial support published until February 24, 2023 @@ -1143,12 +1160,12 @@ $setglobal cm_rcp_scen none !! def = "none" *' * (2021_uncond): all NDCs independent of international financial support published until December 31, 2021 *' * (2018_cond): all NDCs conditional to international financial support published until December 31, 2018 *' * (2018_uncond): all NDCs independent of international financial support published until December 31, 2018 -$setglobal cm_NDC_version 2023_cond !! def = "2023_cond", "2023_uncond", "2022_cond", "2022_uncond", "2021_cond", "2021_uncond", "2018_cond", "2018_uncond" +$setglobal cm_NDC_version 2023_cond !! def = "2023_cond" !! regexp = 20(18|2[1-3])_(un)?cond *** cm_netZeroScen "choose scenario of net zero targets of netZero realization of module 46_carbonpriceRegi" *** (NGFS_v4): settings used for NGFS v4, 2023 *** (NGFS_v4_20pc): settings used for NGFS v4, 2023, with still 20% of 2020 emissions in netZero year *** (ENGAGE4p5_GlP): settings used for ENGAGE 4.5 Glasgow+ scenario -$setglobal cm_netZeroScen NGFS_v4 !! def = "NGFS_v4" +$setglobal cm_netZeroScen NGFS_v4 !! def = "NGFS_v4" !! regexp = NGFS_v4|NGFS_v4_20pc|ENGAGE4p5_GlP *** c_regi_earlyreti_rate "maximum percentage of capital stock that can be retired early (before reaching their expected lifetimes) in one year in specified regions, if they are not economically viable. It is applied to all techs unless otherwise specified in c_tech_earlyreti_rate." *** GLO 0.09, EUR_regi 0.15: default value. (0.09 means full retirement after 11 years, 10% standing after 10 years) $setglobal c_regi_earlyreti_rate GLO 0.09, EUR_regi 0.15 !! def = GLO 0.09, EUR_regi 0.15 @@ -1159,7 +1176,8 @@ $setglobal c_tech_earlyreti_rate GLO.(biodiesel 0.14, bioeths 0.14), EUR_regi.( *** (SSP1): emissions (from SSP1 scenario in MAgPIE) *** (SSP2): emissions (from SSP2 scenario in MAgPIE) *** (SSP5): emissions (from SSP5 scenario in MAgPIE) -$setglobal cm_LU_emi_scen SSP2 !! def = SSP2 +*** (SDP): +$setglobal cm_LU_emi_scen SSP2 !! def = SSP2 !! regexp = SSP(1|2|5)|SDP *** cm_regi_bioenergy_EFTax "region(s) in which bioenergy is charged with an emission-factor-based tax" *** This switch has only an effect if 21_tax is on and cm_bioenergy_EF_for_tax *** is not zero. It reads in the regions that are affected by the emission- @@ -1173,12 +1191,12 @@ $setGlobal cm_regi_bioenergy_EFTax glob !! def = glob *** cm_tradbio_phaseout "Switch that allows for a faster phase out of traditional biomass" *** (default): Default assumption, reaching zero demand in 2100 *** (fast): Fast phase out, starting in 2025 reaching zero demand in 2070 (close to zero in 2060) -$setglobal cm_tradbio_phaseout default !! def = default +$setglobal cm_tradbio_phaseout default !! def = default !! regexp = default|fast *** cm_maxProdBiolc "Bound on global pebiolc production including residues but excluding traditionally used biomass [EJ per yr]" *** (off): (default) no bound *** (100): (e.g.) set maximum to 100 EJ per year *** (any value ge 0): set maximum to that value -$setglobal cm_maxProdBiolc off !! def = off +$setglobal cm_maxProdBiolc off !! def = off !! regexp = off|is.numeric *** cm_bioprod_regi_lim *** limit to total biomass production (including residues) by region to an upper value in EJ/yr from 2035 on *** example: "CHA 20, EUR_regi 7.5" limits total biomass production in China to 20 EJ/yr and @@ -1191,6 +1209,7 @@ $setGLobal cm_bioprod_regi_lim off !! def off *** cm_POPscen "Population growth scenarios from UN data and IIASA projection used in SSP" *** pop_SSP1 "SSP1 population scenario" *** pop_SSP2 "SSP2 population scenario" +*** pop_SSP2EU "SSP2 population scenario" *** pop_SSP3 "SSP3 population scenario" *** pop_SSP4 "SSP4 population scenario" *** pop_SSP5 "SSP5 population scenario" @@ -1198,6 +1217,7 @@ $setglobal cm_POPscen pop_SSP2EU !! def = pop_SSP2EU *** cm_GDPscen "assumptions about future GDP development, linked to population development (cm_POPscen)" *** (gdp_SSP1): SSP1 fastGROWTH medCONV *** (gdp_SSP2): SSP2 medGROWTH medCONV +*** (gdp_SSP2EU): SSP2 medGROWTH medCONV *** (gdp_SSP3): SSP3 slowGROWTH slowCONV *** (gdp_SSP4): SSP4 medGROWTH mixedCONV *** (gdp_SSP5): SSP5 fastGROWTH fastCONV @@ -1207,18 +1227,18 @@ $setglobal cm_GDPscen gdp_SSP2EU !! def = gdp_SSP2EU *** (medOil): medium (this is the new case) *** (highOil): high (formerly this has been the "medium" case; RoSE relevant difference) *** (4): very high (formerly this has been the "high" case; RoSE relevant difference) -$setGlobal cm_oil_scen medOil !! def = medOil +$setGlobal cm_oil_scen medOil !! def = medOil !! regexp = lowOil|medOil|highOil|4 *** cm_gas_scen "assumption on gas availability" *** (lowGas): low *** (medGas): medium *** (highGas): high -$setGlobal cm_gas_scen medGas !! def = medGas +$setGlobal cm_gas_scen medGas !! def = medGas !! regexp = low|medium|high *** cm_coal_scen "assumption on coal availability" *** (0): very low (this has been the "low" case; RoSE relevant difference) *** (lowCoal): low (this is the new case) *** (medCoal): medium *** (highCoal): high -$setGlobal cm_coal_scen medCoal !! def = medCoal +$setGlobal cm_coal_scen medCoal !! def = medCoal !! regexp = 0|lowCoal|medCoal|highCoal *** c_ccsinjecrateRegi "regional upper bound of the CCS injection rate, overwrites for specified regions the settings set with c_ccsinjecratescen" *** ("off") no regional differentiation *** ("GLO 0.005") reproduces c_ccsinjecratescen = 1 @@ -1228,7 +1248,7 @@ $setglobal c_ccsinjecrateRegi off !! def = "off" *** ("forcing_SSP1") settings consistent with SSP 1 *** ("forcing_SSP2") settings consistent with SSP 2 *** ("forcing_SSP5") settings consistent with SSP 5 -$setglobal c_SSP_forcing_adjust forcing_SSP2 !! def = forcing_SSP2 +$setglobal c_SSP_forcing_adjust forcing_SSP2 !! def = forcing_SSP2 !! regexp = forcing_SSP(1|2|5) *** cm_regiExoPrice "set exogenous co2 tax path for specific regions using a switch, require regipol module to be set to regiCarbonPrice (e.g. GLO.(2025 38,2030 49,2035 63,2040 80,2045 102,2050 130,2055 166,2060 212,2070 346,2080 563,2090 917,2100 1494,2110 1494,2130 1494,2150 1494) )" $setGlobal cm_regiExoPrice off !! def = off *** cm_emiMktTarget "set a budget or year emission target, for all (all) or specific emission markets (ETS, ESD or other), and specific regions (e.g. DEU) or region groups (e.g. EU27)" @@ -1274,10 +1294,10 @@ $setGlobal cm_vehiclesSubsidies off !! def = off *** To limit CCS to 8 GtCO2 and BECCS to 5 GtCO2, use "2050.GLO.tax.t.CCS.all 8000, 2050.GLO.tax.t.CCS.biomass 5000" $setGlobal cm_implicitQttyTarget off !! def = off *** cm_loadFromGDX_implicitQttyTargetTax "load p47_implicitQttyTargetTax values from gdx for first iteration. Usefull for policy runs." -$setGlobal cm_loadFromGDX_implicitQttyTargetTax off !! def = off +$setGlobal cm_loadFromGDX_implicitQttyTargetTax off !! def = off !! regexp = off|Initial|HighElectricityPrice|HighGasandLiquidsPrice|HighPrice|LowPrice|LowElectricityPrice" *** cm_implicitPriceTarget "define tax/subsidies to match FE prices defined in the pm_implicitPriceTarget parameter." *** Aceptable values: "off", "Initial", "HighElectricityPrice", "HighGasandLiquidsPrice", "HighPrice", "LowPrice", "LowElectricityPrice" -$setGlobal cm_implicitPriceTarget off !! def = off +$setGlobal cm_implicitPriceTarget off !! def = off !! regexp = off|highFossilPrice *** cm_implicitPePriceTarget "define tax/subsidies to match PE prices defined in the pm_implicitPePriceTarget parameter." *** Aceptable values: "off", "highFossilPrice". $setGlobal cm_implicitPePriceTarget off !! def = off @@ -1293,12 +1313,12 @@ $setGlobal cm_VREminShare off !! def = off *** This switch only works for model native regions. If you want to apply it to a group region use cm_implicitQttyTarget instead. $setGlobal cm_CCSmaxBound off !! def = off *** c_CES_calibration_new_structure <- 0 switch to 1 if you want to calibrate a CES structure different from input gdx -$setglobal c_CES_calibration_new_structure 0 !! def = 0 +$setglobal c_CES_calibration_new_structure 0 !! def = 0 !! regexp = 0|1 *** c_CES_calibration_write_prices <- 0 switch to 1 if you want to generate price file, you can use this as new p29_cesdata_price.cs4r price input file -$setglobal c_CES_calibration_write_prices 0 !! def = 0 +$setglobal c_CES_calibration_write_prices 0 !! def = 0 !! regexp = 0|1 *** cm_CES_calibration_default_prices <- 0.01 # def <- 0.01 lower value if input factors get negative shares (xi), CES prices in the first calibration iteration $setglobal cm_CES_calibration_default_prices 0.01 !! def = 0.01 -*** cm_calibration_string "def = off, else = additional string to include in the calibration name to be used" label for your calibration run to keep calibration files with different setups apart (e.g. with low elasticities, high eleasticies) +*** cm_calibration_string "def = off, else = additional string to include in the calibration name to be used" label for your calibration run to keep calibration files with different setups apart (e.g. with low elasticities, high elasticities) $setglobal cm_calibration_string off !! def = off *** cm_techcosts - use regionalized or globally homogenous technology costs for certain technologies $setglobal cm_techcosts REG !! def = REG @@ -1611,8 +1631,8 @@ $setglobal cm_secondary_steel_bound scenario !! def = scenario $setglobal c_GDPpcScen SSP2EU !! def = gdp_SSP2 (automatically adjusted by start_run() based on GDPscen) $setglobal cm_demScen gdp_SSP2EU !! def = gdp_SSP2EU $setGlobal c_scaleEmiHistorical on !! def = on -$SetGlobal cm_quick_mode off !! def = off -$setGLobal cm_debug_preloop off !! def = off +$SetGlobal cm_quick_mode off !! def = off !! regexp = off|on +$setGLobal cm_debug_preloop off !! def = off !! regexp = off|on $setGlobal cm_APscen SSP2 !! def = SSP2 $setglobal cm_CES_configuration indu_subsectors-buil_simple-tran_edge_esm-POP_pop_SSP2EU-GDP_gdp_SSP2EU-En_gdp_SSP2EU-Kap_debt_limit-Reg_62eff8f7 !! this will be changed by start_run() $setglobal c_CES_calibration_iterations 10 !! def = 10 diff --git a/scripts/start/checkFixCfg.R b/scripts/start/checkFixCfg.R new file mode 100644 index 000000000..be2cbe878 --- /dev/null +++ b/scripts/start/checkFixCfg.R @@ -0,0 +1,50 @@ +checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { + refcfg <- gms::readDefaultConfig(remindPath) + gms::check_config(cfg, reference_file = refcfg, modulepath = file.path(remindPath, "modules"), + settings_config = file.path(remindPath, "config", "settings_config.csv"), + extras = c("backup", "remind_folder", "pathToMagpieReport", "cm_nash_autoconverge_lastrun", + "gms$c_expname", "restart_subsequent_runs", "gms$c_GDPpcScen", + "gms$cm_CES_configuration", "gms$c_description", "model")) + + errorsfound <- 0 + + code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE) + grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)" + + for (n in names(cfg$gms)) { + errormsg <- NULL + paramdef <- paste0("^([ ]*", n, "[ ]*=|\\$setglobal[ ]+", n, " )") + filtered <- grep(paste0(paramdef, ".*regexp[ ]*=[ ]*"), code, value = TRUE) + if (length(filtered) == 1) { + # search for string '!! regexp = whatever', potentially followed by '!! otherstuff' and extract 'whatever' + regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$") + useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE) + if (! grepl(useregexp, cfg$gms[[n]])) { + errormsg <- paste0("Parameter cfg$gms$", n, "=", cfg$gms[[n]], " does not fit this regular expression: ", regexp) + } + } else if (length(filtered) > 1) { + errormsg <- paste0("More than one regexp found for ", n, ". These are the code lines:\n", paste(filtered, collapse = "\n")) + } + if (! is.null(errormsg)) { + errorsfound <- errorsfound + 1 + if (testmode) warning(errormsg) else message(errormsg) + } + } + + # Check for compatibility with subsidizeLearning + if ((cfg$gms$optimization != "nash") && (cfg$gms$subsidizeLearning == "globallyOptimal") ) { + message("Only optimization='nash' is compatible with subsidizeLearning='globallyOptimal'. Switching subsidizeLearning to 'off' now.\n") + cfg$gms$subsidizeLearning <- "off" + } + + # reportCEScalib only works with the calibrate module + if (! isTRUE(cfg$gms$CES_parameters == "calibrate")) { + cfg$output <- setdiff(cfg$output, "reportCEScalib") + } + if (errorsfound > 0) { + if (testmode) warning(errorsfound, " errors found.") + else stop(errorsfound, " errors found, see above. Either adapt the parameter choice or the regexp in main.gms") + } + + return(cfg) +} diff --git a/scripts/start/prepare.R b/scripts/start/prepare.R index ffc47290c..4e4add82b 100644 --- a/scripts/start/prepare.R +++ b/scripts/start/prepare.R @@ -68,28 +68,13 @@ prepare <- function() { # change to REMIND main folder setwd(cfg$remind_folder) - # Check configuration for consistency -# cfg <- check_config(cfg, reference_file="config/default.cfg", -# settings_config = "config/settings_config.csv", -# extras = c("backup", "remind_folder", "pathToMagpieReport", "cm_nash_autoconverge_lastrun", -# "gms$c_expname", "restart_subsequent_runs", "gms$c_GDPpcScen", -# "gms$cm_CES_configuration", "gms$c_description")) - - # Check for compatibility with subsidizeLearning - if ( (cfg$gms$optimization != 'nash') & (cfg$gms$subsidizeLearning == 'globallyOptimal') ) { - cat("Only optimization='nash' is compatible with subsudizeLearning='globallyOptimal'. Switching subsidizeLearning to 'off' now. \n") - cfg$gms$subsidizeLearning = 'off' - } - - # reportCEScalib only works with the calibrate module - if ( cfg$gms$CES_parameters != "calibrate" ) cfg$output <- setdiff(cfg$output,"reportCEScalib") + cfg <- checkFixCfg(cfg, remindPath = cfg$remind_folder) #AJS quit if title is too long - GAMS can't handle that if( nchar(cfg$title) > 75 | grepl("\\.",cfg$title) ) { stop("This title is too long or the name contains dots - GAMS would not tolerate this, and quit working at a point where you least expect it. Stopping now. ") } - # adjust GDPpcScen based on GDPscen cfg$gms$c_GDPpcScen <- gsub("gdp_","",cfg$gms$cm_GDPscen) diff --git a/scripts/start/runGamsCompile.R b/scripts/start/runGamsCompile.R index de2f3ae43..885d112b6 100644 --- a/scripts/start/runGamsCompile.R +++ b/scripts/start/runGamsCompile.R @@ -19,12 +19,12 @@ runGamsCompile <- function(modelFile, cfg, interactive = TRUE) { gcdir <- file.path(dirname(modelFile), "output", "gamscompile") dir.create(gcdir, recursive = TRUE, showWarnings = FALSE) tmpModelFile <- file.path(gcdir, paste0("main_", cfg$title, ".gms")) + tmpModelLst <- gsub("gms$", "lst", tmpModelFile) file.copy(modelFile, tmpModelFile, overwrite = TRUE) lucode2::manipulateConfig(tmpModelFile, cfg$gms) exitcode <- system2( command = cfg$gamsv, - args = paste(tmpModelFile, "-o", gsub("gms$", "lst", tmpModelFile), - "-action=c -errmsg=1 -pw=132 -ps=0 -logoption=0")) + args = paste(tmpModelFile, "-o", tmpModelLst, "-action=c -errmsg=1 -pw=132 -ps=0 -logoption=0")) if (0 < exitcode) { message(red, "FAIL ", NC, gsub("gms$", "lst", tmpModelFile)) if (interactive) { @@ -39,6 +39,9 @@ runGamsCompile <- function(modelFile, cfg, interactive = TRUE) { return(FALSE) } else { message(green, " OK ", NC, gsub("gms$", "lst", tmpModelFile)) + if (isTRUE(grepl("TESTTHAT_scenario_config", cfg$title))) { # for test_04-gamscompile + unlink(c(tmpModelFile, tmpModelLst)) + } return(TRUE) } } diff --git a/scripts/start/selectScenarios.R b/scripts/start/selectScenarios.R index 070a2b9e0..926d48b44 100644 --- a/scripts/start/selectScenarios.R +++ b/scripts/start/selectScenarios.R @@ -13,6 +13,7 @@ #' @return dataframe with scenarios from settings which should be started selectScenarios <- function (settings, interactive, startgroup) { + if (isTRUE(startgroup == "*")) return(settings) scenariosInGroup <- grepl(paste0("(^|,)", startgroup, "($|,)"), as.character(settings$start), perl = TRUE) if (interactive | ! any(scenariosInGroup)) { scenariosInGroup <- gms::chooseFromList(setNames(rownames(settings), settings$start), type = "runs", returnBoolean = TRUE) diff --git a/start.R b/start.R index 44e72ddf3..605a05176 100755 --- a/start.R +++ b/start.R @@ -39,7 +39,7 @@ helpText <- " #' --testOneRegi, -1: starting the REMIND run(s) in testOneRegi mode #' startgroup=MYGROUP when reading a scenario config .csv file, don't start #' everything specified by \"start = 1\", instead start everything -#' specified by \"start = MYGROUP\" +#' specified by \"start = MYGROUP\". Use startgroup=* to start all. #' titletag=MYTAG append \"-MYTAG\" to all titles of all runs that are started #' slurmConfig=CONFIG use the provided CONFIG as slurmConfig: a string, or an integer <= 16 #' to select one of the options shown when running './start.R -t'. @@ -328,6 +328,8 @@ if (any(c("--reprepare", "--restart") %in% flags)) { # abort on too long paths ---- cfg$gms$cm_CES_configuration <- calculate_CES_configuration(cfg, check = TRUE) + cfg <- checkFixCfg(cfg) + # save the cfg object for the later automatic start of subsequent runs (after preceding run finished) if (! "--gamscompile" %in% flags) { filename <- paste0(cfg$title,".RData") diff --git a/start_bundle_coupled.R b/start_bundle_coupled.R index 835792e1a..fc870858f 100755 --- a/start_bundle_coupled.R +++ b/start_bundle_coupled.R @@ -142,7 +142,7 @@ if (length(argv) > 0) { if (sum(file_exists) > 1) stop("Enter only a scenario_config_coupled* file via command line or set all files manually in start_bundle_coupled.R") if (!all(file_exists)) stop("Unknown parameter provided: ", paste(argv[!file_exists], collapse = ", ")) # set config file to not known parameter where the file actually exists - path_settings_coupled <- file.path(path_remind, argv[[1]]) + path_settings_coupled <- normalizePath(argv[[1]]) if (! isTRUE(grepl("scenario_config_coupled", path_settings_coupled))) stop("Enter only a scenario_config_coupled* file via command line or set all files manually in start_bundle_coupled.R.\n", "Your command line arguments were: ", paste0(argv, collapse = " ")) @@ -540,6 +540,8 @@ for(scen in common){ numberOfTasks <- 1 } + cfg_rem <- checkFixCfg(cfg_rem, remindPath = path_remind) + Rdatafile <- paste0(fullrunname, ".RData") message("Save settings to ", Rdatafile) save(path_remind, path_magpie, cfg_rem, cfg_mag, runname, fullrunname, max_iterations, start_iter, diff --git a/tests/testthat/test_01-checkFixCfg.R b/tests/testthat/test_01-checkFixCfg.R new file mode 100644 index 000000000..2bba25d80 --- /dev/null +++ b/tests/testthat/test_01-checkFixCfg.R @@ -0,0 +1,34 @@ +# | (C) 2006-2023 Potsdam Institute for Climate Impact Research (PIK) +# | authors, and contributors see CITATION.cff file. This file is part +# | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of +# | AGPL-3.0, you are granted additional permissions described in the +# | REMIND License Exception, version 1.0 (see LICENSE file). +# | Contact: remind@pik-potsdam.de +test_that("checkFixCfg works", { + remind_folder <- "../.." + savecfg <- cfg <- gms::readDefaultConfig(remind_folder) + # regexp = NA means: no warning + expect_warning(checkFixCfg(cfg, remind_folder, testmode = TRUE), regexp = NA) + + wrongsetting <- c( + "cm_NDC_version" = "2004_cond", + "cm_emiscen" = "123", + "cm_nash_autoconverge" = "NA", + "cm_co2_tax_2020" = "2.2.2", + "cm_co2_tax_growth" = "333++", + "c_macscen" = "-1", + "cm_keep_presolve_gdxes" = "1.1", + "cm_startyear" = "1985", + "cm_netZeroScen" = "NöööGFS_v4", + "cm_rcp_scen" = "apocalypse", + NULL) + + cfg <- savecfg + cfg$gms[names(wrongsetting)] <- wrongsetting + w <- capture_warnings(checkFixCfg(cfg, remind_folder, testmode = TRUE)) + for (n in names(wrongsetting)) { + expect_match(w, paste0(n, "=", wrongsetting[[n]]), all = FALSE, fixed = TRUE) + } + expect_match(w, paste0(length(wrongsetting), " errors found"), all = FALSE, fixed = TRUE) + expect_equal(length(w), length(wrongsetting) + 1) +}) diff --git a/tests/testthat/test_01-readCheckScenarioConfig.R b/tests/testthat/test_01-readCheckScenarioConfig.R index e014de3de..40ba090d6 100644 --- a/tests/testthat/test_01-readCheckScenarioConfig.R +++ b/tests/testthat/test_01-readCheckScenarioConfig.R @@ -10,7 +10,7 @@ if (length(csvfiles) == 0) { file.path("../../config", "*", "scenario_config*.csv"))) } for (csvfile in csvfiles) { - test_that(paste("perform readCheckScenarioConfig with", gsub("../../config/", "", csvfile, fixed = TRUE)), { + test_that(paste("perform readCheckScenarioConfig with", basename(csvfile)), { # regexp = NA means: expect no warning expect_warning(readCheckScenarioConfig(csvfile, remindPath = "../../", testmode = TRUE), regexp = NA) }) @@ -24,15 +24,15 @@ test_that("readCheckScenarioConfig fails on error-loaden config", { "PBScopy;0;PBS;;;mustbedifferenttoPBS"), con = csvfile, sep = "\n") w <- capture_warnings(scenConf <- readCheckScenarioConfig(csvfile, remindPath = "../../", testmode = TRUE)) - expect_match(w, "11 errors found", all = FALSE) - expect_match(w, "These titles are too long", all = FALSE) - expect_match(w, "These titles may be confused with regions", all = FALSE) - expect_match(w, "These titles contain illegal characters", all = FALSE) + expect_match(w, "11 errors found", all = FALSE, fixed = TRUE) + expect_match(w, "These titles are too long", all = FALSE, fixed = TRUE) + expect_match(w, "These titles may be confused with regions", all = FALSE, fixed = TRUE) + expect_match(w, "These titles contain illegal characters", all = FALSE, fixed = TRUE) expect_match(w, "\\.@&", all = FALSE) - expect_match(w, "Outdated column names found that must not be used", all = FALSE) - expect_match(w, "contain whitespaces", all = FALSE) - expect_match(w, "scenario names indicated in copyConfigFrom column were not found", all = FALSE) - expect_match(w, "specify in copyConfigFrom column a scenario name defined below in the file", all = FALSE) + expect_match(w, "Outdated column names found that must not be used", all = FALSE, fixed = TRUE) + expect_match(w, "contain whitespaces", all = FALSE, fixed = TRUE) + expect_match(w, "scenario names indicated in copyConfigFrom column were not found", all = FALSE, fixed = TRUE) + expect_match(w, "specify in copyConfigFrom column a scenario name defined below in the file", all = FALSE, fixed = TRUE) copiedFromPBS <- c("c_budgetCO2", "path_gdx", "path_gdx_ref") expect_identical(unlist(scenConf["PBS", copiedFromPBS]), unlist(scenConf["PBScopy", copiedFromPBS])) diff --git a/tests/testthat/test_01-start.R b/tests/testthat/test_01-start.R index dabc7da1f..5192a2da4 100644 --- a/tests/testthat/test_01-start.R +++ b/tests/testthat/test_01-start.R @@ -14,3 +14,45 @@ test_that("start.R fails on missing path_gdx* files", { expect_true(any(grepl("2 errors were identified", output))) expectFailStatus(output) }) + +test_that("start.R --test startgroup=AMT titletag=AMT config/scenario_config.csv works", { + with_mocked_bindings({ + skipIfPreviousFailed() + output <- localSystem2("Rscript", + c("start.R", "--test", "slurmConfig=16", "startgroup=AMT", "titletag=TESTTHAT", "config/scenario_config.csv")) + printIfFailed(output) + expectSuccessStatus(output) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) + +test_that("start.R --test succeeds on all configs", { + skipIfFast() + skipIfPreviousFailed() + csvfiles <- system("git ls-files ../../config/scenario_config*.csv ../../config/*/scenario_config*.csv", intern = TRUE) + if (length(csvfiles) == 0) { + csvfiles <- Sys.glob(c(file.path("../../config/scenario_config*.csv"), + file.path("../../config", "*", "scenario_config*.csv"))) + } + csvfiles <- normalizePath(grep("scenario_config_coupled", csvfiles, invert = TRUE, value = TRUE)) + skipfiles <- c("scenario_config_21_EU11_ECEMF", "scenario_config_DeepEl", + "scenario_config_EDGE-T_NDC_NPi_pkbudget", "scenario_config_NAVIGATE_300") + csvfiles <- grep(paste(skipfiles, collapse = "|"), csvfiles, invert = TRUE, value = TRUE) + expect_true(length(csvfiles) > 0) + with_mocked_bindings( + for (csvfile in csvfiles) { + test_that(paste("perform start.R --test with", basename(csvfile)), { + output <- localSystem2("Rscript", + c("start.R", "--test", "slurmConfig=16", "startgroup=*", "titletag=TESTTHAT", csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) diff --git a/tests/testthat/test_04-gamscompile.R b/tests/testthat/test_04-gamscompile.R index 3724a854f..817278fa3 100644 --- a/tests/testthat/test_04-gamscompile.R +++ b/tests/testthat/test_04-gamscompile.R @@ -4,10 +4,48 @@ # | AGPL-3.0, you are granted additional permissions described in the # | REMIND License Exception, version 1.0 (see LICENSE file). # | Contact: remind@pik-potsdam.de -test_that("start.R --gamscompile startgroup=AMT titletag=AMT config/scenario_config.csv works", { +test_that("start.R --gamscompile startgroup=AMT config/scenario_config.csv works", { + unlink(paste0("../../output/gamscompile/*TESTTHAT", c(".gms", ".lst"))) + csvfile <- "config/scenario_config.csv" + titletag <- paste0("titletag=TESTTHAT-", gsub(".csv$", "", basename(csvfile))) + with_mocked_bindings({ + skipIfPreviousFailed() + output <- localSystem2("Rscript", + c("start.R", "--gamscompile", "startgroup=AMT", titletag, csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) + +test_that("start.R --gamscompile works on all configs and scenarios", { + skipIfFast() skipIfPreviousFailed() - output <- localSystem2("Rscript", - c("start.R", "--gamscompile", "startgroup=AMT", "titletag=AMT", "config/scenario_config.csv")) - printIfFailed(output) - expectSuccessStatus(output) + csvfiles <- system("git ls-files ../../config/scenario_config*.csv ../../config/*/scenario_config*.csv", intern = TRUE) + if (length(csvfiles) == 0) { + csvfiles <- Sys.glob(c(file.path("../../config/scenario_config*.csv"), + file.path("../../config", "*", "scenario_config*.csv"))) + } + csvfiles <- normalizePath(grep("^scenario_config_coupled.*", csvfiles, invert = TRUE, value = TRUE)) + skipfiles <- c("scenario_config_21_EU11_ECEMF", "scenario_config_EDGE-T_NDC_NPi_pkbudget", + "scenario_config_NAVIGATE_300", "21_regions_EU11", "scenario_config_tradeCap_standalone") + csvfiles <- grep(paste(skipfiles, collapse = "|"), csvfiles, invert = TRUE, value = TRUE) + expect_true(length(csvfiles) > 0) + with_mocked_bindings( + for (csvfile in csvfiles) { + test_that(paste("perform start.R --gamscompile with", basename(csvfile)), { + titletag <- paste0("titletag=TESTTHAT-", gsub(".csv$", "", basename(csvfile))) + output <- localSystem2("Rscript", + c("start.R", "--gamscompile", "startgroup=*", titletag, csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") }) diff --git a/tests/testthat/test_06-output.R b/tests/testthat/test_06-output.R index 4cd51cecb..3f75c9053 100644 --- a/tests/testthat/test_06-output.R +++ b/tests/testthat/test_06-output.R @@ -4,9 +4,10 @@ # | AGPL-3.0, you are granted additional permissions described in the # | REMIND License Exception, version 1.0 (see LICENSE file). # | Contact: remind@pik-potsdam.de +skipIfFast() +skipIfPreviousFailed() + test_that("output.R -> single -> reporting works", { - skipIfFast() - skipIfPreviousFailed() output <- localSystem2("Rscript", c("output.R", "comp=single", "output=reporting", "outputdir=output/testOneRegi", "slurmConfig='--qos=priority --mem=8000 --wait'")) printIfFailed(output) @@ -15,8 +16,6 @@ test_that("output.R -> single -> reporting works", { }) test_that("output.R -> export -> xlsx_IIASA works", { - skipIfFast() - skipIfPreviousFailed() exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*")) unlink(exportfiles) output <- localSystem2("Rscript", c("output.R", "project=TESTTHAT", "filename_prefix=TESTTHAT", @@ -31,8 +30,6 @@ test_that("output.R -> export -> xlsx_IIASA works", { }) test_that("cleanup output.R", { - skipIfFast() - skipIfPreviousFailed() exportfiles <- Sys.glob(file.path("..", "..", "output", "export", "*TESTTHAT*")) unlink(exportfiles) }) diff --git a/tests/testthat/test_20-coupled.R b/tests/testthat/test_20-coupled.R index f90a7314c..ec2342076 100644 --- a/tests/testthat/test_20-coupled.R +++ b/tests/testthat/test_20-coupled.R @@ -239,6 +239,32 @@ test_that("Check path_mif_ghgprice_land with file", { } }) +test_that("start_bundle_coupled.R --test succeeds on all configs", { + skipIfPreviousFailed() + csvfiles <- system("git ls-files ../../config/scenario_config_coupled*.csv ../../config/*/scenario_config_coupled*.csv", intern = TRUE) + if (length(csvfiles) == 0) { + csvfiles <- Sys.glob(c(file.path("../../config/scenario_config_coupled*.csv"), + file.path("../../config", "*", "scenario_config_coupled*.csv"))) + } + skipfiles <- c("scenario_config_coupled_shortCascade", # fails on missing mif file which is present while running other tests + "scenario_config_coupled_GCS") # GCFS setting not in standard MAgPIE release + csvfiles <- normalizePath(grep(paste(skipfiles, collapse = "|"), csvfiles, invert = TRUE, value = TRUE)) + expect_true(length(csvfiles) > 0) + with_mocked_bindings( + for (csvfile in csvfiles) { + test_that(paste("perform start_bundle_coupled.R --test with", basename(csvfile)), { + output <- localSystem2("Rscript", + c("start_bundle_coupled.R", "--test", "startgroup=*", csvfile)) + printIfFailed(output) + expectSuccessStatus(output) + }) + }, + getLine = function() stop("getLine should not called."), + .package = "gms" + ) + unlink("../../*TESTTHAT.RData") +}) + test_that("delete files to leave clean state", { # leave clean state skipIfPreviousFailed() From a4d4be8fd6cd95136481eed26711f566bfe84dd1 Mon Sep 17 00:00:00 2001 From: orichters Date: Fri, 21 Jul 2023 11:08:18 +0200 Subject: [PATCH 2/4] comments for checkFixCfg, in test-coupled send mails only for fails --- .../tests/scenario_config_coupled_shortCascade.csv | 6 +++--- scripts/start/checkFixCfg.R | 11 +++++++++++ scripts/start/combine_slurmConfig.R | 4 ++-- start_bundle_coupled.R | 12 ++++++------ start_coupled.R | 9 +++++---- tests/testthat/test_01-combine_slurmConfig.R | 5 +++++ 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/config/tests/scenario_config_coupled_shortCascade.csv b/config/tests/scenario_config_coupled_shortCascade.csv index 9b3548da7..d9b99f969 100644 --- a/config/tests/scenario_config_coupled_shortCascade.csv +++ b/config/tests/scenario_config_coupled_shortCascade.csv @@ -1,4 +1,4 @@ title;start;qos;sbatch;magpie_scen;magpie_empty;no_ghgprices_land_until;max_iterations;oldrun;path_gdx;path_gdx_ref;path_gdx_bau;path_report;cm_nash_autoconverge_lastrun;path_mif_ghgprice_land -TESTTHAT-SSP2EU-Base;1;auto;--wait;SSP2|NPI;TRUE;y2150;2;;;;;;2; -TESTTHAT-SSP2EU-NDC;1;auto;--wait;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base -TESTTHAT-SSP2EU-Policy;2;auto;--wait;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif +TESTTHAT-SSP2EU-Base;1;auto;--wait --mail-type=FAIL;SSP2|NPI;TRUE;y2150;2;;;;;;2; +TESTTHAT-SSP2EU-NDC;1;auto;--wait --mail-type=FAIL;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base +TESTTHAT-SSP2EU-Policy;2;auto;--wait --mail-type=FAIL;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif diff --git a/scripts/start/checkFixCfg.R b/scripts/start/checkFixCfg.R index be2cbe878..e62f5575e 100644 --- a/scripts/start/checkFixCfg.R +++ b/scripts/start/checkFixCfg.R @@ -8,23 +8,34 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { errorsfound <- 0 + # extract all instances of 'regexp' from main.gms code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE) + # this is used to replace all 'regexp = is.numeric' grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)" + # some simple tests + stopifnot(all(grepl(paste0("^", grepisnum, "$"), c("2", "2.2", "32.", "+32.", "+.05", "-0.5", "-.5", "-5", "-7.")))) + stopifnot(all(! grepl(paste0("^", grepisnum, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2")))) for (n in names(cfg$gms)) { errormsg <- NULL + # how parameter n is defined in main.gms paramdef <- paste0("^([ ]*", n, "[ ]*=|\\$setglobal[ ]+", n, " )") + # filter fitting parameter definition from code snippets containing regexp filtered <- grep(paste0(paramdef, ".*regexp[ ]*=[ ]*"), code, value = TRUE) if (length(filtered) == 1) { # search for string '!! regexp = whatever', potentially followed by '!! otherstuff' and extract 'whatever' regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$") + # replace is.numeric by pattern defined above useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE) + # check whether parameter value fits regular expression if (! grepl(useregexp, cfg$gms[[n]])) { errormsg <- paste0("Parameter cfg$gms$", n, "=", cfg$gms[[n]], " does not fit this regular expression: ", regexp) } } else if (length(filtered) > 1) { + # fail if more than one regexp found for parameter errormsg <- paste0("More than one regexp found for ", n, ". These are the code lines:\n", paste(filtered, collapse = "\n")) } + # count errors if (! is.null(errormsg)) { errorsfound <- errorsfound + 1 if (testmode) warning(errormsg) else message(errormsg) diff --git a/scripts/start/combine_slurmConfig.R b/scripts/start/combine_slurmConfig.R index c061cf783..eab0d526f 100644 --- a/scripts/start/combine_slurmConfig.R +++ b/scripts/start/combine_slurmConfig.R @@ -12,8 +12,8 @@ combine_slurmConfig <- function(original, update_with) { # trim whitespaces - original <- trimws(original) - update_with <- trimws(update_with) + original <- trimws(toString(original)) + update_with <- trimws(toString(update_with)) # remove double whitespaces original <- gsub("\\s+", " ", original) diff --git a/start_bundle_coupled.R b/start_bundle_coupled.R index fc870858f..78987df42 100755 --- a/start_bundle_coupled.R +++ b/start_bundle_coupled.R @@ -620,12 +620,12 @@ for (scen in common) { sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j'"), intern = TRUE) runEnv$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short" } - slurm_command <- paste0("sbatch --qos=", runEnv$qos, " --job-name=", fullrunname, - " --output=", logfile, " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks, - if (runEnv$numberOfTasks == 1) " --mem=8000", " ", runEnv$sbatch, - " ", runEnv$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"") - message(slurm_command) - exitCode <- system(slurm_command) + slurmOptions <- combine_slurmConfig(paste0("--qos=", runEnv$qos, " --job-name=", fullrunname, " --output=", logfile, + " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks, + if (runEnv$numberOfTasks == 1) " --mem=8000"), runEnv$sbatch) + slurmCommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"") + message(slurmCommand) + exitCode <- system(slurmCommand) if (0 < exitCode) { errorsfound <- errorsfound + 1 message("sbatch command failed, check logs.") diff --git a/start_coupled.R b/start_coupled.R index cfe81e670..31a6c3907 100644 --- a/start_coupled.R +++ b/start_coupled.R @@ -18,6 +18,7 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m require(gdx) library(methods) library(remind2) + source("scripts/start/combine_slurmConfig.R") errorsfound <- 0 # delete entries in stack that contain needle and append new @@ -250,10 +251,10 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j' | grep -v ", fullrunname), intern = TRUE) subseq.env$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short" } - subsequentcommand <- paste0("sbatch --qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile, - " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks, - if (subseq.env$numberOfTasks == 1) " --mem=8000", - " ", subseq.env$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"") + slurmOptions <- combine_slurmConfig(paste0("--qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile, + " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks, + if (subseq.env$numberOfTasks == 1) " --mem=8000"), subseq.env$sbatch) + subsequentcommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"") message(subsequentcommand) if (length(needfulldatagdx) > 0) { exitCode <- system(subsequentcommand) diff --git a/tests/testthat/test_01-combine_slurmConfig.R b/tests/testthat/test_01-combine_slurmConfig.R index 89b1e5c5f..5933079a7 100644 --- a/tests/testthat/test_01-combine_slurmConfig.R +++ b/tests/testthat/test_01-combine_slurmConfig.R @@ -6,8 +6,13 @@ # | Contact: remind@pik-potsdam.de test_that("combine_slurmConfig works", { teststring <- "--qos=priority --time=03:30:00" + expect_identical(combine_slurmConfig(teststring, teststring), teststring) + expect_identical(combine_slurmConfig(teststring, NULL), teststring) + expect_identical(combine_slurmConfig(NULL, NULL), "") + expect_identical(combine_slurmConfig(teststring, ""), teststring) expect_identical(combine_slurmConfig(teststring, "--qos=standby"), "--qos=standby --time=03:30:00") expect_identical(combine_slurmConfig(teststring, "--bla=blub"), paste("--bla=blub", teststring)) + expect_identical(combine_slurmConfig(teststring, "--wait"), paste("--wait", teststring)) teststring <- "--qos=priority --wait" expect_identical(combine_slurmConfig(teststring, "--qos=standby"), "--qos=standby --wait") }) From 1b9d2140111f984d7e38a8e3b8741d6f7dbbb7b9 Mon Sep 17 00:00:00 2001 From: orichters Date: Fri, 21 Jul 2023 13:38:07 +0200 Subject: [PATCH 3/4] some more regexp checks, add is.share --- main.gms | 39 ++++++++++++++-------------- scripts/start/checkFixCfg.R | 10 +++++-- tests/testthat/test_01-checkFixCfg.R | 2 ++ 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/main.gms b/main.gms index 5290031eb..9ee75420b 100755 --- a/main.gms +++ b/main.gms @@ -159,7 +159,8 @@ *' * The value behind 'def' contains the default value and is added only for the user to remember if changed manually *' * The value behind regexp is read by scripts/start/checkFixCfg.R to check the validity of the input. *' In this case, it checks whether the value fits this regular expression: ^(0|1)$ -*' Use 'value1|value2' for specific values, use '[1-7]' for a row of integers and use 'is.numeric' for numeric values. +*' Use 'value1|value2' for specific values, use '[1-7]' for a row of integers. +*' Two shortcut are defined: use 'is.numeric' for numeric values and 'is.share' if the value must be >= 0 and <= 1 *' *' *' #### Other general rules: @@ -555,22 +556,22 @@ parameter parameter c_shGreenH2 "lower bound on share of green hydrogen in all hydrogen from 2030 onwards" ; - c_shGreenH2 = 0; !! def = 0 + c_shGreenH2 = 0; !! def = 0 !! regexp = is.share *' (a number between 0 and 1): share parameter c_shBioTrans "upper bound on share of bioliquids in transport from 2025 onwards" ; - c_shBioTrans = 1; !! def = 1 + c_shBioTrans = 1; !! def = 1 !! regexp = is.share *' (a number between 0 and 1): share parameter cm_shSynLiq "lower bound on share of synfuels in SE liquids by 2045, gradual scale-up before" ; - cm_shSynLiq = 0; !! def = 0 + cm_shSynLiq = 0; !! def = 0 !! regexp = is.share *' (a number between 0 and 1): share parameter cm_shSynGas "lower bound on share of synthetic gas in SE gases by 2045, gradual scale-up before" ; - cm_shSynGas = 0; !! def = 0 + cm_shSynGas = 0; !! def = 0 !! regexp = is.share *' parameter cm_IndCCSscen "CCS for Industry" @@ -1566,22 +1567,22 @@ $setGLobal c_agricult_base_shift off !! def off $setglobal cm_wind_offshore 1 !! def = 1 *** cm_INCONV_PENALTY on !! def = on *** *RP* 2012-03-06 Flag to turn on inconvenience penalties, e.g. for air pollution -$setglobal cm_INCONV_PENALTY on !! def = on +$setglobal cm_INCONV_PENALTY on !! def = on !! regexp = off|on *** cm_INCONV_PENALTY_FESwitch off !! def = off *** flag to trun on inconvenience penalty to avoid switching shares on buildings, transport and industry biomass use if costs are relatively close (seLiqbio, sesobio, segabio) -$setglobal cm_INCONV_PENALTY_FESwitch on !! def = on +$setglobal cm_INCONV_PENALTY_FESwitch on !! def = on !! regexp = off|on *** cm_MOFEX off !! def=off *** *JH/LB* Activate MOFEX partial fossil fuel extraction cost minimization model *** * Warning: Use a well-converged run since the model uses vm_prodPe from the input GDX -$setGlobal cm_MOFEX off !! def = off +$setGlobal cm_MOFEX off !! def = off !! regexp = off|on *** cm_Full_Integration *** use "on" to treat wind and solar as fully dispatchable electricity production technologies -$setGlobal cm_Full_Integration off !! def = off +$setGlobal cm_Full_Integration off !! def = off !! regexp = off|on *' MAGICC configuration *' either uncalibrated or calibrate year 2000 temperature to HADCRUT4 data (which is very close to AR5). $setGlobal cm_magicc_calibrateTemperature2000 uncalibrated !! def = uncalibrated *' Derive temperature impulse response to CO2 emissions, based on MAGICC. Adds around 10min runtime. -$setGlobal cm_magicc_temperatureImpulseResponse off !! def = off +$setGlobal cm_magicc_temperatureImpulseResponse off !! def = off !! regexp = off|on *' MAGICC configuration *' roughly comparable to TCRE value, or even more roughly, equivalent climate sensitivity *' choose from OLDDEFAULT (REMIND1.7 legacy file); or different percentiles of RCP26 or generic TCRE outcomes calibrated to CMIP5 (see Schultes et al. (2018) for details) @@ -1589,17 +1590,17 @@ $setGlobal cm_magicc_config OLDDEFAULT !! def = OLDDEFAULT ; {OLDDEFAULT, RC *' climate damages (HowardNonCatastrophic, DICE2013R, DICE2016, HowardNonCatastrophic, HowardInclCatastrophic, KWcross, KWpanelPop} $setGlobal cm_damage_DiceLike_specification HowardNonCatastrophic !! def = HowardNonCatastrophic *** cfg$gms$cm_damage_Labor_exposure <- "low" # def = "low"; {low,high} -$setGlobal cm_damage_Labor_exposure low !!def = low +$setGlobal cm_damage_Labor_exposure low !! def = low !! regexp = low|high *** cfg$gms$cm_TCssp <- "SSP2" #def = "SSP2"; {SSP2,SSP5} the scenario for which the damage function is specified - currently only SSP2 and SSP5 are available -$setGlobal cm_TCssp SSP2 !! def = SSP2 +$setGlobal cm_TCssp SSP2 !! def = SSP2 !! regexp = SSP2|SSP5 *** cfg$gms$cm_TCpers <- 8 #def = 8; {0,1,2,3,4,5,6,7,8,9} the lags taken into account in the damage function -$setGlobal cm_TCpers 8 !! def = 8 -*** cfg$gms$cm_TCspec <- "mean" #def = mean; {mean,median,95,05,83,17} the uncertainty estimate of the TC damage function -$setGlobal cm_TCspec mean !! def = mean +$setGlobal cm_TCpers 8 !! def = 8 !! regexp = [0-9] +*** cfg$gms$cm_TCspec <- "mean" # def = mean; {mean,median,95,05,83,17} the uncertainty estimate of the TC damage function +$setGlobal cm_TCspec mean !! def = mean !! regexp = mean|median|95|05|83|17 *** #cm_transpGDPscale <- "on" # def "on", activate dampening factor to align edge-t non-energy transportation costs with historical GDP data" -$setglobal cm_transpGDPscale off !! def = off +$setglobal cm_transpGDPscale off !! def = off !! regexp = off|on *** This flag turns off output production -$setGlobal c_skip_output off !! def = off +$setGlobal c_skip_output off !! def = off !! regexp = off|on *** cm_CO2TaxSectorMarkup "CO2 tax markup in buildings or transport sector, a value of 0.5 means CO2 tax increased by 50%" *** (off): no markup *** ("GLO.build 1, USA_regi.trans 0.25, EUR_regi.trans 0.25"): "example for CO2 tax markup in transport of 25% in USA and EUR, and CO2eq tax markup in buildings sector of 100 % in all regions. Currently, build and trans are the only two elements of the set emi_sectors that are supported." @@ -1610,7 +1611,7 @@ $setGlobal c_regi_nucscen all !! def = all $setGlobal c_regi_capturescen all !! def = all *** cm_process_based_steel "switch to turn on process-based steel implementation" *** enable process-based implementation of steel in subsectors realisation of industry module -$setglobal cm_process_based_steel off !! off +$setglobal cm_process_based_steel off !! def = off !! regexp = off|on *** c_CO2priceDependent_AdjCosts *** default on changes adjustment costs for advanced vehicles in dependence of CO2 prices $setglobal c_CO2priceDependent_AdjCosts on !! def = on @@ -1637,7 +1638,7 @@ $setGlobal cm_APscen SSP2 !! def = SSP2 $setglobal cm_CES_configuration indu_subsectors-buil_simple-tran_edge_esm-POP_pop_SSP2EU-GDP_gdp_SSP2EU-En_gdp_SSP2EU-Kap_debt_limit-Reg_62eff8f7 !! this will be changed by start_run() $setglobal c_CES_calibration_iterations 10 !! def = 10 $setglobal c_CES_calibration_industry_FE_target 1 -$setglobal c_testOneRegi_region EUR !! def = EUR +$setglobal c_testOneRegi_region EUR !! def = EUR !! regexp = [A-Z]{3} $setglobal cm_fixCO2price off !! def = off *' @stop diff --git a/scripts/start/checkFixCfg.R b/scripts/start/checkFixCfg.R index e62f5575e..dfa0e7f58 100644 --- a/scripts/start/checkFixCfg.R +++ b/scripts/start/checkFixCfg.R @@ -12,9 +12,14 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE) # this is used to replace all 'regexp = is.numeric' grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)" + grepisshare <- "(\\+?0?\\.[0-9]+|0|0\\.0*|1|1\\.0*)" # some simple tests - stopifnot(all(grepl(paste0("^", grepisnum, "$"), c("2", "2.2", "32.", "+32.", "+.05", "-0.5", "-.5", "-5", "-7.")))) - stopifnot(all(! grepl(paste0("^", grepisnum, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2")))) + if (testmode) { + stopifnot(all( grepl(paste0("^", grepisnum, "$"), c("2", "2.2", "32.", "+32.", "+.05", "-0.5", "-.5", "-5", "-7.")))) + stopifnot(all(! grepl(paste0("^", grepisnum, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2")))) + stopifnot(all( grepl(paste0("^", grepisshare, "$"), c("0", "0.0", ".000", "1.0", "1.", "1", "0.12341234")))) + stopifnot(all(! grepl(paste0("^", grepisshare, "$"), c("1.1", "-0.3", "-0", ".")))) + } for (n in names(cfg$gms)) { errormsg <- NULL @@ -27,6 +32,7 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$") # replace is.numeric by pattern defined above useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE) + useregexp <- gsub("is.share", grepisshare, useregexp, fixed = TRUE) # check whether parameter value fits regular expression if (! grepl(useregexp, cfg$gms[[n]])) { errormsg <- paste0("Parameter cfg$gms$", n, "=", cfg$gms[[n]], " does not fit this regular expression: ", regexp) diff --git a/tests/testthat/test_01-checkFixCfg.R b/tests/testthat/test_01-checkFixCfg.R index 2bba25d80..bc7861e44 100644 --- a/tests/testthat/test_01-checkFixCfg.R +++ b/tests/testthat/test_01-checkFixCfg.R @@ -21,6 +21,8 @@ test_that("checkFixCfg works", { "cm_startyear" = "1985", "cm_netZeroScen" = "NöööGFS_v4", "cm_rcp_scen" = "apocalypse", + "c_testOneRegi_region" = "LOONG", + "c_shGreenH2" = "1.5", NULL) cfg <- savecfg From 9e196c69f22c58286560fa65cb667721518ca2f7 Mon Sep 17 00:00:00 2001 From: orichters Date: Fri, 21 Jul 2023 14:26:35 +0200 Subject: [PATCH 4/4] explain checkFixCfg() --- scripts/start/checkFixCfg.R | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/start/checkFixCfg.R b/scripts/start/checkFixCfg.R index dfa0e7f58..d8293e693 100644 --- a/scripts/start/checkFixCfg.R +++ b/scripts/start/checkFixCfg.R @@ -1,3 +1,12 @@ +#' take a REMIND cfg, runs some consistency checks and automatically fix some wrong settings +#' The regexp check loads the code from main.gms and looks for 'regexp = ' patterns. +#' It then checks whether the current cfg matches those patterns. +#' +#' @param cfg list with REMIND setting +#' @param remindPath path to REMIND directory containing the main.gms +#' @param testmode boolean. Default is FALSE which fails on errors, in testmode only raise warnings +#' @author Oliver Richters +#' @return updated cfg checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { refcfg <- gms::readDefaultConfig(remindPath) gms::check_config(cfg, reference_file = refcfg, modulepath = file.path(remindPath, "modules"), @@ -8,6 +17,7 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { errorsfound <- 0 + ## regexp check # extract all instances of 'regexp' from main.gms code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE) # this is used to replace all 'regexp = is.numeric'