From 2eca3a98be3aeca7496dbbf0f750149b50a05eb5 Mon Sep 17 00:00:00 2001 From: David Klein Date: Mon, 25 Mar 2024 10:09:24 +0100 Subject: [PATCH 1/7] Convert cm_nash_mode from $setglobal into parameter (to be able to switch from parallel to debug mode at runtime) --- core/loop.gms | 13 ++++++---- main.gms | 15 +++++++----- modules/80_optimization/nash/solve.gms | 33 +++++++++++++------------- scripts/start/run.R | 2 +- start.R | 4 ++-- start_bundle_coupled.R | 2 +- 6 files changed, 37 insertions(+), 32 deletions(-) diff --git a/core/loop.gms b/core/loop.gms index 78271e37b..c75d49c67 100644 --- a/core/loop.gms +++ b/core/loop.gms @@ -71,9 +71,12 @@ if( (cm_startyear gt 2005), ***-------------------------------------------------------------------------- *** SOLVE ***-------------------------------------------------------------------------- -***this disables solprint in cm_nash_mode=debug case by default. It is switched on in case of infes in nash/solve.gms -*RP* for faster debugging, turn solprint immediately on -$IF %cm_nash_mode% == "debug" option solprint = on ; +*** This disables solprint in cm_nash_mode=1 (debug) case by default. It is switched on in case of infes in nash/solve.gms +*** For faster debugging, turn solprint immediately on +if (cm_nash_mode eq 1, + option solprint = on; +); + o_modelstat = 100; loop(sol_itr$(sol_itr.val <= cm_solver_try_max), @@ -153,8 +156,8 @@ $batinclude "./modules/include.gms" postsolve *** save gdx *-------------------------------------------------------------------------- *** write the fulldata.gdx file after each optimal iteration -*AJS* in Nash status 7 is considered optimal in that respect (see definition of -*** o_modelstat in solve.gms) +*** In Nash status 7 is considered optimal in that respect (see definition of +*** o_modelstat in solve.gms) logfile.nr = 1; if (o_modelstat le 2, execute_unload "fulldata"; diff --git a/main.gms b/main.gms index 0a90fa489..39aef401b 100755 --- a/main.gms +++ b/main.gms @@ -443,7 +443,14 @@ $setGlobal codePerformance off !! def = off *' #### SWITCHES ***----------------------------------------------------------------------------- parameter - cm_iteration_max "number of iterations, if optimization is set to negishi or testOneRegi; used in nash mode only with cm_nash_autoconverge = 0" + cm_nash_mode "mode for solving nash problem" +; + cm_nash_mode = 2; !! def = 2 !! regexp = 1|2 +*' * (1): debug - all regions are run in a sequence and the lst-file will contain information on infeasiblities +*' * (2): parallel - all regions are run in parallel +*' +parameter + cm_iteration_max "number of iterations, if optimization is set to negishi or testOneRegi; is overwritten in Nash mode, except for cm_nash_autoconverge = 0" ; cm_iteration_max = 1; !! def = 1 *' @@ -1638,11 +1645,7 @@ $setGlobal cm_conoptv conopt3 !! def = conopt3 *' (off): normal model operation, default *' (on): no model operation, instead input.gdx is copied to fulldata.gdx $setGlobal c_empty_model off !! def = off !! regexp = off|on -*' mode for solving nash problem -*' -*' * parallel - all regions are run an parallel -*' * debug - all regions are run in a sequence and the lst-file will contain information on infeasiblities -$setGlobal cm_nash_mode parallel !! def = parallel !! regexp = debug|parallel|serial + $setglobal cm_secondary_steel_bound scenario !! def = scenario $setglobal c_GDPpcScen SSP2EU !! def = gdp_SSP2 (automatically adjusted by start_run() based on GDPscen) diff --git a/modules/80_optimization/nash/solve.gms b/modules/80_optimization/nash/solve.gms index 5aa4f69e1..edc14d7eb 100644 --- a/modules/80_optimization/nash/solve.gms +++ b/modules/80_optimization/nash/solve.gms @@ -6,20 +6,20 @@ *** | Contact: remind@pik-potsdam.de *** SOF ./modules/80_optimization/nash/solve.gms -regi(all_regi) = no; -hybrid.solvelink = 3; +regi(all_regi) = NO; +hybrid.solvelink = 3; !! activate multiple-CPU mode for GAMS hybrid.optfile = 9; -$ifthene.debug (sameas("%cm_nash_mode%","serial"))OR(sameas("%cm_nash_mode%","debug")) -hybrid.solvelink = 0; -$endif.debug +if(cm_nash_mode eq 1, + hybrid.solvelink = 0; !! activate single-CPU mode for GAMS +); loop (all_regi, !! only solve for regions that do not have a valid solution for this nash iteration if ( sol_itr.val gt 1 AND ( p80_repy(all_regi,"modelstat") eq 2 $ifthen.repeatNonOpt "%cm_repeatNonOpt%" == "off" - OR p80_repy(all_regi,"modelstat") eq 7 + OR p80_repy(all_regi,"modelstat") eq 7 $endif.repeatNonOpt ), @@ -27,7 +27,7 @@ $endif.repeatNonOpt continue; ); - regi(all_regi) = yes; + regi(all_regi) = YES; if (execError > 0, execute_unload "abort.gdx"; @@ -50,7 +50,7 @@ $endif.repeatNonOpt solve hybrid using nlp maximizing vm_welfareGlob; -$ifthene.debug (sameas("%cm_nash_mode%","serial"))OR(sameas("%cm_nash_mode%","debug")) +if(cm_nash_mode eq 1, p80_repy_thisSolitr(all_regi,"solvestat") = hybrid.solvestat; p80_repy_thisSolitr(all_regi,"modelstat") = hybrid.modelstat; p80_repy_thisSolitr(all_regi,"resusd") = hybrid.resusd; @@ -58,13 +58,13 @@ $ifthene.debug (sameas("%cm_nash_mode%","serial"))OR(sameas("%cm_nash_mode%","de if (p80_repy_thisSolitr(all_regi,"modelstat") eq 2, p80_repyLastOptim(all_regi,"objval") = p80_repy(all_regi,"objval"); ); -$endif.debug +); regi(all_regi) = NO; p80_handle(all_regi) = hybrid.handle; ); !! close regi loop -$ifthen.parallel %cm_nash_mode% == "parallel" +if(cm_nash_mode eq 2, repeat loop (all_regi$handlecollect(p80_handle(all_regi)), p80_repy_thisSolitr(all_regi,"solvestat") = hybrid.solvestat; @@ -72,8 +72,6 @@ repeat p80_repy_thisSolitr(all_regi,"resusd") = hybrid.resusd; p80_repy_thisSolitr(all_regi,"objval") = hybrid.objval; -* p80_repyLatestSolve(all_regi,solveinfo80) = p80_repy(all_regi,solveinfo80); - if (p80_repy_thisSolitr(all_regi,"modelstat") eq 2, p80_repyLastOptim(all_regi,"objval") = p80_repy_thisSolitr(all_regi,"objval"); ); @@ -83,7 +81,7 @@ repeat ); display$sleep(5) "sleep some time"; until card(p80_handle) = 0; -$endif.parallel +); regi(all_regi) = YES; @@ -110,7 +108,8 @@ loop (regi, if (p80_repy(regi,"modelstat") eq 2 OR p80_repy(regi,"modelstat") eq 7, pm_SolNonInfes(regi) = 1; ); - if (p80_repy(regi,"modelstat") eq 7, p80_SolNonOpt(regi) = 1); + if (p80_repy(regi,"modelstat") eq 7, + p80_SolNonOpt(regi) = 1); ); *** set o_modelstat to the highest value across all regions @@ -121,13 +120,13 @@ $else.repeatNonOpt = smax(regi, p80_repy(regi,"modelstat")); !! also taking into account status 7 $endif.repeatNonOpt -*** in cm_nash_mode=debug mode, enable solprint for next sol_itr when last +*** in cm_nash_mode=1 (debug) mode, enable solprint for next sol_itr when last *** iteration was non-optimal: -$ifthen.solprint %cm_nash_mode% == "debug" +if(cm_nash_mode eq 1, if (o_modelstat ne 2, option solprint = on; ); -$endif.solprint +); !! add information if this region was solved in this iteration p80_repy_iteration(regi,solveinfo80,iteration)$( diff --git a/scripts/start/run.R b/scripts/start/run.R index c493a3d96..fffbda1a8 100644 --- a/scripts/start/run.R +++ b/scripts/start/run.R @@ -121,7 +121,7 @@ run <- function() { cat("\nREMIND run finished!\n\n") # Create solution report for Nash runs - if (cfg$gms$optimization == "nash" && cfg$gms$cm_nash_mode != "debug" && file.exists("fulldata.gdx")) { + if (cfg$gms$optimization == "nash" && cfg$gms$cm_nash_mode != 1 && file.exists("fulldata.gdx")) { system("gdxdump fulldata.gdx Format=gamsbas Delim=comma Output=output_nash.gms") file.append("full.lst", "output_nash.gms") file.remove("output_nash.gms") diff --git a/start.R b/start.R index e8d6bb51a..5505d21a2 100755 --- a/start.R +++ b/start.R @@ -167,7 +167,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) { # for debug, testOneRegi, quick: save original settings to cfg$backup; restore them from there if not set. if ("--debug" %in% flags) { if (is.null(cfg[["backup"]][["cm_nash_mode"]])) cfg$backup$cm_nash_mode <- cfg$gms$cm_nash_mode - cfg$gms$cm_nash_mode <- "debug" + cfg$gms$cm_nash_mode <- 1 } else { if (! is.null(cfg[["backup"]][["cm_nash_mode"]])) cfg$gms$cm_nash_mode <- cfg$backup$cm_nash_mode } @@ -312,7 +312,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) { } if ("--debug" %in% flags) { - cfg$gms$cm_nash_mode <- "debug" + cfg$gms$cm_nash_mode <- 1 cfg$slurmConfig <- slurmConfig } diff --git a/start_bundle_coupled.R b/start_bundle_coupled.R index 4e8705967..8e7c9a1bc 100755 --- a/start_bundle_coupled.R +++ b/start_bundle_coupled.R @@ -539,7 +539,7 @@ for(scen in common){ } } - if (cfg_rem$gms$optimization == "nash" && cfg_rem$gms$cm_nash_mode == "parallel" && isFALSE(magpie_empty)) { + if (cfg_rem$gms$optimization == "nash" && cfg_rem$gms$cm_nash_mode == 2 && isFALSE(magpie_empty)) { # for nash: set the number of CPUs per node to number of regions + 1 numberOfTasks <- length(unique(read.csv2(cfg_rem$regionmapping)$RegionCode)) + 1 } else { From 3c3b3e02c1ec8ab0d47e195e55cb437e93e8f324 Mon Sep 17 00:00:00 2001 From: David Klein Date: Mon, 25 Mar 2024 10:20:22 +0100 Subject: [PATCH 2/7] Remove legacy code --- config/default.cfg | 3 --- modules/80_optimization/nash/sets.gms | 4 ---- 2 files changed, 7 deletions(-) diff --git a/config/default.cfg b/config/default.cfg index a5c6376d5..5cace5476 100644 --- a/config/default.cfg +++ b/config/default.cfg @@ -100,9 +100,6 @@ cfg$gamsv <- "gams" # def = gams # Action that should be performed ("ce": compile and run, "c": compile) cfg$action <- "ce" #def <- ce -# Tar-ball all the put files to save space? -cfg$tarball_puts <- "on" # def <- on - # Provide additional SLURM parameters (e.g. "--time=03:30:00") not included in # the list of following parameters that are set automatically in start.R: # --qos, --nodes, --tasks-per-node, -job-name, --output, --mail-type, --comment. diff --git a/modules/80_optimization/nash/sets.gms b/modules/80_optimization/nash/sets.gms index 03f254235..c49a78fd1 100644 --- a/modules/80_optimization/nash/sets.gms +++ b/modules/80_optimization/nash/sets.gms @@ -32,10 +32,6 @@ convMessage80 "contains all convergence criteria" / infes,surplus,nonopt,taxconv,anticip,target,regiTarget,implicitEnergyTarget,cm_implicitPriceTarget,cm_implicitPePriceTarget,damage,DevPriceAnticip / -nash_sol_itr80 "nash iterations" -/ - 1*10 -/ ; teLearn(learnte_dyn80) = YES; From e9111b3ff5ceb90caf9ca4f1df0d8a64cb518af8 Mon Sep 17 00:00:00 2001 From: David Klein Date: Mon, 25 Mar 2024 11:14:41 +0100 Subject: [PATCH 3/7] Start Nash in debug mode instead of aborting after maximum number of consecutive failures was reached in at least one region. --- modules/80_optimization/nash/declarations.gms | 2 +- modules/80_optimization/nash/postsolve.gms | 49 +++++++++++-------- modules/80_optimization/nash/solve.gms | 10 ++-- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/modules/80_optimization/nash/declarations.gms b/modules/80_optimization/nash/declarations.gms index 869bb4cba..3f811052f 100644 --- a/modules/80_optimization/nash/declarations.gms +++ b/modules/80_optimization/nash/declarations.gms @@ -129,7 +129,7 @@ s80_numberIterations "display helper" s80_bool "helper" s80_converged "if nash converged, this is 1" s80_cnptfile "parameter that indicates which optimality tolerance will be used" /1/ - +s80_runInDebug "Is 1 if regions stayed infeasible in nash and start in debug mode automatically following the parallel mode" /0/ ; *' defining specific output formats: diff --git a/modules/80_optimization/nash/postsolve.gms b/modules/80_optimization/nash/postsolve.gms index 54d504aed..bcd09e911 100644 --- a/modules/80_optimization/nash/postsolve.gms +++ b/modules/80_optimization/nash/postsolve.gms @@ -39,7 +39,7 @@ loop(ttot$(ttot.val ge 2005), ( sm_fadeoutPriceAnticip*p80_etaXp(trade) * ( (pm_Xport0(ttot,regi,trade) - p80_Mport0(ttot,regi,trade)) - (vm_Xport.l(ttot,regi,trade) - vm_Mport.l(ttot,regi,trade)) - p80_taxrev0(ttot,regi)$(ttot.val gt 2005)$(sameas(trade,"good")) + vm_taxrev.l(ttot,regi)$(ttot.val gt 2005)$(sameas(trade,"good")) - ) + ) / (p80_normalize0(ttot,regi,trade) + sm_eps) ) ; @@ -126,9 +126,9 @@ if(iteration.val > 2, if(iteration.val > 2, loop(ttot$(ttot.val ge 2005), loop(trade$(tradePe(trade) OR sameas(trade,"good") ), - if( ( Sign(p80_surplus(ttot,trade,iteration) ) eq Sign(p80_surplus(ttot,trade,iteration-1) ) ) AND - ( abs(p80_surplus(ttot,trade,iteration)) gt p80_surplusMaxTolerance(trade) ) , - o80_trackSurplusSign(ttot,trade,iteration) = o80_trackSurplusSign(ttot,trade,iteration-1) +1; + if( ( Sign(p80_surplus(ttot,trade,iteration) ) eq Sign(p80_surplus(ttot,trade,iteration-1) ) ) + AND ( abs(p80_surplus(ttot,trade,iteration)) gt p80_surplusMaxTolerance(trade) ) , + o80_trackSurplusSign(ttot,trade,iteration) = o80_trackSurplusSign(ttot,trade,iteration-1) +1; else o80_trackSurplusSign(ttot,trade,iteration) = 0; ); @@ -191,7 +191,7 @@ loop(trade$(NOT tradeSe(trade)), pm_pvp(ttot,trade) = p80_pvp_itr(ttot,trade,iteration+1); pm_Xport0(ttot,regi,trade)$(pm_SolNonInfes(regi) eq 1) = vm_Xport.l(ttot,regi,trade); p80_Mport0(ttot,regi,trade)$(pm_SolNonInfes(regi) eq 1) = vm_Mport.l(ttot,regi,trade); - p80_Mport0(ttot,regi,trade)$(pm_SolNonInfes(regi) eq 0) = 1.2 * vm_Mport.l(ttot,regi,trade); + p80_Mport0(ttot,regi,trade)$(pm_SolNonInfes(regi) eq 0) = 1.2 * vm_Mport.l(ttot,regi,trade); ); ); ); @@ -268,7 +268,7 @@ loop(trade$(NOT tradeSe(trade)), p80_messageShow("surplus") = YES; loop(ttot$((ttot.val ge cm_startyear) and (ttot.val gt 2100)), if( (abs(p80_surplus(ttot,trade,iteration)) gt p80_surplusMaxTolerance(trade) ), - p80_messageFailedMarket(ttot,trade) = YES; + p80_messageFailedMarket(ttot,trade) = YES; ); ); ); @@ -648,8 +648,8 @@ if (cm_abortOnConsecFail gt 0, loop (regi, if ( ( p80_repy_iteration(regi,"solvestat",iteration) eq 1 AND p80_repy_iteration(regi,"modelstat",iteration) eq 2) - OR ( p80_repy_iteration(regi,"solvestat",iteration) eq 4 - AND p80_repy_iteration(regi,"modelstat",iteration) eq 7), + OR ( p80_repy_iteration(regi,"solvestat",iteration) eq 4 + AND p80_repy_iteration(regi,"modelstat",iteration) eq 7), !! region was solved successfully p80_trackConsecFail(regi) = 0; else @@ -659,16 +659,25 @@ if (cm_abortOnConsecFail gt 0, ); if (smax(regi, p80_trackConsecFail(regi)) >= cm_abortOnConsecFail, - execute_unload "abort.gdx"; - display p80_trackConsecFail; - abort "Run was aborted because the maximum number of consecutive failures was reached in at least one region!"; + if (s80_runInDebug eq 0, + s80_runInDebug = 1; + cm_nash_mode = 1; + display "Starting nash in debug mode after maximum number of consecutive failures was reached in at least one region."; + else + execute_unload "abort.gdx"; + display p80_trackConsecFail; + abort "Run was aborted because the maximum number of consecutive failures was reached in at least one region!"; + ); + else + !! Set nash mode back to parallel because all regions got feasible after they have been automatically restarted as debug + if (s80_runInDebug eq 1, + s80_runInDebug = 0; + cm_nash_mode = 2; + display "Set nash mode back to parallel after regions got feasible in auto-debug mode."; + ); ); ); -***Fade out LT correction terms, they should only be important in the first iterations and might interfere with ST corrections. -***p80_etaLT(trade) = p80_etaLT(trade)*0.5; - - ***-------------------------- *** EMIOPT implementation @@ -676,11 +685,11 @@ if (cm_abortOnConsecFail gt 0, $ifthen.emiopt %emicapregi% == 'none' if(cm_emiscen eq 6, -*mlb 20150609* nash emiopt algorithm -***we iteratively reach the point where these two marginals are equal for each region by adjusting regional permit budgets: -***marginal of cumulative emissions: +*** nash emiopt algorithm +*** we iteratively reach the point where these two marginals are equal for each region by adjusting regional permit budgets: +*** marginal of cumulative emissions: p80_eoMargEmiCum(regi) = 5*(abs(qm_co2eqCum.m(regi)))$(pm_SolNonInfes(regi) eq 1); -***marginal of permit budget : +*** marginal of permit budget : p80_eoMargPermBudg(regi) = 5*(abs(q80_budgetPermRestr.m(regi)))$(pm_SolNonInfes(regi) eq 1); display pm_budgetCO2eq; @@ -689,7 +698,7 @@ display pm_budgetCO2eq; loop(regi, p80_eoWeights(regi) = 1/max(abs(qm_budget.m("2050",regi)),1E-9); ); -***normalize sum to unity +***normalize sum to unity p80_eoWeights(regi) = p80_eoWeights(regi) / sum(regi2, p80_eoWeights(regi2) ); diff --git a/modules/80_optimization/nash/solve.gms b/modules/80_optimization/nash/solve.gms index edc14d7eb..6d93144dc 100644 --- a/modules/80_optimization/nash/solve.gms +++ b/modules/80_optimization/nash/solve.gms @@ -15,11 +15,13 @@ if(cm_nash_mode eq 1, ); loop (all_regi, - !! only solve for regions that do not have a valid solution for this nash iteration - if ( sol_itr.val gt 1 - AND ( p80_repy(all_regi,"modelstat") eq 2 + !! only solve for regions that do not have a valid solution from the + !! last solver iteration + if ( ( sol_itr.val gt 1 + OR s80_runInDebug eq 1) + AND ( p80_repy(all_regi,"modelstat") eq 2 $ifthen.repeatNonOpt "%cm_repeatNonOpt%" == "off" - OR p80_repy(all_regi,"modelstat") eq 7 + OR p80_repy(all_regi,"modelstat") eq 7 $endif.repeatNonOpt ), From a492d92a4f5d22ff7a186499e5a4002cb3e7bde0 Mon Sep 17 00:00:00 2001 From: David Klein Date: Mon, 25 Mar 2024 11:35:40 +0100 Subject: [PATCH 4/7] Use new type (parameter instead of $setglobal) and value (1,2 instead of "debug","parallel") in the remaining places. --- config/scenario_config_21_EU11_ECEMF.csv | 2 +- config/tests/scenario_config_quick.csv | 2 +- standalone/MOFEX/MOFEX.gms | 3 ++- standalone/template.gms | 3 ++- standalone/trade/trade.gms | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/scenario_config_21_EU11_ECEMF.csv b/config/scenario_config_21_EU11_ECEMF.csv index 3464e2112..3af043562 100644 --- a/config/scenario_config_21_EU11_ECEMF.csv +++ b/config/scenario_config_21_EU11_ECEMF.csv @@ -47,7 +47,7 @@ xx_WP5_RAP-REG-LimCCS;0,WP5p3;;;;./config/regionmapping_21_EU11.csv;./config/ext xx_WP5_RAP-REG-LimNuclear;0,WP5p3;;;;./config/regionmapping_21_EU11.csv;./config/extramapping_EU27.csv;rcp20;9;globallyOptimal;650;diffCurvPhaseIn2Lin;100;2080;;9;;;;;;;;;;;;2050.GLO 0.9;NDC;regiCarbonPrice;2030.EU27_regi.tax.t.FE_wo_n_e.all 1.1235, (2035,2040,2045,2050).EU27_regi.tax.t.PE.biomass 0.237825;2020.2030.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 2.122, 2035.2040.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 0.321, 2045.2050.EU27_regi.all.year.netGHG_LULUCFGrassi 0.01;on;on;EUR_regi, NEU_regi;;;;;;;;5;;;;;;;;heat;Mix4;2025;xx_WP5_RAP-REG;xx_DIAG-NPI;xx_DIAG-NPI;;;on;;;;ENC,EWN,ECS,ESC,ECE,FRA,DEU,UKI,ESW;;;;;;;; # xx_WP5_RAP-REG-LimRES;0;;;;;;;;;;;;;;;;;;;;;;;;;;2050.GLO 0.9;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; # _____tests_____;0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;35.43046358;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -test_WP1_Nzero_DEU;0,WP1;;testOneRegi;;./config/regionmapping_21_EU11.csv;./config/extramapping_EU27.csv;rcp20;9;globallyOptimal;650;diffCurvPhaseIn2Lin;100;2080;;9;;;;;;;;;;;;2050.GLO 0.9;NDC;regiCarbonPrice;(2035,2040,2045,2050).EU27_regi.tax.t.PE.biomass 0.237825;2020.2030.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 2.122, 2035.2050.EU27_regi.all.year.netGHG_LULUCFGrassi 0.01;on;on;EUR_regi, NEU_regi;;;;;;;;;;;;;;;;heat;Mix4;2025;xx_DIAG-NPI;xx_DIAG-NPI;xx_DIAG-NPI;;same as xx_DIAG-Nzero;;;;;;;;;DEU;debug;1;1; +test_WP1_Nzero_DEU;0,WP1;;testOneRegi;;./config/regionmapping_21_EU11.csv;./config/extramapping_EU27.csv;rcp20;9;globallyOptimal;650;diffCurvPhaseIn2Lin;100;2080;;9;;;;;;;;;;;;2050.GLO 0.9;NDC;regiCarbonPrice;(2035,2040,2045,2050).EU27_regi.tax.t.PE.biomass 0.237825;2020.2030.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 2.122, 2035.2050.EU27_regi.all.year.netGHG_LULUCFGrassi 0.01;on;on;EUR_regi, NEU_regi;;;;;;;;;;;;;;;;heat;Mix4;2025;xx_DIAG-NPI;xx_DIAG-NPI;xx_DIAG-NPI;;same as xx_DIAG-Nzero;;;;;;;;;DEU;1;1;1; xx_WP1_Nzero_highPrecision_10tCO2eq;0,test;;;;./config/regionmapping_21_EU11.csv;./config/extramapping_EU27.csv;rcp20;9;globallyOptimal;650;diffCurvPhaseIn2Lin;100;2080;;9;;;;;;;;;;;;2050.GLO 0.9;NDC;regiCarbonPrice;(2035,2040,2045,2050).EU27_regi.tax.t.PE.biomass 0.237825;2020.2030.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 2.122, 2035.2050.EU27_regi.all.year.netGHG_LULUCFGrassi 0.01;on;on;EUR_regi, NEU_regi;;;;;;;;;;;;;;;;heat;Mix4;2025;xx_WP1_Nzero;xx_DIAG-NPI;xx_DIAG-NPI;;same as xx_DIAG-Nzero;;;;;;;;;;;;;0.002 xx_WP1_Nzero_highPrecision_5tCO2eq;0,test;;;;./config/regionmapping_21_EU11.csv;./config/extramapping_EU27.csv;rcp20;9;globallyOptimal;650;diffCurvPhaseIn2Lin;100;2080;;9;;;;;;;;;;;;2050.GLO 0.9;NDC;regiCarbonPrice;(2035,2040,2045,2050).EU27_regi.tax.t.PE.biomass 0.237825;2020.2030.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 2.122, 2035.2050.EU27_regi.all.year.netGHG_LULUCFGrassi 0.01;on;on;EUR_regi, NEU_regi;;;;;;;;;;;;;;;;heat;Mix4;2025;xx_WP1_Nzero;xx_DIAG-NPI;xx_DIAG-NPI;;same as xx_DIAG-Nzero;;;;;;;;;;;;;0.001 xx_WP1_Nzero_highPrecision_1tCO2eq;0,test;;;;./config/regionmapping_21_EU11.csv;./config/extramapping_EU27.csv;rcp20;9;globallyOptimal;650;diffCurvPhaseIn2Lin;100;2080;;9;;;;;;;;;;;;2050.GLO 0.9;NDC;regiCarbonPrice;(2035,2040,2045,2050).EU27_regi.tax.t.PE.biomass 0.237825;2020.2030.EU27_regi.all.year.netGHG_LULUCFGrassi_intraRegBunker 2.122, 2035.2050.EU27_regi.all.year.netGHG_LULUCFGrassi 0.01;on;on;EUR_regi, NEU_regi;;;;;;;;;;;;;;;;heat;Mix4;2025;xx_WP1_Nzero;xx_DIAG-NPI;xx_DIAG-NPI;;same as xx_DIAG-Nzero;;;;;;;;;;;;;0.0002 diff --git a/config/tests/scenario_config_quick.csv b/config/tests/scenario_config_quick.csv index 378c987aa..0dc8d6f3c 100644 --- a/config/tests/scenario_config_quick.csv +++ b/config/tests/scenario_config_quick.csv @@ -1,2 +1,2 @@ title;start;cm_nash_mode;cm_iteration_max;optimization;results_folder;output;cm_quick_mode;force_replace;slurmConfig -testOneRegi;1;debug;1;testOneRegi;output/testOneRegi;NA;on;TRUE;--qos=priority --nodes=1 --tasks-per-node=1 --mem=8000 --time=60 --wait +testOneRegi;1;1;1;testOneRegi;output/testOneRegi;NA;on;TRUE;--qos=priority --nodes=1 --tasks-per-node=1 --mem=8000 --time=60 --wait diff --git a/standalone/MOFEX/MOFEX.gms b/standalone/MOFEX/MOFEX.gms index 5829681f7..bb5eba53a 100644 --- a/standalone/MOFEX/MOFEX.gms +++ b/standalone/MOFEX/MOFEX.gms @@ -118,6 +118,7 @@ $setGlobal codePerformance off !! def = off ***----------------------------------------------------------------------------- ***--------------- declaration of parameters for switches ---------------------- parameters +cm_nash_mode "mode for solving nash problem, (1): debug (2): parallel" cm_iteration_max "number of Negishi iterations" c_solver_try_max "maximum number of inner iterations within one Negishi iteration (<10)" c_keep_iteration_gdxes "save intermediate iteration gdxes" @@ -233,6 +234,7 @@ cm_deuCDRmax "switch to limit maximum annual CDR amount in Germa *** YOU ARE IN THE WARNING ZONE (DON'T DO CHANGES HERE) *** -------------------------------------------------------------------------------------------------------------------------------------------------------------------- +cm_nash_mode = 2; !! def = 2 !! regexp = 1|2 cm_iteration_max = 1; !! def = 1 c_solver_try_max = 2; !! def = 2 c_keep_iteration_gdxes = 0; !! def = 0 @@ -421,7 +423,6 @@ cm_deuCDRmax = -1; !! def = -1 *** -------------------------------------------------------------------------------------------------------------------------------------------------------------------- *--------------------flags------------------------------------------------------------ $SETGLOBAL cm_SlowConvergence off !! def = off -$setGlobal cm_nash_mode parallel !! def = parallel $setGLobal cm_debug_preloop off !! def = off $setglobal cm_INCONV_PENALTY on !! def = on $setglobal cm_INCONV_PENALTY_bioSwitch off !! def = off diff --git a/standalone/template.gms b/standalone/template.gms index 37f2910d0..e1c7e0172 100644 --- a/standalone/template.gms +++ b/standalone/template.gms @@ -118,6 +118,7 @@ $setGlobal codePerformance off !! def = off ***--------------- declaration of parameters for switches ---------------------- parameters +cm_nash_mode "mode for solving nash problem, (1): debug (2): parallel" cm_iteration_max "number of Negishi iterations (up to 49)" cm_solver_try_max "maximum number of inner iterations within one Negishi iteration (<10)" c_keep_iteration_gdxes "save intermediate iteration gdxes" @@ -187,6 +188,7 @@ cm_DiscRateScen "Scenario for the implicit discount rate applied to the *** YOU ARE IN THE WARNING ZONE (DON'T DO CHANGES HERE) *** -------------------------------------------------------------------------------------------------------------------------------------------------------------------- +cm_nash_mode = 2; !! def = 2 !! regexp = 1|2 cm_iteration_max = 1; !! def = 1 cm_solver_try_max = 2; !! def = 2 c_keep_iteration_gdxes = 0; !! def = 0 @@ -279,7 +281,6 @@ cm_DiscRateScen = 0;!! def = 0 *** -------------------------------------------------------------------------------------------------------------------------------------------------------------------- *--------------------flags------------------------------------------------------------ $SETGLOBAL cm_SlowConvergence off !! def = off -$setGlobal cm_nash_mode parallel !! def = parallel $setglobal cm_INCONV_PENALTY on !! def = on $setGlobal c_skip_output off !! def = off $setGlobal cm_MOFEX off !! def = off diff --git a/standalone/trade/trade.gms b/standalone/trade/trade.gms index fa2f78aa2..01cbd0b99 100644 --- a/standalone/trade/trade.gms +++ b/standalone/trade/trade.gms @@ -122,6 +122,7 @@ $setGlobal codePerformance off !! def = off ***----------------------------------------------------------------------------- ***--------------- declaration of parameters for switches ---------------------- parameters +cm_nash_mode "mode for solving nash problem, (1): debug (2): parallel" cm_iteration_max "number of Negishi iterations" c_solver_try_max "maximum number of inner iterations within one Negishi iteration (<10)" c_keep_iteration_gdxes "save intermediate iteration gdxes" @@ -237,6 +238,7 @@ cm_deuCDRmax "switch to limit maximum annual CDR amount in Germa *** YOU ARE IN THE WARNING ZONE (DON'T DO CHANGES HERE) *** -------------------------------------------------------------------------------------------------------------------------------------------------------------------- +cm_nash_mode = 2; !! def = 2 !! regexp = 1|2 cm_iteration_max = 1; !! def = 1 c_solver_try_max = 2; !! def = 2 c_keep_iteration_gdxes = 0; !! def = 0 @@ -428,7 +430,6 @@ cm_deuCDRmax = -1; !! def = -1 *** -------------------------------------------------------------------------------------------------------------------------------------------------------------------- *--------------------flags------------------------------------------------------------ $SETGLOBAL cm_SlowConvergence off !! def = off -$setGlobal cm_nash_mode parallel !! def = parallel $setGLobal cm_debug_preloop off !! def = off $setglobal cm_INCONV_PENALTY on !! def = on $setglobal cm_INCONV_PENALTY_bioSwitch off !! def = off From 23eed3630506fe7c7826780f862d80cf24e29583 Mon Sep 17 00:00:00 2001 From: David Klein Date: Tue, 26 Mar 2024 12:16:16 +0100 Subject: [PATCH 5/7] Update CHANGELOG.md and comment. --- CHANGELOG.md | 4 +++- main.gms | 2 +- modules/80_optimization/nash/solve.gms | 6 ++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 324df04d0..7c2d862d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **scripts** add MAGICCv7.5.3 with AR6 settings as output script, add compareScenarios2 option [[#1475](https://github.com/remindmodel/remind/pull/1475), [[#1615](https://github.com/remindmodel/remind/pull/1615)] - **scripts** add 'make test-fix' which runs codeCheck in interactive mode, adjusting not_used.txt files - [[#1625](https://github.com/remindmodel/remind/pull/1625) + [[#1625](https://github.com/remindmodel/remind/pull/1625)] +- **80_optimization** For Nash mode: after infeasibilities continue in debug mode before aborting + [[#1636](https://github.com/remindmodel/remind/pull/1636)] ### fixed - fixed weights of energy carriers in `pm_IndstCO2Captured` diff --git a/main.gms b/main.gms index 39aef401b..09de085aa 100755 --- a/main.gms +++ b/main.gms @@ -455,7 +455,7 @@ parameter cm_iteration_max = 1; !! def = 1 *' parameter - cm_abortOnConsecFail "number of iterations of consecutive infeasibilities/failures to solve for one region, after which the run is automatically aborted" + cm_abortOnConsecFail "number of iterations of consecutive infeasibilities/failures to solve for one region, after which the run automatically switches to serial debug mode" ; cm_abortOnConsecFail = 5; !! def = 5 *' diff --git a/modules/80_optimization/nash/solve.gms b/modules/80_optimization/nash/solve.gms index 6d93144dc..b9ef1f1b0 100644 --- a/modules/80_optimization/nash/solve.gms +++ b/modules/80_optimization/nash/solve.gms @@ -124,10 +124,8 @@ $endif.repeatNonOpt *** in cm_nash_mode=1 (debug) mode, enable solprint for next sol_itr when last *** iteration was non-optimal: -if(cm_nash_mode eq 1, -if (o_modelstat ne 2, - option solprint = on; -); +if(cm_nash_mode eq 1 AND o_modelstat ne 2, + option solprint = on; ); !! add information if this region was solved in this iteration From cae76290f089428ddda50f31d27ea94f85c58583 Mon Sep 17 00:00:00 2001 From: David Klein Date: Tue, 26 Mar 2024 13:54:35 +0100 Subject: [PATCH 6/7] Add cm_nash_mode to the appropriate not_used.txt --- modules/80_optimization/nash/not_used.txt | 2 +- modules/80_optimization/negishi/not_used.txt | 2 +- modules/80_optimization/testOneRegi/not_used.txt | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/80_optimization/nash/not_used.txt b/modules/80_optimization/nash/not_used.txt index 2ea425572..487488b88 100644 --- a/modules/80_optimization/nash/not_used.txt +++ b/modules/80_optimization/nash/not_used.txt @@ -4,7 +4,7 @@ # | AGPL-3.0, you are granted additional permissions described in the # | REMIND License Exception, version 1.0 (see LICENSE file). # | Contact: remind@pik-potsdam.de -name, type, reason +name,type,reason cm_solver_try_max, switch, not needed pm_prtp, parameter, ??? pm_taxCO2eq, parameter, ??? diff --git a/modules/80_optimization/negishi/not_used.txt b/modules/80_optimization/negishi/not_used.txt index 1d9bc57e6..e539f5901 100644 --- a/modules/80_optimization/negishi/not_used.txt +++ b/modules/80_optimization/negishi/not_used.txt @@ -10,6 +10,7 @@ cm_iteration_max, switch, ??? cm_keep_presolve_gdxes, switch, not needed cm_nash_autoconverge, switch, ??? cm_solver_try_max, switch, not needed +cm_nash_mode, switch, not needed cm_TaxConvCheck, variable, ?? cm_emiMktTarget_tolerance, switch, not needed cm_implicitQttyTarget_tolerance, switch, not needed @@ -66,5 +67,4 @@ vm_perm, variable, ??? vm_prodPe, variable, ?? sm_CES_calibration_iteration, scalar, only needed during calibration which is not support in negishi mode cm_maxFadeOutPriceAnticip, parameter, ??? - cm_LearningSpillover,input,learning spillover only implemented in nash diff --git a/modules/80_optimization/testOneRegi/not_used.txt b/modules/80_optimization/testOneRegi/not_used.txt index a89fb3579..bdcc17f03 100644 --- a/modules/80_optimization/testOneRegi/not_used.txt +++ b/modules/80_optimization/testOneRegi/not_used.txt @@ -14,6 +14,7 @@ pm_welf, parameter, ??? pm_prtp, parameter, ??? cm_iteration_max, switch, ??? cm_abortOnConsecFail, switch, ??? +cm_nash_mode, switch, not needed cm_nash_autoconverge, switch, ??? cm_emiMktTarget_tolerance, switch, not needed cm_implicitQttyTarget_tolerance,switch, not needed From 0818b58766d074f734fd403cdfec1ed65a787faa Mon Sep 17 00:00:00 2001 From: David Klein Date: Wed, 27 Mar 2024 12:07:59 +0100 Subject: [PATCH 7/7] Nash: Add two conditions for auto-debug mode. Only start auto-debug mode if 1) not all regions are infeasible 2) if not already in manual debug mode Activate equation and column listing in debug mode Remove solprint statement in solve.gms that was always overwritten in loop.gms --- core/loop.gms | 11 +++++++---- modules/80_optimization/nash/postsolve.gms | 18 +++++++++++------- modules/80_optimization/nash/solve.gms | 22 ++++++++-------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/core/loop.gms b/core/loop.gms index c75d49c67..5e3e6fa54 100644 --- a/core/loop.gms +++ b/core/loop.gms @@ -61,7 +61,7 @@ $batinclude "./modules/include.gms" presolve *cb 20140305 submit.R looks for the unique string in the following line and replaces it with the offlisting include into the full.gms at this position ***cb20140305readinpositionforfinxingfiles -*AJS* In case of fixing, fix to prices from input_ref.gdx (t < cm_startyear). +*** In case of fixing, fix to prices from input_ref.gdx (t < cm_startyear). *** Parameters are not automatically treated by the fixing mechanism above. if( (cm_startyear gt 2005), Execute_Loadpoint "input_ref" p_pvpRef = pm_pvp; @@ -71,10 +71,13 @@ if( (cm_startyear gt 2005), ***-------------------------------------------------------------------------- *** SOLVE ***-------------------------------------------------------------------------- -*** This disables solprint in cm_nash_mode=1 (debug) case by default. It is switched on in case of infes in nash/solve.gms -*** For faster debugging, turn solprint immediately on +*** Set options for debugging if (cm_nash_mode eq 1, - option solprint = on; + option + solprint = on + limcol = 2147483647 + limrow = 2147483647 + ; ); diff --git a/modules/80_optimization/nash/postsolve.gms b/modules/80_optimization/nash/postsolve.gms index bcd09e911..620eb7054 100644 --- a/modules/80_optimization/nash/postsolve.gms +++ b/modules/80_optimization/nash/postsolve.gms @@ -659,14 +659,18 @@ if (cm_abortOnConsecFail gt 0, ); if (smax(regi, p80_trackConsecFail(regi)) >= cm_abortOnConsecFail, - if (s80_runInDebug eq 0, - s80_runInDebug = 1; - cm_nash_mode = 1; - display "Starting nash in debug mode after maximum number of consecutive failures was reached in at least one region."; - else + if ((s80_runInDebug eq 0) AND (cm_nash_mode ne 1), !! auto-start debug only if not already in debug mode + if (sum(regi, pm_SolNonInfes(regi) ne 0) eq 0, !! if all regions are infeasible debug makes no sense + execute_unload "abort.gdx"; + abort "Run was aborted because the maximum number of consecutive failures was reached in at least one region! No debug started since all regions are infeasible."; + else !! start debug mode only if at leat one region was feasible + s80_runInDebug = 1; + cm_nash_mode = 1; + display "Starting nash in debug mode after maximum number of consecutive failures was reached in at least one region."; + ); + else !! s80_runInDebug eq 1 AND/OR cm_nash_mode eq 1 execute_unload "abort.gdx"; - display p80_trackConsecFail; - abort "Run was aborted because the maximum number of consecutive failures was reached in at least one region!"; + abort "After debug mode run was aborted because the maximum number of consecutive failures was still reached in at least one region!"; ); else !! Set nash mode back to parallel because all regions got feasible after they have been automatically restarted as debug diff --git a/modules/80_optimization/nash/solve.gms b/modules/80_optimization/nash/solve.gms index b9ef1f1b0..dd1e874b0 100644 --- a/modules/80_optimization/nash/solve.gms +++ b/modules/80_optimization/nash/solve.gms @@ -52,15 +52,15 @@ $endif.repeatNonOpt solve hybrid using nlp maximizing vm_welfareGlob; -if(cm_nash_mode eq 1, - p80_repy_thisSolitr(all_regi,"solvestat") = hybrid.solvestat; - p80_repy_thisSolitr(all_regi,"modelstat") = hybrid.modelstat; - p80_repy_thisSolitr(all_regi,"resusd") = hybrid.resusd; - p80_repy_thisSolitr(all_regi,"objval") = hybrid.objval; - if (p80_repy_thisSolitr(all_regi,"modelstat") eq 2, - p80_repyLastOptim(all_regi,"objval") = p80_repy(all_regi,"objval"); + if(cm_nash_mode eq 1, + p80_repy_thisSolitr(all_regi,"solvestat") = hybrid.solvestat; + p80_repy_thisSolitr(all_regi,"modelstat") = hybrid.modelstat; + p80_repy_thisSolitr(all_regi,"resusd") = hybrid.resusd; + p80_repy_thisSolitr(all_regi,"objval") = hybrid.objval; + if (p80_repy_thisSolitr(all_regi,"modelstat") eq 2, + p80_repyLastOptim(all_regi,"objval") = p80_repy(all_regi,"objval"); + ); ); -); regi(all_regi) = NO; p80_handle(all_regi) = hybrid.handle; @@ -122,12 +122,6 @@ $else.repeatNonOpt = smax(regi, p80_repy(regi,"modelstat")); !! also taking into account status 7 $endif.repeatNonOpt -*** in cm_nash_mode=1 (debug) mode, enable solprint for next sol_itr when last -*** iteration was non-optimal: -if(cm_nash_mode eq 1 AND o_modelstat ne 2, - option solprint = on; -); - !! add information if this region was solved in this iteration p80_repy_iteration(regi,solveinfo80,iteration)$( p80_repy_thisSolitr(regi,solveinfo80) )