comments for checkFixCfg, in test-coupled send mails only for fails

config/tests/scenario_config_coupled_shortCascade.csv

@@ -1,4 +1,4 @@
 title;start;qos;sbatch;magpie_scen;magpie_empty;no_ghgprices_land_until;max_iterations;oldrun;path_gdx;path_gdx_ref;path_gdx_bau;path_report;cm_nash_autoconverge_lastrun;path_mif_ghgprice_land
-TESTTHAT-SSP2EU-Base;1;auto;--wait;SSP2|NPI;TRUE;y2150;2;;;;;;2;
-TESTTHAT-SSP2EU-NDC;1;auto;--wait;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base
-TESTTHAT-SSP2EU-Policy;2;auto;--wait;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif
+TESTTHAT-SSP2EU-Base;1;auto;--wait --mail-type=FAIL;SSP2|NPI;TRUE;y2150;2;;;;;;2;
+TESTTHAT-SSP2EU-NDC;1;auto;--wait --mail-type=FAIL;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base
+TESTTHAT-SSP2EU-Policy;2;auto;--wait --mail-type=FAIL;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif

scripts/start/checkFixCfg.R

@@ -8,23 +8,34 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) {
 
   errorsfound <- 0
 
+  # extract all instances of 'regexp' from main.gms
   code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE)
+  # this is used to replace all 'regexp = is.numeric'
   grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)"
+  # some simple tests
+  stopifnot(all(grepl(paste0("^", grepisnum, "$"), c("2", "2.2", "32.", "+32.", "+.05", "-0.5", "-.5", "-5", "-7."))))
+  stopifnot(all(! grepl(paste0("^", grepisnum, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2"))))
 
   for (n in names(cfg$gms)) {
     errormsg <- NULL
+    # how parameter n is defined in main.gms
     paramdef <- paste0("^([ ]*", n, "[ ]*=|\\$setglobal[ ]+", n, " )")
+    # filter fitting parameter definition from code snippets containing regexp
     filtered <- grep(paste0(paramdef, ".*regexp[ ]*=[ ]*"), code, value = TRUE)
     if (length(filtered) == 1) {
       # search for string '!! regexp = whatever', potentially followed by '!! otherstuff' and extract 'whatever'
       regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$")
+      # replace is.numeric by pattern defined above
       useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE)
+      # check whether parameter value fits regular expression
       if (! grepl(useregexp, cfg$gms[[n]])) {
         errormsg <- paste0("Parameter cfg$gms$", n, "=", cfg$gms[[n]], " does not fit this regular expression: ", regexp)
       }
     } else if (length(filtered) > 1) {
+      # fail if more than one regexp found for parameter
       errormsg <- paste0("More than one regexp found for ", n, ". These are the code lines:\n", paste(filtered, collapse = "\n"))
     }
+    # count errors
     if (! is.null(errormsg)) {
       errorsfound <- errorsfound + 1
       if (testmode) warning(errormsg) else message(errormsg)

scripts/start/combine_slurmConfig.R

@@ -12,8 +12,8 @@
 combine_slurmConfig <- function(original, update_with) {
 
   # trim whitespaces
-  original <- trimws(original)
-  update_with <- trimws(update_with)
+  original <- trimws(toString(original))
+  update_with <- trimws(toString(update_with))
 
   # remove double whitespaces
   original <- gsub("\\s+", " ", original)

start_bundle_coupled.R

@@ -620,12 +620,12 @@ for (scen in common) {
           sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j'"), intern = TRUE)
           runEnv$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short"
         }
-        slurm_command <- paste0("sbatch --qos=", runEnv$qos, " --job-name=", fullrunname,
-        " --output=", logfile, " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks,
-        if (runEnv$numberOfTasks == 1) " --mem=8000", " ", runEnv$sbatch,
-        " ", runEnv$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"")
-        message(slurm_command)
-        exitCode <- system(slurm_command)
+        slurmOptions <- combine_slurmConfig(paste0("--qos=", runEnv$qos, " --job-name=", fullrunname, " --output=", logfile,
+          " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks,
+          if (runEnv$numberOfTasks == 1) " --mem=8000"), runEnv$sbatch)
+        slurmCommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"")
+        message(slurmCommand)
+        exitCode <- system(slurmCommand)
         if (0 < exitCode) {
           errorsfound <- errorsfound + 1
           message("sbatch command failed, check logs.")

start_coupled.R

@@ -18,6 +18,7 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
   require(gdx)
   library(methods)
   library(remind2)
+  source("scripts/start/combine_slurmConfig.R")
 
   errorsfound <- 0
   # delete entries in stack that contain needle and append new
@@ -250,10 +251,10 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m
           sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j' | grep -v ", fullrunname), intern = TRUE)
           subseq.env$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short"
         }
-        subsequentcommand <- paste0("sbatch --qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile,
-        " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks,
-        if (subseq.env$numberOfTasks == 1) " --mem=8000",
-        " ", subseq.env$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"")
+        slurmOptions <- combine_slurmConfig(paste0("--qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile,
+           " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks,
+          if (subseq.env$numberOfTasks == 1) " --mem=8000"), subseq.env$sbatch)
+        subsequentcommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"")
         message(subsequentcommand)
         if (length(needfulldatagdx) > 0) {
           exitCode <- system(subsequentcommand)

tests/testthat/test_01-combine_slurmConfig.R

@@ -6,8 +6,13 @@
 # |  Contact: [email protected]
 test_that("combine_slurmConfig works", {
   teststring <- "--qos=priority --time=03:30:00"
+  expect_identical(combine_slurmConfig(teststring, teststring), teststring)
+  expect_identical(combine_slurmConfig(teststring, NULL), teststring)
+  expect_identical(combine_slurmConfig(NULL, NULL), "")
+  expect_identical(combine_slurmConfig(teststring, ""), teststring)
   expect_identical(combine_slurmConfig(teststring, "--qos=standby"), "--qos=standby --time=03:30:00")
   expect_identical(combine_slurmConfig(teststring, "--bla=blub"), paste("--bla=blub", teststring))
+  expect_identical(combine_slurmConfig(teststring, "--wait"), paste("--wait", teststring))
   teststring <- "--qos=priority --wait"
   expect_identical(combine_slurmConfig(teststring, "--qos=standby"), "--qos=standby --wait")
 })