From a4d4be8fd6cd95136481eed26711f566bfe84dd1 Mon Sep 17 00:00:00 2001 From: orichters Date: Fri, 21 Jul 2023 11:08:18 +0200 Subject: [PATCH] comments for checkFixCfg, in test-coupled send mails only for fails --- .../tests/scenario_config_coupled_shortCascade.csv | 6 +++--- scripts/start/checkFixCfg.R | 11 +++++++++++ scripts/start/combine_slurmConfig.R | 4 ++-- start_bundle_coupled.R | 12 ++++++------ start_coupled.R | 9 +++++---- tests/testthat/test_01-combine_slurmConfig.R | 5 +++++ 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/config/tests/scenario_config_coupled_shortCascade.csv b/config/tests/scenario_config_coupled_shortCascade.csv index 9b3548da7..d9b99f969 100644 --- a/config/tests/scenario_config_coupled_shortCascade.csv +++ b/config/tests/scenario_config_coupled_shortCascade.csv @@ -1,4 +1,4 @@ title;start;qos;sbatch;magpie_scen;magpie_empty;no_ghgprices_land_until;max_iterations;oldrun;path_gdx;path_gdx_ref;path_gdx_bau;path_report;cm_nash_autoconverge_lastrun;path_mif_ghgprice_land -TESTTHAT-SSP2EU-Base;1;auto;--wait;SSP2|NPI;TRUE;y2150;2;;;;;;2; -TESTTHAT-SSP2EU-NDC;1;auto;--wait;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base -TESTTHAT-SSP2EU-Policy;2;auto;--wait;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif +TESTTHAT-SSP2EU-Base;1;auto;--wait --mail-type=FAIL;SSP2|NPI;TRUE;y2150;2;;;;;;2; +TESTTHAT-SSP2EU-NDC;1;auto;--wait --mail-type=FAIL;SSP2|NDC;TRUE;y2150;2;;;;;;2;TESTTHAT-SSP2EU-Base +TESTTHAT-SSP2EU-Policy;2;auto;--wait --mail-type=FAIL;SSP2|NDC;TRUE;y2150;2;TESTTHAT-SSP2EU-Base;;;;;;output/C_TESTTHAT-SSP2EU-Base-rem-1/REMIND_generic_C_TESTTHAT-SSP2EU-Base-rem-1.mif diff --git a/scripts/start/checkFixCfg.R b/scripts/start/checkFixCfg.R index be2cbe878..e62f5575e 100644 --- a/scripts/start/checkFixCfg.R +++ b/scripts/start/checkFixCfg.R @@ -8,23 +8,34 @@ checkFixCfg <- function(cfg, remindPath = ".", testmode = FALSE) { errorsfound <- 0 + # extract all instances of 'regexp' from main.gms code <- system(paste0("grep regexp ", file.path(remindPath, "main.gms")), intern = TRUE) + # this is used to replace all 'regexp = is.numeric' grepisnum <- "((\\+|-)?[0-9]*([0-9]\\.?|\\.?[0-9])[0-9]*)" + # some simple tests + stopifnot(all(grepl(paste0("^", grepisnum, "$"), c("2", "2.2", "32.", "+32.", "+.05", "-0.5", "-.5", "-5", "-7.")))) + stopifnot(all(! grepl(paste0("^", grepisnum, "$"), c("2.2.", "0a", "1e1", ".2.", "ab", "2.3a", "--a", "++2")))) for (n in names(cfg$gms)) { errormsg <- NULL + # how parameter n is defined in main.gms paramdef <- paste0("^([ ]*", n, "[ ]*=|\\$setglobal[ ]+", n, " )") + # filter fitting parameter definition from code snippets containing regexp filtered <- grep(paste0(paramdef, ".*regexp[ ]*=[ ]*"), code, value = TRUE) if (length(filtered) == 1) { # search for string '!! regexp = whatever', potentially followed by '!! otherstuff' and extract 'whatever' regexp <- paste0("^(", trimws(gsub("!!.*", "", gsub("^.*regexp[ ]*=", "", filtered))), ")$") + # replace is.numeric by pattern defined above useregexp <- gsub("is.numeric", grepisnum, regexp, fixed = TRUE) + # check whether parameter value fits regular expression if (! grepl(useregexp, cfg$gms[[n]])) { errormsg <- paste0("Parameter cfg$gms$", n, "=", cfg$gms[[n]], " does not fit this regular expression: ", regexp) } } else if (length(filtered) > 1) { + # fail if more than one regexp found for parameter errormsg <- paste0("More than one regexp found for ", n, ". These are the code lines:\n", paste(filtered, collapse = "\n")) } + # count errors if (! is.null(errormsg)) { errorsfound <- errorsfound + 1 if (testmode) warning(errormsg) else message(errormsg) diff --git a/scripts/start/combine_slurmConfig.R b/scripts/start/combine_slurmConfig.R index c061cf783..eab0d526f 100644 --- a/scripts/start/combine_slurmConfig.R +++ b/scripts/start/combine_slurmConfig.R @@ -12,8 +12,8 @@ combine_slurmConfig <- function(original, update_with) { # trim whitespaces - original <- trimws(original) - update_with <- trimws(update_with) + original <- trimws(toString(original)) + update_with <- trimws(toString(update_with)) # remove double whitespaces original <- gsub("\\s+", " ", original) diff --git a/start_bundle_coupled.R b/start_bundle_coupled.R index fc870858f..78987df42 100755 --- a/start_bundle_coupled.R +++ b/start_bundle_coupled.R @@ -620,12 +620,12 @@ for (scen in common) { sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j'"), intern = TRUE) runEnv$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short" } - slurm_command <- paste0("sbatch --qos=", runEnv$qos, " --job-name=", fullrunname, - " --output=", logfile, " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks, - if (runEnv$numberOfTasks == 1) " --mem=8000", " ", runEnv$sbatch, - " ", runEnv$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"") - message(slurm_command) - exitCode <- system(slurm_command) + slurmOptions <- combine_slurmConfig(paste0("--qos=", runEnv$qos, " --job-name=", fullrunname, " --output=", logfile, + " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", runEnv$numberOfTasks, + if (runEnv$numberOfTasks == 1) " --mem=8000"), runEnv$sbatch) + slurmCommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", Rdatafile, "\"") + message(slurmCommand) + exitCode <- system(slurmCommand) if (0 < exitCode) { errorsfound <- errorsfound + 1 message("sbatch command failed, check logs.") diff --git a/start_coupled.R b/start_coupled.R index cfe81e670..31a6c3907 100644 --- a/start_coupled.R +++ b/start_coupled.R @@ -18,6 +18,7 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m require(gdx) library(methods) library(remind2) + source("scripts/start/combine_slurmConfig.R") errorsfound <- 0 # delete entries in stack that contain needle and append new @@ -250,10 +251,10 @@ start_coupled <- function(path_remind, path_magpie, cfg_rem, cfg_mag, runname, m sq <- system(paste0("squeue -u ", Sys.info()[["user"]], " -o '%q %j' | grep -v ", fullrunname), intern = TRUE) subseq.env$qos <- if (is.null(attr(sq, "status")) && sum(grepl("^priority ", sq)) < 4) "priority" else "short" } - subsequentcommand <- paste0("sbatch --qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile, - " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks, - if (subseq.env$numberOfTasks == 1) " --mem=8000", - " ", subseq.env$sbatch, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"") + slurmOptions <- combine_slurmConfig(paste0("--qos=", subseq.env$qos, " --job-name=", subseq.env$fullrunname, " --output=", logfile, + " --mail-type=END --comment=REMIND-MAgPIE --tasks-per-node=", subseq.env$numberOfTasks, + if (subseq.env$numberOfTasks == 1) " --mem=8000"), subseq.env$sbatch) + subsequentcommand <- paste0("sbatch ", slurmOptions, " --wrap=\"Rscript start_coupled.R coupled_config=", RData_file, "\"") message(subsequentcommand) if (length(needfulldatagdx) > 0) { exitCode <- system(subsequentcommand) diff --git a/tests/testthat/test_01-combine_slurmConfig.R b/tests/testthat/test_01-combine_slurmConfig.R index 89b1e5c5f..5933079a7 100644 --- a/tests/testthat/test_01-combine_slurmConfig.R +++ b/tests/testthat/test_01-combine_slurmConfig.R @@ -6,8 +6,13 @@ # | Contact: remind@pik-potsdam.de test_that("combine_slurmConfig works", { teststring <- "--qos=priority --time=03:30:00" + expect_identical(combine_slurmConfig(teststring, teststring), teststring) + expect_identical(combine_slurmConfig(teststring, NULL), teststring) + expect_identical(combine_slurmConfig(NULL, NULL), "") + expect_identical(combine_slurmConfig(teststring, ""), teststring) expect_identical(combine_slurmConfig(teststring, "--qos=standby"), "--qos=standby --time=03:30:00") expect_identical(combine_slurmConfig(teststring, "--bla=blub"), paste("--bla=blub", teststring)) + expect_identical(combine_slurmConfig(teststring, "--wait"), paste("--wait", teststring)) teststring <- "--qos=priority --wait" expect_identical(combine_slurmConfig(teststring, "--qos=standby"), "--qos=standby --wait") })