diff --git a/DESCRIPTION b/DESCRIPTION index ed5811928..9af8641c3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -54,7 +54,7 @@ Imports: piamInterfaces (>= 0.12.18), plotly, purrr, - quitte (>= 0.3123.0), + quitte (>= 0.3128.4), R.utils, raster, readr, diff --git a/config/default.cfg b/config/default.cfg index 8e6ac91d1..daf8a5ae2 100644 --- a/config/default.cfg +++ b/config/default.cfg @@ -49,6 +49,12 @@ cfg$runstatistics <- "/p/projects/rd3mod/models/statistics/remind" # copied into remind when starting a run and required to calculate climate responses (e.g. temperature). cfg$magicc_template <- "/p/projects/rd3mod/magicc/" +#### Folder containing files for climate-assessment that cannot be published with REMIND +# These files currently all have permissive licenses and can be easily found online +cfg$climate_assessment_root <- "/p/projects/rd3mod/python/climate-assessment/src/" +cfg$climate_assessment_files_dir <- "/p/projects/rd3mod/climate-assessment-files/" +cfg$climate_assessment_magicc_bin <- "/p/projects/rd3mod/climate-assessment-files/magicc-v7.5.3/bin/magicc" + #### Output folder of the modeltests generated by running config/scenario_config_AMT.csv # start.R searches in this folder for finished runs to be used as path_gdx... cfg$modeltests_folder <- "/p/projects/remind/modeltests/remind/output" @@ -110,7 +116,7 @@ cfg$logoption <- 2 # Just list the name of the output scripts that should be used by output.R # At the moment there are several R-scripts located in scripts/output/ -cfg$output <- c("reporting","reportCEScalib","rds_report","fixOnRef") #,"validation","emulator","reportCEScalib","validationSummary","dashboard") +cfg$output <- c("reporting","reportCEScalib","rds_report","fixOnRef") #"ar6Climate","validation","emulator","reportCEScalib","validationSummary","dashboard") # Set the format for the results folder, type string :date: in order to use the current time stamp in the folder name (e.g. "results:date:") use :title: to use the current title in the folder name cfg$results_folder <- "output/:title::date:" diff --git a/output.R b/output.R index b8030a4c4..946901025 100755 --- a/output.R +++ b/output.R @@ -67,7 +67,8 @@ library(lucode2) library(gms) require(stringr, quietly = TRUE) -source("scripts/start/isSlurmAvailable.R") +# Import all functions from the scripts/start folder +invisible(sapply(list.files("scripts/start", pattern = "\\.R$", full.names = TRUE), source)) flags <- NULL ### Define arguments that can be read from command line @@ -83,7 +84,7 @@ if ("--help" %in% flags) { q() } -choose_slurmConfig_output <- function(slurmExceptions = NULL) { +choose_slurmConfig_output <- function(output) { slurm_options <- c("--qos=priority", "--qos=short", "--qos=standby", "--qos=priority --mem=16000", "--qos=short --mem=16000", "--qos=standby --mem=16000", "--qos=priority --mem=32000") @@ -91,9 +92,11 @@ choose_slurmConfig_output <- function(slurmExceptions = NULL) { if (!isSlurmAvailable()) return("direct") - if (!is.null(slurmExceptions)) { - slurm_options <- grep(slurmExceptions, slurm_options, value = TRUE) - } + # Modify slurm options for ar6 reporting, since we want to run MAGICC in parallel and we'll need a lot of memory + if ("ar6Climate" %in% output) slurm_options <- paste(slurm_options[1:3], "--tasks-per-node=12 --mem=32000") + # reporting.R, in particular remind2::convGDX2MIF, requires at least --mem=8000 of memory + if ("reporting" %in% output) slurm_options <- grep("--mem=[0-9]*[0-9]{3}", slurm_options, value = TRUE) + if (length(slurm_options) == 1) { return(slurm_options[[1]]) } @@ -176,7 +179,7 @@ if (comp %in% c("comparison", "export")) { # choose the slurm options. If you use command line arguments, use slurmConfig=priority or standby modules_using_slurmConfig <- c("compareScenarios2") if (!exists("slurmConfig") && any(modules_using_slurmConfig %in% output)) { - slurmConfig <- choose_slurmConfig_output() + slurmConfig <- choose_slurmConfig_output(output = output) } if (exists("slurmConfig")) { if (slurmConfig %in% c("priority", "short", "standby")) { @@ -207,11 +210,9 @@ if (comp %in% c("comparison", "export")) { } } } else { # comp = single - # define slurm class or direct execution + # define slurm class or direct execution outputInteractive <- c("plotIterations", "fixOnRef", "integratedDamageCosts") if (! exists("source_include")) { - # for selected output scripts, only slurm configurations matching these regex are available - slurmExceptions <- if ("reporting" %in% output) "--mem=[0-9]*[0-9]{3}" else NULL if (any(output %in% outputInteractive)) { slurmConfig <- "direct" flags <- c(flags, "--interactive") # to tell scripts they can run in interactive mode @@ -219,8 +220,8 @@ if (comp %in% c("comparison", "export")) { # if this script is not being sourced by another script but called from the command line via Rscript let the user # choose the slurm options if (!exists("slurmConfig")) { - slurmConfig <- choose_slurmConfig_output(slurmExceptions = slurmExceptions) - if (slurmConfig != "direct") slurmConfig <- paste(slurmConfig, "--nodes=1 --tasks-per-node=1") + slurmConfig <- choose_slurmConfig_output(output = output) + if (slurmConfig != "direct") slurmConfig <- combine_slurmConfig("--nodes=1 --tasks-per-node=1", slurmConfig) } if (slurmConfig %in% c("priority", "short", "standby")) { slurmConfig <- paste0("--qos=", slurmConfig, " --nodes=1 --tasks-per-node=1") diff --git a/scripts/output/single/ar6Climate.R b/scripts/output/single/ar6Climate.R new file mode 100644 index 000000000..ad1ddd5d9 --- /dev/null +++ b/scripts/output/single/ar6Climate.R @@ -0,0 +1,232 @@ +# | (C) 2006-2023 Potsdam Institute for Climate Impact Research (PIK) +# | authors, and contributors see CITATION.cff file. This file is part +# | of REMIND and licensed under AGPL-3.0-or-later. Under Section 7 of +# | AGPL-3.0, you are granted additional permissions described in the +# | REMIND License Exception, version 1.0 (see LICENSE file). +# | Contact: remind@pik-potsdam.de + +#' @title AR6 climate assessment +#' @description Assessment of new emissions pathways consistent with the climate variable data from the working group +#' III contribution to the IPCC Sixth Assessment (AR6) report. The script uses REMIND emissions data to perform +#' harmonization and infilling based on the IIASA [climate-assessment](https://climate-assessment.readthedocs.io) +#' package. The harmonized and infilled emissions are then used as input to the +#' [MAGICC7 simple climate model](https://magicc.org/). Results are postprocessed and appended to the REMIND model +#' intercomparison file (MIF). +#' +#' @author Gabriel Abrahao, Oliver Richters, Tonn Rüter +#' +#' @param outputdir Directory where REMIND MIF file is located. Output files generated in the process will be written +#' to a subfolder "climate-assessment-data" in this directory. Defaults to "." +#' @param gdxName Name of the REMIND GDX file. Defaults to "fulldata.gdx" + +library(madrat) +library(remind2) +library(quitte) +library(piamInterfaces) +library(lucode2) +library(yaml) +library(tidyverse) +library(readr) +library(stringr) + +############################# BASIC CONFIGURATION ############################# + +gdxName <- "fulldata.gdx" # name of the gdx +cfgName <- "cfg.txt" # cfg file for getting file paths + +if (!exists("source_include")) { + # Define arguments that can be read from command line + outputdir <- "." + readArgs("outputdir", "gdxName", "gdx_ref_name", "gdx_refpolicycost_name") +} + +cfgPath <- file.path(outputdir, cfgName) +logFile <- file.path(outputdir, "log_climate.txt") # specific log for python steps +scenario <- getScenNames(outputdir) +remindReportingFile <- file.path(outputdir, paste0("REMIND_generic_", scenario, ".mif")) +climateAssessmentEmi <- normalizePath(file.path(outputdir, paste0("ar6_climate_assessment_", scenario, ".csv")), + mustWork = FALSE) +climateAssessmentYaml <- file.path(system.file(package = "piamInterfaces"), + "iiasaTemplates", "climate_assessment_variables.yaml") + +############################# PREPARING EMISSIONS INPUT ############################# + +logmsg <- paste0( + date(), " =================== CONFIGURATION STARTED ==================================\n", + " outputdir = '", outputdir, "' exists? ", file.exists(outputdir), "\n", + " cfgPath = '", cfgPath, "' exists? ", file.exists(cfgPath), "\n", + " logFile = '", logFile, "' exists? ", file.exists(logFile), "\n", + " scenario = '", scenario, "'\n", + " remindReportingFile = '", remindReportingFile, "' exists? ", file.exists(remindReportingFile), "\n", + " climateAssessmentYaml = '", climateAssessmentYaml, "' exists? ", file.exists(climateAssessmentYaml), "\n", + " climateAssessmentEmi = '", climateAssessmentEmi, "' exists? ", file.exists(climateAssessmentEmi), "\n", + date(), " =================== EXTRACT REMIND emission data ===========================\n", + " ar6Climate.R: Extracting REMIND emission data\n" +) +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +climateAssessmentInputData <- as.quitte(remindReportingFile) %>% + # Consider only the global region + filter(region %in% c("GLO", "World")) %>% + # Extract only the variables needed for climate-assessment. These are provided from the iiasaTemplates in the + # piamInterfaces package. See also: + # https://github.com/pik-piam/piamInterfaces/blob/master/inst/iiasaTemplates/climate_assessment_variables.yaml + generateIIASASubmission( + mapping = "AR6", + outputFilename = NULL, + iiasatemplate = climateAssessmentYaml, + logFile = logFile + ) %>% + mutate(region = factor("World")) %>% + # Rename the columns using str_to_title which capitalizes the first letter of each word + rename_with(str_to_title) %>% + # Transforms the yearly values for each variable from a long to a wide format. The resulting data frame then has + # one column for each year and one row for each variable + pivot_wider(names_from = "Period", values_from = "Value") %>% + write_csv(climateAssessmentEmi, quote = "none") + +logmsg <- paste0( + date(), " ar6Climate.R: Wrote REMIND emission data to '", climateAssessmentEmi, "' for climate-assessment\n" +) +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +############################# PYTHON/MAGICC SETUP ############################# + +# Read the cfg to get the location of MAGICC-related files +cfg <- read_yaml(cfgPath) + +if (is.null(cfg$climate_assessment_root)) cfg$climate_assessment_root <- "/p/projects/rd3mod/python/climate-assessment/src/" +if (is.null(cfg$climate_assessment_files_dir)) cfg$climate_assessment_files_dir <- "/p/projects/rd3mod/climate-assessment-files/" +if (is.null(cfg$cfg$climate_assessment_magicc_bin)) cfg$climate_assessment_magicc_bin <- "/p/projects/rd3mod/climate-assessment-files/magicc-v7.5.3/bin/magicc" + +# All climate-assessment files will be written to this folder +climateAssessmentFolder <- normalizePath(file.path(outputdir, "climate-assessment-data")) +dir.create(climateAssessmentFolder, showWarnings = FALSE) + +# The base name, that climate-assessment uses to derive it's output names +baseFileName <- sub("\\.csv$", "", basename(climateAssessmentEmi)) + +# These files are supposed to be all inside cfg$climate_assessment_files_dir in a certain structure +probabilisticFile <- normalizePath(file.path( + cfg$climate_assessment_files_dir, + "parsets", "0fd0f62-derived-metrics-id-f023edb-drawnset.json" +)) +infillingDatabaseFile <- normalizePath(file.path( + cfg$climate_assessment_files_dir, + "1652361598937-ar6_emissions_vetted_infillerdatabase_10.5281-zenodo.6390768.csv" +)) + +# Extract the location of the climate-assessment scripts and the MAGICC binary from cfg.txt +scriptsFolder <- normalizePath(file.path(cfg$climate_assessment_root, "scripts")) +magiccBinFile <- normalizePath(file.path(cfg$climate_assessment_magicc_bin)) +magiccWorkersFolder <- file.path(normalizePath(climateAssessmentFolder), "workers") + +# Read parameter sets file to ascertain how many parsets there are +allparsets <- read_yaml(probabilisticFile) +nparsets <- length(allparsets$configurations) + +logmsg <- paste0(date(), " =================== SET UP climate-assessment scripts environment ===================\n") +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +# Create working folder for climate-assessment files +dir.create(magiccWorkersFolder, recursive = TRUE, showWarnings = FALSE) + +# Set relevant environment variables and create a MAGICC worker directory +Sys.setenv(MAGICC_EXECUTABLE_7 = magiccBinFile) +Sys.setenv(MAGICC_WORKER_ROOT_DIR = magiccWorkersFolder) # Has to be an absolute path +Sys.setenv(MAGICC_WORKER_NUMBER = 1) # TODO: Get this from slurm or nproc + +# Specify the commands to (de-)activate the venv & run the harmonization/infilling/model scripts +# TODO: This makes assumptions about the users climate-assessment installation. There are a couple of options: +# A) Remove entirely and assume that the user has set up their environment correctly +# B) Make this more flexible by explictly setting them in default.cfg +#activate_venv_cmd <- paste("source", normalizePath(file.path(cfg$climate_assessment_root, "..", "venv", "bin", "activate"))) +#deactivate_venv_cmd <- "deactivate" + +run_harm_inf_cmd <- paste( + "python", file.path(scriptsFolder, "run_harm_inf.py"), + climateAssessmentEmi, + climateAssessmentFolder, + "--no-inputcheck", + "--infilling-database", infillingDatabaseFile +) + +run_clim_cmd <- paste( + "python", file.path(scriptsFolder, "run_clim.py"), + normalizePath(file.path(climateAssessmentFolder, paste0(baseFileName, "_harmonized_infilled.csv"))), + climateAssessmentFolder, + "--num-cfgs", nparsets, + "--scenario-batch-size", 1, + "--probabilistic-file", probabilisticFile +) + +logmsg <- paste0( + " climateAssessmentFolder = '", climateAssessmentFolder, "' exists? ", dir.exists(climateAssessmentFolder), "\n", + " baseFileName = '", baseFileName, "'\n", + " probabilisticFile = '", probabilisticFile, "' exists? ", file.exists(probabilisticFile), "\n", + " infillingDatabaseFile = '", infillingDatabaseFile, "' exists? ", file.exists(infillingDatabaseFile), "\n", + " scriptsFolder = '", scriptsFolder, "' exists? ", dir.exists(scriptsFolder), "\n", + " magiccBinFile = '", magiccBinFile, "' exists? ", file.exists(magiccBinFile), "\n", + " magiccWorkersFolder = '", magiccWorkersFolder, "' exists? ", dir.exists(magiccWorkersFolder), "\n\n", + " ENVIRONMENT VARIABLES:\n", + " MAGICC_EXECUTABLE_7 = ", Sys.getenv("MAGICC_EXECUTABLE_7") ,"\n", + " MAGICC_WORKER_ROOT_DIR = ", Sys.getenv("MAGICC_WORKER_ROOT_DIR") ,"\n", + " MAGICC_WORKER_NUMBER = ", Sys.getenv("MAGICC_WORKER_NUMBER") ,"\n", + date(), " =================== RUN climate-assessment infilling & harmonization ===================\n", + run_harm_inf_cmd, "'\n" +) +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +############################# HARMONIZATION/INFILLING ############################# + +system(run_harm_inf_cmd) + +logmsg <- paste0(date(), " Done with harmonization & infilling\n") +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +############################# RUNNING MODEL ############################# + +logmsg <- paste0( + date(), " Found ", nparsets, " nparsets, start climate-assessment model runs\n", run_harm_inf_cmd, "\n", + date(), " =================== RUN climate-assessment model ============================\n", + run_clim_cmd, "'\n" +) +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +system(run_clim_cmd) + +############################# POSTPROCESS CLIMATE OUTPUT ############################# +climateAssessmentOutput <- file.path( + climateAssessmentFolder, + paste0(baseFileName, "_harmonized_infilled_IAMC_climateassessment.xlsx") +) + +logmsg <- paste0( + date(), " climate-assessment finished\n", + date(), " =================== POSTPROCESS climate-assessment output ==================\n", + " climateAssessmentOutput = '", climateAssessmentOutput, "'\n" +) +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE) + +############################# APPEND TO REMIND MIF ############################# +# Filter only periods used in REMIND, so that it doesn't expand the original mif +usePeriods <- as.numeric(grep("[0-9]+", quitte::read_mif_header(remindReportingFile)$header, value = TRUE)) + +climateAssessmentData <- read.quitte(climateAssessmentOutput) %>% + filter(period %in% usePeriods) %>% + interpolate_missing_periods(usePeriods, expand.values = FALSE) %>% + write.mif(remindReportingFile, append = TRUE) + +logmsg <- paste0( + date(), " postprocessing done! Results appended to REMIND mif '", remindReportingFile, "'\n", + "ar6Climate.R finished\n" +) +cat(logmsg) +capture.output(cat(logmsg), file = logFile, append = TRUE)