merge trunk

pik-piam · Sep 28, 2023 · 2dd9a32 · 2dd9a32
2 parents 0f82b62 + c11cd64
commit 2dd9a32
Show file tree

Hide file tree

Showing 9 changed files with 202 additions and 42 deletions.
diff --git a/.buildlibrary b/.buildlibrary
@@ -1,9 +1,10 @@
-ValidationKey: '219183712'
+ValidationKey: '219654192'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'
 - .*qpdf.* is needed for checks on size reduction of PDFs
 - .*following variables are expected in the piamInterfaces.*
+- Summation checks have revealed some gaps!*
 AcceptedNotes:
 - Imports includes .* non-default packages.
 - unable to verify current time

diff --git a/CITATION.cff b/CITATION.cff
@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'remind2: The REMIND R package (2nd generation)'
-version: 1.117.6
-date-released: '2023-09-12'
+version: 1.119.2
+date-released: '2023-09-26'
 abstract: Contains the REMIND-specific routines for data and model output manipulation.
 authors:
 - family-names: Rodrigues

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Type: Package
 Package: remind2
 Title: The REMIND R package (2nd generation)
-Version: 1.117.6
-Date: 2023-09-12
+Version: 1.119.2
+Date: 2023-09-26
 Authors@R: c(
     person("Renato", "Rodrigues", , "[email protected]", role = c("aut", "cre")),
     person("Lavinia", "Baumstark", role = "aut"),
@@ -63,7 +63,7 @@ Imports:
     madrat,
     mip (>= 0.139.1),
     openxlsx,
-    piamInterfaces (>= 0.0.102),
+    piamInterfaces (>= 0.1.2),
     plotly,
     quitte,
     readr,

diff --git a/NAMESPACE b/NAMESPACE
@@ -250,6 +250,7 @@ importFrom(openxlsx,createStyle)
 importFrom(openxlsx,createWorkbook)
 importFrom(openxlsx,saveWorkbook)
 importFrom(openxlsx,writeData)
+importFrom(piamInterfaces,checkSummations)
 importFrom(plotly,config)
 importFrom(plotly,ggplotly)
 importFrom(plotly,hide_legend)
@@ -309,6 +310,7 @@ importFrom(utils,globalVariables)
 importFrom(utils,read.csv)
 importFrom(utils,read.csv2)
 importFrom(utils,tail)
+importFrom(utils,write.csv)
 importFrom(utils,write.table)
 importFrom(withr,defer)
 importFrom(withr,local_connection)
diff --git a/R/compareScenarios2.R b/R/compareScenarios2.R
@@ -150,15 +150,23 @@ compareScenarios2 <- function(
   } else if (outputFormat == "rmd") {
     return(.compareScenarios2Rmd(yamlParams, outputDir, outputFile))
   }
+
+  # copy the template directory from the package to the outputDir because rmarkdown writes to the folder
+  # containing the template.
+  templateInOutputDir <- file.path(outputDir, "compareScenarios2", "cs2_main.Rmd")
+  file.copy(system.file("markdown/compareScenarios2/", package = "remind2"),
+            outputDir, recursive = TRUE)
+
   rmarkdown::render(
-    system.file("markdown/compareScenarios2/cs2_main.Rmd", package = "remind2"),
+    templateInOutputDir,
     intermediates_dir = outputDir,
     output_dir = outputDir,
     output_file = outputFile,
     output_format = outputFormat,
     params = yamlParams,
     envir = envir,
     quiet = quiet)
+  unlink(file.path(outputDir, "compareScenarios2"), recursive = TRUE)
 }
 
 # Copies the CompareScenarios2-Rmds to the specified location and modifies

diff --git a/R/convGDX2MIF.R b/R/convGDX2MIF.R
@@ -19,8 +19,11 @@
 #' \dontrun{convGDX2MIF(gdx,gdx_refpolicycost,file="REMIND_generic_default.csv",scenario="default")}
 #'
 #' @export
+#' @importFrom dplyr %>% bind_rows filter
 #' @importFrom gdx readGDX
 #' @importFrom magclass mbind write.report
+#' @importFrom piamInterfaces checkSummations
+#' @importFrom utils write.csv
 
 convGDX2MIF <- function(gdx, gdx_ref = NULL, file = NULL, scenario = "default",
                         t = c(seq(2005, 2060, 5), seq(2070, 2110, 10), 2130, 2150),
@@ -117,11 +120,31 @@ convGDX2MIF <- function(gdx, gdx_ref = NULL, file = NULL, scenario = "default",
 
   checkVariableNames(getNames(output, dim = 3))
 
-  sumChecks <- piamInterfaces::checkSummations(
+  .reportSummationErrors <- function(msg) {
+    if (!any(grepl('All summation checks were fine', msg)))
+      message(paste(msg, collapse = '\n'))
+  }
+
+  capture.output(
+    sumChecks <- checkSummations(
+      mifFile = output, outputDirectory = NULL,
+      summationsFile = "extractVariableGroups",
+      absDiff = 1.5e-8, relDiff = 1e-8, roundDiff = TRUE
+    ) %>%
+      filter(abs(.data$diff) >= 1.5e-8),
+    type = 'message') %>%
+    .reportSummationErrors()
+
+  capture.output(sumChecks <- checkSummations(
     mifFile = output, outputDirectory = NULL,
-    summationsFile = "extractVariableGroups",
-    absDiff = 1.5e-8, relDiff = 1e-8, roundDiff = TRUE
-  ) %>% filter(abs(!!sym("diff")) >= 1.5e-8)
+    summationsFile = system.file('extdata/additional_summation_checks.csv',
+                                 package = 'remind2'),
+    absDiff = 1.5e-8, relDiff = 1e-8, roundDiff = TRUE) %>%
+      filter(abs(.data$diff) >= 1.5e-8) %>%
+      bind_rows(sumChecks),
+    type = 'message'
+  ) %>%
+    .reportSummationErrors()
 
   # either write the *.mif or return the magpie object
   if (!is.null(file)) {
@@ -131,10 +154,10 @@ convGDX2MIF <- function(gdx, gdx_ref = NULL, file = NULL, scenario = "default",
 
     # write additional file on summation errors if needed
     if (nrow(sumChecks) > 0) {
-      summation_errors_file <- sub('(\\.[^.]+)$', '_summation_errors\\1', file)
+      summation_errors_file <- sub('(\\.[^.]+)$', '_summation_errors.csv', file)
       warning("Summation checks have revealed some gaps! See file ",
               summation_errors_file)
-      write.table(sumChecks, summation_errors_file, quote = FALSE, sep = ';')
+      write.csv(sumChecks, summation_errors_file, quote = FALSE, row.names = FALSE)
     }
   }
   else {

diff --git a/R/reportEmi.R b/R/reportEmi.R
@@ -626,30 +626,67 @@ reportEmi <- function(gdx, output = NULL, regionSubsetList = NULL, t = c(seq(200
     variable_postfix <- ' (Mt CO2/yr)'
 
     mixer <- tribble(
-      ~variable,                    ~secInd37,     ~all_enty1,
-      '+|Solids',                   NULL,          'fesos',
-      '+|Liquids',                  NULL,          'fehos',
-      '+|Gases',                    NULL,          'fegas',
-
-      '++|Cement',                  'cement',      NULL,
-      'Cement|+|Solids',            'cement',      'fesos',
-      'Cement|+|Liquids',           'cement',      'fehos',
-      'Cement|+|Gases',             'cement',      'fegas',
-
-      '++|Chemicals',               'chemicals',   NULL,
-      'Chemicals|+|Solids',         'chemicals',   'fesos',
-      'Chemicals|+|Liquids',        'chemicals',   'fehos',
-      'Chemicals|+|Gases',          'chemicals',   'fegas',
-
-      '++|Steel',                   'steel',       NULL,
-      'Steel|+|Solids',             'steel',       'fesos',
-      'Steel|+|Liquids',            'steel',       'fehos',
-      'Steel|+|Gases',              'steel',       'fegas',
-
-      '++|Other Industry',          'otherInd',    NULL,
-      'Other Industry|+|Solids',    'otherInd',    'fesos',
-      'Other Industry|+|Liquids',   'otherInd',    'fehos',
-      'Other Industry|+|Gases',     'otherInd',    'fegas') %>%
+      ~variable,                            ~secInd37,     ~all_enty,    ~all_enty1,
+      '+|Solids',                           NULL,          NULL,         'fesos',
+      '+|Liquids',                          NULL,          NULL,         'fehos',
+      '+|Gases',                            NULL,          NULL,         'fegas',
+
+      'Solids|+|Fossil',                    NULL,          'sesofos',    'fesos',
+      'Solids|+|Biomass',                   NULL,          'sesobio',    'fesos',
+      'Liquids|+|Fossil',                   NULL,          'seliqfos',   'fehos',
+      'Liquids|+|Biomass',                  NULL,          'seliqbio',   'fehos',
+      'Liquids|+|Synfuel',                  NULL,          'seliqsyn',   'fehos',
+      'Gases|+|Fossil',                     NULL,          'segafos',    'fegas',
+      'Gases|+|Biomass',                    NULL,          'segabio',    'fegas',
+      'Gases|+|Synfuel',                    NULL,          'segasyn',    'fegas',
+
+      '++|Cement',                          'cement',      NULL,         NULL,
+      'Cement|+|Solids',                    'cement',      NULL,         'fesos',
+      'Cement|Solids|+|Fossil',             'cement',      'sesofos',    'fesos',
+      'Cement|Solids|+|Biomass',            'cement',      'sesobio',    'fesos',
+      'Cement|+|Liquids',                   'cement',      NULL,         'fehos',
+      'Cement|Liquids|+|Fossil',            'cement',      'seliqfos',   'fehos',
+      'Cement|Liquids|+|Biomass',           'cement',      'seliqbio',   'fehos',
+      'Cement|Liquids|+|Synfuel',           'cement',      'seliqsyn',   'fehos',
+      'Cement|+|Gases',                     'cement',      NULL,         'fegas',
+      'Cement|Gases|+|Fossil',              'cement',      'segafos',    'fegas',
+      'Cement|Gases|+|Biomass',             'cement',      'segabio',    'fegas',
+      'Cement|Gases|+|Synfuel',             'cement',      'segasyn',    'fegas',
+
+      '++|Chemicals',                       'chemicals',   NULL,         NULL,
+      'Chemicals|+|Solids',                 'chemicals',   NULL,         'fesos',
+      'Chemicals|Solids|+|Fossil',          'chemicals',   'sesofos',    'fesos',
+      'Chemicals|Solids|+|Biomass',         'chemicals',   'sesobio',    'fesos',
+      'Chemicals|+|Liquids',                'chemicals',   NULL,         'fehos',
+      'Chemicals|Liquids|+|Fossil',         'chemicals',   'seliqfos',   'fehos',
+      'Chemicals|Liquids|+|Biomass',        'chemicals',   'seliqbio',   'fehos',
+      'Chemicals|Liquids|+|Synfuel',        'chemicals',   'seliqsyn',   'fehos',
+      'Chemicals|+|Gases',                  'chemicals',   NULL,         'fegas',
+      'Chemicals|Gases|+|Fossil',           'chemicals',   'segafos',    'fegas',
+      'Chemicals|Gases|+|Biomass',          'chemicals',   'segabio',    'fegas',
+      'Chemicals|Gases|+|Synfuel',          'chemicals',   'segasyn',    'fegas',
+
+      '++|Steel',                           'steel',       NULL,         NULL,
+      'Steel|+|Solids',                     'steel',       NULL,         'fesos',
+      'Steel|Solids|+|Fossil',              'steel',       'sesofos',    'fesos',
+      'Steel|Solids|+|Biomass',             'steel',       'sesobio',    'fesos',
+      'Steel|+|Liquids',                    'steel',       NULL,         'fehos',
+      'Steel|Liquids|+|Fossil',             'steel',       'seliqfos',   'fehos',
+      'Steel|Liquids|+|Biomass',            'steel',       'seliqbio',   'fehos',
+      'Steel|Liquids|+|Synfuel',            'steel',       'seliqsyn',   'fehos',
+      'Steel|+|Gases',                      'steel',       NULL,         'fegas',
+      'Steel|Gases|+|Fossil',               'steel',       'segafos',    'fegas',
+      'Steel|Gases|+|Biomass',              'steel',       'segabio',    'fegas',
+      'Steel|Gases|+|Synfuel',              'steel',       'segasyn',    'fegas',
+
+      '++|Other Industry',                  'otherInd',    NULL,         NULL,
+      'Other Industry|+|Solids',            'otherInd',    NULL,         'fesos',
+      'Other Industry|Solids|+|Fossil',     'otherInd',    'sesofos',    'fesos',
+      'Other Industry|+|Liquids',           'otherInd',    NULL,         'fehos',
+      'Other Industry|Liquids|+|Fossil',    'otherInd',    'seliqfos',   'fehos',
+      'Other Industry|+|Gases',             'otherInd',    NULL,         'fegas',
+      'Other Industry|Gases|+|Fossil',      'otherInd',    'segafos',    'fegas'
+) %>%
       mutate(
         variable = paste0(variable_prefix, .data$variable, variable_postfix))
 
@@ -663,14 +700,15 @@ reportEmi <- function(gdx, output = NULL, regionSubsetList = NULL, t = c(seq(200
           ( dimSums(mselect(EmiIndSubSec, x[setdiff(names(x), 'variable')]),
                     dim = 3)
           - dimSums(
-              ( mselect(vm_emiIndCCS_Sub, x[setdiff(names(x), 'variable')])
+              ( mselect(pm_IndstCO2Captured, x[setdiff(names(x), 'variable')])
               * p_share_CCS
               ),
               dim = 3)
           ) * GtC_2_MtCO2,
           x[['variable']])
       }) %>%
-        mbind())
+        mbind()
+      )
 
     # Baseline emission before CCS, corresponds to energy carbon content
     out <- mbind(

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # The REMIND R package (2nd generation)
 
-R package **remind2**, version **1.117.6**
+R package **remind2**, version **1.119.2**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/remind2)](https://cran.r-project.org/package=remind2)  [![R build status](https://github.com/pik-piam/remind2/workflows/check/badge.svg)](https://github.com/pik-piam/remind2/actions) [![codecov](https://codecov.io/gh/pik-piam/remind2/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/remind2) [![r-universe](https://pik-piam.r-universe.dev/badges/remind2)](https://pik-piam.r-universe.dev/builds)
 
@@ -49,7 +49,7 @@ In case of questions / problems please contact Renato Rodrigues <renato.rodrigue
 
 To cite package **remind2** in publications use:
 
-Rodrigues R, Baumstark L, Benke F, Dietrich J, Dirnaichner A, Führlich P, Giannousakis A, Hasse R, Hilaire J, Klein D, Koch J, Kowalczyk K, Levesque A, Malik A, Merfort A, Merfort L, Morena-Leiva S, Pehl M, Pietzcker R, Rauner S, Richters O, Rottoli M, Schötz C, Schreyer F, Siala K, Sörgel B, Spahr M, Strefler J, Verpoort P (2023). _remind2: The REMIND R package (2nd generation)_. R package version 1.117.6, <URL: https://github.com/pik-piam/remind2>.
+Rodrigues R, Baumstark L, Benke F, Dietrich J, Dirnaichner A, Führlich P, Giannousakis A, Hasse R, Hilaire J, Klein D, Koch J, Kowalczyk K, Levesque A, Malik A, Merfort A, Merfort L, Morena-Leiva S, Pehl M, Pietzcker R, Rauner S, Richters O, Rottoli M, Schötz C, Schreyer F, Siala K, Sörgel B, Spahr M, Strefler J, Verpoort P (2023). _remind2: The REMIND R package (2nd generation)_. R package version 1.119.2, <URL: https://github.com/pik-piam/remind2>.
 
 A BibTeX entry for LaTeX users is
 
@@ -58,7 +58,7 @@ A BibTeX entry for LaTeX users is
   title = {remind2: The REMIND R package (2nd generation)},
   author = {Renato Rodrigues and Lavinia Baumstark and Falk Benke and Jan Philipp Dietrich and Alois Dirnaichner and Pascal Führlich and Anastasis Giannousakis and Robin Hasse and Jérome Hilaire and David Klein and Johannes Koch and Katarzyna Kowalczyk and Antoine Levesque and Aman Malik and Anne Merfort and Leon Merfort and Simón Morena-Leiva and Michaja Pehl and Robert Pietzcker and Sebastian Rauner and Oliver Richters and Marianna Rottoli and Christof Schötz and Felix Schreyer and Kais Siala and Björn Sörgel and Mike Spahr and Jessica Strefler and Philipp Verpoort},
   year = {2023},
-  note = {R package version 1.117.6},
+  note = {R package version 1.119.2},
   url = {https://github.com/pik-piam/remind2},
 }
 ```
diff --git a/inst/extdata/additional_summation_checks.csv b/inst/extdata/additional_summation_checks.csv
@@ -0,0 +1,88 @@
+parent;                           child;                                     factor
+;;
+# total FE primary steel production;;
+FE|Industry|Steel|++|Primary;     FE|Industry|Steel|+|Solids;1
+FE|Industry|Steel|++|Primary;     FE|Industry|Steel|+|Liquids;1
+FE|Industry|Steel|++|Primary;     FE|Industry|Steel|+|Gases;1
+FE|Industry|Steel|++|Primary;     FE|Industry|Steel|+|Hydrogen;1
+FE|Industry|Steel|++|Primary;     FE|Industry|Steel|Primary|Electricity;1
+;;
+# total FE secondary steel production;;
+FE|Industry|Steel|++|Secondary;   FE|Industry|Steel|Secondary|Electricity;1
+;;
+Emi|GHG;Emi|GHG|Gross|Energy|Supply|Electricity;1
+Emi|GHG;Emi|GHG|Gross|Energy|Supply|Non-electric;1
+Emi|GHG;Emi|GHG|Energy|Demand|+|Buildings;1
+Emi|GHG;Emi|GHG|Gross|Energy|Demand|+|Industry;1
+Emi|GHG;Emi|GHG|Energy|Demand|+|Transport;1
+Emi|GHG;Emi|GHG|Energy|Demand|+|CDR;1
+Emi|GHG;Emi|GHG|+++|Industrial Processes;1
+Emi|GHG;Emi|GHG|+++|Waste;1
+Emi|GHG;Emi|GHG|+++|Agriculture;1
+Emi|GHG;Emi|GHG|+++|Land-Use Change;1
+Emi|GHG;Emi|CO2|CDR|BECCS;1
+Emi|GHG;Emi|CO2|CDR|DACCS;1
+Emi|GHG;Emi|CO2|CDR|Industry CCS|Synthetic Fuels;1
+Emi|GHG;Emi|CO2|CDR|EW;1
+;;
+Emi|CO2;Emi|CO2|+|Land-Use Change;1
+Emi|CO2;Emi|CO2|+|Industrial Processes;1
+Emi|CO2;Emi|CO2|Energy|Demand|+|Transport;1
+Emi|CO2;Emi|CO2|Gross|Energy|Demand|+|Industry;1
+Emi|CO2;Emi|CO2|Energy|Demand|+|Buildings;1
+Emi|CO2;Emi|CO2|Energy|Demand|+|CDR;1
+Emi|CO2;Emi|CO2|Gross|Energy|Supply|Non-electric;1
+Emi|CO2;Emi|CO2|Gross|Energy|Supply|+|Electricity;1
+Emi|CO2;Emi|CO2|CDR|BECCS;1
+Emi|CO2;Emi|CO2|CDR|Industry CCS|Synthetic Fuels;1
+Emi|CO2;Emi|CO2|CDR|DACCS;1
+Emi|CO2;Emi|CO2|CDR|EW;1
+Emi|GHG;Emi|CO2|+|Energy;1
+;;
+Emi|GHG 1;Emi|CO2|+|Industrial Processes;1
+Emi|GHG 1;Emi|CO2|+|Land-Use Change;1
+Emi|GHG 1;Emi|GHG|+|CH4;1
+Emi|GHG 1;Emi|GHG|+|N2O;1
+Emi|GHG 1;Emi|GHG|+|F-Gases;1
+Emi|GHG 1;Emi|CO2|+|non-BECCS CDR;1
+;;
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Land-Use Change;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Industrial Processes;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Energy|Demand|Transport;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Gross|Energy|Demand|Industry;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Energy|Demand|Buildings;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Energy|Demand|CDR;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Gross|Energy|Supply|Non-electric;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|Gross|Energy|Supply|Electricity;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|CDR|BECCS;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|CDR|Industry CCS|Synthetic Fuels;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|CDR|DACCS;1
+Emi|CO2|Cumulated;Emi|CO2|Cumulated|CDR|EW;1
+;;
+Emi|CO2|CDR;Emi|CO2|CDR|Land-Use Change;1
+Emi|CO2|CDR;Emi|CO2|CDR|BECCS|Pe2Se;1
+Emi|CO2|CDR;Emi|CO2|CDR|BECCS|Industry;1
+Emi|CO2|CDR;Emi|CO2|CDR|Industry CCS|Synthetic Fuels;1
+Emi|CO2|CDR;Emi|CO2|CDR|DACCS;1
+Emi|CO2|CDR;Emi|CO2|CDR|EW;1
+;;
+SE|Electricity;SE|Input|Electricity|Hydrogen|Synthetic Fuels|+|Liquids;1
+SE|Electricity;SE|Input|Electricity|Hydrogen|Synthetic Fuels|+|Gases;1
+SE|Electricity;SE|Input|Electricity|Hydrogen|direct FE H2;1
+SE|Electricity;SE|Input|Electricity|Hydrogen|Electricity Storage;1
+SE|Electricity;SE|Input|Electricity|Buildings;1
+SE|Electricity;SE|Input|Electricity|Industry;1
+SE|Electricity;SE|Input|Electricity|Transport;1
+SE|Electricity;SE|Input|Electricity|CDR;1
+SE|Electricity;SE|Input|Electricity|Self Consumption Energy System;1
+SE|Hydrogen;FE|Industry|+|Hydrogen;1
+SE|Hydrogen;FE|Buildings|+|Hydrogen;1
+SE|Hydrogen;FE|Transport|+|Hydrogen;1
+SE|Hydrogen;FE|CDR|+|Hydrogen;1
+SE|Hydrogen;SE|Input|Hydrogen|Electricity|+|Normal Turbines;1
+SE|Hydrogen;SE|Input|Hydrogen|Electricity|+|Forced VRE Turbines;1
+SE|Hydrogen;SE|Input|Hydrogen|Synthetic Fuels|+|Liquids;1
+SE|Hydrogen;SE|Input|Hydrogen|Synthetic Fuels|+|Gases;1
+;;
+Carbon Management|Carbon Capture;Carbon Management|Storage;1
+Carbon Management|Carbon Capture;Carbon Management|Usage;1