Merge branch 'remindmodel:develop' into deltaCap_for_capCum

.Rprofile

@@ -114,4 +114,14 @@ if (use_cluster_defaults &&
     "/p/projects/remind/inputdata/CESparametersAndGDX" = NULL))
 }
 
+# Include local calibration results, if they exist, from either the main
+# directory or output directories.
+path <- file.path(
+    c('.', file.path('..', '..')),
+    'calibration_results', '.Rprofile_calibration_results')
+
+path <- head(path[file.exists(path)], 1)
+
+if (!rlang::is_empty(path))
+    source(path)
 })

.github/workflows/size-check

@@ -0,0 +1,17 @@
+#!/usr/bin/env Rscript
+
+# make sure that files are not too big
+maxfilesize <- function(max) {
+  files <- union(system("git ls-tree -r HEAD --name-only", intern = TRUE),
+                 system("git diff --cached --name-only", intern = TRUE))
+  out <- data.frame(files = files, size = round(file.size(files) / 1024, 2))
+  out <- out[!is.na(out$size), ]
+  out <- out[out$size > max, ]
+  if (nrow(out) > 0) {
+    stop(nrow(out), " files with size > ", max, "kB detected: \n",
+         paste0(out$files, " (", out$size, "kB)", collapse = "\n"))
+  }
+  message("All files are smaller than ", max, "kB\n")
+}
+
+maxfilesize(250)

.github/workflows/test-code.yaml

@@ -18,22 +18,24 @@ jobs:
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Remove .Rprofile
         run: rm .Rprofile
 
       - uses: r-lib/actions/setup-r@v2
         with:
-          extra-repositories: https://cloud.r-project.org https://rse.pik-potsdam.de/r/packages/
+          extra-repositories: https://rse.pik-potsdam.de/r/packages/
 
       - name: Install R dependencies
         uses: r-lib/actions/setup-r-dependencies@v2
         with:
           packages: gms
-          extra-packages: gms
 
       - name: codeCheck
         run: null <- gms::codeCheck(strict=TRUE)
         shell: Rscript {0}
 
+      - name: fileSizeCheck
+        run: source(".github/workflows/size-check")
+        shell: Rscript {0}

.gitignore

@@ -79,3 +79,4 @@ runtime.rds
 # ignore slurm logs
 slurm-[0-9]*.log
 slurm-[0-9]*.out
+calibration_results/

CHANGELOG.md

@@ -11,7 +11,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **config** regex tests for many parameters
 - **scripts** add script to check fixing of runs to reference run
     [[#1410](https://github.com/remindmodel/remind/pull/1410)]
+- **scripts** add script for cost decomposition of integrated damage runs
+    [[#1445](https://github.com/remindmodel/remind/pull/1445)]
 - **testthat** test and compile all config files
+- **37_Industry** add feedstocks for chemicals subsector and plastics production
+- **core** add process emissions from chemicals subsector and from plastics incineration
 
 ### fixed
 - fixed weights of energy carriers in `pm_IndstCO2Captured`
@@ -21,7 +25,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - let preempted and resumed runs start their subsequent runs
     [[#1414](https://github.com/remindmodel/remind/pull/1414)]
 - correctly report `Tech|*|Capital Costs|w/ Adj Costs` for t < cm_startyear
-    [[#1429](https://github.com/remindmodel/remind/pull/1429)]
+    [[#1429](https://github.com/remindmodel/remind/pull/1429), [#1476](https://github.com/remindmodel/remind/pull/1476)]
+
+### removed
+- **45_carbonprice** remove outdated realizations:
+    NDC2constant, NPi2018, diffPhaseIn2Constant, diffPhaseIn2Lin, diffPhaseInLin2LinFlex, diffPriceSameCost
+    [[#1480](https://github.com/remindmodel/remind/pull/1480)]
 
 ## [3.2.1] - 2023-07-13 (incomplete)
 

CITATION.cff

@@ -246,8 +246,8 @@ authors:
   affiliation: "Potsdam Institute for Climate Impact Research"
 
 title: REMIND - REgional Model of INvestments and Development
-version: "3.2.1.dev212"
-date-released: 2023-10-18
+version: "3.2.1.dev634"
+date-released: 2024-01-03
 repository-code: https://github.com/remindmodel/remind
 keywords:
   - energy

DESCRIPTION

@@ -59,7 +59,7 @@ Imports:
     raster,
     readr,
     readxl,
-    remind2 (>= 1.112.0),
+    remind2 (>= 1.125.0),
     renv,
     reshape2,
     reticulate,
@@ -69,7 +69,7 @@ Imports:
     SPEI,
     stringr,
     terra,
-    testthat (>= 3.1.10),
+    testthat (>= 3.2.0),
     tibble,
     tidyr,
     tidyverse,

Makefile

@@ -1,12 +1,17 @@
-.PHONY: help docs update-renv update-renv-all archive-renv restore-renv check check-fix test test-coupled test-full
+.PHONY: help docs update-renv update-renv-all archive-renv restore-renv check \
+	check-fix test test-coupled test-full set-local-calibration
 .DEFAULT_GOAL := help
 
 # extracts the help text and formats it nicely
-HELP_PARSING = 'm <- readLines("Makefile");\
-				m <- grep("\#\#", m, value=TRUE);\
-				command <- sub("^([^ ]*) *\#\#(.*)", "\\1", m);\
-				help <- sub("^([^ ]*) *\#\#(.*)", "\\2", m);\
-				cat(sprintf("%-18s%s", command, help), sep="\n")'
+HELP_PARSING = 'm <- grep("\#\#", readLines("Makefile"), value = TRUE);\
+                parse <- "^([^[[:space:]]*)[[:space:]]*\#\#[[:space:]]*(.*)";\
+                command <- sub(parse, "\\1", m, perl = TRUE);\
+                help <- sub(parse, "\\2",  m, perl = TRUE);\
+                i <- grep("^$$", command, invert = TRUE)[-1];\
+                command[i] <- paste0("\n", command[i]);\
+                help[i] <- paste0(" ", help[i]);\
+                cat(sprintf("%-*s%s", max(nchar(command)), command, help),\
+                    sep = "\n")'
 
 help:            ## Show this help.
 	@Rscript -e $(HELP_PARSING)
@@ -22,15 +27,27 @@ update-renv:     ## Upgrade all pik-piam packages in your renv to the respective
 update-renv-all: ## Upgrade all packages (including CRAN packages) in your renv
                  ## to the respective latest release, write renv.lock archive
                  ## Upgrade all packages in python venv, if python venv exists
-	Rscript -e 'renv::update(exclude = "renv"); piamenv::archiveRenv()'
-	[ -e ".venv/bin/python" ] && .venv/bin/python -mpip install --upgrade pip wheel
-	[ -e ".venv/bin/python" ] && .venv/bin/python -mpip install --upgrade --upgrade-strategy eager -r requirements.txt
+	@Rscript -e 'renv::update(exclude = "renv"); piamenv::archiveRenv()'
+	@if [ -e "./venv/bin/python" ]; then \
+		pv_maj=$$( .venv/bin/python -V | sed 's/^Python \([0-9]\).*/\1/' ); \
+		pv_min=$$( .venv/bin/python -V | sed 's/^Python [0-9]\.\([0-9]\+\).*/\1/' ); \
+		if (( 3 == $$pv_maj )) && (( 7 <= $$pv_min )) && (( $pv_min < 11 )); then \
+			.venv/bin/python -mpip install --upgrade pip wheel; \
+			.venv/bin/python -mpip install --upgrade --upgrade-strategy eager -r requirements.txt; \
+		fi \
+	fi
 
 ensure-reqs:     ## Ensure the REMIND library requirements are fulfilled
                  ## by installing updates and new libraries as necessary. Does not
                  ## install updates unless it is required.
 	@Rscript -e 'source("scripts/start/ensureRequirementsInstalled.R"); ensureRequirementsInstalled(rerunPrompt="make ensure-reqs")'
-	@[ -e ".venv/bin/python" ] && .venv/bin/python -mpip -qq install -r requirements.txt
+	@if [ -e "./venv/bin/python" ]; then \
+		pv_maj=$$( .venv/bin/python -V | sed 's/^Python \([0-9]\).*/\1/' ); \
+		pv_min=$$( .venv/bin/python -V | sed 's/^Python [0-9]\.\([0-9]\+\).*/\1/' ); \
+		if (( 3 == $$pv_maj )) && (( 7 <= $$pv_min )) && (( $pv_min < 11 )); then \
+			.venv/bin/python -mpip -qq install -r requirements.txt; \
+		fi \
+	fi
 
 archive-renv:    ## Write renv.lock into archive.
 	Rscript -e 'piamenv::archiveRenv()'
@@ -50,14 +67,14 @@ check-fix:       ## Check if the GAMS code follows the coding etiquette
 
 test:            ## Test if the model compiles and runs without running a full
                  ## scenario. Tests take about 10 minutes to run.
-	$(info Tests take about 10 minutes to run, please be patient)
-	@R_PROFILE_USER= Rscript -e 'testthat::test_dir("tests/testthat")'
+	$(info Tests take about 20 minutes to run, please be patient)
+	@Rscript -e 'testthat::test_dir("tests/testthat")'
 
 test-coupled:    ## Test if the coupling with MAgPIE works. Takes significantly
                  ## longer than 60 minutes to run and needs slurm and magpie
                  ## available
 	$(info Coupling tests take around 75 minutes to run, please be patient)
-	@R_PROFILE_USER= TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_file("tests/testthat/test_20-coupled.R")'
+	@TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_file("tests/testthat/test_20-coupled.R")'
 
 test-coupled-slurm: ## test-coupled, but on slurm
 	$(info Coupling tests take around 75 minutes to run. Sent to slurm, find log in test-coupled.log)
@@ -66,7 +83,11 @@ test-coupled-slurm: ## test-coupled, but on slurm
 test-full:       ## Run all tests, including coupling tests and a default
                  ## REMIND scenario. Takes significantly longer than 10 minutes to run.
 	$(info Full tests take more than an hour to run, please be patient)
-	@R_PROFILE_USER= TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_dir("tests/testthat")'
+	@TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_dir("tests/testthat")'
 test-validation: ## Run validation tests, requires a full set of runs in the output folder
 	$(info Run validation tests, requires a full set of runs in the output folder)
-	@R_PROFILE_USER= TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_dir("tests/testthat/validation")'	
+	@TESTTHAT_RUN_SLOW=TRUE Rscript -e 'testthat::test_dir("tests/testthat/validation")'	
+
+set-local-calibration:		## set up local calibration results directory
+	@./scripts/utils/set-local-calibration.sh
+	$(info use `collect_calibration` script in calibration_results/ directory )