-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Creation of users scripts for OM3, see COSIMA/access-om3#182 There are three processes here: 1. Per COSIMA/access-om3#81 initial conditions file for CICE are converted from symlink to files/hardlinks 2. CICE daily output in concatenated into one file per month 3. An intake esm datastore for the run is generated --------- Co-authored-by: Dougie Squire <[email protected]> Co-authored-by: dougiesquire <[email protected]>
- Loading branch information
1 parent
c62452e
commit fdb5172
Showing
15 changed files
with
459 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
# This workflow will install dependencies, run tests and the black-formatter | ||
|
||
name: om3_scripts | ||
|
||
on: | ||
push: | ||
branches: [ "main" ] | ||
pull_request: | ||
branches: [ "main" ] | ||
|
||
permissions: | ||
contents: read | ||
|
||
jobs: | ||
formatting: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Code formatting | ||
uses: psf/black@stable | ||
with: | ||
options: "--check --verbose --diff" | ||
|
||
test: | ||
needs: formatting | ||
runs-on: ubuntu-latest | ||
strategy: | ||
matrix: | ||
python-version: ['3.10', '3.11', '3.12'] | ||
|
||
steps: | ||
- uses: actions/checkout@v3 | ||
- name: Set up Python ${{ matrix.python-version }} | ||
uses: actions/setup-python@v3 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
- name: Install nco | ||
run: | | ||
sudo apt-get update | ||
sudo apt-get -y install nco | ||
- name: Install python dependencies | ||
run: | | ||
python -m pip install --upgrade pip | ||
pip install -r test/test_requirements.txt | ||
- name: Test with pytest | ||
run: | | ||
python -m pytest -m "not broken" | ||
- name: Upload coverage reports to Codecov with GitHub Action | ||
uses: codecov/codecov-action@v3 | ||
env: | ||
CODEOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# See https://pre-commit.com for more information | ||
# See https://pre-commit.com/hooks.html for more hooks | ||
repos: | ||
- repo: local | ||
hooks: | ||
- id: black | ||
name: black | ||
entry: black | ||
language: system | ||
types: [python] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
This directory contains Payu setup and archive userscripts for ACCESS-OM3. | ||
|
||
To use, add these to config.yaml: | ||
|
||
```yaml | ||
userscripts: | ||
setup: /usr/bin/bash /g/data/vk83/apps/om3-scripts/payu_config/setup.sh | ||
archive: /usr/bin/bash /g/data/vk83/apps/om3-scripts/payu_config/archive.sh | ||
|
||
modules: | ||
use: | ||
- /g/data/hh5/public/modules | ||
load: | ||
- conda/analysis | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
#!/usr/bin/bash -i | ||
|
||
source $(dirname "$0")/archive_scripts/archive_cice_restarts.sh | ||
source $(dirname "$0")/archive_scripts/concat_ice_daily.sh | ||
python3 $(dirname "$0")/archive_scripts/build_intake_ds.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
#!/usr/bin/bash -i | ||
# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# clean up cice_restarts.sh | ||
|
||
if [ -f archive/output*/input/iced.1900-01-01-10800.nc ] | ||
then | ||
rm archive/output*/input/iced.1900-01-01-10800.nc | ||
return 0 | ||
fi | ||
|
||
latest_o=$(ls -drv archive/output*[0-9] | head -1) | ||
|
||
#initial restart was copied from the previous run | ||
ic_restart=$(ls -dv $latest_o/access-om3.cice.r.* | head -1) | ||
|
||
if [ -f $latest_o/access-om3.cice.r.* ] | ||
then | ||
rm $latest_o/access-om3.cice.r.* | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
#!python3 | ||
# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# modules: | ||
# use: | ||
# - /g/data/hh5/public/modules | ||
# load: | ||
# - conda/analysis | ||
|
||
from access_nri_intake.source import builders | ||
import os | ||
import sys | ||
from pathlib import Path | ||
from warnings import warn | ||
from ruamel.yaml import YAML | ||
from ruamel.yaml.comments import CommentedMap | ||
|
||
path_root = Path(__file__).parents[2] | ||
sys.path.append(str(path_root)) | ||
|
||
from scripts_common import get_provenance_metadata, md5sum | ||
|
||
METADATA_FILENAME = "metadata.yaml" | ||
UUID_FIELD = "experiment_uuid" | ||
ARCHIVE_PATH = "archive" | ||
|
||
|
||
def description(): | ||
|
||
# Get experiment uuid | ||
# follows https://github.com/payu-org/payu/blob/ef55e93fe23fcde19024479c0dc4112dcdf6603f/payu/metadata.py#L90 | ||
metadata_filename = Path(METADATA_FILENAME) | ||
if metadata_filename.exists(): | ||
metadata = CommentedMap() | ||
metadata = YAML().load(metadata_filename) | ||
uuid = metadata.get(UUID_FIELD, None) | ||
else: | ||
warn(f"{METADATA_FILENAME} not found in archive folder") | ||
uuid = False | ||
|
||
# Check git status of this .py file | ||
this_file = os.path.normpath(__file__) | ||
|
||
runcmd = f"python3 {os.path.basename(this_file)}" | ||
|
||
# Get string "Created using $file: $command" | ||
provenance = get_provenance_metadata(this_file, runcmd) | ||
|
||
if uuid: | ||
description = f"intake-esm datastore for experiment {uuid}, in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})" | ||
else: | ||
description = f"intake-esm datastore for experiment in folder {os.getcwd()}. {provenance}. (md5 hash: {md5sum(this_file)})" | ||
|
||
return description | ||
|
||
|
||
if __name__ == "__main__": | ||
|
||
builder = builders.AccessOm3Builder(path=ARCHIVE_PATH) | ||
|
||
print("LOG: Building intake-esm datastore") | ||
|
||
builder.build() | ||
|
||
# Log invalid assets | ||
builder.invalid_assets | ||
|
||
builder.save( | ||
name="intake_esm_ds", | ||
description=description(), | ||
directory=ARCHIVE_PATH, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
#!/usr/bin/bash -i | ||
# Copyright 2024 ACCESS-NRI and contributors. See the top-level COPYRIGHT file for details. | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# Concatenate sea-ice daily output from access-om3 | ||
# this was written assuming it would be used as a payu "userscript" at the "archive" stage, but alternatively a path to an "archive" directory can be provided | ||
# Script inspired from https://github.com/COSIMA/1deg_jra55_ryf/blob/master/sync_data.sh#L87-L108 | ||
# | ||
# This script uses "ncrcat". Load this through either 'module use /g/data/vk83/modules; module load payu' or 'module load nco'. | ||
|
||
shopt -s extglob | ||
|
||
Help() | ||
{ | ||
# Display Help | ||
echo "Concatenante daily history output from the (sea) ice model to a single file" | ||
echo | ||
echo "Syntax: scriptTemplate [-h|d DIRECTORY]" | ||
echo "options:" | ||
echo "h Print this Help." | ||
echo "d Process "name" directory rather than latest output in archive folder." | ||
echo | ||
} | ||
|
||
# Get the options | ||
while getopts ":hd:" option; do | ||
case $option in | ||
h) # display Help | ||
Help | ||
exit;; | ||
d) # Enter a directory | ||
out_dir=$OPTARG | ||
if [ ! -d $out_dir ]; then | ||
echo $out_dir Does not exist | ||
exit | ||
fi;; | ||
\?) # Invalid option | ||
echo "Error: Invalid option" | ||
exit;; | ||
esac | ||
done | ||
|
||
#If no directory option provided , then use latest | ||
if [ -z $out_dir ]; then | ||
#latest output dir only | ||
out_dir=$(ls -drv archive/output*[0-9] | head -1) | ||
fi | ||
|
||
if ! command -v -- "ncrcat" > /dev/null 2>&1; then | ||
echo "ncrcat not available, trying module load nco" | ||
module load nco | ||
fi | ||
|
||
for f in $out_dir/access-om3.cice.h.????-??-01.nc ; do | ||
#if the 1st and the 28th of that month exists, then assume its a whole month and concatenate | ||
if [ -f $f ] && [ -f ${f/-01.nc/-28.nc} ]; then | ||
|
||
output_f=${f/-01.nc/.nc} #remove day in date string | ||
|
||
#concat daily files for this month | ||
echo LOG: concatenating daily sea ice files in $out_dir | ||
echo doing ncrcat -O -L 5 -4 ${f/-01.nc/-??.nc} $output_f | ||
ncrcat -O -L 5 -4 ${f/-01.nc/-??.nc} $output_f | ||
|
||
if [[ $? == 0 ]]; then | ||
rm ${f/-01.nc/-??.nc} #delete individual dailys on success | ||
fi | ||
fi | ||
done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
#!/usr/bin/bash -i | ||
|
||
source $(dirname "$0")/setup_scripts/setup_cice_restarts.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
#!/bin/bash | ||
# patch for https://github.com/open-mpi/ompi/issues/12141 | ||
if ! [ -f work/access-om3.cice.r.* ] | ||
then | ||
# no restart files yet, use initial conditions | ||
IC=$(readlink work/input/iced.1900-01-01-10800.nc) | ||
rm work/input/iced.1900-01-01-10800.nc | ||
cp $IC work/input/iced.1900-01-01-10800.nc | ||
else | ||
# change restart symlink to hardlink | ||
RESTART=$(echo work/access-om3.cice.r.*) | ||
ln -f $(readlink $RESTART) $RESTART | ||
fi |
Oops, something went wrong.