diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a817e4..eb99ba6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,17 +34,18 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install nco + - name: Install sys dependencies run: | sudo apt-get update sudo apt-get -y install nco + sudo apt-get -y install ncal - name: Install python dependencies run: | python -m pip install --upgrade pip pip install -r test/test_requirements.txt - name: Test with pytest run: | - python -m pytest -m "not broken" + python -m pytest -m "not broken" -v -s - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v3 env: diff --git a/payu_config/archive_scripts/archive_cice_restarts.sh b/payu_config/archive_scripts/archive_cice_restarts.sh index 9e254f9..bec7e04 100755 --- a/payu_config/archive_scripts/archive_cice_restarts.sh +++ b/payu_config/archive_scripts/archive_cice_restarts.sh @@ -3,10 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # clean up cice_restarts.sh -if [ -f archive/output*/INPUT/iced.1900-01-01-10800.nc ] -then -rm archive/output*/INPUT/iced.1900-01-01-10800.nc -return 0 +if [ -f archive/output*/INPUT/iced.1900-01-01-10800.nc ]; then + rm archive/output*/INPUT/iced.1900-01-01-10800.nc + return 0 fi latest_o=$(ls -drv archive/output*[0-9] | head -1) @@ -14,7 +13,6 @@ latest_o=$(ls -drv archive/output*[0-9] | head -1) #initial restart was copied from the previous run ic_restart=$(ls -dv $latest_o/access-om3.cice.r.* | head -1) -if [ -f $latest_o/access-om3.cice.r.* ] -then -rm $latest_o/access-om3.cice.r.* +if [ -f $latest_o/access-om3.cice.r.* ]; then + rm $latest_o/access-om3.cice.r.* fi diff --git a/payu_config/archive_scripts/concat_ice_daily.sh b/payu_config/archive_scripts/concat_ice_daily.sh index bd02b4d..6882a26 100755 --- a/payu_config/archive_scripts/concat_ice_daily.sh +++ b/payu_config/archive_scripts/concat_ice_daily.sh @@ -51,12 +51,22 @@ if ! command -v -- "ncrcat" > /dev/null 2>&1; then module load nco fi -for f in $out_dir/access-om3.cice.h.????-??-01.nc ; do - #if the 1st and the 28th of that month exists, then assume its a whole month and concatenate - if [ -f $f ] && [ -f ${f/-01.nc/-28.nc} ]; then +for f in $out_dir/access-om3.cice*.????-??-01.nc ; do + # extract the year and month from existing files + year_month=$(echo "$f" | sed -E "s/.*\.([0-9]{4}-[0-9]{2})-[0-9]{2}\.nc/\1/") + year=$(echo "$year_month" | cut -d- -f1) + month=$(echo "$year_month" | cut -d- -f2) + + # calculate the expected end day for the given year and month + end_day=$(cal $month $year | awk "NF {end_day=\$NF}; END {print end_day}") + end_day_file=${f/-01.nc/-$end_day.nc} + + output_f=${f/-01.nc/.nc} #remove day in date string + + if [ -f $output_f ]; then + echo WARN: $output_f exists, skipping concatenation daily sea ice files + elif [ -f $f ] && [ -f $end_day_file ] ; then - output_f=${f/-01.nc/.nc} #remove day in date string - #concat daily files for this month echo LOG: concatenating daily sea ice files in $out_dir echo doing ncrcat -O -L 5 -4 ${f/-01.nc/-??.nc} $output_f @@ -65,5 +75,7 @@ for f in $out_dir/access-om3.cice.h.????-??-01.nc ; do if [[ $? == 0 ]]; then rm ${f/-01.nc/-??.nc} #delete individual dailys on success fi + else + echo "LOG: skipping concatenating daily sea ice files (incomplete month)" fi done diff --git a/payu_config/setup_scripts/setup_cice_restarts.sh b/payu_config/setup_scripts/setup_cice_restarts.sh index 941e8c7..c1e7367 100755 --- a/payu_config/setup_scripts/setup_cice_restarts.sh +++ b/payu_config/setup_scripts/setup_cice_restarts.sh @@ -1,13 +1,12 @@ #!/bin/bash # patch for https://github.com/open-mpi/ompi/issues/12141 -if ! [ -f work/access-om3.cice.r.* ] -then -# no restart files yet, use initial conditions -IC=$(readlink work/INPUT/iced.1900-01-01-10800.nc) -rm work/INPUT/iced.1900-01-01-10800.nc -cp $IC work/INPUT/iced.1900-01-01-10800.nc +if ! [ -f work/access-om3.cice.r.* ]; then + # no restart files yet, use initial conditions + IC=$(readlink work/INPUT/iced.1900-01-01-10800.nc) + rm work/INPUT/iced.1900-01-01-10800.nc + cp $IC work/INPUT/iced.1900-01-01-10800.nc else -# change restart symlink to hardlink -RESTART=$(echo work/access-om3.cice.r.*) -ln -f $(readlink $RESTART) $RESTART + # change restart symlink to hardlink + RESTART=$(echo work/access-om3.cice.r.*) + ln -f $(readlink $RESTART) $RESTART fi diff --git a/test/test_payu_conf/test_concat_ice_daily.py b/test/test_payu_conf/test_concat_ice_daily.py index 1fbf3d7..af0af26 100644 --- a/test/test_payu_conf/test_concat_ice_daily.py +++ b/test/test_payu_conf/test_concat_ice_daily.py @@ -21,8 +21,7 @@ def assert_f_not_exists(p): raise AssertionError("File exists and should not: %s" % str(p)) -@pytest.fixture -def daily_files(request, tmp_path): +def daily_files(dir_name, hist_base, ndays, tmp_path): """ Make 365 days of fake data, and then write it into 365 files @@ -31,12 +30,8 @@ def daily_files(request, tmp_path): """ - if request.param[0] == "Default": + if dir_name == "Default": dir_name = "archive/output000" - else: - dir_name = str(request.param[0]) - - ndays = request.param[1] nx = 30 ny = 50 @@ -58,7 +53,7 @@ def daily_files(request, tmp_path): # ds.time.encoding['dtype'] = 'float' out_dir = str(tmp_path) + "/" + dir_name + "/" - paths = [f"{out_dir}access-om3.cice.h.{str(t.values)[0:10]}.nc" for t in ds.time] + paths = [f"{out_dir}{hist_base}.{str(t.values)[0:10]}.nc" for t in ds.time] datasets = [ds.sel(time=slice(t, t)) for t in ds.time] makedirs(out_dir) @@ -67,23 +62,30 @@ def daily_files(request, tmp_path): return paths +@pytest.fixture( + params=["access-om3.cice.h", "access-om3.cice", "access-om3.cice.1day.mean"] +) +def hist_base(request): + return str(request.param) + + @pytest.mark.parametrize( - "daily_files, use_dir, nmonths", + "hist_dir, ndays, use_dir, nmonths", [ - (("archive/output000", 365), False, 12), - (("archive/output999", 31), False, 1), - (("archive/output9999", 31), False, 1), - (("archive/output574", 365), True, 12), + ("Default", 365, False, 12), + ("archive/output999", 31, False, 1), + ("archive/output9999", 31, False, 1), + ("archive/output574", 365, True, 12), ], - indirect=["daily_files"], ) # run this test with a several folder names and lengths, provide the directory as an argument sometimes -def test_true_case(daily_files, use_dir, nmonths, tmp_path): +def test_true_case(hist_dir, ndays, use_dir, nmonths, hist_base, tmp_path): """ Run the script to convert the daily data into monthly files, and check the monthly files and the daily files dont exist. """ + daily_paths = daily_files(hist_dir, hist_base, ndays, tmp_path) chdir(tmp_path) - output_dir = Path(daily_files[0]).parents[0] + output_dir = Path(daily_paths[0]).parents[0] if not use_dir: # default path run([run_str]) @@ -94,13 +96,13 @@ def test_true_case(daily_files, use_dir, nmonths, tmp_path): run_str, "-d", output_dir, - ] + ], ) expected_months = pd.date_range("2010-01-01", freq="ME", periods=nmonths + 1) # valid output filenames monthly_paths = [ - f"{output_dir}/access-om3.cice.h.{str(t)[0:7]}.nc" for t in expected_months + f"{output_dir}/{hist_base}.{str(t)[0:7]}.nc" for t in expected_months ] for p in monthly_paths[0:nmonths]: @@ -109,30 +111,58 @@ def test_true_case(daily_files, use_dir, nmonths, tmp_path): for p in monthly_paths[nmonths]: assert_f_not_exists(p) - for p in daily_files: + for p in daily_paths: assert_f_not_exists(p) -@pytest.mark.parametrize( - "daily_files", [("Default", 1), ("Default", 27)], indirect=True -) -def test_no_concat_case(daily_files, tmp_path): +@pytest.mark.parametrize("hist_dir, ndays", [("Default", 1), ("Default", 30)]) +def test_incomplete_month(hist_dir, ndays, hist_base, tmp_path): """ Run the script to convert the daily data into monthly files, with less than 28 days data, and check no things happen. """ + daily_paths = daily_files(hist_dir, hist_base, ndays, tmp_path) + chdir(tmp_path) - output_dir = Path(daily_files[0]).parents[0] + output_dir = Path(daily_paths[0]).parents[0] run([run_str]) expected_months = pd.date_range("2010-01-01", freq="ME", periods=1) monthly_paths = [ - f"{output_dir}/access-om3.cice.h.{str(t)[0:7]}.nc" for t in expected_months + f"{output_dir}/{hist_base}.{str(t)[0:7]}.nc" for t in expected_months ] - for p in daily_files: + for p in daily_paths: assert_file_exists(p) for p in monthly_paths: assert_f_not_exists(p) + + +@pytest.mark.parametrize("hist_dir, ndays", [("Default", 31), ("Default", 27)]) +def test_no_override(hist_dir, ndays, hist_base, tmp_path): + """ + Run the script to convert the daily data into monthly files, but the output filename already exists, and check nothing happens. + """ + + daily_paths = daily_files(hist_dir, hist_base, ndays, tmp_path) + + chdir(tmp_path) + output_dir = Path(daily_paths[0]).parents[0] + + expected_months = pd.date_range("2010-01-01", freq="ME", periods=1) + + monthly_paths = [ + f"{output_dir}/{hist_base}.{str(t)[0:7]}.nc" for t in expected_months + ] + for p in monthly_paths: + Path(p).touch() + + run([run_str]) + + for p in daily_paths: + assert_file_exists(p) + + for p in monthly_paths: + assert_file_exists(p)