From d7dcad891e78a9a2b72efc8fc818eb3bd114ee89 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 10:41:24 -0700 Subject: [PATCH 1/6] Update esmf and scorpio For spack builds, update esmf to 8.4.0 and scorpio to 1.4.1 --- conda/default.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/default.cfg b/conda/default.cfg index a5cde0183a..627f52c6cc 100644 --- a/conda/default.cfg +++ b/conda/default.cfg @@ -21,11 +21,11 @@ mpi = nompi # the version of various packages to include if using spack albany = develop -esmf = 8.2.0 +esmf = 8.4.0 hdf5 = 1.12.1 lapack = 3.9.1 netcdf_c = 4.8.1 netcdf_fortran = 4.5.3 petsc = 3.16.1 pnetcdf = 1.12.2 -scorpio = 1.3.2 +scorpio = 1.4.1 \ No newline at end of file From b79ffcb34e019f946bb66a5e48c4add17983eaeb Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 10:44:40 -0700 Subject: [PATCH 2/6] Update to 1.2.0-alpha.5 --- compass/version.py | 2 +- conda/recipe/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compass/version.py b/compass/version.py index 84f88e39e7..17663d7594 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '1.2.0-alpha.4' +__version__ = '1.2.0-alpha.5' \ No newline at end of file diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 2d77664ab3..742e1927e0 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "compass" %} -{% set version = "1.2.0alpha.4" %} +{% set version = "1.2.0alpha.5" %} {% set build = 0 %} {% if mpi == "nompi" %} From 2fb37f44f598ca11083a701755b836235f42c399 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 13:19:44 -0600 Subject: [PATCH 3/6] Switch esmf spec from ~pio to ~parallelio We need to build with internal parallelio in esmf. The variant name has been changed for recent esmf. --- conda/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 925177929e..5f1b6eaa32 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -386,7 +386,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'parallel-netcdf@{pnetcdf}+cxx+fortran']) if esmf != 'None': - specs.append(f'esmf@{esmf}+mpi+netcdf~pio+pnetcdf') + specs.append(f'esmf@{esmf}+mpi+netcdf~parallelio+pnetcdf') if lapack != 'None': specs.append(f'netlib-lapack@{lapack}') include_e3sm_lapack = False From 19d1220011b012046ac0b334664cfe168d6ea991 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 23:04:48 -0700 Subject: [PATCH 4/6] Enable easier mache testing The `configure_compass_env.py` script now takes two flags, `--mache_fork` and `--mache_branch` that are used to clone the fork and branch locally, then install mache from it. This saves the trouble of developers cloning it themselves and building the mache conda package. --- conda/bootstrap.py | 23 ++++++++++++++++++----- conda/compass_env/spec-file.template | 2 ++ conda/configure_compass_env.py | 26 +++++++++++++++++++++++--- conda/shared.py | 8 ++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 5f1b6eaa32..c029f66b41 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -231,10 +231,10 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, activ_path, env_path, env_name, activate_env, spack_env -def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, +def build_conda_env(env_type, recreate, mpi, conda_mpi, version, python, source_path, conda_template_path, conda_base, env_name, env_path, activate_base, use_local, - local_conda_build, logger): + local_conda_build, logger, local_mache): if env_type != 'dev': install_miniconda(conda_base, activate_base, logger) @@ -276,7 +276,8 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, conda_openmp = '' spec_file = template.render(supports_otps=supports_otps, mpi=conda_mpi, openmp=conda_openmp, - mpi_prefix=mpi_prefix) + mpi_prefix=mpi_prefix, + include_mache=not local_mache) spec_filename = f'spec-file-{conda_mpi}.txt' with open(spec_filename, 'w') as handle: @@ -777,6 +778,8 @@ def main(): compass_version = get_version() + local_mache = args.mache_fork is not None and args.mache_branch is not None + machine = None if not args.env_only: if args.machine is None: @@ -864,10 +867,20 @@ def main(): if previous_conda_env != conda_env_name: build_conda_env( - env_type, recreate, machine, mpi, conda_mpi, compass_version, + env_type, recreate, mpi, conda_mpi, compass_version, python, source_path, conda_template_path, conda_base, conda_env_name, conda_env_path, activate_base, args.use_local, - args.local_conda_build, logger) + args.local_conda_build, logger, local_mache) + + if local_mache: + print('Install local mache\n') + commands = f'source {conda_base}/etc/profile.d/conda.sh; ' \ + f'source {conda_base}/etc/profile.d/mamba.sh; ' \ + f'conda activate {conda_env_name}; ' \ + 'cd ../build_mache/mache; ' \ + 'python -m pip install .' + check_call(commands, logger=logger) + previous_conda_env = conda_env_name if env_type != 'dev': diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index 7915fa8231..00e73448e4 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -15,7 +15,9 @@ jigsaw=0.9.14 jigsawpy=0.3.3 jupyter lxml +{% if include_mache %} mache=1.10.0 +{% endif %} matplotlib-base metis mpas_tools=0.17.0 diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index a7fd4938c7..3ee8f412eb 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -47,13 +47,13 @@ def bootstrap(activate_install_env, source_path, local_conda_build): def setup_install_env(env_name, activate_base, use_local, logger, recreate, - conda_base): + conda_base, mache): env_path = os.path.join(conda_base, 'envs', env_name) if use_local: channels = '--use-local' else: channels = '' - packages = 'progressbar2 jinja2 "mache=1.10.0"' + packages = 'progressbar2 jinja2 {}'.format(mache) if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing compass\n') commands = '{}; ' \ @@ -101,8 +101,28 @@ def main(): # install miniconda if needed install_miniconda(conda_base, activate_base, logger) + local_mache = args.mache_fork is not None and args.mache_branch is not None + if local_mache: + mache = '' + else: + mache = '"mache=1.10.0"' + setup_install_env(env_name, activate_base, args.use_local, logger, - args.recreate, conda_base) + args.recreate, conda_base, mache) + + if local_mache: + print('Clone and install local mache\n') + commands = '{}; ' \ + 'rm -rf conda/build_mache; ' \ + 'mkdir -p conda/build_mache; ' \ + 'cd conda/build_mache; ' \ + 'git clone -b {} git@github.com:{}.git mache; ' \ + 'cd mache; ' \ + 'python -m pip install .'.format(activate_install_env, + args.mache_branch, + args.mache_fork) + + check_call(commands, logger=logger) env_type = config.get('deploy', 'env_type') if env_type not in ['dev', 'test_release', 'release']: diff --git a/conda/shared.py b/conda/shared.py index 0ec9498d79..9bbd8d3639 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -47,6 +47,10 @@ def parse_args(bootstrap): "packages") parser.add_argument("--use_local", dest="use_local", action='store_true', help="Use locally built conda packages (for testing).") + parser.add_argument("--mache_fork", dest="mache_fork", + help="Point to a mache fork (and branch) for testing") + parser.add_argument("--mache_branch", dest="mache_branch", + help="Point to a mache branch (and fork) for testing") parser.add_argument("--update_spack", dest="update_spack", action='store_true', help="If the shared spack environment should be " @@ -78,6 +82,10 @@ def parse_args(bootstrap): args = parser.parse_args(sys.argv[1:]) + if (args.mache_fork is None) != (args.mache_branch is None): + raise ValueError('You must supply both or neither of ' + '--mache_fork and --mache_branch') + return args From 1a4b8a4942235afd73ce2f252d31e46da62d2694 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 2 Feb 2023 23:18:27 -0700 Subject: [PATCH 5/6] Require python 3 for deployment I believe all systems have python3 available. This allows us to remove some awkward imports and to use f-strings instead of format commands. --- conda/bootstrap.py | 4 +- conda/configure_compass_env.py | 71 +++++++++++++--------------------- conda/shared.py | 41 +++++++++----------- 3 files changed, 45 insertions(+), 71 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index c029f66b41..bde0546521 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python - -from __future__ import print_function +#!/usr/bin/env python3 import os import platform diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 3ee8f412eb..e3ceaebf33 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -1,21 +1,9 @@ -#!/usr/bin/env python - -from __future__ import print_function +#!/usr/bin/env python3 import os import sys -try: - from configparser import ConfigParser -except ImportError: - from six.moves import configparser - import six - - if six.PY2: - ConfigParser = configparser.SafeConfigParser - else: - ConfigParser = configparser.ConfigParser - +from configparser import ConfigParser from shared import parse_args, get_conda_base, check_call, install_miniconda, \ get_logger @@ -36,13 +24,11 @@ def get_config(config_file): def bootstrap(activate_install_env, source_path, local_conda_build): print('Creating the compass conda environment\n') - bootstrap_command = '{}/conda/bootstrap.py'.format(source_path) - command = '{}; ' \ - '{} {}'.format(activate_install_env, bootstrap_command, - ' '.join(sys.argv[1:])) + bootstrap_command = f'{source_path}/conda/bootstrap.py' + command = f'{activate_install_env}; ' \ + f'{bootstrap_command} {" ".join(sys.argv[1:])}' if local_conda_build is not None: - command = '{} --local_conda_build {}'.format(command, - local_conda_build) + command = f'{command} --local_conda_build {local_conda_build}' check_call(command) @@ -53,19 +39,15 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate, channels = '--use-local' else: channels = '' - packages = 'progressbar2 jinja2 {}'.format(mache) + packages = f'progressbar2 jinja2 {mache}' if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing compass\n') - commands = '{}; ' \ - 'mamba create -y -n {} {} {}'.format(activate_base, - env_name, channels, - packages) + commands = f'{activate_base}; ' \ + f'mamba create -y -n {env_name} {channels} {packages}' else: print('Updating conda environment for installing compass\n') - commands = '{}; ' \ - 'mamba install -y -n {} {} {}'.format(activate_base, - env_name, channels, - packages) + commands = f'{activate_base}; ' \ + f'mamba install -y -n {env_name} {channels} {packages}' check_call(commands, logger=logger) @@ -82,14 +64,14 @@ def main(): env_name = 'compass_bootstrap' source_activation_scripts = \ - 'source {}/etc/profile.d/conda.sh; ' \ - 'source {}/etc/profile.d/mamba.sh'.format(conda_base, conda_base) + f'source {conda_base}/etc/profile.d/conda.sh; ' \ + f'source {conda_base}/etc/profile.d/mamba.sh' - activate_base = '{}; conda activate'.format(source_activation_scripts) + activate_base = f'{source_activation_scripts}; conda activate' activate_install_env = \ - '{}; ' \ - 'conda activate {}'.format(source_activation_scripts, env_name) + f'{source_activation_scripts}; ' \ + f'conda activate {env_name}' try: os.makedirs('conda/logs') except OSError: @@ -112,24 +94,23 @@ def main(): if local_mache: print('Clone and install local mache\n') - commands = '{}; ' \ - 'rm -rf conda/build_mache; ' \ - 'mkdir -p conda/build_mache; ' \ - 'cd conda/build_mache; ' \ - 'git clone -b {} git@github.com:{}.git mache; ' \ - 'cd mache; ' \ - 'python -m pip install .'.format(activate_install_env, - args.mache_branch, - args.mache_fork) + commands = f'{activate_install_env}; ' \ + f'rm -rf conda/build_mache; ' \ + f'mkdir -p conda/build_mache; ' \ + f'cd conda/build_mache; ' \ + f'git clone -b {args.mache_branch} ' \ + f'git@github.com:{args.mache_fork}.git mache; ' \ + f'cd mache; ' \ + f'python -m pip install .' check_call(commands, logger=logger) env_type = config.get('deploy', 'env_type') if env_type not in ['dev', 'test_release', 'release']: - raise ValueError('Unexpected env_type: {}'.format(env_type)) + raise ValueError(f'Unexpected env_type: {env_type}') if env_type == 'test_release' and args.use_local: - local_conda_build = os.path.abspath('{}/conda-bld'.format(conda_base)) + local_conda_build = os.path.abspath(f'{conda_base}/conda-bld') else: local_conda_build = None diff --git a/conda/shared.py b/conda/shared.py index 9bbd8d3639..6c3e73d777 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import sys import argparse @@ -8,10 +6,7 @@ import logging import shutil -try: - from urllib.request import urlopen, Request -except ImportError: - from urllib2 import urlopen, Request +from urllib.request import urlopen, Request def parse_args(bootstrap): @@ -100,9 +95,9 @@ def get_conda_base(conda_base, config, shared=False, warn=False): conda_base = os.path.abspath( os.path.join(conda_exe, '..', '..')) if warn: - print('\nWarning: --conda path not supplied. Using conda ' - 'installed at:\n' - ' {}\n'.format(conda_base)) + print(f'\nWarning: --conda path not supplied. Using conda ' + f'installed at:\n' + f' {conda_base}\n') else: raise ValueError('No conda base provided with --conda and ' 'none could be inferred.') @@ -126,9 +121,9 @@ def get_spack_base(spack_base, config): def check_call(commands, env=None, logger=None): print_command = '\n '.join(commands.split('; ')) if logger is None: - print('\n Running:\n {}\n'.format(print_command)) + print(f'\n Running:\n {print_command}\n') else: - logger.info('\nrunning:\n {}\n'.format(print_command)) + logger.info(f'\nrunning:\n {print_command}\n') if logger is None: process = subprocess.Popen(commands, env=env, executable='/bin/bash', @@ -162,8 +157,8 @@ def install_miniconda(conda_base, activate_base, logger): system = 'MacOSX' else: system = 'Linux' - miniconda = 'Mambaforge-{}-x86_64.sh'.format(system) - url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda) + miniconda = f'Mambaforge-{system}-x86_64.sh' + url = f'https://github.com/conda-forge/miniforge/releases/latest/download/{miniconda}' print(url) req = Request(url, headers={'User-Agent': 'Mozilla/5.0'}) f = urlopen(req) @@ -172,19 +167,19 @@ def install_miniconda(conda_base, activate_base, logger): outfile.write(html) f.close() - command = '/bin/bash {} -b -p {}'.format(miniconda, conda_base) + command = f'/bin/bash {miniconda} -b -p {conda_base}' check_call(command, logger=logger) os.remove(miniconda) backup_bashrc() print('Doing initial setup\n') - commands = '{}; ' \ - 'conda config --add channels conda-forge; ' \ - 'conda config --set channel_priority strict; ' \ - 'conda install -y boa; ' \ - 'conda update -y --all; ' \ - 'mamba init'.format(activate_base) + commands = f'{activate_base}; ' \ + f'conda config --add channels conda-forge; ' \ + f'conda config --set channel_priority strict; ' \ + f'conda install -y boa; ' \ + f'conda update -y --all; ' \ + f'mamba init' check_call(commands, logger=logger) @@ -196,7 +191,7 @@ def backup_bashrc(): files = ['.bashrc', '.bash_profile'] for filename in files: src = os.path.join(home_dir, filename) - dst = os.path.join(home_dir, '{}.conda_bak'.format(filename)) + dst = os.path.join(home_dir, f'{filename}.conda_bak') if os.path.exists(src): shutil.copyfile(src, dst) @@ -205,14 +200,14 @@ def restore_bashrc(): home_dir = os.path.expanduser('~') files = ['.bashrc', '.bash_profile'] for filename in files: - src = os.path.join(home_dir, '{}.conda_bak'.format(filename)) + src = os.path.join(home_dir, f'{filename}.conda_bak') dst = os.path.join(home_dir, filename) if os.path.exists(src): shutil.move(src, dst) def get_logger(name, log_filename): - print('Logging to: {}\n'.format(log_filename)) + print(f'Logging to: {log_filename}\n') try: os.remove(log_filename) except OSError: From 14ba59e2b826b9f38f08a534e36747035be3aa90 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 3 Feb 2023 14:10:14 -0600 Subject: [PATCH 6/6] Remove reference to albany+cxx17 This spack variant no longer exists, as albany always uses c++17 now. --- conda/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index bde0546521..188c390785 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -398,7 +398,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, specs.append(f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc') if albany != 'None': - specs.append(f'albany@{albany}+mpas+cxx17') + specs.append(f'albany@{albany}+mpas') yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template):