diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d477f799025..5a73761c573 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -118,6 +118,8 @@ jobs: # from 'entrypoint.sh', create and activate new environment create_environment ${{ matrix.python-version }} + pip install -r test-requirements.txt + # GitHub runner home is different than container cp -rf /root/.cime /github/home/ @@ -162,6 +164,8 @@ jobs: CIME_DRIVER: ${{ matrix.driver }} CIME_TEST_PLATFORM: ubuntu-latest run: | + pip install -r test-requirements.txt + export CIME_REMOTE=https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }} export CIME_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}} diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 4efe3ee48cf..60f863a2670 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -9,23 +9,100 @@ import os import json import logging - from shutil import copytree -import CIME.test_status -import CIME.utils +from CIME import test_status +from CIME import utils from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines - +from CIME.config import ConfigBase +from CIME.SystemTests import test_mods +from CIME.namelist import Namelist import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error -evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) +version = evv4esm.__version_info__ + +assert version >= (0, 5, 0), "Please install evv4esm greater or equal to 0.5.0" + +EVV_LIB_DIR = os.path.abspath(os.path.dirname(evv4esm.__file__)) + logger = logging.getLogger(__name__) -NINST = 30 + + +class MVKConfig(ConfigBase): + def __init__(self): + super().__init__() + + if self.loaded: + return + + self._set_attribute("component", "", "The main component.") + self._set_attribute( + "components", [], "Components that require namelist customization." + ) + self._set_attribute("ninst", 30, "The number of instances.") + self._set_attribute( + "var_set", "default", "Name of the variable set to analyze." + ) + self._set_attribute("ref_case", "Baseline", "Name of the reference case.") + self._set_attribute("test_case", "Test", "Name of the test case.") + + def generate_namelist( + self, case, component, i, filename + ): # pylint: disable=unused-argument + """Generate per instance namelist. + + This method is called for each instance to generate the desired + modifications. + + Args: + case (CIME.case.case.Case): The case instance. + component (str): Component the namelist belongs to. + i (int): Instance unique number. + filename (str): Name of the namelist that needs to be created. + """ + namelist = Namelist() + + with namelist(filename) as nml: + nml.set_variable_value("", "new_random", True) + nml.set_variable_value("", "pertlim", "1.0e-10") + nml.set_variable_value("", "seed_custom", f"{i}") + nml.set_variable_value("", "seed_clock", True) + + def evv_test_config(self, case, config): # pylint: disable=unused-argument + """Customize the evv4esm configuration. + + This method is used to customize the default evv4esm configuration + or generate a completely new one. + + The return configuration will be written to `$RUNDIR/$CASE.json`. + + Args: + case (CIME.case.case.Case): The case instance. + config (dict): Default evv4esm configuration. + + Returns: + dict: Dictionary with test configuration. + """ + return config + + def _default_evv_test_config(self, run_dir, base_dir, evv_lib_dir): + config = { + "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), + "test-case": self.test_case, + "test-dir": run_dir, + "ref-case": self.ref_case, + "ref-dir": base_dir, + "var-set": self.var_set, + "ninst": self.ninst, + "component": self.component, + } + + return config class MVK(SystemTestsCommon): @@ -33,12 +110,37 @@ def __init__(self, case, **kwargs): """ initialize an object interface to the MVK test """ + self._config = None + SystemTestsCommon.__init__(self, case, **kwargs) - if self._case.get_value("MODEL") == "e3sm": - self.component = "eam" - else: - self.component = "cam" + *_, case_test_mods = utils.parse_test_name(self._casebaseid) + + test_mods_paths = test_mods.find_test_mods( + case.get_value("COMP_INTERFACE"), case_test_mods + ) + + for test_mods_path in test_mods_paths: + self._config = MVKConfig.load(os.path.join(test_mods_path, "params.py")) + + if self._config is None: + self._config = MVKConfig() + + # Use old behavior for component + if self._config.component == "": + # TODO remove model specific + if self._case.get_value("MODEL") == "e3sm": + self._config.component = "eam" + else: + self._config.component = "cam" + + if len(self._config.components) == 0: + self._config.components = [self._config.component] + elif ( + self._config.component != "" + and self._config.component not in self._config.components + ): + self._config.components.extend([self._config.component]) if ( self._case.get_value("RESUBMIT") == 0 @@ -53,27 +155,28 @@ def build_phase(self, sharedlib_only=False, model_only=False): # so it has to happen there. if not model_only: logging.warning("Starting to build multi-instance exe") + for comp in self._case.get_values("COMP_CLASSES"): self._case.set_value("NTHRDS_{}".format(comp), 1) ntasks = self._case.get_value("NTASKS_{}".format(comp)) - self._case.set_value("NTASKS_{}".format(comp), ntasks * NINST) + self._case.set_value( + "NTASKS_{}".format(comp), ntasks * self._config.ninst + ) + if comp != "CPL": - self._case.set_value("NINST_{}".format(comp), NINST) + self._case.set_value("NINST_{}".format(comp), self._config.ninst) self._case.flush() case_setup(self._case, test_mode=False, reset=True) - for iinst in range(1, NINST + 1): - with open( - "user_nl_{}_{:04d}".format(self.component, iinst), "w" - ) as nl_atm_file: - nl_atm_file.write("new_random = .true.\n") - nl_atm_file.write("pertlim = 1.0e-10\n") - nl_atm_file.write("seed_custom = {}\n".format(iinst)) - nl_atm_file.write("seed_clock = .true.\n") + for i in range(1, self._config.ninst + 1): + for component in self._config.components: + filename = "user_nl_{}_{:04d}".format(component, i) + + self._config.generate_namelist(self._case, component, i, filename) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) @@ -83,7 +186,7 @@ def _generate_baseline(self): """ super(MVK, self)._generate_baseline() - with CIME.utils.SharedArea(): + with utils.SharedArea(): basegen_dir = os.path.join( self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"), @@ -94,17 +197,20 @@ def _generate_baseline(self): env_archive = self._case.get_env("archive") hists = env_archive.get_all_hist_files( - self._case.get_value("CASE"), self.component, rundir, ref_case=ref_case + self._case.get_value("CASE"), + self._config.component, + rundir, + ref_case=ref_case, ) logger.debug("MVK additional baseline files: {}".format(hists)) hists = [os.path.join(rundir, hist) for hist in hists] for hist in hists: - basename = hist[hist.rfind(self.component) :] + basename = hist[hist.rfind(self._config.component) :] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) - CIME.utils.safe_copy(hist, baseline, preserve_meta=False) + utils.safe_copy(hist, baseline, preserve_meta=False) def _compare_baseline(self): with self._test_status: @@ -113,12 +219,12 @@ def _compare_baseline(self): # and we only want to compare once the whole run is finished. We # need to return a pass here to continue the submission process. self._test_status.set_status( - CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_PASS_STATUS + test_status.BASELINE_PHASE, test_status.TEST_PASS_STATUS ) return self._test_status.set_status( - CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_FAIL_STATUS + test_status.BASELINE_PHASE, test_status.TEST_FAIL_STATUS ) run_dir = self._case.get_value("RUNDIR") @@ -129,78 +235,102 @@ def _compare_baseline(self): ) test_name = "{}".format(case_name.split(".")[-1]) - evv_config = { - test_name: { - "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), - "test-case": "Test", - "test-dir": run_dir, - "ref-case": "Baseline", - "ref-dir": base_dir, - "var-set": "default", - "ninst": NINST, - "critical": 13, - "component": self.component, - } - } - - json_file = os.path.join(run_dir, ".".join([case_name, "json"])) + + default_config = self._config._default_evv_test_config( + run_dir, + base_dir, + EVV_LIB_DIR, + ) + + test_config = self._config.evv_test_config( + self._case, + default_config, + ) + + evv_config = {test_name: test_config} + + json_file = os.path.join(run_dir, f"{case_name}.json") with open(json_file, "w") as config_file: json.dump(evv_config, config_file, indent=4) - evv_out_dir = os.path.join(run_dir, ".".join([case_name, "evv"])) + evv_out_dir = os.path.join(run_dir, f"{case_name}.evv") evv(["-e", json_file, "-o", evv_out_dir]) - with open(os.path.join(evv_out_dir, "index.json")) as evv_f: - evv_status = json.load(evv_f) - - comments = "" - for evv_ele in evv_status["Page"]["elements"]: - if "Table" in evv_ele: - comments = "; ".join( - "{}: {}".format(key, val[0]) - for key, val in evv_ele["Table"]["data"].items() - ) - if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": - self._test_status.set_status( - CIME.test_status.BASELINE_PHASE, - CIME.test_status.TEST_PASS_STATUS, - ) - break - - status = self._test_status.get_status(CIME.test_status.BASELINE_PHASE) - mach_name = self._case.get_value("MACH") - mach_obj = Machines(machine=mach_name) - htmlroot = CIME.utils.get_htmlroot(mach_obj) - urlroot = CIME.utils.get_urlroot(mach_obj) - if htmlroot is not None: - with CIME.utils.SharedArea(): - copytree( - evv_out_dir, - os.path.join(htmlroot, "evv", case_name), - ) - if urlroot is None: - urlroot = "[{}_URL]".format(mach_name.capitalize()) - viewing = "{}/evv/{}/index.html".format(urlroot, case_name) - else: - viewing = ( - "{}\n" - " EVV viewing instructions can be found at: " - " https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/" - "climate_reproducibility/README.md#test-passfail-and-extended-output" - "".format(evv_out_dir) - ) + self.update_testlog(test_name, case_name, evv_out_dir) + + def update_testlog(self, test_name, case_name, evv_out_dir): + comments = self.process_evv_output(evv_out_dir) + + status = self._test_status.get_status(test_status.BASELINE_PHASE) + + mach_name = self._case.get_value("MACH") - comments = ( - "{} {} for test '{}'.\n" - " {}\n" - " EVV results can be viewed at:\n" - " {}".format( - CIME.test_status.BASELINE_PHASE, - status, - test_name, - comments, - viewing, + mach_obj = Machines(machine=mach_name) + + htmlroot = utils.get_htmlroot(mach_obj) + + if htmlroot is not None: + urlroot = utils.get_urlroot(mach_obj) + + with utils.SharedArea(): + copytree( + evv_out_dir, + os.path.join(htmlroot, "evv", case_name), ) + + if urlroot is None: + urlroot = "[{}_URL]".format(mach_name.capitalize()) + + viewing = "{}/evv/{}/index.html".format(urlroot, case_name) + else: + viewing = ( + "{}\n" + " EVV viewing instructions can be found at: " + " https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/" + "climate_reproducibility/README.md#test-passfail-and-extended-output" + "".format(evv_out_dir) + ) + + comments = ( + "{} {} for test '{}'.\n" + " {}\n" + " EVV results can be viewed at:\n" + " {}".format( + test_status.BASELINE_PHASE, + status, + test_name, + comments, + viewing, ) + ) + + append_testlog(comments, self._orig_caseroot) + + def process_evv_output(self, evv_out_dir): + with open(os.path.join(evv_out_dir, "index.json")) as evv_f: + evv_status = json.load(evv_f) + + comments = "" + + for evv_ele in evv_status["Page"]["elements"]: + if "Table" in evv_ele: + comments = "; ".join( + "{}: {}".format(key, val[0]) + for key, val in evv_ele["Table"]["data"].items() + ) + + if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": + with self._test_status: + self._test_status.set_status( + test_status.BASELINE_PHASE, + test_status.TEST_PASS_STATUS, + ) + + break + + return comments + - append_testlog(comments, self._orig_caseroot) +if __name__ == "__main__": + _config = MVKConfig() + _config.print_rst_table() diff --git a/CIME/SystemTests/test_mods.py b/CIME/SystemTests/test_mods.py new file mode 100644 index 00000000000..0f163280099 --- /dev/null +++ b/CIME/SystemTests/test_mods.py @@ -0,0 +1,81 @@ +import logging +import os + +from CIME.utils import CIMEError +from CIME.XML.files import Files + +logger = logging.getLogger(__name__) + +MODS_DIR_VARS = ("TESTS_MODS_DIR", "USER_MODS_DIR") + + +def find_test_mods(comp_interface, test_mods): + """Finds paths from names of testmods. + + Testmod format is `${component}-${testmod}`. Each testmod is search for + it it's component respective `TESTS_MODS_DIR` and `USER_MODS_DIR`. + + Args: + comp_interface (str): Name of the component interface. + test_mods (list): List of testmods names. + + Returns: + List of paths for each testmod. + + Raises: + CIMEError: If a testmod is not in correct format. + CIMEError: If testmod could not be found. + """ + if test_mods is None: + return [] + + files = Files(comp_interface=comp_interface) + + test_mods_paths = [] + + logger.debug("Checking for testmods {}".format(test_mods)) + + for test_mod in test_mods: + if test_mod.find("/") != -1: + component, mod_path = test_mod.split("/", 1) + else: + raise CIMEError( + f"Invalid testmod, format should be `${{component}}-${{testmod}}`, got {test_mod!r}" + ) + + logger.info( + "Searching for testmod {!r} for component {!r}".format(mod_path, component) + ) + + test_mod_path = None + + for var in MODS_DIR_VARS: + mods_dir = files.get_value(var, {"component": component}) + + try: + candidate_path = os.path.join(mods_dir, component, mod_path) + except TypeError: + # mods_dir is None + continue + + logger.debug( + "Checking for testmod {!r} in {!r}".format(test_mod, candidate_path) + ) + + if os.path.exists(candidate_path): + test_mod_path = candidate_path + + logger.info( + "Found testmod {!r} for component {!r} in {!r}".format( + mod_path, component, test_mod_path + ) + ) + + break + + if test_mod_path is None: + raise CIMEError(f"Could not locate testmod {mod_path!r}") + + test_mods_paths.append(test_mod_path) + + return test_mods_paths diff --git a/CIME/config.py b/CIME/config.py index 076e5b9fc38..f63a4bea78d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -1,10 +1,11 @@ import os import re import sys -import glob import logging import importlib.machinery import importlib.util +import inspect +from pathlib import Path from CIME import utils @@ -13,6 +14,45 @@ DEFAULT_CUSTOMIZE_PATH = os.path.join(utils.get_src_root(), "cime_config", "customize") +def print_rst_header(header, anchor=None, separator='"'): + n = len(header) + if anchor is not None: + print(f".. _{anchor}\n") + print(separator * n) + print(header) + print(separator * n) + + +def print_rst_table(headers, *rows): + column_widths = [] + + columns = [[rows[y][x] for y in range(len(rows))] for x in range(len(rows[0]))] + + for header, column in zip(headers, columns): + column_widths.append( + max( + [ + len(x) + for x in [ + header, + ] + + column + ] + ) + ) + + divider = " ".join([f"{'=' * x}" for x in column_widths]) + + print(divider) + print(" ".join(f"{y}{' ' * (x - len(y))}" for x, y in zip(column_widths, headers))) + print(divider) + + for row in rows: + print(" ".join([f"{y}{' ' * (x-len(y))}" for x, y in zip(column_widths, row)])) + + print(divider) + + class ConfigBase: def __new__(cls): if not hasattr(cls, "_instance"): @@ -41,14 +81,19 @@ def load(cls, customize_path): logger.debug("Searching %r for files to load", customize_path) - customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) + customize_path = Path(customize_path) - ignore_pattern = re.compile(f"{customize_path}/(?:tests|conftest|test_)") + if customize_path.is_file(): + customize_files = [f"{customize_path}"] + else: + ignore_pattern = re.compile(f"{customize_path}/(?:tests|conftest|test_)") - # filter out any tests - customize_files = [ - x for x in customize_files if ignore_pattern.search(x) is None - ] + # filter out any tests + customize_files = [ + f"{x}" + for x in customize_path.glob("**/*.py") + if ignore_pattern.search(f"{x}") is None + ] customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) @@ -99,39 +144,56 @@ def _set_attribute(self, name, value, desc=None): } def print_rst_table(self): - max_variable = max([len(x) for x in self._attribute_config.keys()]) - max_default = max( - [len(str(x["default"])) for x in self._attribute_config.values()] - ) - max_type = max( - [len(type(x["default"]).__name__) for x in self._attribute_config.values()] - ) - max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) + self.print_variable_rst() + + print("") + + self.print_method_rst() + + def print_variable_rst(self): + print_rst_header("Variables", anchor=f"{self.__class__.__name__} Variables:") - divider_row = ( - f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" + headers = ("Variable", "Default", "Type", "Description") + + rows = ( + (x, str(y["default"]), type(y["default"]).__name__, y["desc"]) + for x, y in self._attribute_config.items() ) - rows = [ - divider_row, - f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", - divider_row, - ] + print_rst_table(headers, *rows) - for variable, value in sorted( - self._attribute_config.items(), key=lambda x: x[0] - ): - variable_fill = max_variable - len(variable) - default_fill = max_default - len(str(value["default"])) - type_fill = max_type - len(type(value["default"]).__name__) + def print_method_rst(self): + print_rst_header("Methods", anchor=f"{self.__class__.__name__} Methods:") - rows.append( - f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" - ) + methods = inspect.getmembers(self, inspect.ismethod) - rows.append(divider_row) + ignore = ( + "__init__", + "loaded", + "load", + "instance", + "_load_file", + "_set_attribute", + "print_rst_table", + "print_method_rst", + "print_variable_rst", + ) + + child_methods = [ + (x[0], inspect.signature(x[1]), inspect.getdoc(x[1])) + for x in methods + if x[1].__class__ != Config and x[0] not in ignore + ] - print("\n".join(rows)) + for (name, sig, doc) in child_methods: + if doc is None: + continue + print(".. code-block::\n") + print(f" def {name}{sig!s}:") + print(' """') + for line in doc.split("\n"): + print(f" {line}") + print(' """') class Config(ConfigBase): diff --git a/CIME/namelist.py b/CIME/namelist.py index 43907a9fbd1..562f0ca55ac 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -102,6 +102,7 @@ import re import collections +from contextlib import contextmanager # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import @@ -156,6 +157,17 @@ FORTRAN_REPEAT_PREFIX_REGEX = re.compile(r"^[0-9]*[1-9]+[0-9]*\*") +def convert_bool(value): + if isinstance(value, bool): + value = f".{str(value).lower()}." + elif isinstance(value, str): + value = f".{value.lower()}." + else: + raise ValueError("Unable to convert {}".format(value)) + + return value + + def is_valid_fortran_name(string): """Check that a variable name is allowed in Fortran. @@ -926,6 +938,13 @@ def __init__(self, groups=None): variable_name ] + @contextmanager + def __call__(self, filename): + try: + yield self + finally: + self.write(filename) + def clean_groups(self): self._groups = collections.OrderedDict() @@ -1044,6 +1063,11 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): >>> x.get_variable_value('foo', 'red') ['', '2', '', '4', '', '6'] """ + if not isinstance(value, (set, list)): + value = [ + value, + ] + minindex, maxindex, step = get_fortran_variable_indices(variable_name, var_size) variable_name = get_fortran_name_only(variable_name) @@ -1054,7 +1078,7 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): ), ) gn = string_in_list(group_name, self._groups) - if not gn: + if gn is None: gn = group_name self._groups[gn] = {} @@ -1211,7 +1235,7 @@ def _write(self, out_file, groups, format_, sorted_groups): else: group_names = groups for group_name in group_names: - if format_ == "nml": + if group_name != "" and format_ == "nml": out_file.write("&{}\n".format(group_name)) # allow empty group if group_name in self._groups: @@ -1227,18 +1251,23 @@ def _write(self, out_file, groups, format_, sorted_groups): # To prettify things for long lists of values, build strings # line-by-line. - if values[0] == "True" or values[0] == "False": - values[0] = ( - values[0] - .replace("True", ".true.") - .replace("False", ".false.") - ) - lines = [" {}{} {}".format(name, equals, values[0])] + if isinstance(values[0], bool) or values[0].lower() in ( + "true", + "false", + ): + values[0] = convert_bool(values[0]) + + if group_name == "": + lines = ["{}{} {}".format(name, equals, values[0])] + else: + lines = [" {}{} {}".format(name, equals, values[0])] for value in values[1:]: - if value == "True" or value == "False": - value = value.replace("True", ".true.").replace( - "False", ".false." - ) + if isinstance(value, bool) or value.lower() in ( + "true", + "false", + ): + value = convert_bool(value) + if len(lines[-1]) + len(value) <= 77: lines[-1] += ", " + value else: @@ -1247,7 +1276,7 @@ def _write(self, out_file, groups, format_, sorted_groups): lines[-1] += "\n" for line in lines: out_file.write(line) - if format_ == "nml": + if group_name != "" and format_ == "nml": out_file.write("/\n") if format_ == "nmlcontents": out_file.write("\n") diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index 179402c7501..8e3b77603cb 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -29,6 +29,7 @@ get_timestamp, get_cime_default_driver, clear_folder, + CIMEError, ) from CIME.config import Config from CIME.test_status import * @@ -46,6 +47,7 @@ from CIME.cs_status_creator import create_cs_status from CIME.hist_utils import generate_teststatus from CIME.build import post_build +from CIME.SystemTests.test_mods import find_test_mods logger = logging.getLogger(__name__) @@ -687,34 +689,18 @@ def _create_newcase_phase(self, test): if test_mods is not None: create_newcase_cmd += " --user-mods-dir " - for one_test_mod in test_mods: # pylint: disable=not-an-iterable - if one_test_mod.find("/") != -1: - (component, modspath) = one_test_mod.split("/", 1) - else: - error = "Missing testmod component. Testmods are specified as '${component}-${testmod}'" - self._log_output(test, error) - return False, error + try: + test_mods_paths = find_test_mods(self._cime_driver, test_mods) + except CIMEError as e: + error = f"{e}" - files = Files(comp_interface=driver) - testmods_dir = files.get_value( - "TESTS_MODS_DIR", {"component": component} - ) - test_mod_file = os.path.join(testmods_dir, component, modspath) - # if no testmod is found check if a usermod of the same name exists and - # use it if it does. - if not os.path.exists(test_mod_file): - usermods_dir = files.get_value( - "USER_MODS_DIR", {"component": component} - ) - test_mod_file = os.path.join(usermods_dir, modspath) - if not os.path.exists(test_mod_file): - error = "Missing testmod file '{}', checked {} and {}".format( - modspath, testmods_dir, usermods_dir - ) - self._log_output(test, error) - return False, error + self._log_output(test, error) + + return False, error + else: + test_mods_paths = " ".join(test_mods_paths) - create_newcase_cmd += "{} ".format(test_mod_file) + create_newcase_cmd += f"{test_mods_paths}" # create_test mpilib option overrides default but not explicitly set case_opt mpilib if mpilib is None and self._mpilib is not None: diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index 3dfd4b62124..d4822f93e32 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import re import glob import logging import os @@ -14,6 +15,22 @@ class TestTestScheduler(base.BaseTestCase): + def get_default_tests(self): + # exclude the MEMLEAK tests here. + return get_tests.get_full_test_names( + [ + "cime_test_only", + "^TESTMEMLEAKFAIL_P1.f09_g16.X", + "^TESTMEMLEAKPASS_P1.f09_g16.X", + "^TESTRUNSTARCFAIL_P1.f19_g16_rx1.A", + "^TESTTESTDIFF_P1.f19_g16_rx1.A", + "^TESTBUILDFAILEXC_P1.f19_g16_rx1.A", + "^TESTRUNFAILEXC_P1.f19_g16_rx1.A", + ], + self._machine, + self._compiler, + ) + @mock.patch("time.strftime", return_value="00:00:00") def test_chksum(self, strftime): # pylint: disable=unused-argument if self._config.test_mode == "e3sm": @@ -38,21 +55,77 @@ def test_chksum(self, strftime): # pylint: disable=unused-argument from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.perlmutter_gnu.00:00:00", ) - def test_a_phases(self): - # exclude the MEMLEAK tests here. - tests = get_tests.get_full_test_names( - [ - "cime_test_only", - "^TESTMEMLEAKFAIL_P1.f09_g16.X", - "^TESTMEMLEAKPASS_P1.f09_g16.X", - "^TESTRUNSTARCFAIL_P1.f19_g16_rx1.A", - "^TESTTESTDIFF_P1.f19_g16_rx1.A", - "^TESTBUILDFAILEXC_P1.f19_g16_rx1.A", - "^TESTRUNFAILEXC_P1.f19_g16_rx1.A", - ], - self._machine, - self._compiler, + def test_testmods(self): + if self._config.test_mode == "cesm": + self.skipTest("Skipping testmods test. Depends on E3SM settings") + + tests = self.get_default_tests() + ct = test_scheduler.TestScheduler( + tests, + test_root=self._testroot, + output_root=self._testroot, + compiler=self._compiler, + mpilib=self.TEST_MPILIB, + machine_name=self.MACHINE.get_machine_name(), ) + + with mock.patch.object(ct, "_shell_cmd_for_phase"): + ct._create_newcase_phase( + "TESTRUNPASS_P1.f19_g16_rx1.A.docker_gnu.eam-rrtmgp" + ) + + create_newcase_cmd = ct._shell_cmd_for_phase.call_args.args[1] + + assert ( + re.search(r"--user-mods-dir .*eam/rrtmgp", create_newcase_cmd) + is not None + ), create_newcase_cmd + + def test_testmods_malformed(self): + tests = self.get_default_tests() + ct = test_scheduler.TestScheduler( + tests, + test_root=self._testroot, + output_root=self._testroot, + compiler=self._compiler, + mpilib=self.TEST_MPILIB, + machine_name=self.MACHINE.get_machine_name(), + ) + + with mock.patch.object(ct, "_shell_cmd_for_phase"): + success, message = ct._create_newcase_phase( + "TESTRUNPASS_P1.f19_g16_rx1.A.docker_gnu.notacomponent?fun" + ) + + assert not success + assert ( + message + == "Invalid testmod, format should be `${component}-${testmod}`, got 'notacomponent?fun'" + ), message + + def test_testmods_missing(self): + tests = self.get_default_tests() + ct = test_scheduler.TestScheduler( + tests, + test_root=self._testroot, + output_root=self._testroot, + compiler=self._compiler, + mpilib=self.TEST_MPILIB, + machine_name=self.MACHINE.get_machine_name(), + ) + + with mock.patch.object(ct, "_shell_cmd_for_phase"): + success, message = ct._create_newcase_phase( + "TESTRUNPASS_P1.f19_g16_rx1.A.docker_gnu.notacomponent-fun" + ) + + assert not success + assert ( + re.search("Could not locate testmod 'fun'", message) is not None + ), message + + def test_a_phases(self): + tests = self.get_default_tests() self.assertEqual(len(tests), 3) ct = test_scheduler.TestScheduler( tests, diff --git a/CIME/tests/test_unit_case_setup.py b/CIME/tests/test_unit_case_setup.py index fe5fa7308c1..a00dcf1b413 100644 --- a/CIME/tests/test_unit_case_setup.py +++ b/CIME/tests/test_unit_case_setup.py @@ -8,6 +8,7 @@ from unittest import mock from CIME.case import case_setup +from CIME.tests.utils import chdir @contextlib.contextmanager @@ -23,17 +24,6 @@ def create_machines_dir(): yield temp_path -@contextlib.contextmanager -def chdir(path): - old_path = os.getcwd() - os.chdir(path) - - try: - yield - finally: - os.chdir(old_path) - - # pylint: disable=protected-access class TestCaseSetup(unittest.TestCase): @mock.patch("CIME.case.case_setup.copy_depends_files") diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py new file mode 100644 index 00000000000..56eaa39f67e --- /dev/null +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 + +import os +import json +import unittest +import tempfile +import contextlib +import sysconfig +from pathlib import Path +from unittest import mock + +from CIME.SystemTests.mvk import MVK +from CIME.SystemTests.mvk import MVKConfig +from CIME.tests.utils import chdir + + +def create_complex_case( + case_name, + temp_dir, + run_dir, + baseline_dir, + compare_baseline=False, + mock_evv_output=False, +): + case = mock.MagicMock() + + side_effect = [ + str(temp_dir), # CASEROOT + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "mct", # COMP_INTERFACE + "mct", # COMP_INTERFACE + ] + + # single extra call for _compare_baseline + if compare_baseline: + side_effect.append("e3sm") # MODEL + + side_effect.extend( + [ + 0, # RESUBMIT + False, # GENERATE_BASELINE + 0, # RESUBMIT + str(run_dir), # RUNDIR + case_name, # CASE + str(baseline_dir), # BASELINE_ROOT + "", # BASECMP_CASE + "docker", # MACH + ] + ) + + case.get_value.side_effect = side_effect + + run_dir.mkdir(parents=True, exist_ok=True) + + evv_output = run_dir / f"{case_name}.evv" / "index.json" + + evv_output.parent.mkdir(parents=True, exist_ok=True) + + write_evv_output(evv_output, mock_evv_output=mock_evv_output) + + return case + + +def write_evv_output(evv_output_path, mock_evv_output): + if mock_evv_output: + evv_output_data = { + "Page": { + "elements": [ + { + "Table": { + "data": { + "Test status": ["pass"], + "Variables analyzed": ["v1", "v2"], + "Rejecting": [2], + "Critical value": [12], + } + } + } + ] + } + } + else: + evv_output_data = {"Page": {"elements": []}} + + with open(evv_output_path, "w") as fd: + fd.write(json.dumps(evv_output_data)) + + +def create_simple_case(model="e3sm", resubmit=0, generate_baseline=False): + case = mock.MagicMock() + + case.get_value.side_effect = ( + "/tmp/case", # CASEROOT + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "mct", # COMP_INTERFACE + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + model, + resubmit, + generate_baseline, + ) + + return case + + +class TestSystemTestsMVK(unittest.TestCase): + def tearDown(self): + # reset singleton + try: + delattr(MVKConfig, "_instance") + except: + pass + + @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") + @mock.patch("CIME.SystemTests.mvk.evv") + def test_testmod_complex(self, evv, find_test_mods): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + print(temp_dir) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + testmods_dir = temp_dir / "testmods" / "eam" + + testmods_dir.mkdir(parents=True) + + find_test_mods.return_value = [str(testmods_dir)] + + with open(testmods_dir / "params.py", "w") as fd: + fd.write( + """ +import os +from CIME.namelist import Namelist +from CIME.SystemTests.mvk import EVV_LIB_DIR + +component = "new-comp" +components = ["new-comp", "secondary-comp"] +ninst = 8 + +def generate_namelist(case, component, i, filename): + nml = Namelist() + + if component == "new-comp": + nml.set_variable_value("", "var1", "value1") + elif component == "secondary-comp": + nml.set_variable_value("", "var2", "value2") + + nml.write(filename) + +def evv_test_config(case, config): + config["module"] = os.path.join(EVV_LIB_DIR, "extensions", "kso.py") + config["component"] = "someother-comp" + + return config + """ + ) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test.build_phase(False, True) + test._compare_baseline() + + with open(run_dir / f"{case_name}.json", "r") as fd: + config = json.load(fd) + + expected_config = { + "20240515_212034_41b5u2": { + "component": "someother-comp", + "ninst": 8, + "ref-case": "Baseline", + "ref-dir": f"{temp_dir}/baselines/", + "test-case": "Test", + "test-dir": f"{temp_dir}/run", + "var-set": "default", + } + } + + module = config["20240515_212034_41b5u2"].pop("module") + + assert ( + f'{sysconfig.get_paths()["purelib"]}/evv4esm/extensions/kso.py' + == module + ) + + assert config == expected_config + + nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] + + assert len(nml_files) == 16 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == ["var1 = value1\n"] + + with open(sorted(nml_files)[-1], "r") as fd: + lines = fd.readlines() + + assert lines == ["var2 = value2\n"] + + @mock.patch("CIME.SystemTests.mvk.append_testlog") + @mock.patch("CIME.SystemTests.mvk.Machines") + def test_update_testlog(self, machines, append_testlog): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + run_dir.mkdir(parents=True) + + evv_output_path = run_dir / "index.json" + + write_evv_output(evv_output_path, True) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + machines.return_value.get_value.return_value = "docker" + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + test.update_testlog("test1", case_name, str(run_dir)) + + append_testlog.assert_any_call( + """BASELINE PASS for test 'test1'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + docker/evv/MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2/index.html""", + str(temp_dir), + ) + + @mock.patch("CIME.SystemTests.mvk.utils.get_urlroot") + @mock.patch("CIME.SystemTests.mvk.append_testlog") + @mock.patch("CIME.SystemTests.mvk.Machines") + def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + run_dir.mkdir(parents=True) + + evv_output_path = run_dir / "index.json" + + write_evv_output(evv_output_path, True) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + machines.return_value.get_value.return_value = "docker" + + get_urlroot.return_value = None + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + test.update_testlog("test1", case_name, str(run_dir)) + + print(append_testlog.call_args_list) + append_testlog.assert_any_call( + f"""BASELINE PASS for test 'test1'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + [{run_dir!s}_URL]/evv/MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2/index.html""", + str(temp_dir), + ) + + @mock.patch("CIME.SystemTests.mvk.utils.get_htmlroot") + @mock.patch("CIME.SystemTests.mvk.append_testlog") + @mock.patch("CIME.SystemTests.mvk.Machines") + def test_update_testlog_htmlroot(self, machines, append_testlog, get_htmlroot): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + run_dir.mkdir(parents=True) + + evv_output_path = run_dir / "index.json" + + write_evv_output(evv_output_path, True) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + machines.return_value.get_value.return_value = "docker" + + get_htmlroot.return_value = None + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + test.update_testlog("test1", case_name, str(run_dir)) + + append_testlog.assert_any_call( + f"""BASELINE PASS for test 'test1'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + {run_dir!s} + EVV viewing instructions can be found at: https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/climate_reproducibility/README.md#test-passfail-and-extended-output""", + str(temp_dir), + ) + + @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") + @mock.patch("CIME.SystemTests.mvk.evv") + def test_testmod_simple(self, evv, find_test_mods): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + testmods_dir = temp_dir / "testmods" / "eam" + + testmods_dir.mkdir(parents=True) + + find_test_mods.return_value = [str(testmods_dir)] + + with open(testmods_dir / "params.py", "w") as fd: + fd.write( + """ +component = "new-comp" +components = ["new-comp", "second-comp"] +ninst = 8 +var_set = "special" +ref_case = "Reference" +test_case = "Default" + """ + ) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test.build_phase(False, True) + test._compare_baseline() + + with open(run_dir / f"{case_name}.json", "r") as fd: + config = json.load(fd) + + expected_config = { + "20240515_212034_41b5u2": { + "test-case": "Default", + "test-dir": f"{run_dir}", + "ref-case": "Reference", + "ref-dir": f"{baseline_dir}/", + "var-set": "special", + "ninst": 8, + "component": "new-comp", + } + } + + module = config["20240515_212034_41b5u2"].pop("module") + + assert ( + f'{sysconfig.get_paths()["purelib"]}/evv4esm/extensions/ks.py' == module + ) + + assert config == expected_config + + nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] + + assert len(nml_files) == 16 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_clock = .true.\n", + "seed_custom = 1\n", + ] + + with open(sorted(nml_files)[-1], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_clock = .true.\n", + "seed_custom = 8\n", + ] + + @mock.patch("CIME.SystemTests.mvk.case_setup") + @mock.patch("CIME.SystemTests.mvk.MVK.build_indv") + def test_build_phase(self, build_indv, case_setup): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + case.get_values.side_effect = (("CPL", "LND"),) + + side_effect = [x for x in case.get_value.side_effect] + + n = 7 + side_effect.insert(n, 8) + side_effect.insert(n, 16) + + case.get_value.side_effect = side_effect + + test = MVK(case) + + test.build_phase(sharedlib_only=True) + + case.set_value.assert_any_call("NTHRDS_CPL", 1) + case.set_value.assert_any_call("NTASKS_CPL", 480) + case.set_value.assert_any_call("NTHRDS_LND", 1) + case.set_value.assert_any_call("NTASKS_LND", 240) + case.set_value.assert_any_call("NINST_LND", 30) + + case.flush.assert_called() + + case_setup.assert_any_call(case, test_mode=False, reset=True) + + @mock.patch("CIME.SystemTests.mvk.SystemTestsCommon._generate_baseline") + @mock.patch("CIME.SystemTests.mvk.append_testlog") + @mock.patch("CIME.SystemTests.mvk.evv") + def test__generate_baseline(self, evv, append_testlog, _generate_baseline): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + # use original 5 args + side_effect = [x for x in case.get_value.side_effect][:7] + + side_effect.extend( + [ + str(baseline_dir), + "MVK.f19_g16.S", + str(run_dir), + "MVK.f19_g16.S", + case_name, + ] + ) + + case.get_value.side_effect = side_effect + + case_baseline_dir = baseline_dir / "MVK.f19_g16.S" / "eam" + + case_baseline_dir.mkdir(parents=True, exist_ok=True) + + (run_dir / "eam").mkdir(parents=True, exist_ok=True) + + (run_dir / "eam" / "test1.nc").touch() + (run_dir / "eam" / "test2.nc").touch() + + case.get_env.return_value.get_all_hist_files.return_value = ( + "eam/test1.nc", + "eam/test2.nc", + ) + + test = MVK(case) + + test._generate_baseline() + + files = os.listdir(case_baseline_dir) + + assert sorted(files) == sorted(["test1.nc", "test2.nc"]) + + # reset side_effect + case.get_value.side_effect = side_effect + + test = MVK(case) + + # test baseline_dir already exists + test._generate_baseline() + + files = os.listdir(case_baseline_dir) + + assert sorted(files) == sorted(["test1.nc", "test2.nc"]) + + @mock.patch("CIME.SystemTests.mvk.append_testlog") + @mock.patch("CIME.SystemTests.mvk.evv") + def test__compare_baseline_resubmit(self, evv, append_testlog): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + side_effect = [x for x in case.get_value.side_effect][:-8] + + side_effect.extend([1, 1]) + + case.get_value.side_effect = side_effect + + test = MVK(case) + + with mock.patch.object(test, "_test_status") as _test_status: + test._compare_baseline() + + _test_status.set_status.assert_any_call("BASELINE", "PASS") + + @mock.patch("CIME.SystemTests.mvk.append_testlog") + @mock.patch("CIME.SystemTests.mvk.evv") + def test__compare_baseline(self, evv, append_testlog): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + test = MVK(case) + + test._compare_baseline() + + with open(run_dir / f"{case_name}.json", "r") as fd: + config = json.load(fd) + + expected_config = { + "20240515_212034_41b5u2": { + "test-case": "Test", + "test-dir": f"{run_dir}", + "ref-case": "Baseline", + "ref-dir": f"{baseline_dir}/", + "var-set": "default", + "ninst": 30, + "component": "eam", + } + } + + module = config["20240515_212034_41b5u2"].pop("module") + + assert ( + f'{sysconfig.get_paths()["purelib"]}/evv4esm/extensions/ks.py' == module + ) + + assert config == expected_config + + expected_comments = f"""BASELINE PASS for test '20240515_212034_41b5u2'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + {run_dir}/MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2.evv + EVV viewing instructions can be found at: https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/climate_reproducibility/README.md#test-passfail-and-extended-output""" + + append_testlog.assert_any_call( + expected_comments, str(temp_dir) + ), append_testlog.call_args.args + + def test_generate_namelist_multiple_components(self): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + case = create_simple_case() + + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test._config.components = ["eam", "elm"] + + test.build_phase(False, True) + + nml_files = os.listdir(temp_dir) + + assert len(nml_files) == 60 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_clock = .true.\n", + "seed_custom = 1\n", + ] + + def test_generate_namelist(self): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + case = create_simple_case() + + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test.build_phase(False, True) + + nml_files = os.listdir(temp_dir) + + assert len(nml_files) == 30 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_clock = .true.\n", + "seed_custom = 1\n", + ] + + def test_compare_baseline(self): + case = create_simple_case() + + MVK(case) + + case.set_value.assert_any_call("COMPARE_BASELINE", True) + + case = create_simple_case(generate_baseline=True) + + MVK(case) + + case.set_value.assert_any_call("COMPARE_BASELINE", False) + + case = create_simple_case(resubmit=1, generate_baseline=True) + + MVK(case) + + case.set_value.assert_any_call("COMPARE_BASELINE", False) + + def test_mvk(self): + case = create_simple_case() + + test = MVK(case) + + assert test._config.component == "eam" + assert test._config.components == ["eam"] + + case = create_simple_case("cesm") + + test = MVK(case) + + assert test._config.component == "cam" + assert test._config.components == ["cam"] diff --git a/CIME/tests/utils.py b/CIME/tests/utils.py index 719aaaff290..0f75fa5ad24 100644 --- a/CIME/tests/utils.py +++ b/CIME/tests/utils.py @@ -5,6 +5,7 @@ import shutil import sys import time +import contextlib from collections.abc import Iterable from CIME import utils @@ -50,6 +51,17 @@ ] +@contextlib.contextmanager +def chdir(path): + old_path = os.getcwd() + os.chdir(path) + + try: + yield + finally: + os.chdir(old_path) + + def parse_test_status(line): status, test = line.split()[0:2] return test, status diff --git a/CIME/utils.py b/CIME/utils.py index 59cbf3c7666..9f6f0dfd2c9 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -1579,7 +1579,7 @@ def get_charge_account(machobj=None, project=None): >>> import CIME >>> import CIME.XML.machines - >>> machobj = CIME.XML.machines.Machines(machine="theta") + >>> machobj = CIME.XML.machines.Machines(machine="docker") >>> project = get_project(machobj) >>> charge_account = get_charge_account(machobj, project) >>> project == charge_account diff --git a/doc/requirements.txt b/doc/requirements.txt index 956df97689b..e6edcf66b06 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,5 @@ sphinx sphinxcontrib-programoutput sphinx-rtd-theme +sphinx-copybutton evv4esm diff --git a/doc/source/conf.py b/doc/source/conf.py index 0239006c530..c6a1de2db7e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -* coding: utf-8 -*- # # on documentation build configuration file, created by # sphinx-quickstart on Tue Jan 31 19:46:36 2017. @@ -46,6 +46,7 @@ "sphinx.ext.todo", "sphinxcontrib.programoutput", "sphinx_rtd_theme", + "sphinx_copybutton", ] todo_include_todos = True diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index ed15a849e00..f604e93f7d8 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -4,32 +4,34 @@ Testing Cases ************** -`create_test <../Tools_user/create_test.html>`_ -is a powerful system testing capability provided by the CIME Case Control System. -create_test can, in one command, create a case, setup, build and run the case -according to the test type and return a PASS or FAIL for the test result. +The `create_test <../Tools_user/create_test.html>`_ command provides +a powerful tool capable of testing a Case. The command can create, +setup, build and run a case according to the :ref:`testname ` syntax, returning +a PASS or FAIL result. .. _individual: An individual test can be run as:: - $CIMEROOT/scripts/create_test $test_name + $CIMEROOT/scripts/create_test -Everything the test will do is controlled by parsing the test name. +Everything the test will do is controlled by the :ref:`testname `. -================= +.. _`testname syntax`: + +================ Testname syntax -================= -.. _`Test naming`: +================ -Tests must be named with the following forms, [ ]=optional:: +Tests are defined by the following format, where anything enclosed in ``[]`` is optional:: TESTTYPE[_MODIFIERS].GRID.COMPSET[.MACHINE_COMPILER][.GROUP-TESTMODS] -For example using the minimum required elements of a testname:: +For example using the minimum TESTTYPE_, `GRID <../users_guide/grids.html>`_, and `COMPSET <../users_guide/compsets.html>`_:: - $CIMEROOT/scripts/create_test ERP.ne4pg2_oQU480.F2010 + ERP.ne4pg2_oQU480.F2010 +Below is a break-down of the different parts of the ``testname`` syntax. ================= ===================================================================================== NAME PART @@ -165,7 +167,7 @@ elements of the test through a test type modifier. .. _MODIFIERS: ------------------- -Testtype Modifiers +MODIFIERS ------------------- ============ ===================================================================================== @@ -199,72 +201,255 @@ MODIFIERS Description For example, this will run the ERP test with debugging turned on during compilation:: - CIMEROOT/scripts/create_test ERP_D.ne4pg2_oQU480.F2010 + $CIMEROOT/scripts/create_test ERP_D.ne4pg2_oQU480.F2010 This will run the ERP test for 3 days instead of the default 11 days:: - CIMEROOT/scripts/create_test ERP_Ld3.ne4pg2_oQU480.F2010 + $CIMEROOT/scripts/create_test ERP_Ld3.ne4pg2_oQU480.F2010 You can combine testtype modifiers:: - CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010 - -------------------- -Test Case Modifiers -------------------- + $CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010 .. _GROUP-TESTMODS: -create_test runs with out-of-the-box compsets and grid sets. Sometimes you may want to run a test with -modification to a namelist or other setting without creating an entire compset. CCS provides the testmods -capability for this situation. +------------------- +GROUP-TESTMODS +------------------- -A testmod is a string at the end of the full testname (including machine and compiler) -with the form GROUP-TESTMODS which are parsed by create_test as follows: +The `create_test <../Tools_user/create_test.html>`_ command runs with out-of-the-box compsets and grid sets. +Sometimes you may want to run a test with modification to a namelist or other setting without creating an +entire compset. Case Control System (CCS) provides the testmods capability for this situation. +The ``GROUP-TESTMODS`` string is at the end of the full :ref:`testname ` (including machine and compiler). +The form ``GROUP-TESTMODS`` are parsed as follows. ============ ===================================================================================== -TESTMOD Description +PART Description ============ ===================================================================================== -GROUP Define the subdirectory of testmods_dirs and the parent directory of various testmods. - -TESTMODS A subdirectory of GROUP containing files which set non-default values - of the set-up and run-time variables via namelists or xml_change commands. - Example: - - | GROUP-TESTMODS = cam-outfrq9s points to - | $cesm/components/cam/cime_config/testdefs/testmods_dirs/cam/outfrq9s - | while allactive-defaultio points to - | $cesm/cime_config/testmods_dirs/allactive/defaultio +GROUP Name of the directory under ``TESTS_MODS_DIR`` that contains ``TESTMODS``. +TESTMODS Any combination of `user_nl_* `_, `shell_commands `_, + `user_mods `_, or `params.py `_ in a directory under the + ``GROUP`` directory. ============ ===================================================================================== -For example, the ERP test for an E3SM F-case can be modified to use a different radiation scheme:: +For example, the *ERP* test for an E3SM *F-case* can be modified to use a different radiation scheme by using ``eam-rrtmgp``:: + + ERP_D_Ld3.ne4pg2_oQU480.F2010.pm-cpu_intel.eam-rrtmgp - CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010.pm-cpu_intel.eam-rrtmgp +If ``TESTS_MODS_DIR`` was set to ``$E3SM/components/eam/cime_config/testdefs/testmods_dirs`` then the +directory containg the testmods woulc be ``$E3SM/components/eam/cime_config/testdefs/testmods_dirs/eam/rrtmpg``. -This tells create_test to look in $e3sm/components/eam/cime_config/testdefs/testmods_dirs/eam/rrtmpg -where it finds the following lines in the shell_commands file:: +In this directory you'd find a `shell_commands`` file containing the following:: - #!/bin/bash - ./xmlchange --append CAM_CONFIG_OPTS='-rad rrtmgp' + #!/bin/bash + ./xmlchange --append CAM_CONFIG_OPTS='-rad rrtmgp' These commands are applied after the testcase is created and case.setup is called. -The contents of each testmods directory can include +Note; do not use '-' in the testmods directory name because it has a special meaning to create_test. + +.. _USER_NL: + +```````` +Example *user_nl_* +```````` + +A components namelist can be modified by providing a ``user_nl_*`` file in a GROUP-TESTMODS_ directory. +For example, to change the namelist for the *eam* component a file name ``user_nl_eam`` could be used. + :: - user_nl_$components namelist variable=value pairs - shell_commands xmlchange commands - user_mods a list of other GROUP-TESTMODS which should be imported - but at a lower precedence than the local testmods. + # user_nl_eam + deep_scheme = 'off', + zmconv_microp = .false. + shallow_scheme = 'CLUBB_SGS', + l_tracer_aero = .false. + l_rayleigh = .false. + l_gw_drag = .false. + l_ac_energy_chk = .true. + l_bc_energy_fix = .true. + l_dry_adj = .false. + l_st_mac = .true. + l_st_mic = .false. + l_rad = .false. -eam/cime_config/testdefs/testmods_dirs/eam contains modifications for eam in an F-case test. You -might make a directory called eam/cime_config/testdefs/testmods_dirs/elm to modify the land model -in an F-case test. +.. _SHELL_COMMANDS: + +`````````````` +Example *shell_commands* +`````````````` + +A test can be modified by providing a ``shell_commands`` file in a GROUP-TESTMODS_ directory. +This shell file can contain any arbitrary commands, for example:: + + # shell_commands + #!/bin/bash + + # Remove exe if chem pp exe (campp) already exists (it ensures that exe is always built) + /bin/rm -f $CIMEROOT/../components/eam/chem_proc/campp + + # Invoke campp (using v3 mechanism file) + ./xmlchange --append CAM_CONFIG_OPTS='-usr_mech_infile $CIMEROOT/../components/eam/chem_proc/inputs/pp_chemUCI_linozv3_mam5_vbs.in' + + # Assuming atmchange is available via $PATH + atmchange initial_conditions::perturbation_random_seed = 32 + +.. _USER_MODS: + +````````` +Example *user_mods* +````````` + +Additional GROUP_TESTMODS_ can be applied by providing a list in a ``user_mods`` file in a GROUP-TESTMODS_ directory. + +:: + + # user_mods + eam/cosp + eam/hommexx + +.. _TESTYPE_MOD: + +`````````````````````` +Example *params.py* +`````````````````````` + +Supported TESTYPES_ can further be modified by providing a ``params.py`` file in the GROUP-TESTMODS_ directory. + +^^^^^^^^^^^^ +MVK +^^^^^^^^^^^^ +The `MVK` system test can be configured by defining :ref:`variables ` and :ref:`methods ` in ``params.py``. + +See :ref:`examples ` for a simple and complex use case. + +.. _MVKConfig Variables: + +""""""""" +Variables +""""""""" +========== ======== ==== =============================================== +Variable Default Type Description +========== ======== ==== =============================================== +component str The main component. +components [] list Components that require namelist customization. +ninst 30 int The number of instances. +var_set default str Name of the variable set to analyze. +ref_case Baseline str Name of the reference case. +test_case Test str Name of the test case. +========== ======== ==== =============================================== + +.. _MVKConfig Methods: + +""""""" +Methods +""""""" +.. code-block:: + + def evv_test_config(case, config): + """ + Customize the evv4esm configuration. + + This method is used to customize the default evv4esm configuration + or generate a completely new one. + + The return configuration will be written to `$RUNDIR/$CASE.json`. + + Args: + case (CIME.case.case.Case): The case instance. + config (dict): Default evv4esm configuration. + + Returns: + dict: Dictionary with test configuration. + """ +.. code-block:: + + def generate_namelist(case, component, i, filename): + """ + Generate per instance namelist. + + This method is called for each instance to generate the desired + modifications. + + Args: + case (CIME.case.case.Case): The case instance. + component (str): Component the namelist belongs to. + i (int): Instance unique number. + filename (str): Name of the namelist that needs to be created. + """ + +.. _MVK Examples: + +"""""""""" +Examples +"""""""""" +.. _MVK Simple: +In the simplest form just :ref:`variables ` need to be defined in ``params.py``. + +For this case the default ``evv_test_config`` and ``generate_namelist`` functions will be called. + +.. code-block:: + + component = "eam" + # components = [] can be omitted when modifying a single component + ninst = 10 + +.. _MVK Complex: + +If more control over the evv4esm configuration file or the per instance configuration is desired then +the ``evv_test_config`` and ``generate_namelist`` functions can be overridden in the ``params.py`` file. + +The :ref:`variables ` will still need to be defined to generate the default +evv4esm config or ``config`` in the ``evv_test_config`` function can be ignored and a completely new +dictionary can be returned. + +In the following example, the default ``module`` is changed as well as ``component`` and ``ninst``. +The ``generate_namelist`` function creates namelists for certain components while running a shell +command to customize others. + +Note; this is a toy example, no scientific usage. + +.. code-block:: + + import os + from CIME.SystemTests.mvk import EVV_LIB_DIR + from CIME.namelist import Namelist + from CIME.utils import safe_copy + from CIME.utils import run_cmd + + component "eam" + # The generate_namelist function will be called `ninst` times per component + components = ["eam", "clm", "eamxx"] + ninst = 30 + + # This can be omitted if the default evv4esm configuration is sufficient + def evv_test_config(case, config): + config["module"] = os.path.join(EVV_LIB_DIR, "extensions", "kso.py") + config["component"] = "clm" + config["ninst"] = 20 + + return config + + def generate_namelist(case, component, i, filename): + namelist = Namelist() + + if component in ["eam", "clm"]: + with namelist(filename) as nml: + if component == "eam": + # arguments group, key, value + nml.set_variable_value("", "eam_specific", f"perturn-{i}") + elif component == "clm": + if i % 2 == 0: + nml.set_variable_value("", "clm_specific", "even") + else: + nml.set_variable_value("", "clm_specific", "odd") + else: + stat, output, err = run_cmd(f"atmchange initial_conditions::perturbation_random_seed = {i*32}") + + safe_copy("namelist_scream.xml", f"namelist_scream_{i:04}.xml") -The "rrtmpg" directory contains the actual testmods to apply. -Note; do not use '-' in the testmods directory name because it has a special meaning to create_test. ======================== Test progress and output diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000000..d4abdec4d0f --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +evv4esm