diff --git a/.conda/meta.yaml b/.conda/meta.yaml index abc97f5..529ba01 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -29,5 +29,5 @@ requirements: - cerberus >=1.3.5 - gitpython - jinja2 - - hpcpy>=0.3.0 + - hpcpy>=0.5.0 - meorg_client diff --git a/docs/user_guide/config_options.md b/docs/user_guide/config_options.md index fdc76b5..b8d7515 100644 --- a/docs/user_guide/config_options.md +++ b/docs/user_guide/config_options.md @@ -68,6 +68,7 @@ fluxsite: walltime: 06:00:00 storage: [scratch/a00, gdata/xy11] multiprocess: True + meorg_model_output_id: XXXXXXXX ``` ### [experiment](#experiment) @@ -154,7 +155,7 @@ fluxsite: ### [multiprocess](#multiprocess) -: **Default:** True, _optional key_. :octicons-dash-24: Enables or disables multiprocessing for executing embarrassingly parallel tasks. + ```yaml @@ -163,6 +164,14 @@ fluxsites: ``` +### [meorg_model_output_id](#meorg_model_output_id) + +: **Default:** False, _optional key_. :octicons-dash-24: The unique Model Output ID from modelevaluation.org to which output files will be automatically uploaded for analysis. + +A separate upload job will be submitted at the successful completion of benchcab tasks if this key is present, however, the validity is not checked by benchcab at this stage. + +Note: It is the user's responsbility to ensure the model output is configured on modelevaluation.org. + ## spatial Contains settings specific to spatial tests. @@ -493,4 +502,14 @@ codecov: [f90nml-github]: https://github.com/marshallward/f90nml [environment-modules]: https://modules.sourceforge.net/ [nci-pbs-directives]: https://opus.nci.org.au/display/Help/PBS+Directives+Explained -[cable-github]: https://github.com/CABLE-LSM/CABLE \ No newline at end of file +[cable-github]: https://github.com/CABLE-LSM/CABLE + +## meorg_bin + +: **Default:** False, _optional key. :octicons-dash-24: Specifies the absolute system path to the ME.org client executable. In the absence of this key it will be inferred from the same directory as benchcab should `meorg_model_output_id` be set in `fluxsite` above. + +``` yaml + +meorg_bin: /path/to/meorg + +``` \ No newline at end of file diff --git a/src/benchcab/benchcab.py b/src/benchcab/benchcab.py index 4eec1af..c4d1b9b 100644 --- a/src/benchcab/benchcab.py +++ b/src/benchcab/benchcab.py @@ -11,6 +11,7 @@ from subprocess import CalledProcessError from typing import Optional +import benchcab.utils.meorg as bm from benchcab import fluxsite, internal, spatial from benchcab.comparison import run_comparisons, run_comparisons_in_parallel from benchcab.config import read_config @@ -24,7 +25,6 @@ from benchcab.model import Model from benchcab.utils import is_verbose, task_summary from benchcab.utils.fs import mkdir, next_path -import benchcab.utils.meorg as bm from benchcab.utils.pbs import render_job_script from benchcab.utils.repo import create_repo from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface @@ -245,15 +245,14 @@ def fluxsite_submit_job(self, config_path: str, skip: list[str]) -> None: logger.info(f"{internal.FLUXSITE_DIRS['TASKS']}//out.txt") logger.info("The NetCDF output for each task is written to:") logger.info(f"{internal.FLUXSITE_DIRS['OUTPUT']}/_out.nc") - + # Upload to meorg by default bm.do_meorg( config, - upload_dir=internal.FLUXSITE_DIRS['OUTPUT'], + upload_dir=internal.FLUXSITE_DIRS["OUTPUT"], benchcab_bin=str(self.benchcab_exe_path), - benchcab_job_id=job_id + benchcab_job_id=job_id, ) - def gen_codecov(self, config_path: str): """Endpoint for `benchcab codecov`.""" @@ -360,7 +359,7 @@ def fluxsite_run_tasks(self, config_path: str): else: fluxsite.run_tasks(tasks) - n_tasks, n_success, n_failed, all_complete = task_summary(tasks) + _, n_success, n_failed, _ = task_summary(tasks) logger.info(f"{n_failed} failed, {n_success} passed") def fluxsite_bitwise_cmp(self, config_path: str): @@ -386,9 +385,9 @@ def fluxsite_bitwise_cmp(self, config_path: str): ncpus = config["fluxsite"]["pbs"]["ncpus"] run_comparisons_in_parallel(comparisons, n_processes=ncpus) else: - run_comparisons(comparisons) - - n_tasks, n_success, n_failed, all_complete = task_summary(comparisons) + run_comparisons(comparisons) + + _, n_success, n_failed, _ = task_summary(comparisons) logger.info(f"{n_failed} failed, {n_success} passed") def fluxsite(self, config_path: str, no_submit: bool, skip: list[str]): diff --git a/src/benchcab/config.py b/src/benchcab/config.py index 102a9bb..14766e1 100644 --- a/src/benchcab/config.py +++ b/src/benchcab/config.py @@ -119,6 +119,9 @@ def read_optional_key(config: dict): config["fluxsite"]["pbs"] = internal.FLUXSITE_DEFAULT_PBS | config["fluxsite"].get( "pbs", {} ) + config["fluxsite"]["meorg_model_output_id"] = config["fluxsite"].get( + "meorg_model_output_id", internal.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID + ) config["codecov"] = config.get("codecov", False) diff --git a/src/benchcab/data/config-schema.yml b/src/benchcab/data/config-schema.yml index ee83fdf..654d1a5 100644 --- a/src/benchcab/data/config-schema.yml +++ b/src/benchcab/data/config-schema.yml @@ -108,8 +108,11 @@ fluxsite: type: "string" required: false meorg_model_output_id: - type: "string" + type: + - "boolean" + - "string" required: false + default: false spatial: type: "dict" @@ -137,4 +140,11 @@ spatial: codecov: type: "boolean" - required: false \ No newline at end of file + required: false + +meorg_bin: + type: + - "boolean" + - "string" + required: False + default: False \ No newline at end of file diff --git a/src/benchcab/data/test/config-optional.yml b/src/benchcab/data/test/config-optional.yml index f4605e8..e36c436 100644 --- a/src/benchcab/data/test/config-optional.yml +++ b/src/benchcab/data/test/config-optional.yml @@ -3,6 +3,7 @@ project: hh5 fluxsite: experiment: AU-Tum + meorg_model_output_id: False multiprocess: False pbs: ncpus: 6 diff --git a/src/benchcab/data/test/integration_meorg.sh b/src/benchcab/data/test/integration_meorg.sh index 26b1d1e..e90c531 100644 --- a/src/benchcab/data/test/integration_meorg.sh +++ b/src/benchcab/data/test/integration_meorg.sh @@ -46,6 +46,7 @@ fluxsite: storage: - scratch/$PROJECT - gdata/$PROJECT + # This ID is currently configured on the me.org server. meorg_model_output_id: Sss7qupAHEZ8ovbCv EOL diff --git a/src/benchcab/internal.py b/src/benchcab/internal.py index be56982..66af72d 100644 --- a/src/benchcab/internal.py +++ b/src/benchcab/internal.py @@ -252,6 +252,7 @@ } FLUXSITE_DEFAULT_EXPERIMENT = "forty-two-site-test" +FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID = False OPTIONAL_COMMANDS = ["fluxsite-bitwise-cmp", "gen_codecov"] @@ -275,11 +276,12 @@ def get_met_forcing_file_names(experiment: str) -> list[str]: return file_names + # Configuration for the client upload MEORG_CLIENT = dict( - num_threads=1, # Parallel uploads over 4 cores - cache_delay=60*5, # 5mins between upload and analysis triggering + num_threads=1, # Parallel uploads over 4 cores + cache_delay=60 * 5, # 5mins between upload and analysis triggering mem="8G", walltime="01:00:00", - storage=["gdata/ks32", "gdata/hh5", "gdata/wd9", "gdata/rp23"] -) \ No newline at end of file + storage=["gdata/ks32", "gdata/hh5", "gdata/wd9", "gdata/rp23"], +) diff --git a/src/benchcab/utils/__init__.py b/src/benchcab/utils/__init__.py index b652c4f..b628e6e 100644 --- a/src/benchcab/utils/__init__.py +++ b/src/benchcab/utils/__init__.py @@ -11,7 +11,7 @@ import sys from importlib import resources from pathlib import Path -from typing import Union, Iterable +from typing import Iterable, Union import yaml from jinja2 import BaseLoader, Environment @@ -162,9 +162,10 @@ def task_summary(tasks: Iterable) -> tuple: ------- tuple num_tasks, num_complete, num_failed, all_complete + """ num_tasks = len(tasks) num_complete = len([task for task in tasks if task.is_done()]) num_failed = num_tasks - num_complete - - return num_tasks, num_complete, num_failed, num_complete == num_tasks \ No newline at end of file + + return num_tasks, num_complete, num_failed, num_complete == num_tasks diff --git a/src/benchcab/utils/meorg.py b/src/benchcab/utils/meorg.py index 9359ccb..9ee35df 100644 --- a/src/benchcab/utils/meorg.py +++ b/src/benchcab/utils/meorg.py @@ -1,10 +1,13 @@ """Utility methods for interacting with the ME.org client.""" -from benchcab.internal import MEORG_CLIENT -from meorg_client.client import Client as MeorgClient + +import os + from hpcpy import get_client +from meorg_client.client import Client as MeorgClient + import benchcab.utils as bu -import os -from glob import glob +from benchcab.internal import MEORG_CLIENT + def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: str): """Perform the upload of model outputs to modelevaluation.org @@ -22,29 +25,29 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: ------- bool True if successful, False otherwise - """ + """ logger = bu.get_logger() - model_output_id = config.get("fluxsite").get("meorg_model_output_id", False) + model_output_id = config["fluxsite"]["meorg_model_output_id"] num_threads = MEORG_CLIENT["num_threads"] - + # Check if a model output id has been assigned if model_output_id == False: logger.info("No model_output_id found in fluxsite configuration.") logger.info("NOT uploading to modelevaluation.org") return False - + # Allow the user to specify an absolute path to the meorg bin in config meorg_bin = config.get("meorg_bin", False) - + # Otherwise infer the path from the benchcab installation if meorg_bin == False: logger.debug(f"Inferring meorg bin from {benchcab_bin}") bin_segments = benchcab_bin.split("/") bin_segments[-1] = "meorg" meorg_bin = "/".join(bin_segments) - + logger.debug(f"meorg_bin = {meorg_bin}") # Now, check if that actually exists @@ -56,28 +59,34 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: # Also only run if the client is initialised if MeorgClient().is_initialised() == False: - logger.warn("A model_output_id has been supplied, but the meorg_client is not initialised.") - logger.warn("To initialise, run `meorg initialise` in the installation environment.") - logger.warn("Once initialised, the outputs from this run can be uploaded with the following command:") - logger.warn(f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}") + logger.warn( + "A model_output_id has been supplied, but the meorg_client is not initialised." + ) + logger.warn( + "To initialise, run `meorg initialise` in the installation environment." + ) + logger.warn( + "Once initialised, the outputs from this run can be uploaded with the following command:" + ) + logger.warn( + f"meorg file upload {upload_dir}/*.nc -n {num_threads} --attach_to {model_output_id}" + ) logger.warn("Then the analysis can be triggered with:") logger.warn(f"meorg analysis start {model_output_id}") return False # Finally, attempt the upload! else: - + logger.info("Uploading outputs to modelevaluation.org") # Submit the outputs client = get_client() meorg_jobid = client.submit( - bu.get_installed_root() / "data" / "meorg_jobscript.j2", render=True, dry_run=False, depends_on=benchcab_job_id, - # Interpolate into the job script model_output_id=model_output_id, data_dir=upload_dir, @@ -85,12 +94,12 @@ def do_meorg(config: dict, upload_dir: str, benchcab_bin: str, benchcab_job_id: mem=MEORG_CLIENT["mem"], num_threads=MEORG_CLIENT["num_threads"], walltime=MEORG_CLIENT["walltime"], - storage=MEORG_CLIENT['storage'], - project=config['project'], - modules=config['modules'], + storage=MEORG_CLIENT["storage"], + project=config["project"], + modules=config["modules"], purge_outputs=True, - meorg_bin=meorg_bin + meorg_bin=meorg_bin, ) logger.info(f"Upload job submitted: {meorg_jobid}") - return True \ No newline at end of file + return True diff --git a/tests/test_comparison.py b/tests/test_comparison.py index cec5125..568b1c2 100644 --- a/tests/test_comparison.py +++ b/tests/test_comparison.py @@ -8,6 +8,7 @@ from pathlib import Path import pytest + from benchcab import internal from benchcab.comparison import ComparisonTask diff --git a/tests/test_config.py b/tests/test_config.py index 5f741be..e995d99 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -75,6 +75,7 @@ def all_optional_default_config(no_optional_config) -> dict: "experiment": bi.FLUXSITE_DEFAULT_EXPERIMENT, "multiprocess": bi.FLUXSITE_DEFAULT_MULTIPROCESS, "pbs": bi.FLUXSITE_DEFAULT_PBS, + "meorg_model_output_id": bi.FLUXSITE_DEFAULT_MEORG_MODEL_OUTPUT_ID }, "science_configurations": bi.DEFAULT_SCIENCE_CONFIGURATIONS, "spatial": { @@ -106,6 +107,7 @@ def all_optional_custom_config(no_optional_config) -> dict: "walltime": "10:00:00", "storage": ["scratch/$PROJECT"], }, + "meorg_model_output_id": False }, "science_configurations": [ { diff --git a/tests/test_fluxsite.py b/tests/test_fluxsite.py index e4732ba..a7558d5 100644 --- a/tests/test_fluxsite.py +++ b/tests/test_fluxsite.py @@ -10,6 +10,7 @@ import f90nml import netCDF4 import pytest + from benchcab import __version__, internal from benchcab.fluxsite import ( CableError, diff --git a/tests/test_fs.py b/tests/test_fs.py index 23d95d0..3101699 100644 --- a/tests/test_fs.py +++ b/tests/test_fs.py @@ -10,6 +10,7 @@ from pathlib import Path import pytest + from benchcab.utils.fs import chdir, mkdir, next_path, prepend_path diff --git a/tests/test_spatial.py b/tests/test_spatial.py index aed4283..7a0d950 100644 --- a/tests/test_spatial.py +++ b/tests/test_spatial.py @@ -10,6 +10,7 @@ import f90nml import pytest import yaml + from benchcab import internal from benchcab.model import Model from benchcab.spatial import SpatialTask, get_spatial_tasks diff --git a/tests/test_state.py b/tests/test_state.py index 7c77f33..cda2e73 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -1,9 +1,10 @@ import time from pathlib import Path +from tempfile import TemporaryDirectory import pytest + from benchcab.utils.state import State, StateAttributeError -from tempfile import TemporaryDirectory def test_state_is_set(): @@ -39,4 +40,4 @@ def test_state_get_raises_exception(): with TemporaryDirectory() as tmp_dir: state = State(state_dir=Path(tmp_dir)) with pytest.raises(StateAttributeError): - state.get() \ No newline at end of file + state.get() diff --git a/tests/test_utils.py b/tests/test_utils.py index 37284ba..d6ce264 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -64,7 +64,7 @@ def test_get_logger_singleton_fail(): def test_task_summary(): - + # Create some mocked tasks t1 = ComparisonTask(files=(), task_name="t1") t2 = ComparisonTask(files=(), task_name="t2") @@ -72,7 +72,7 @@ def test_task_summary(): # Inject success/fail cases t1.is_done = lambda: True t2.is_done = lambda: False - + # Run the function n_tasks, n_success, n_failed, all_complete = bu.task_summary([t1, t2]) @@ -80,4 +80,4 @@ def test_task_summary(): assert n_tasks == 2 assert n_success == 1 assert n_failed == 1 - assert all_complete == False \ No newline at end of file + assert all_complete == False diff --git a/versioneer.py b/versioneer.py index 8851eaa..0ae83db 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,4 +1,3 @@ - # Version: 0.29 """The Versioneer - like a rocketeer, but for versions. @@ -310,15 +309,14 @@ import configparser import errno +import functools import json import os import re import subprocess import sys from pathlib import Path -from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Union -from typing import NoReturn -import functools +from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, Union, cast have_tomllib = True if sys.version_info >= (3, 11): @@ -367,11 +365,13 @@ def get_root() -> str: or os.path.exists(pyproject_toml) or os.path.exists(versioneer_py) ): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") + err = ( + "Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND')." + ) raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools @@ -384,8 +384,10 @@ def get_root() -> str: me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py)) + print( + "Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(my_path), versioneer_py) + ) except NameError: pass return root @@ -403,9 +405,9 @@ def get_config_from_root(root: str) -> VersioneerConfig: section: Union[Dict[str, Any], configparser.SectionProxy, None] = None if pyproject_toml.exists() and have_tomllib: try: - with open(pyproject_toml, 'rb') as fobj: + with open(pyproject_toml, "rb") as fobj: pp = tomllib.load(fobj) - section = pp['tool']['versioneer'] + section = pp["tool"]["versioneer"] except (tomllib.TOMLDecodeError, KeyError) as e: print(f"Failed to load config from {pyproject_toml}: {e}") print("Try to load it from setup.cfg") @@ -422,7 +424,7 @@ def get_config_from_root(root: str) -> VersioneerConfig: # `None` values elsewhere where it matters cfg = VersioneerConfig() - cfg.VCS = section['VCS'] + cfg.VCS = section["VCS"] cfg.style = section.get("style", "") cfg.versionfile_source = cast(str, section.get("versionfile_source")) cfg.versionfile_build = section.get("versionfile_build") @@ -450,10 +452,12 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f + return decorate @@ -480,10 +484,14 @@ def run_command( try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) + process = subprocess.Popen( + [command] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + **popen_kwargs, + ) break except OSError as e: if e.errno == errno.ENOENT: @@ -505,7 +513,9 @@ def run_command( return stdout, process.returncode -LONG_VERSION_PY['git'] = r''' +LONG_VERSION_PY[ + "git" +] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -1250,7 +1260,7 @@ def git_versions_from_keywords( # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1259,7 +1269,7 @@ def git_versions_from_keywords( # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1267,32 +1277,36 @@ def git_versions_from_keywords( for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') - if not re.match(r'\d', r): + if not re.match(r"\d", r): continue if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs( - tag_prefix: str, - root: str, - verbose: bool, - runner: Callable = run_command + tag_prefix: str, root: str, verbose: bool, runner: Callable = run_command ) -> Dict[str, Any]: """Get version from 'git describe' in the root of the source tree. @@ -1311,8 +1325,7 @@ def git_pieces_from_vcs( env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -1320,10 +1333,19 @@ def git_pieces_from_vcs( # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) + describe_out, rc = runner( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + f"{tag_prefix}[[:digit:]]*", + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -1338,8 +1360,7 @@ def git_pieces_from_vcs( pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") @@ -1379,17 +1400,16 @@ def git_pieces_from_vcs( dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -1398,10 +1418,12 @@ def git_pieces_from_vcs( if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -1479,15 +1501,21 @@ def versions_from_parentdir( for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -1516,11 +1544,13 @@ def versions_from_file(filename: str) -> Dict[str, Any]: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) + mo = re.search( + r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S + ) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1528,8 +1558,7 @@ def versions_from_file(filename: str) -> Dict[str, Any]: def write_to_version_file(filename: str, versions: Dict[str, Any]) -> None: """Write the given version number to the given _version.py file.""" - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) + contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) @@ -1561,8 +1590,7 @@ def render_pep440(pieces: Dict[str, Any]) -> str: rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1591,8 +1619,7 @@ def render_pep440_branch(pieces: Dict[str, Any]) -> str: rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -1753,11 +1780,13 @@ def render_git_describe_long(pieces: Dict[str, Any]) -> str: def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -1781,9 +1810,13 @@ def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } class VersioneerBadRootError(Exception): @@ -1806,8 +1839,9 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or bool(cfg.verbose) # `bool()` used to avoid `None` - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" + assert ( + cfg.versionfile_source is not None + ), "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) @@ -1861,9 +1895,13 @@ def get_versions(verbose: bool = False) -> Dict[str, Any]: if verbose: print("unable to compute version") - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } def get_version() -> str: @@ -1916,6 +1954,7 @@ def run(self) -> None: print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version # we override "build_py" in setuptools @@ -1937,8 +1976,8 @@ def run(self) -> None: # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py: Any = cmds['build_py'] + if "build_py" in cmds: + _build_py: Any = cmds["build_py"] else: from setuptools.command.build_py import build_py as _build_py @@ -1955,14 +1994,14 @@ def run(self) -> None: # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py - if 'build_ext' in cmds: - _build_ext: Any = cmds['build_ext'] + if "build_ext" in cmds: + _build_ext: Any = cmds["build_ext"] else: from setuptools.command.build_ext import build_ext as _build_ext @@ -1982,19 +2021,22 @@ def run(self) -> None: # it with an updated value if not cfg.versionfile_build: return - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) + target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): - print(f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running build_ext " - "without first running build_py.") + print( + f"Warning: {target_versionfile} does not exist, skipping " + "version update. This can happen if you are running build_ext " + "without first running build_py." + ) return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # type: ignore + # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ @@ -2015,17 +2057,21 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["build_exe"] = cmd_build_exe del cmds["build_py"] - if 'py2exe' in sys.modules: # py2exe enabled? + if "py2exe" in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe # type: ignore except ImportError: @@ -2044,18 +2090,22 @@ def run(self) -> None: os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info - if 'egg_info' in cmds: - _egg_info: Any = cmds['egg_info'] + if "egg_info" in cmds: + _egg_info: Any = cmds["egg_info"] else: from setuptools.command.egg_info import egg_info as _egg_info @@ -2068,7 +2118,7 @@ def find_sources(self) -> None: # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) - self.filelist.append('versioneer.py') + self.filelist.append("versioneer.py") if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit @@ -2081,18 +2131,21 @@ def find_sources(self) -> None: # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils - normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') - for f in self.filelist.files] - manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') - with open(manifest_filename, 'w') as fobj: - fobj.write('\n'.join(normalized)) + normalized = [ + unicode_utils.filesys_decode(f).replace(os.sep, "/") + for f in self.filelist.files + ] - cmds['egg_info'] = cmd_egg_info + manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") + with open(manifest_filename, "w") as fobj: + fobj.write("\n".join(normalized)) + + cmds["egg_info"] = cmd_egg_info # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist: Any = cmds['sdist'] + if "sdist" in cmds: + _sdist: Any = cmds["sdist"] else: from setuptools.command.sdist import sdist as _sdist @@ -2114,8 +2167,10 @@ def make_release_tree(self, base_dir: str, files: List[str]) -> None: # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) + write_to_version_file( + target_versionfile, self._versioneer_generated_versions + ) + cmds["sdist"] = cmd_sdist return cmds @@ -2175,11 +2230,9 @@ def do_setup() -> int: root = get_root() try: cfg = get_config_from_root(root) - except (OSError, configparser.NoSectionError, - configparser.NoOptionError) as e: + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) + print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) @@ -2188,15 +2241,18 @@ def do_setup() -> int: print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") + f.write( + LONG + % { + "DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + } + ) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") maybe_ipy: Optional[str] = ipy if os.path.exists(ipy): try: @@ -2275,4 +2331,3 @@ def setup_command() -> NoReturn: cmd = sys.argv[1] if cmd == "setup": setup_command() -