diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..af554be4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +payu/_version.py export-subst diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 351c8454..f3b8be68 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -1,47 +1,24 @@ name: CD -on: [push, pull_request] +on: + push: + tags: + - '*' -jobs: - conda: - name: build and deploy to conda - if: github.repository == 'payu-org/payu' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - - name: Checkout source - uses: actions/checkout@v4 - - - name: Setup conda environment - uses: conda-incubator/setup-miniconda@v2 - with: - miniconda-version: "latest" - python-version: 3.11 - environment-file: conda/environment.yml - auto-update-conda: false - auto-activate-base: false - show-channel-urls: true - - - name: Build and upload the conda package - uses: uibcdf/action-build-and-upload-conda-packages@v1.2.0 - with: - meta_yaml_dir: conda - python-version: 3.11 - user: accessnri - label: main - token: ${{ secrets.anaconda_token }} +env: + PY_VERSION: 3.11 +jobs: pypi-build: name: Build package for PyPI - if: github.repository == 'payu-org/payu' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + if: github.repository == 'payu-org/payu' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: ${{ env.PY_VERSION }} - run: | python3 -m pip install --upgrade build && python3 -m build @@ -54,7 +31,6 @@ jobs: # Split build and publish to restrict trusted publishing to just this workflow needs: ['pypi-build'] name: Publish to PyPI.org - if: github.repository == 'payu-org/payu' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest permissions: # IMPORTANT: this permission is mandatory for trusted publishing @@ -65,7 +41,34 @@ jobs: path: artifact/ - name: Publish package distributions to PyPI - # This is version v1.8.10 - uses: pypa/gh-action-pypi-publish@b7f401de30cb6434a1e19f805ff006643653240e + uses: pypa/gh-action-pypi-publish@2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf # v1.8.11 + with: + packages-dir: artifact/ + + conda: + name: Build with conda and upload + if: github.repository == 'payu-org/payu' + runs-on: ubuntu-latest + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Setup conda environment + uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 with: - packages_dir: artifact/ + miniconda-version: "latest" + python-version: ${{ env.PY_VERSION }} + environment-file: conda/environment.yml + auto-update-conda: false + auto-activate-base: false + show-channel-urls: true + + - name: Build and upload the conda package + uses: uibcdf/action-build-and-upload-conda-packages@c6e7a90ad5e599d6cde76e130db4ee52ad733ecf # v1.2.0 + with: + meta_yaml_dir: conda + python-version: ${{ env.PY_VERSION }} + user: accessnri + label: main + token: ${{ secrets.anaconda_token }} diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index db3e050f..e48f0a75 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -12,8 +12,28 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +env: + PY_VERSION: 3.11 + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: + pypa-build: + name: PyPA build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PY_VERSION }} + + - run: | + python3 -m pip install --upgrade build && python3 -m build + + - uses: actions/upload-artifact@v3 + with: + path: ./dist + conda-build: name: Conda Build runs-on: ubuntu-latest @@ -21,10 +41,10 @@ jobs: - uses: actions/checkout@v4 - name: Setup conda environment - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 with: miniconda-version: "latest" - python-version: 3.11 + python-version: ${{ env.PY_VERSION }} environment-file: conda/environment.yml auto-update-conda: false auto-activate-base: false @@ -32,14 +52,10 @@ jobs: - name: Run conda build shell: bash -el {0} - # For the build, these environment variables would usually be set based on repo tags, - # but they can be these defaults instead since we don't use the build - env: - GIT_DESCRIBE_TAG: test - GIT_DESCRIBE_NUMBER: 0 run: conda build . --no-anaconda-upload - build: + tests: + name: Tests runs-on: ubuntu-latest # Run the job for different versions of python @@ -76,7 +92,7 @@ jobs: PYTHONPATH=. pytest --cov=payu -s test; - name: Coveralls - uses: AndreMiras/coveralls-python-action@develop + uses: AndreMiras/coveralls-python-action@f5fd5c309b39d01599fb92c72d4f7409ea78aec9 # v20201129 with: parallel: true @@ -84,10 +100,10 @@ jobs: run: cd docs && make html coveralls_finish: - needs: build + name: Coveralls Finished + needs: tests runs-on: ubuntu-latest steps: - - name: Coveralls Finished - uses: AndreMiras/coveralls-python-action@develop + - uses: AndreMiras/coveralls-python-action@f5fd5c309b39d01599fb92c72d4f7409ea78aec9 # v20201129 with: parallel-finished: true diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..b63bb646 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,5 @@ +[MAIN] + +# Files or directories to be skipped. They should be base names, not +# paths. +ignore=_version.py diff --git a/bin/payu b/bin/payu deleted file mode 100755 index 466f5007..00000000 --- a/bin/payu +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from payu import cli -cli.parse() diff --git a/bin/payu-collate b/bin/payu-collate deleted file mode 100755 index f4229c8d..00000000 --- a/bin/payu-collate +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from payu.subcommands import collate_cmd -collate_cmd.runscript() diff --git a/bin/payu-profile b/bin/payu-profile deleted file mode 100755 index bf03dad5..00000000 --- a/bin/payu-profile +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from payu.subcommands import profile_cmd -profile_cmd.runscript() diff --git a/bin/payu-run b/bin/payu-run deleted file mode 100755 index 6278f2f2..00000000 --- a/bin/payu-run +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from payu.subcommands import run_cmd -run_cmd.runscript() diff --git a/bin/payu-sync b/bin/payu-sync deleted file mode 100644 index 644453d2..00000000 --- a/bin/payu-sync +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env python - -from payu.subcommands import sync_cmd -sync_cmd.runscript() \ No newline at end of file diff --git a/conda/environment.yml b/conda/environment.yml index 069e0f96..110a4043 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -7,3 +7,4 @@ dependencies: - anaconda-client - conda-build - conda-verify + - versioneer diff --git a/conda/meta.yaml b/conda/meta.yaml index 7a39f4df..c3d8879b 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -1,44 +1,35 @@ +{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %} +{% set version = data.get('version') %} +{% set pyproj = load_file_data('../pyproject.toml', from_recipe_dir=True) %} +{% set project = pyproj.get('project') %} + package: name: payu - version: {{ GIT_DESCRIBE_TAG }} + version: "{{ version }}" build: noarch: python - number: {{ GIT_DESCRIBE_NUMBER }} - script: "{{ PYTHON }} -m pip install . --no-deps" + number: 0 + script: "{{ PYTHON }} -m pip install . -vv" entry_points: - - payu = payu.cli:parse - - payu-run = payu.subcommands.run_cmd:runscript - - payu-collate = payu.subcommands.collate_cmd:runscript - - payu-profile = payu.subcommands.profile_cmd:runscript - - payu-sync = payu.subcommands.sync_cmd:runscript - - payu-branch = payu.subcommands.branch_cmd:runscript - - payu-clone = payu.subcommands.clone_cmd:runscript - - payu-checkout = payu.subcommands.checkout_cmd:runscript + {% for name, script in project.get('scripts', {}).items() %} + - {{ name }} = {{ script }} + {% endfor %} source: - git_url: ../ + path: ../ requirements: - build: - - python <=3.11 - - pbr - - setuptools + host: + - python + - pip + - setuptools >=61.0.0 + - versioneer run: - - python <=3.11 - - f90nml >=0.16 - - yamanifest >=0.3.4 - - PyYAML - - python-dateutil - - tenacity - - requests - - cftime - # The following two requirements are from the security - # extra for the pypi package - - pyOpenSSL >=0.14 - - cryptography>=1.3.4 - - GitPython >=3.1.40 - - ruamel.yaml >=0.18.5 + - python >=3.9 + {% for dep in project.get('dependencies', []) %} + - {{ dep }} + {% endfor %} test: imports: @@ -46,6 +37,9 @@ test: commands: - payu list - about: home: https://github.com/payu-org/payu/ + license: Apache Software + license_family: APACHE + summary: "A climate model workflow manager for supercomputing environments" + doc_url: https://payu.readthedocs.io/en/latest/ diff --git a/payu/__init__.py b/payu/__init__.py index f9dad5cf..84a52efd 100644 --- a/payu/__init__.py +++ b/payu/__init__.py @@ -3,4 +3,5 @@ :copyright: Copyright 2011 Marshall Ward, see AUTHORS for details. :license: Apache License, Version 2.0, see LICENSE for details. """ -__version__ = '1.0.19' +from . import _version +__version__ = _version.get_versions()['version'] diff --git a/payu/_version.py b/payu/_version.py new file mode 100644 index 00000000..58c60cfc --- /dev/null +++ b/payu/_version.py @@ -0,0 +1,683 @@ + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. +# Generated by versioneer-0.29 +# https://github.com/python-versioneer/python-versioneer + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys +from typing import Any, Callable, Dict, List, Optional, Tuple +import functools + + +def get_keywords() -> Dict[str, str]: + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + VCS: str + style: str + tag_prefix: str + parentdir_prefix: str + versionfile_source: str + verbose: bool + + +def get_config() -> VersioneerConfig: + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "v" + cfg.parentdir_prefix = "payu-" + cfg.versionfile_source = "payu/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY: Dict[str, str] = {} +HANDLERS: Dict[str, Dict[str, Callable]] = {} + + +def register_vcs_handler(vcs: str, method: str) -> Callable: # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f: Callable) -> Callable: + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command( + commands: List[str], + args: List[str], + cwd: Optional[str] = None, + verbose: bool = False, + hide_stderr: bool = False, + env: Optional[Dict[str, str]] = None, +) -> Tuple[Optional[str], Optional[int]]: + """Call the given command(s).""" + assert isinstance(commands, list) + process = None + + popen_kwargs: Dict[str, Any] = {} + if sys.platform == "win32": + # This hides the console window if pythonw.exe is used + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + popen_kwargs["startupinfo"] = startupinfo + + for command in commands: + try: + dispcmd = str([command] + args) + # remember shell=False, so use git.cmd on windows, not just git + process = subprocess.Popen([command] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None), **popen_kwargs) + break + except OSError as e: + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = process.communicate()[0].strip().decode() + if process.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, process.returncode + return stdout, process.returncode + + +def versions_from_parentdir( + parentdir_prefix: str, + root: str, + verbose: bool, +) -> Dict[str, Any]: + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for _ in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs: str) -> Dict[str, str]: + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords: Dict[str, str] = {} + try: + with open(versionfile_abs, "r") as fobj: + for line in fobj: + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + except OSError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords( + keywords: Dict[str, str], + tag_prefix: str, + verbose: bool, +) -> Dict[str, Any]: + """Get version information from git keywords.""" + if "refnames" not in keywords: + raise NotThisMethod("Short version file found") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = {r.strip() for r in refnames.strip("()").split(",")} + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = {r for r in refs if re.search(r'\d', r)} + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + # Filter out refs that exactly match prefix or that don't start + # with a number once the prefix is stripped (mostly a concern + # when prefix is '') + if not re.match(r'\d', r): + continue + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs( + tag_prefix: str, + root: str, + verbose: bool, + runner: Callable = run_command +) -> Dict[str, Any]: + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + # GIT_DIR can interfere with correct operation of Versioneer. + # It may be intended to be passed to the Versioneer-versioned project, + # but that should not change where we get our version from. + env = os.environ.copy() + env.pop("GIT_DIR", None) + runner = functools.partial(runner, env=env) + + _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=not verbose) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = runner(GITS, [ + "describe", "--tags", "--dirty", "--always", "--long", + "--match", f"{tag_prefix}[[:digit:]]*" + ], cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces: Dict[str, Any] = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) + # --abbrev-ref was added in git-1.6.3 + if rc != 0 or branch_name is None: + raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") + branch_name = branch_name.strip() + + if branch_name == "HEAD": + # If we aren't exactly on a branch, pick a branch which represents + # the current commit. If all else fails, we are on a branchless + # commit. + branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) + # --contains was added in git-1.5.4 + if rc != 0 or branches is None: + raise NotThisMethod("'git branch --contains' returned error") + branches = branches.split("\n") + + # Remove the first line if we're running detached + if "(" in branches[0]: + branches.pop(0) + + # Strip off the leading "* " from the list of branches. + branches = [branch[2:] for branch in branches] + if "master" in branches: + branch_name = "master" + elif not branches: + branch_name = None + else: + # Pick the first branch that is returned. Good or bad. + branch_name = branches[0] + + pieces["branch"] = branch_name + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparsable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) + pieces["distance"] = len(out.split()) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces: Dict[str, Any]) -> str: + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces: Dict[str, Any]) -> str: + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_branch(pieces: Dict[str, Any]) -> str: + """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . + + The ".dev0" means not master branch. Note that .dev0 sorts backwards + (a feature branch will appear "older" than the master branch). + + Exceptions: + 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0" + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def pep440_split_post(ver: str) -> Tuple[str, Optional[int]]: + """Split pep440 version string at the post-release segment. + + Returns the release segments before the post-release and the + post-release version number (or -1 if no post-release segment is present). + """ + vc = str.split(ver, ".post") + return vc[0], int(vc[1] or 0) if len(vc) == 2 else None + + +def render_pep440_pre(pieces: Dict[str, Any]) -> str: + """TAG[.postN.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + if pieces["distance"]: + # update the post release segment + tag_version, post_version = pep440_split_post(pieces["closest-tag"]) + rendered = tag_version + if post_version is not None: + rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) + else: + rendered += ".post0.dev%d" % (pieces["distance"]) + else: + # no commits, use the tag as the version + rendered = pieces["closest-tag"] + else: + # exception #1 + rendered = "0.post0.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_post_branch(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . + + The ".dev0" means not master branch. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["branch"] != "master": + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_old(pieces: Dict[str, Any]) -> str: + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces: Dict[str, Any]) -> str: + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces: Dict[str, Any]) -> str: + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces: Dict[str, Any], style: str) -> Dict[str, Any]: + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-branch": + rendered = render_pep440_branch(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-post-branch": + rendered = render_pep440_post_branch(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions() -> Dict[str, Any]: + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for _ in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/payu/models/um.py b/payu/models/um.py index 778a0d7a..f5ecb740 100644 --- a/payu/models/um.py +++ b/payu/models/um.py @@ -12,7 +12,8 @@ import datetime import fileinput import glob -import imp +from importlib.machinery import SourceFileLoader +from importlib.util import spec_from_loader, module_from_spec import os import shutil import string @@ -108,8 +109,14 @@ def setup(self): # Set up environment variables needed to run UM. # Look for a python file in the config directory. - um_env = imp.load_source('um_env', - os.path.join(self.control_path, 'um_env.py')) + loader = SourceFileLoader( + 'um_env', + os.path.join(self.control_path, 'um_env.py') + ) + um_env = module_from_spec( + spec_from_loader(loader.name, loader) + ) + loader.exec_module(um_env) um_vars = um_env.vars # Stage the UM restart file. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..e1c7582c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[project] +name = "payu" +authors = [ + { name = "Marshall Ward", email = "python@marshallward.org" } +] +maintainers = [ + { name = "ACCESS-NRI", email = "access.nri@anu.edu.au" } +] +description = "A climate model workflow manager for supercomputing environments" +readme = "README.rst" +license = { text = "Apache-2.0" } +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Topic :: Utilities", +] +keywords = ["climate model", "workflow"] +dynamic = ["version"] +# The dependencies here are also used by the conda build and so must follow +# conda package match specifications, see here: +# https://docs.conda.io/projects/conda-build/en/stable/resources/package-spec.html#package-match-specifications +dependencies = [ + "f90nml >=0.16", + "yamanifest >=0.3.4", + "PyYAML", + "requests", + "python-dateutil", + "tenacity >=8.0.0", + "cftime", + "GitPython >=3.1.40", + "ruamel.yaml >=0.18.5" +] + +[project.optional-dependencies] +test = [ + "pytest", + "pylint", + "Sphinx" +] +mitgcm = ["mnctools>=0.2"] + +[project.scripts] +payu = "payu.cli:parse" +payu-run = "payu.subcommands.run_cmd:runscript" +payu-collate = "payu.subcommands.collate_cmd:runscript" +payu-profile = "payu.subcommands.profile_cmd:runscript" +payu-sync = "payu.subcommands.sync_cmd:runscript" +payu-branch = "payu.subcommands.branch_cmd:runscript" +payu-clone = "payu.subcommands.clone_cmd:runscript" +payu-checkout = "payu.subcommands.checkout_cmd:runscript" + +[build-system] +build-backend = "setuptools.build_meta" +requires = [ + "setuptools >= 61.0.0", + "versioneer[toml]" +] + +[tool.setuptools.packages.find] +include = ["payu*"] +namespaces = false + +[tool.versioneer] +VCS = "git" +style = "pep440" +versionfile_source = "payu/_version.py" +versionfile_build = "payu/_version.py" +tag_prefix = "" +parentdir_prefix = "payu-" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2a9acf13..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 1 diff --git a/setup.py b/setup.py index ebdff68c..9ed5e21c 100644 --- a/setup.py +++ b/setup.py @@ -1,87 +1,7 @@ -"""setup.py - Installation script for payu - - Additional configuration settings are in ``setup.cfg``. -""" - -import os -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -PKG_NAME = 'payu' -PKG_VERSION = __import__(PKG_NAME).__version__ -PKG_PKGS = [path for (path, dirs, files) in os.walk(PKG_NAME) - if '__init__.py' in files] - -with open('README.rst') as f: - README_RST = f.read() +from setuptools import setup +import versioneer setup( - name=PKG_NAME, - version=PKG_VERSION, - description='A climate model workflow manager for supercomputing ' - 'environments.', - long_description=README_RST, - author='Marshall Ward', - author_email='python@marshallward.org', - url='http://github.com/marshallward/payu', - - packages=PKG_PKGS, - requires=[ - 'f90nml', - 'PyYAML', - 'requests', - 'yamanifest', - 'dateutil', - 'tenacity', - 'cftime', - 'GitPython', - 'ruamel.yaml' - ], - install_requires=[ - 'f90nml >= 0.16', - 'yamanifest >= 0.3.4', - 'PyYAML', - 'requests[security]', - 'python-dateutil', - 'tenacity!=7.0.0', - 'cftime', - 'GitPython >= 3.1.40', - 'ruamel.yaml >= 0.18.5' - ], - tests_require=[ - 'pytest', - 'pylint', - 'Sphinx', - ], - entry_points={ - 'console_scripts': [ - 'payu = payu.cli:parse', - 'payu-run = payu.subcommands.run_cmd:runscript', - 'payu-collate = payu.subcommands.collate_cmd:runscript', - 'payu-profile = payu.subcommands.profile_cmd:runscript', - 'payu-sync = payu.subcommands.sync_cmd:runscript', - 'payu-branch = payu.subcommands.branch_cmd:runscript', - 'payu-clone = payu.subcommands.clone_cmd:runscript', - 'payu-checkout = payu.subcommands.checkout_cmd:runscript' - ] - }, - classifiers=[ - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Topic :: Utilities', - ], - extras_require={ - 'mitgcm': ['mnctools>=0.2'] - }, - keywords='{0} supercomputer model climate workflow'.format(PKG_NAME) + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ) diff --git a/tools/README.rst b/tools/README.rst deleted file mode 100644 index 6e741130..00000000 --- a/tools/README.rst +++ /dev/null @@ -1,14 +0,0 @@ -Support tools and install scripts ---------------------------------- - -These are various scripts which aid for installation on Raijin. - -- ``nci_install`` is for the newer versions of Payu, which use a fixed Python - executable and rely on entry points for command line execution. - -- ``legacy_install`` is for older versions, which have less strict install - requirements (mostly due to internal bootstrapping techniques which re-apply - similar constraint) - -- ``get-pip.py`` is a small script available publicly which install pip for - Python 2.6. diff --git a/tools/legacy_install b/tools/legacy_install deleted file mode 100755 index 17d7d9dc..00000000 --- a/tools/legacy_install +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -set -x - -module purge -#module load python/2.7.6 -#module load python/2.7.6-matplotlib -module load python/2.7.11 -module load python/2.7.11-matplotlib -#module load python/2.7.15 -#module load python3/3.6.2 - -# TODO: Set conditionally -#PYTHON=python3 -#PIP=pip3 -PYTHON=python -PIP=pip - -# TODO: getopts -PKG_NAME="payu" -PKG_VERSION="" -FORCE="y" - -# System Environment -PROJECT_MODULEFILES="/apps/Modules/restricted-modulefiles/fp0" -#PROJECT_MODULEFILES="/apps/Modules/modulefiles" -PROJECT_APPS="/apps" - -#PROJECT_ROOT="/projects/v45" -#PROJECT_MODULEFILES="${PROJECT_ROOT}/modules" -#PROJECT_APPS="${PROJECT_ROOT}/apps" - -AUTHOR_UID=$(whoami) -MOD_AUTHOR=$(getent passwd ${AUTHOR_UID} | cut -d: -f5) -MOD_EMAIL=$(ldapsearch -LLL -x "(uid=${AUTHOR_UID})" mail | \ - grep -oP "(?<=mail: ).*$") - -# Get package version (or use a custom version) -if [[ ! -n ${PKG_VERSION} ]]; then - PKG_VERSION=$(${PYTHON} -c "print(__import__('${PKG_NAME}').__version__)") -fi - -# Get application paths -PKG_ROOT="${PROJECT_APPS}/${PKG_NAME}/${PKG_VERSION}" -PKG_BIN="${PKG_ROOT}/bin" -PKG_LIB="${PKG_ROOT}/lib" - -# Check if application is already installed -if [[ -d ${PKG_ROOT} && ! -n ${FORCE} ]]; then - echo "There is already an application installed at ${PKG_ROOT}." - exit -1 -fi - -rm -rf ${PKG_ROOT} -mkdir -p ${PKG_BIN} -mkdir -p ${PKG_LIB} - -${PYTHON} --version -${PIP} --version - -# First install importlib under 2.6 -${PIP} install --no-cache-dir -t ${PKG_LIB} importlib - -# Now setup a "modern" Python environment -module load python/2.7.15 - -#${PYTHON} setup.py install --prefix=${PKG_ROOT} - -# TODO: Set source directory -${PYTHON} setup.py install_lib --install-dir ${PKG_LIB} -${PYTHON} setup.py install_scripts --install-dir ${PKG_BIN} -#${PYTHON} setup.py build_scripts \ -# --build-dir ${PKG_BIN} \ -# --executable $(which ${PYTHON}) - -# Install dependencies -${PIP} install --no-cache-dir -t ${PKG_LIB} python-dateutil -${PIP} install --no-cache-dir -t ${PKG_LIB} f90nml -${PIP} install --no-cache-dir -t ${PKG_LIB} pyyaml -${PIP} install --no-cache-dir -t ${PKG_LIB} requests[security] - -# Generate the modulefile -PKG_APP_MODS=${PROJECT_MODULEFILES}/${PKG_NAME} -mkdir -p ${PKG_APP_MODS} - -PKG_MOD=${PKG_APP_MODS}/${PKG_VERSION} -printf "#%%Module1.0\n" > ${PKG_MOD} -printf "# ${PKG_NAME} ${PKG_VERSION} environment module\n" >> ${PKG_MOD} -printf "\n" >> ${PKG_MOD} -printf "set install-contact \"${MOD_AUTHOR} <${MOD_EMAIL}>\"\n" >> ${PKG_MOD} -printf "set install-date \"$(date +%Y-%m-%d)\"\n" >> ${PKG_MOD} -printf "source /opt/Modules/extensions/extensions.tcl\n" >> ${PKG_MOD} -printf "docommon\n" >> ${PKG_MOD} - -# Update default version -printf "#%%Module1.0\n" > ${PKG_APP_MODS}/.version -printf "set ModulesVersion ${PKG_VERSION}\n" >> ${PKG_APP_MODS}/.version - -# Set permissions -chmod g=rwX ${PROJECT_APPS}/${PKG_NAME} -chmod o=rX ${PROJECT_APPS}/${PKG_NAME} - -chgrp -R apps ${PKG_ROOT} -chmod -R g=rwX ${PKG_ROOT} -chmod -R o=rX ${PKG_ROOT} - -chmod g=rwX ${PKG_APP_MODS} -chmod o=rX ${PKG_APP_MODS} - -chgrp apps ${PKG_MOD} -chmod g=rwX ${PKG_MOD} -chmod o=rX ${PKG_MOD} diff --git a/tools/nci_install b/tools/nci_install deleted file mode 100755 index e05f4bfb..00000000 --- a/tools/nci_install +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash - -# Testing -set -ex - -PKG_NAME="payu" -PKG_VERSION="" -FORCE="y" - -module purge # Remove existing modules -rm -rf build # Delete old scripts -unset PYTHONPATH -export PYTHONNOUSERSITE='y' - -# Application name and version -if [[ ! -n ${PKG_VERSION} ]]; then - PKG_VERSION=$(python -c "print(__import__('${PKG_NAME}').__version__)") -fi - -# User information -AUTHOR_UID=$(whoami) -MOD_AUTHOR=$(getent passwd ${AUTHOR_UID} | cut -d: -f5) -MOD_EMAIL=$(ldapsearch -LLL -x "(uid=${AUTHOR_UID})" mail | \ - grep -oP "(?<=mail: ).*$") - -# Module environment -PROJECT_APPS="/apps" -PROJECT_MODULEFILES="/apps/Modules/modulefiles" -#PROJECT_MODULEFILES="/apps/Modules/restricted-modulefiles/fp0" - -PKG_ROOT="${PROJECT_APPS}/${PKG_NAME}/${PKG_VERSION}" -#PKG_ROOT=tmp - -# Check if application is already installed -if [[ -d ${PKG_ROOT} && ! -n ${FORCE} ]]; then - echo "There is already an application installed at ${PKG_ROOT}." - exit -1 -fi - -# Delete existing app if already installed -rm -rf ${PKG_ROOT} - -# Python 3.6 install -module purge -module load python3/3.6.2 -module load netcdf/4.2.1.1 - -PAYU_LIB=${PKG_ROOT}/lib/python3.6/site-packages - -mkdir -p ${PAYU_LIB} - -# cftime install is broken, used by netCDF4 -# Install this one separately via pip -PYTHONPATH=${PAYU_LIB} \ - pip3 install cftime --prefix=${PKG_ROOT} - -#PYTHONPATH=${PAYU_LIB} \ -# python3 setup.py install --prefix=${PKG_ROOT} -PYTHONPATH=${PAYU_LIB} \ - pip3 install -e .[mitgcm] --prefix=${PKG_ROOT} - -#------------------------------------------------------------------------------ -# Generate the modulefile - -PKG_APP_MODS=${PROJECT_MODULEFILES}/${PKG_NAME} -mkdir -p ${PKG_APP_MODS} - -PKG_MOD=${PKG_APP_MODS}/${PKG_VERSION} -printf "#%%Module1.0\n" > ${PKG_MOD} -printf "# ${PKG_NAME} ${PKG_VERSION} environment module\n" >> ${PKG_MOD} -printf "\n" >> ${PKG_MOD} -printf "set install-contact \"${MOD_AUTHOR} <${MOD_EMAIL}>\"\n" >> ${PKG_MOD} -printf "set install-date \"$(date +%Y-%m-%d)\"\n" >> ${PKG_MOD} -printf "source /opt/Modules/extensions/extensions.tcl\n" >> ${PKG_MOD} -printf "docommon\n" >> ${PKG_MOD} - -## Update default version -#printf "#%%Module1.0\n" > ${PKG_APP_MODS}/.version -#printf "set ModulesVersion ${PKG_VERSION}\n" >> ${PKG_APP_MODS}/.version - -# Set permissions -chmod g=rwX ${PROJECT_APPS}/${PKG_NAME} -chmod o=rX ${PROJECT_APPS}/${PKG_NAME} - -chgrp -R apps ${PKG_ROOT} -chmod -R g=rwX ${PKG_ROOT} -chmod -R o=rX ${PKG_ROOT} - -chmod g=rwX ${PKG_APP_MODS} -chmod o=rX ${PKG_APP_MODS} - -chgrp apps ${PKG_MOD} -chmod g=rwX ${PKG_MOD} -chmod o=rX ${PKG_MOD}