diff --git a/docs/api.rst b/docs/api.rst index 26ca3bd6..d8b8bf8c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -27,3 +27,9 @@ Mask classes .. automodule:: xija.component.mask :members: + +Get model spec +-------------- +.. automodule:: xija.get_model_spec + :members: + diff --git a/xija/get_model_spec.py b/xija/get_model_spec.py new file mode 100644 index 00000000..cbff42f6 --- /dev/null +++ b/xija/get_model_spec.py @@ -0,0 +1,218 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +Get Chandra model specifications +""" +import json +import tempfile +import os +import re +import warnings +from pathlib import Path +from typing import List, Optional, Union + +import git +import requests +from Ska.File import get_globfiles + +__all__ = ['get_xija_model_spec', 'get_xija_model_names', 'get_repo_version', + 'get_github_version'] + +REPO_PATH = Path(os.environ['SKA'], 'data', 'chandra_models') +MODELS_PATH = REPO_PATH / 'chandra_models' / 'xija' +CHANDRA_MODELS_URL = 'https://api.github.com/repos/sot/chandra_models/releases' + + +def _models_path(repo_path=REPO_PATH) -> Path: + return Path(repo_path) / 'chandra_models' / 'xija' + + +def get_xija_model_spec(model_name, version=None, repo_path=REPO_PATH, + check_version=False, timeout=5) -> dict: + """ + Get Xija model specification for the specified ``model_name``. + + Supported model names include (but are not limited to): ``'aca'``, + ``'acisfp'``, ``'dea'``, ``'dpa'``, ``'psmc'``, ``'minusyz'``, and + ``'pftank2t'``. + + Use ``get_xija_model_names()`` for the full list. + + Examples + -------- + Get the latest version of the ``acisfp`` model spec from the local Ska data + directory ``$SKA/data/chandra_models``, checking that the version matches + the latest release tag on GitHub. + + >>> import xija + >>> from xija.get_model_spec import get_xija_model_spec + >>> model_spec, version = get_xija_model_spec('acisfp', check_version=True) + >>> model = xija.XijaModel('acisfp', model_spec=model_spec, + ... start='2020:001', stop='2020:010') + >>> model.make() + >>> model.calc() + + Get the ``aca`` model spec from release version 3.30 of chandra_models from + GitHub. + + >>> repo_path = 'https://github.com/sot/chandra_models.git' + >>> model_spec, version = get_xija_model_spec('aca', version='3.30', + ... repo_path=repo_path) + + Parameters + ---------- + model_name : str + Name of model + version : str + Tag, branch or commit of chandra_models to use (default=latest tag from + repo) + repo_path : str, Path + Path to directory or URL containing chandra_models repository (default + is $SKA/data/chandra_models) + check_version : bool + Check that ``version`` matches the latest release on GitHub + timeout : int, float + Timeout (sec) for querying GitHub for the expected chandra_models version. + Default = 5 sec. + + Returns + ------- + dict, str + Xija model specification dict, chandra_models version + """ + with tempfile.TemporaryDirectory() as repo_path_local: + repo = git.Repo.clone_from(repo_path, repo_path_local) + if version is not None: + repo.git.checkout(version) + model_spec, version = _get_xija_model_spec(model_name, version, repo_path_local, + check_version, timeout) + return model_spec, version + + +def _get_xija_model_spec(model_name, version=None, repo_path=REPO_PATH, + check_version=False, timeout=5) -> dict: + + models_path = _models_path(repo_path) + + if not models_path.exists(): + raise FileNotFoundError(f'xija models directory {models_path} does not exist') + + file_glob = str(models_path / '*' / f'{model_name.lower()}_spec.json') + try: + # get_globfiles() default requires exactly one file match and returns a list + file_name = get_globfiles(file_glob)[0] + except ValueError: + names = get_xija_model_names() + raise ValueError(f'no models matched {model_name}. Available models are: ' + f'{", ".join(names)}') + + model_spec = json.load(open(file_name, 'r')) + + # Get version and ensure that repo is clean and tip is at latest tag + if version is None: + version = get_repo_version(repo_path) + + if check_version: + gh_version = get_github_version(timeout=timeout) + if gh_version is None: + warnings.warn('Could not verify GitHub chandra_models release tag ' + f'due to timeout ({timeout} sec)') + elif version != gh_version: + raise ValueError(f'version mismatch: local repo {version} vs ' + f'github {gh_version}') + + return model_spec, version + + +def get_xija_model_names(repo_path=REPO_PATH) -> List[str]: + """Return list of available xija model names. + + Examples + -------- + >>> from xija.get_model_spec import get_xija_model_names + >>> names = get_xija_model_names() + ['aca', + 'acisfp', + 'dea', + 'dpa', + '4rt700t', + 'minusyz', + 'pm1thv2t', + 'pm2thv1t', + 'pm2thv2t', + 'pftank2t', + 'pline03t_model', + 'pline04t_model', + 'psmc', + 'tcylaft6'] + + Parameters + ---------- + repo_path : str, Path + Path to directory containing chandra_models repository (default is + $SKA/data/chandra_models) + + Returns + ------- + list + List of available xija model names + """ + models_path = _models_path(repo_path) + + fns = get_globfiles(str(models_path / '*' / '*_spec.json'), minfiles=0, maxfiles=None) + names = [re.sub(r'_spec\.json', '', Path(fn).name) for fn in sorted(fns)] + + return names + + +def get_repo_version(repo_path: Path = REPO_PATH) -> str: + """Return version (most recent tag) of models repository. + + Returns + ------- + str + Version (most recent tag) of models repository + """ + repo = git.Repo(repo_path) + + if repo.is_dirty(): + raise ValueError('repo is dirty') + + tags = sorted(repo.tags, key=lambda tag: tag.commit.committed_datetime) + tag_repo = tags[-1] + if tag_repo.commit != repo.head.commit: + raise ValueError(f'repo tip is not at tag {tag_repo}') + + return tag_repo.name + + +def get_github_version(url: str = CHANDRA_MODELS_URL, + timeout: Union[int, float] = 5) -> Optional[bool]: + """Get latest chandra_models GitHub repo release tag (version). + + This queries GitHub for the latest release of chandra_models. + + Parameters + ---------- + url : str + URL for chandra_models releases on GitHub API + timeout : int, float + Request timeout (sec, default=5) + + Returns + ------- + str, None + Tag name (str) of latest chandra_models release on GitHub. + None if the request timed out, indicating indeterminate answer. + """ + try: + req = requests.get(url, timeout=timeout) + except requests.ConnectTimeout: + return None + + if req.status_code != requests.codes.ok: + req.raise_for_status() + + tags_gh = sorted(req.json(), key=lambda tag: tag['published_at']) + tag_gh_name = tags_gh[-1]['tag_name'] + + return tag_gh_name diff --git a/xija/tests/test_get_model_spec.py b/xija/tests/test_get_model_spec.py new file mode 100644 index 00000000..c50fb01a --- /dev/null +++ b/xija/tests/test_get_model_spec.py @@ -0,0 +1,81 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import re + +import pytest +import requests +import git + +from ..get_model_spec import (get_xija_model_spec, get_xija_model_names, + get_repo_version, get_github_version) + +try: + # Fast request to see if GitHub is available + req = requests.get('https://raw.githubusercontent.com/sot/chandra_models/master/README', + timeout=5) + HAS_GITHUB = req.status_code == 200 +except Exception: + HAS_GITHUB = False + + +def test_get_model_spec_aca_3_30(): + # Version 3.30 + spec, version = get_xija_model_spec('aca', version='3.30') + assert spec['name'] == 'aacccdpt' + assert 'comps' in spec + assert spec["datestop"] == "2018:305:11:52:30.816" + + +def test_get_model_spec_aca_latest(): + # Latest version + spec, version = get_xija_model_spec('aca', check_version=HAS_GITHUB) + assert spec['name'] == 'aacccdpt' + # version 3.30 value, changed in 3.31.1/3.32 + assert spec["datestop"] != "2018:305:11:52:30.816" + assert 'comps' in spec + + +@pytest.mark.skipif('not HAS_GITHUB') +def test_get_model_spec_aca_from_github(): + # Latest version + repo_path = 'https://github.com/sot/chandra_models.git' + spec, version = get_xija_model_spec('aca', repo_path=repo_path, version='3.30') + assert spec['name'] == 'aacccdpt' + assert spec["datestop"] == "2018:305:11:52:30.816" + assert 'comps' in spec + + +def test_get_model_file_fail(): + with pytest.raises(ValueError, match='no models matched xxxyyyzzz'): + get_xija_model_spec('xxxyyyzzz') + + with pytest.raises(git.GitCommandError, match='does not exist'): + get_xija_model_spec('aca', repo_path='__NOT_A_DIRECTORY__') + + +def test_get_xija_model_names(): + names = get_xija_model_names() + assert all(name in names for name in ('aca', 'acisfp', 'dea', 'dpa', 'pftank2t')) + + +def test_get_repo_version(): + version = get_repo_version() + assert isinstance(version, str) + assert re.match(r'^[0-9.]+$', version) + + +@pytest.mark.skipif('not HAS_GITHUB') +def test_check_github_version(): + version = get_repo_version() + status = get_github_version() == version + assert status is True + + status = get_github_version() == 'asdf' + assert status is False + + # Force timeout + status = get_github_version(timeout=0.00001) + assert status is None + + with pytest.raises(requests.ConnectionError): + get_github_version(url='https://______bad_url______')