diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 0000000..376fe93 --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,47 @@ +name: Codecov + +on: + push: + branches: [main, master] + +jobs: + run: + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 3 + matrix: + os: [ubuntu-latest] + env: + OS: ${{ matrix.os }} + PYTHON: '3.9' + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + fetch-tags: true + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: install dependencies + run: | + python -m pip install --upgrade pip + conda install hdf4 + python -m pip install pyhdf + python -m pip install pytest + python -m pip install pytest-cov + python -m pip install . + - name: Generate coverage report + run: | + python -m pytest --cov=./ --cov-report=xml:coverage.xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false + verbose: true diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml new file mode 100644 index 0000000..f96cad0 --- /dev/null +++ b/.github/workflows/gh_pages.yml @@ -0,0 +1,37 @@ +name: Documentation + +on: + push: + branches: [main, master] + +jobs: + make-pages: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + fetch-depth: 0 + fetch-tags: true + - name: select python version + uses: actions/setup-python@v4 + with: + python-version: 3.9 + - name: install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install . + python -m pip install sphinx + python -m pip install sphinx_rtd_theme + python -m pip install sphinx-click + - name: build documentation + run: | + cd docs + make html + - name: deploy + uses: peaceiris/actions-gh-pages@v3.6.1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/_build/html + force_orphan: true + full_commit_message: ${{ github.event.head_commit.message }} diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 8de5301..8c0dff9 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,30 +1,19 @@ name: Lint Code Base on: + workflow_dispatch: push: branches-ignore: - 'gh-pages' jobs: - build: - name: Lint Code Base + ruff: + name: Ruff runs-on: ubuntu-latest steps: - - name: Checkout Code - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: chartboost/ruff-action@v1 with: - fetch-depth: 0 - - name: Lint Code Base - uses: super-linter/super-linter@v4 - env: - VALIDATE_ALL_CODEBASE: false - VALIDATE_PYTHON_BLACK: false - VALIDATE_PYTHON_ISORT: false - VALIDATE_PYTHON_MYPY: false - VALIDATE_DOCKERFILE_HADOLINT: false - VALIDATE_JSCPD: false - VALIDATE_JSON: false - VALIDATE_MARKDOWN: false - VALIDATE_YAML: false - DEFAULT_BRANCH: main - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + version: 0.4.10 + args: check --output-format=github + src: "./mlclouds ./tests" diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 448d966..d5df4c2 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -25,7 +25,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools build - python -m build --sdist --wheel --outdir dist/ . + python -m build --wheel --outdir dist/ . - name: Check distribution files run: | pip install twine diff --git a/.github/workflows/pull_request_tests.yml b/.github/workflows/pull_request_tests.yml index b1221b3..81fe634 100644 --- a/.github/workflows/pull_request_tests.yml +++ b/.github/workflows/pull_request_tests.yml @@ -2,6 +2,7 @@ name: Pytests on: pull_request + jobs: build: runs-on: ${{ matrix.os }} @@ -9,55 +10,28 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ['3.10'] + python-version: ['3.11'] include: - os: ubuntu-latest - python-version: 3.9 + python-version: '3.10' + - os: ubuntu-latest + python-version: '3.9' steps: - - name: checkout mlclouds - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.ref }} - path: mlclouds - - name: checkout rest2 - uses: actions/checkout@v2 - with: - repository: nrel/rest2 - ssh-key: ${{ secrets.SSH_KEY }} - path: rest2 - - name: checkout nsrdb - uses: actions/checkout@v2 - with: - repository: nrel/nsrdb - ssh-key: ${{ secrets.SSH_KEY }} - path: nsrdb + fetch-depth: 0 + fetch-tags: true - name: Set up Python ${{ matrix.python-version }} - uses: conda-incubator/setup-miniconda@v2 + uses: actions/setup-python@v5 with: - auto-update-conda: true python-version: ${{ matrix.python-version }} - - name: Install rest2 dependencies - working-directory: ./rest2 - shell: bash -l {0} - run: | - conda install pip - pip install -e . - - name: Install nsrdb dependencies - working-directory: ./nsrdb - shell: bash -l {0} - run: | - conda install hdf4 - conda install -c conda-forge pyhdf - pip install -e . - - name: Install mlclouds dependencies - working-directory: ./mlclouds - shell: bash -l {0} + cache: 'pip' + - name: Install dependencies run: | - pip install -e . - pip install pytest - - name: Run mlclouds pytest - working-directory: ./mlclouds - shell: bash -l {0} + python -m pip install --upgrade pip + python -m pip install .[test] + - name: Run pytest run: | - pytest -v --disable-warnings + python -m pytest -v --disable-warnings \ No newline at end of file diff --git a/.github/workflows/release_drafter.yml b/.github/workflows/release_drafter.yml new file mode 100644 index 0000000..71f9dcc --- /dev/null +++ b/.github/workflows/release_drafter.yml @@ -0,0 +1,14 @@ +name: Release Drafter + +on: + push: + branches: [main, master] + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - name: Release Drafter + uses: release-drafter/release-drafter@v5.15.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index bf6b4c7..d39720c 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/source/_autosummary/ # PyBuilder target/ diff --git a/README.rst b/README.rst index be2c965..1683be0 100644 --- a/README.rst +++ b/README.rst @@ -1,17 +1,68 @@ +#################### +Welcome to MLClouds! +#################### +|Docs| |Tests| |Linter| |PyPi| |PythonV| |Codecov| |Zenodo| + +.. |Docs| image:: https://github.com/NREL/mlclouds/workflows/Documentation/badge.svg + :target: https://nrel.github.io/mlclouds/ + +.. |Tests| image:: https://github.com/NREL/mlclouds/workflows/Pytests/badge.svg + :target: https://github.com/NREL/mlclouds/actions?query=workflow%3A%22Pytests%22 + +.. |Linter| image:: https://github.com/NREL/mlclouds/workflows/Lint%20Code%20Base/badge.svg + :target: https://github.com/NREL/mlclouds/actions?query=workflow%3A%22Lint+Code+Base%22 + +.. |PyPi| image:: https://img.shields.io/pypi/pyversions/NREL-mlclouds.svg + :target: https://pypi.org/project/NREL-mlclouds/ + +.. |PythonV| image:: https://badge.fury.io/py/NREL-mlclouds.svg + :target: https://badge.fury.io/py/NREL-mlclouds + +.. |Codecov| image:: https://codecov.io/gh/nrel/mlclouds/branch/main/graph/badge.svg + :target: https://codecov.io/gh/nrel/mlclouds + +.. |Zenodo| image:: https://zenodo.org/badge/340209614.svg + :target: https://zenodo.org/badge/latestdoi/340209614 + +.. inclusion-intro + + A machine learning approach to predicting missing cloud properties in the National Solar Radiation Database (NSRDB) -==================================================================================================================== +------------------------------------------------------------------------------------------------------------------- -The National Solar Radiation Database (NSRDB) is NREL’s flagship solar data resource. With over 20 years of high-resolution surface irradiance +The National Solar Radiation Database (NSRDB) is NREL's flagship solar data resource. With over 20 years of high-resolution surface irradiance data covering most of the western hemisphere, the NSRDB is a crucial public data asset. A fundamental input to accurate surface irradiance in the NSRDB is high quality cloud property data. Cloud properties are used in radiative transfer calculations and are sourced from satellite imagery. Improving the accuracy of cloud property inputs is a tractable method for improving the accuracy of the irradiance data in the NSRDB. For example, in July of 2018, an average location in the Continental United States is missing cloud property data for nearly one quarter of all daylight cloudy timesteps. -This project aims to improve the cloud data inputs to the NSRDB by using machine learning techniques to exploit the NSRDB’s massive data resources. +This project aims to improve the cloud data inputs to the NSRDB by using machine learning techniques to exploit the NSRDB's massive data resources. More accurate cloud property input data will yield more accurate surface irradiance data in the NSRDB, providing direct benefit to researchers at NREL and to public data users everywhere. Installation ------------- +============ + It is recommended that you first follow the `install instructions for the NSRDB `_. -Then run `pip install -e .` from the mlclouds directory containing setup.py. -If you are a developer, also run `pre-commit install` in the same directory. \ No newline at end of file +Then run ``pip install -e .`` from the mlclouds directory containing ``setup.py``. +If you are a developer, also run ``pre-commit install`` in the same directory. + + +Acknowledgments +=============== + +This work (SWR-23-77) was authored by the National Renewable Energy Laboratory, +operated by Alliance for Sustainable Energy, LLC, for the U.S. Department of +Energy (DOE) under Contract No. DE-AC36-08GO28308. Funding provided by the DOE +Grid Deployment Office (GDO), the DOE Advanced Scientific Computing Research +(ASCR) program, the DOE Solar Energy Technologies Office (SETO), the DOE Wind +Energy Technologies Office (WETO), the United States Agency for International +Development (USAID), and the Laboratory Directed Research and Development +(LDRD) program at the National Renewable Energy Laboratory. The research was +performed using computational resources sponsored by the Department of Energy's +Office of Energy Efficiency and Renewable Energy and located at the National +Renewable Energy Laboratory. The views expressed in the article do not +necessarily represent the views of the DOE or the U.S. Government. The U.S. +Government retains and the publisher, by accepting the article for publication, +acknowledges that the U.S. Government retains a nonexclusive, paid-up, +irrevocable, worldwide license to publish or reproduce the published form of +this work, or allow others to do so, for U.S. Government purposes. \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..f6d7249 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,29 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + rm -rf _build + rm -rf source/_autosummary + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +github: html + -git branch -D gh-pages + -git push origin --delete gh-pages + ghp-import -n -b gh-pages -m "Update documentation" ./_build/html + git checkout gh-pages + git push --set-upstream origin gh-pages + git checkout ${BRANCH} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c48ed89 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,83 @@ +# Sphinx Documentation + +The documentation is built with [Sphinx](http://sphinx-doc.org/index.html). +See their documentation for (a lot) more details. + +## Installation + +To generate the docs yourself, you'll need the appropriate package: + +``` +conda install sphinx +conda install sphinx_rtd_theme + +pip install ghp-import +pip install sphinx-click +``` + +## Add any new CLI docs + +- Create a new file .rest file in source/_cli +- Add the following to the top of the new CLI module's .rst file: +``` +.. click:: module_path:main + :prog: CLI-Alias # e.g. NSRDB + :show-nested: +``` +- `git push` changes to the documentation source code as needed. +- Make the documentation per below + +## Building HTML Docs + +### Mac/Linux + +``` +make html +``` + +### Windows + +``` +make.bat html +``` + +## Building PDF Docs + +To build a PDF, you'll need a latex distribution for your system. + +### Mac/Linux + +``` +make latexpdf +``` + +### Windows + +``` +make.bat latexpdf +``` + +## Pushing to GitHub Pages + +### Mac/Linux + +``` +make github +``` + +### Windows + +``` +make.bat html +``` + +Then run the github-related commands by hand: + +``` +git branch -D gh-pages +git push origin --delete gh-pages +ghp-import -n -b gh-pages -m "Update documentation" ./_build/html +git checkout gh-pages +git push origin gh-pages +git checkout master # or whatever branch you were on +``` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..7843185 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..41bb417 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +ghp-import +sphinx +sphinx-click +sphinx_rtd_theme diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000..bb92b93 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,14 @@ +.wy-nav-content { + max-width: 60% !important; +} + +.wy-side-nav-search { + display: block; + width: 300px; + padding: 0.809em; + margin-bottom: 0.809em; + z-index: 200; + background-color: #fcfcfc; + text-align: center; + color: #fcfcfc; +} diff --git a/docs/source/_templates/custom-class-template.rst b/docs/source/_templates/custom-class-template.rst new file mode 100644 index 0000000..e51461e --- /dev/null +++ b/docs/source/_templates/custom-class-template.rst @@ -0,0 +1,33 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :show-inheritance: + :inherited-members: + :special-members: __call__, __add__, __mul__ + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + {% for item in methods %} + {%- if not item.startswith('_') %} + ~{{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/source/_templates/custom-module-template.rst b/docs/source/_templates/custom-module-template.rst new file mode 100644 index 0000000..a785139 --- /dev/null +++ b/docs/source/_templates/custom-module-template.rst @@ -0,0 +1,64 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: Module attributes + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :toctree: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :toctree: + :template: custom-class-template.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :toctree: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..fb00cc1 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,6 @@ +.. autosummary:: + :toctree: _autosummary + :template: custom-module-template.rst + :recursive: + + mlclouds diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..2a9411c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +""" +Documentation config file +""" +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath('../../')) + +# -- Project information ----------------------------------------------------- + +project = 'mlclouds' +copyright = '2022, Alliance for Sustainable Energy, LLC' +author = 'Grant Buster, Brandon Benton' + +pkg = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) +pkg = os.path.dirname(pkg) +sys.path.append(pkg) + +from mlclouds import __version__ as v + +# The short X.Y version +version = v +# The full version, including alpha/beta/rc tags +release = v + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx.ext.napoleon", + "sphinx_rtd_theme", + 'sphinx_click.ext', +] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = '.rst' + +# source_parsers = {'.md': MarkdownParser} + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [ + "**.ipynb_checkpoints", + "**__pycache__**", + # to ensure that include files (partial pages) aren't built, exclude them + # https://github.com/sphinx-doc/sphinx/issues/1965#issuecomment-124732907 + "**/includes/**", + "**version.py" +] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = {"navigation_depth": 4, "collapse_navigation": False} +html_css_file = ["custom.css"] + +html_context = { + "display_github": True, + "github_user": "nrel", + "github_repo": "mlclouds", + "github_version": "main", + "conf_py_path": "/docs/source/", + "source_suffix": source_suffix, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'rexdoc' + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'mlclouds.tex', 'mlclouds Documentation', + author, 'manual'), +] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'mlclouds', 'mlclouds Documentation', + [author], 1) +] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'mlclouds', 'mlclouds Documentation', + author, 'mlclouds', 'Miscellaneous'), +] + +# -- Extension configuration ------------------------------------------------- + +autosummary_generate = True # Turn on sphinx.ext.autosummary +autoclass_content = "both" # Add __init__ doc (ie. params) to class summaries +autodoc_member_order = 'bysource' +autodoc_inherit_docstrings = True # If no docstring, inherit from base class +add_module_names = False # Remove namespaces from class/method signatures +# Remove 'view source code' from top of page (for html, not python) +html_show_sourcelink = False +numpy_show_class_member = True +napoleon_google_docstring = False +napoleon_use_param = False +napoleon_use_ivar = False +napoleon_use_rtype = False diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..9332e78 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,13 @@ +.. toctree:: + :hidden: + + Home page + Installation and Usage + API reference <_autosummary/mlclouds> + +#################### +Welcome to MLClouds! +#################### + +.. include:: ../../README.rst + :start-after: inclusion-intro diff --git a/docs/source/misc/installation.rst b/docs/source/misc/installation.rst new file mode 100644 index 0000000..2caa3cb --- /dev/null +++ b/docs/source/misc/installation.rst @@ -0,0 +1,6 @@ +Installation +============ + +.. include:: ../../../README.rst + :start-after: Installation + :end-before: Acknowledgments diff --git a/docs/source/misc/installation_usage.rst b/docs/source/misc/installation_usage.rst new file mode 100644 index 0000000..68895fa --- /dev/null +++ b/docs/source/misc/installation_usage.rst @@ -0,0 +1,6 @@ +Installation and Usage +====================== + +.. toctree:: + + installation diff --git a/mlclouds/__init__.py b/mlclouds/__init__.py index dc03e66..8a9bdd8 100644 --- a/mlclouds/__init__.py +++ b/mlclouds/__init__.py @@ -1,7 +1,7 @@ """MLClouds library.""" import os -from .version import __version__ # noqa: F401 +from ._version import __version__ # noqa: F401 MLCLOUDSDIR = os.path.dirname(os.path.realpath(__file__)) TESTDATADIR = os.path.join(os.path.dirname(MLCLOUDSDIR), 'tests', 'data') diff --git a/mlclouds/model/__init__.py b/mlclouds/model/__init__.py new file mode 100644 index 0000000..34c069a --- /dev/null +++ b/mlclouds/model/__init__.py @@ -0,0 +1 @@ +"""MLClouds model module""" diff --git a/mlclouds/model/experimental/__init__.py b/mlclouds/model/experimental/__init__.py new file mode 100644 index 0000000..8e0f74d --- /dev/null +++ b/mlclouds/model/experimental/__init__.py @@ -0,0 +1 @@ +"""MLClouds experimental model module.""" diff --git a/mlclouds/model/experimental/make_plots.py b/mlclouds/model/experimental/make_plots.py deleted file mode 100644 index 8763673..0000000 --- a/mlclouds/model/experimental/make_plots.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Make plots of the statistics -""" -import os -import pandas as pd -import seaborn as sns -import matplotlib.pyplot as plt - -plot_dir = './plots' - -if not os.path.exists(plot_dir): - os.makedirs(plot_dir) - -west_sites = ['TBL', 'DRA', 'FPK', 'SRRL'] -east_sites = ['BON', 'GWN', 'PSU', 'SXF', 'SGP'] -df_east = pd.read_csv('./outputs/validation_stats_east.csv') -df_west = pd.read_csv('./outputs/validation_stats_west.csv') -df_east = df_east[df_east.Site.isin(east_sites)] -df_west = df_west[df_west.Site.isin(west_sites)] -df = pd.concat([df_east, df_west], ignore_index=True) -df = df.sort_values(['Site', 'Model']) - -variables = df.Variable.unique() -conditions = df.Condition.unique() -metrics = ('MAE (%)', 'MBE (%)', 'RMSE (%)') -models = ('Baseline', 'MLClouds') - -assert all([m in df for m in metrics]), 'Could not find: {}'.format(metrics) -assert all([m in df.Model.unique() for m in models]), 'Could not find: {}'.format(models) - -for var in variables: - for condition in conditions: - for metric in metrics: - mask = ((df.Variable == var) - & (df.Condition == condition) - & df.Model.isin(models)) - df_plot = df[mask] - sns.barplot(x='Site', y=metric, hue='Model', data=df_plot, - errorbar=None) - fname = 'stats_{}_{}_{}.png'.format(metric, var, condition) - fname = fname.lower().replace(' (%)', '') - fname = fname.replace('-', '_').replace(' ', '_') - plt.title(fname.replace('.png', '')) - fp = os.path.join(plot_dir, fname) - plt.savefig(fp) - print('Saved: {}'.format(fname)) - plt.close() diff --git a/mlclouds/model/experimental/run_sbatch.sh b/mlclouds/model/experimental/run_sbatch.sh deleted file mode 100644 index d03b7ce..0000000 --- a/mlclouds/model/experimental/run_sbatch.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Run cross validation against all sites on eagle - -rm -r ./outputs -rm -r ./stdout -mkdir ./outputs -mkdir ./stdout - -echo Running production model train and test -sbatch run_train_test.sh diff --git a/mlclouds/model/experimental/run_train_test.sh b/mlclouds/model/experimental/run_train_test.sh deleted file mode 100755 index a65bb6f..0000000 --- a/mlclouds/model/experimental/run_train_test.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -#SBATCH --account=pxs -#SBATCH --output=./stdout/stdout_%A.txt -#SBATCH --error=./stdout/stdout_%A.txt -#SBATCH --time=60 -#SBATCH --partition=debug -#SBATCH --mem=178000 -#SBATCH --qos=high -python train_n_val.py diff --git a/mlclouds/model/k_fold/__init__.py b/mlclouds/model/k_fold/__init__.py new file mode 100644 index 0000000..a8c3f48 --- /dev/null +++ b/mlclouds/model/k_fold/__init__.py @@ -0,0 +1 @@ +"""MLClouds k_fold model module.""" diff --git a/mlclouds/model/k_fold/make_plots.py b/mlclouds/model/k_fold/make_plots.py index f95747d..9ac9c76 100644 --- a/mlclouds/model/k_fold/make_plots.py +++ b/mlclouds/model/k_fold/make_plots.py @@ -2,10 +2,10 @@ Make plots of the statistics """ import os + +import matplotlib.pyplot as plt import pandas as pd import seaborn as sns -import matplotlib.pyplot as plt - from rex import Resource plot_dir = './plots' diff --git a/mlclouds/model/production_model/__init__.py b/mlclouds/model/production_model/__init__.py new file mode 100644 index 0000000..f806d1f --- /dev/null +++ b/mlclouds/model/production_model/__init__.py @@ -0,0 +1 @@ +"""MLClouds production model module.""" diff --git a/mlclouds/model/production_model/make_plots.py b/mlclouds/model/production_model/make_plots.py index 8763673..277baaa 100644 --- a/mlclouds/model/production_model/make_plots.py +++ b/mlclouds/model/production_model/make_plots.py @@ -2,9 +2,10 @@ Make plots of the statistics """ import os + +import matplotlib.pyplot as plt import pandas as pd import seaborn as sns -import matplotlib.pyplot as plt plot_dir = './plots' diff --git a/mlclouds/scripts/__init__.py b/mlclouds/scripts/__init__.py new file mode 100644 index 0000000..cb4303c --- /dev/null +++ b/mlclouds/scripts/__init__.py @@ -0,0 +1 @@ +"""MLClouds scripts.""" diff --git a/mlclouds/version.py b/mlclouds/version.py deleted file mode 100644 index c918124..0000000 --- a/mlclouds/version.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -mlcouds version number -""" - -__version__ = "0.0.2" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..127d33a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,176 @@ +[build-system] +requires = [ + "setuptools >= 61.0", + "setuptools_scm[toml] >= 8", +] +build-backend = "setuptools.build_meta" + +[project] +name = "NREL-mlclouds" +dynamic = ["version"] +description = "A machine learning approach to predicting missing cloud properties in the National Solar Radiation Database (mlclouds)" +keywords = ["mlclouds", "NREL"] +readme = "README.rst" +authors = [ + {name = "Grant Buster", email = "grant.buster@nrel.gov"}, + {name = "Brandon Benton", email = "brandon.benton@nrel.gov"}, +] +license = {text = "BSD-3-Clause"} +requires-python = ">= 3.9" +classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] + +dependencies = [ + "NREL-nsrdb>=3.0", + "NREL-phygnn>=0.0.24", + "tensorflow>=2.12.0,<=2.15.1" +] + +[project.optional-dependencies] +dev = [ + "build>=0.5", + "pre-commit", + "ruff>=0.5.0" +] +doc = [ + "sphinx>=7.0", + "sphinx_rtd_theme>=2.0", + "sphinx-click>=4.0", +] +test = [ + "pytest>=5.2", + "pytest-env>=1.1.3", + "pytest-cov>=5.0.0" +] + +[project.urls] +homepage = "https://github.com/NREL/mlclouds" +documentation = "https://nrel.github.io/mlclouds/" +repository = "https://github.com/NREL/mlclouds" + +[tool.ruff] +line-length = 79 +indent-width = 4 + +target-version = "py39" + +[tool.ruff.lint] +fixable = [] +# preview = true +# logger-objects = [] +task-tags = ["TODO", "FIXME", "XXX"] +select = [ + "A", # flake8-builtins + "ARG", # flake8-unused-arguments + "C", + "C4", # flake8-comprehensions + "C90", # mccabe + "COM", # flake8-commas + "D", # pydocstyle + "E", # pycodestyle + "F", # Pyflakes + "G", # flake8-logging-format + "I", # isort + "LOG", # flake8-logging + "N", # pep8-naming + "NPY", # numpy-specific + "PERF", # Perflint + "PL", # Pylint + "Q", # flake8-quotes + "SIM", # flake8-simplify + "UP", # pyupgrade + "W", # Warning +] + +ignore = [ + # Currently don't conform but we might want to reconsider + "A001", # builtin-variable-shadowing + "A002", # builtin-argument-shadowing + # Currently don't conform but we might want to reconsider + "ARG005", # unused-lambda-argument + "C408", # unnecessary-collection-call + "C414", # unnecessary-double-cast-or-process + "COM812", # missing-trailing-comma + "D105", # undocumented-magic-method + "D200", # fits-on-one-line + "D202", # no-blank-line-after-function + "D204", # one-blank-line-after-class + "D205", # blank-line-after-summary + "D207", # under-indentation + "D209", # new-line-after-last-paragraph + "D400", # ends-in-period + "D401", # non-imperative-mood + "D404", # docstring-starts-with-this + "E402", # import not at top of file + "FIX001", # line-contains-fixme + "G004", # f-string logging + "G001", # str.format logging + "N802", # invalid-function-name + "N803", # invalid-argument-name + "N806", # non-lowercase-variable-in-function + "N811", # constant imported as non constant + "N817", # imported as acronym + "PERF102", # incorrect-dict-iterator + "PERF203", # try-except-in-loop + "PERF401", # manual-list-comprehension + "PLC0415", # import not at top of file + "PLR0904", # too-many-public-methods + "PLR0912", # too-many-branches + "PLR0913", # too-many-arguments + "PLR0914", # too-many-locals + "PLR0915", # too-many-statements + "PLR1702", # too-many-nested-blocks + "PLR1704", # redefined-argument-from-local + "PLR2004", # magic-value-comparison + "PLW1514", # unspecified-encoding + "PLW2901", # redefined-loop-name + "Q000", # bad-quotes-inline-string + "Q004", # unnecessary-escaped-quote + "SIM108", # if-else-block-instead-of-if-exp + "SIM117", #multiple-with-statements + "SIM118", # in-dict-keys + "SIM211", # if-expr-with-false-true + "UP009", # utf8-encoding-declaration + "UP015", # redundant-open-modes + "UP032", # f-string +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] # unused imports + +[tool.ruff.lint.pylint] +max-args = 5 # (PLR0913) Maximum number of arguments for function / method +max-bool-expr = 5 # ( PLR0916) Boolean in a single if statement +max-branches=12 # (PLR0912) branches allowed for a function or method body +max-locals=15 # (PLR0912) local variables allowed for a function or method body +max-nested-blocks = 5 # (PLR1702) nested blocks within a function or method body +max-public-methods=20 # (R0904) public methods allowed for a class +max-returns=6 # (PLR0911) return statements for a function or method body +max-statements=50 # (PLR0915) statements allowed for a function or method body + +[tool.ruff.format] +quote-style = "single" +indent-style = "space" +# Consider adopting "lf" instead +line-ending = "auto" + +[tool.ruff.lint.pydocstyle] +convention = "numpy" + +[tool.setuptools] +include-package-data = true +packages = ["mlclouds"] + +[tool.setuptools_scm] +version_file = "mlclouds/_version.py" + +[tool.pytest_env] +CUDA_VISIBLE_DEVICES = "-1" +TF_ENABLE_ONEDNN_OPTS = "0" \ No newline at end of file diff --git a/setup.py b/setup.py index af8bb23..745017c 100644 --- a/setup.py +++ b/setup.py @@ -1,55 +1,36 @@ """ setup.py """ -import os -from codecs import open -from setuptools import find_packages, setup +import shlex +from subprocess import check_call +from warnings import warn -try: - from pypandoc import convert_text -except ImportError: - convert_text = lambda string, *args, **kwargs: string +from setuptools import setup +from setuptools.command.develop import develop -here = os.path.abspath(os.path.dirname(__file__)) -with open("README.rst", encoding="utf-8") as readme_file: - readme = convert_text(readme_file.read(), "rst", format="rst") +class PostDevelopCommand(develop): + """ + Class to run post setup commands + """ -with open("requirements.txt") as f: - install_requires = f.readlines() + def run(self): + """ + Run method that tries to install pre-commit hooks + """ + try: + check_call(shlex.split('pre-commit install')) + except Exception as e: + warn("Unable to run 'pre-commit install': {}".format(e)) -with open(os.path.join(here, "mlclouds", "version.py"), encoding="utf-8") as f: - version = f.read() + develop.run(self) -version = version.split('=')[-1].strip().strip('"').strip("'") setup( - name="NREL-mlclouds", - version=version, - description="Machines Learning Clouds", - long_description=readme, - author="Grant Buster", - author_email="grant.buster@nrel.gov", - url="https://github.com/NREL", - packages=find_packages(), - package_dir={"mlclouds": "mlclouds"}, package_data={'mlclouds': ['model/production_model/outputs/mlclouds_model.pkl', 'model/production_model/outputs/mlclouds_model.json']}, - include_package_data=True, - license="BSD license", - zip_safe=False, - keywords="mlclouds", - python_requires='>=3.9', - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Natural Language :: English", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - test_suite="tests", - install_requires=install_requires, - ) + test_suite='tests', + cmdclass={'develop': PostDevelopCommand}, +)