From d3ebac4ccdb46a173ef97370d2056200fb275011 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Thu, 4 Nov 2021 10:48:17 -0700 Subject: [PATCH 01/18] Initial draft --- .../example-components-connector/MANIFEST.in | 39 +++++++ .../example-components-connector/Makefile | 43 +++++++ .../example-components-connector/README.md | 63 ++++++++++ .../elyra_examples_connector/__init__.py | 15 +++ .../elyra_examples_connector/_version.py | 16 +++ .../elyra-examples-catalog.json | 71 ++++++++++++ .../examples_connector.py | 109 ++++++++++++++++++ .../examples_schema_provider.py | 49 ++++++++ .../kfp_example_components/download_file.yaml | 24 ++++ .../example-components-connector/setup.cfg | 53 +++++++++ .../example-components-connector/setup.py | 71 ++++++++++++ .../test_requirements.txt | 1 + 12 files changed, 554 insertions(+) create mode 100644 component-catalog-connectors/example-components-connector/MANIFEST.in create mode 100644 component-catalog-connectors/example-components-connector/Makefile create mode 100644 component-catalog-connectors/example-components-connector/README.md create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/__init__.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/_version.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml create mode 100644 component-catalog-connectors/example-components-connector/setup.cfg create mode 100644 component-catalog-connectors/example-components-connector/setup.py create mode 100644 component-catalog-connectors/example-components-connector/test_requirements.txt diff --git a/component-catalog-connectors/example-components-connector/MANIFEST.in b/component-catalog-connectors/example-components-connector/MANIFEST.in new file mode 100644 index 0000000..8660f5e --- /dev/null +++ b/component-catalog-connectors/example-components-connector/MANIFEST.in @@ -0,0 +1,39 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# exclude from ANY directory +global-exclude *.ipynb +global-exclude *.py[cod] +global-exclude __pycache__ +global-exclude .git +global-exclude .ipynb_checkpoints +global-exclude .DS_Store +global-exclude *.sh +global-exclude docs +global-exclude tests + +# explicit includes +include CONTRIBUTING.md +include README.md +include LICENSE +include dist/*.tgz + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +include elyra_examples_connector/elyra-examples-catalog.json +include elyra_examples_connector/kfp_example_components/*.yaml +recursive-include elyra_examples_connector diff --git a/component-catalog-connectors/example-components-connector/Makefile b/component-catalog-connectors/example-components-connector/Makefile new file mode 100644 index 0000000..ed0d0c3 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/Makefile @@ -0,0 +1,43 @@ +# +# Copyright 2021-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: help clean lint test-dependencies source-install install dist + +SHELL:=/bin/bash + +help: +# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +clean: + rm -rf dist/ + rm -rf build/ + rm -rf *.egg-info + +test-dependencies: + @pip install -q -r test_requirements.txt + +lint: test-dependencies + flake8 elyra_examples_connector + +dist: clean lint ## Build distribution + python setup.py bdist_wheel sdist + +source-install: dist ## Install example pipeline components package from source + pip install . + +install: ## Install example pipeline components package from PyPI + pip install elyra-example-components-catalog diff --git a/component-catalog-connectors/example-components-connector/README.md b/component-catalog-connectors/example-components-connector/README.md new file mode 100644 index 0000000..79320a4 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/README.md @@ -0,0 +1,63 @@ +## Machine Learning Exchange catalog connector + +This catalog connector enables Elyra to load Kubeflow Pipelines components from [Machine Learning Exchange](https://github.com/machine-learning-exchange) (MLX) deployments. + +### Install the connector + +You can install the MLX catalog connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. + +**Prerequisites** + +- [Install Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above). +- [Machine Learning Exchange deployment](https://github.com/machine-learning-exchange/mlx) ([quickstart guide](https://github.com/machine-learning-exchange/mlx/tree/main/quickstart)) + +**Install from PyPI** + + ``` + $ pip install mlx-component-catalog-connector + ``` + +**Install from source code** + + ``` + $ git clone https://github.com/elyra-ai/examples.git + $ cd examples/component-catalog-connectors/mlx-connector/ + $ make source-install + ``` + +### Use the connector + +1. Launch JupyterLab. +1. [Open the `Manage Components` panel]( +https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). +1. Add a new MLX component catalog ('`+`' > '`New Machine Learning Exchange Component Catalog`'). +1. Specify a catalog name, e.g. '`MLX dev catalog`'. +1. Select Kubeflow Pipelines as runtime. +1. (Optional) Specify a category under which the catalog's component will be organized in the palette. +1. Configure the `MLX API URL`, e.g. '`http://my-mlx-server.mydomain:8080/`'. + > Note: The Machine Learning Exchange API URL is different from the GUI URL! +1. Apply an optional filter expression to the component names. The `*` (zero or more characters) and `?` (zero or one character) wildcards are supported. +1. Save the catalog entry. +1. Open the Kubeflow Pipelines Visual Pipeline Editor and expand the palette. The components that were loaded from the specified MLX URL are displayed. + +### Uninstall the connector + +1. Remove all MLX catalog entries from the '`Manage Components`' panel. +1. Stop JupyterLab. +1. Uninstall the `mlx-component-catalog-connector` package. + ``` + $ pip uninstall -y mlx-component-catalog-connector + ``` + +### Troubleshooting + +**Problem: The palette does not diplay any components from the configured catalog.** + +**Solution:** If the the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. + +Example error message (The specified MLX URL is invalid): + +``` +Error fetching component list from MLX catalog http://localhost:8080: ... Failed to establish a new connection: [Errno 61] Connection refused' +``` + diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/__init__.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/__init__.py new file mode 100644 index 0000000..febc0a2 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/_version.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/_version.py new file mode 100644 index 0000000..4abcc16 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/_version.py @@ -0,0 +1,16 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = '0.0.1' diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json b/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json new file mode 100644 index 0000000..d723390 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json", + "title": "Elyra examples component catalog", + "name": "elyra-examples-catalog", + "display_name": "Elyra examples component catalog", + "schemaspace": "component-registries", + "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", + "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", + "uihints": { + "title": "Elyra examples component catalog", + "icon": "", + "reference_url": "https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html" + }, + "properties": { + "schema_name": { + "title": "Schema Name", + "description": "The schema associated with this instance", + "type": "string", + "const": "elyra-examples-catalog" + }, + "display_name": { + "title": "Display Name", + "description": "Display name of this Component Catalog", + "type": "string", + "minLength": 1 + }, + "version": { + "title": "Version", + "description": "The version associated with this instance", + "type": "integer", + "const": 1 + }, + "metadata": { + "description": "Additional data specific to this metadata", + "type": "object", + "properties": { + "description": { + "title": "Description", + "description": "Description of this Component Catalog", + "type": "string" + }, + "runtime": { + "title": "Runtime", + "description": "The runtime associated with this Component Catalog", + "type": "string", + "enum": ["kfp", "airflow"], + "default": "kfp", + "uihints": { + "field_type": "dropdown" + } + }, + "categories": { + "title": "Category Names", + "description": "Category names associated with this Component Catalog (the components defined in this registry will be organized in the component palette according to these categories)", + "type": "array", + "items": { + "type": "string", + "maxLength": 18 + }, + "uihints": { + "field_type": "array", + "category": "Component Categories" + } + } + }, + "required": ["runtime"] + } + }, + "required": ["schema_name", "display_name", "version", "metadata"] +} diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py new file mode 100644 index 0000000..d415c6f --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py @@ -0,0 +1,109 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from glob import glob +from pathlib import Path +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + + +from elyra.pipeline.catalog_connector import ComponentCatalogConnector + + +class ExamplesCatalogConnector(ComponentCatalogConnector): + """ + Makes example components for Kubeflow Pipelines and Apache Airflow + available to Elyra. + """ + config = { + 'kfp': { + 'root_dir': 'kfp_examples_components', + 'file_filter': '*.yaml' + } + } + + def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str, Any]]: + + """ + Returns Elyra example custom components for the selected runtime (Kubeflow Pipelines + and Apache Airflow only) + :param registry_metadata: the dictionary-form of the Metadata instance for a single registry + """ + component_list = [] + + if catalog_metadata.get('runtime') not in ExamplesCatalogConnector.config.keys(): + self.log.error('Example custom components are only available for Kubeflow Pipelines ' + 'and Apache Airflow.') + # return empty component specification list + return component_list + + try: + root_dir = Path(__file__).parent / ExamplesCatalogConnector.config[catalog_metadata.get('runtime')] + self.log.info(f"Retrieving component list for runtime '{catalog_metadata.get('runtime')}' from " + f'{root_dir}') + file_spec = ExamplesCatalogConnector.config[catalog_metadata.get('runtime')].get('file_filter', '*') + for file in glob.iglob(root_dir / '**' / file_spec): + component_list.append({'component-id': file}) + + self.log.info(f'Component list: {component_list}') + + except Exception as ex: + self.log.warning(f"Error retrieving component list for runtime '{catalog_metadata.get('runtime')}'" + f" from {root_dir}: {ex}") + + return component_list + + def read_catalog_entry(self, + catalog_entry_data: Dict[str, Any], + catalog_metadata: Dict[str, Any]) -> Optional[str]: + """ + Read a component definition for a single catalog entry using the its data (as returned from + get_catalog_entries()) and the catalog metadata, if needed + + :param catalog_entry_data: a dictionary that contains the information needed to read the content + of the component definition + :param catalog_metadata: the metadata associated with the catalog in which this catalog entry is + stored; in addition to catalog_entry_data, catalog_metadata may also be + needed to read the component definition for certain types of catalogs + + :returns: the requested component specification, if it exists + """ + + component_id = catalog_entry_data['component-id'] + if component_id is None: + self.log.error('Cannot retrieve example component specification: ' + 'A component id must be provided.') + return None + + try: + with open(component_id, 'r') as fp: + return fp.read() + except Exception as e: + self.log.error(f'Failed to download component specification {component_id} ' + f': {str(e)}') + return None + + def get_hash_keys(self) -> List[Any]: + """ + Identifies the key(s) that read_catalog_entry method + requires to be present in the catalog_entry_data parameter + to allow for retrieval of a component. + + :returns: a list of keys + """ + return ['component-id'] diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py new file mode 100644 index 0000000..8a5a46e --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py @@ -0,0 +1,49 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +from pathlib import Path +from typing import Dict +from typing import List + +from elyra.metadata.schema import SchemasProvider + + +class ExamplesSchemasProvider(SchemasProvider): + """ + Provides access to example custom components + for Kubeflow Pipelines and Apache Airflow. + """ + + def get_schemas(self) -> List[Dict]: + """ + Return the example schema + """ + # use Elyra logger + log = logging.getLogger('ElyraApp') + examples_catalog_schema_defs = [] + try: + # load examples schema definition + examples_catalog_connector_schema_file = Path(__file__).parent / 'elyra-examples-catalog.json' + log.debug(f'Reading examples catalog connector schema from {examples_catalog_connector_schema_file}') + with open(examples_catalog_connector_schema_file, 'r') as fp: + examples_catalog_connector_schema = json.load(fp) + examples_catalog_schema_defs.append(examples_catalog_connector_schema) + except Exception as ex: + log.error(f'Error reading examples catalog connector schema {examples_catalog_connector_schema_file}: {ex}') + + return examples_catalog_schema_defs diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml new file mode 100644 index 0000000..8ddd982 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml @@ -0,0 +1,24 @@ +name: Download File +description: Downloads a file from a public HTTP/S URL using a GET request. + +inputs: +- {name: URL, type: String, optional: false, description: 'File URL'} + +outputs: +- {name: downloaded file, type: String, description: 'Content of the downloaded file.'} + +implementation: + container: + image: quay.io/elyra/kfp-tutorial-download-file-component@sha256:499c6e687ab94babb67664b99562f199289e081edb2f38f26a13bc198aa5cfb9 + # command is a list of strings (command-line arguments). + # The YAML language has two syntaxes for lists and you can use either of them. + # Here we use the "flow syntax" - comma-separated strings inside square brackets. + command: [ + python3, + # Path of the program inside the container + /pipelines/component/src/download-file.py, + --file-url, + {inputValue: URL}, + --downloaded-file-path, + {outputPath: downloaded file} + ] diff --git a/component-catalog-connectors/example-components-connector/setup.cfg b/component-catalog-connectors/example-components-connector/setup.cfg new file mode 100644 index 0000000..e35c831 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/setup.cfg @@ -0,0 +1,53 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[bdist_wheel] +universal=0 + +[metadata] +description_file=README.md + +[flake8] +application-import-names = catalog_connector +application-package-names = catalog_connector +enable-extensions = G +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # File contains nothing but comments + H104, + # Include name with TODOs as in # TODO(yourname) + H101, + # Enable mocking + H216, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405, + # Allow breaks after binary operators + W504 +import-order-style = google +max-line-length = 120 diff --git a/component-catalog-connectors/example-components-connector/setup.py b/component-catalog-connectors/example-components-connector/setup.py new file mode 100644 index 0000000..2aadcc7 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/setup.py @@ -0,0 +1,71 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from setuptools import find_packages, setup + +long_desc = """ + Elyra component catalog for example Kubeflow Pipelines + and Apache Airflow components. + """ + +here = os.path.abspath(os.path.dirname(__file__)) + +version_ns = {} +with open(os.path.join(here, 'elyra_examples_connector', '_version.py')) as f: + exec(f.read(), {}, version_ns) + +setup_args = dict( + name="elyra-example-components-catalog", + version=version_ns['__version__'], + url="https://github.com/elyra-ai/examples", + description="Kubeflow Pipelines and Airflow example components for Elyra", + long_description=long_desc, + author="Elyra Maintainers", + license="Apache License Version 2.0", + packages=find_packages(), + install_requires=[ + 'elyra==3.3.0.dev0' + ], + setup_requires=['flake8'], + include_package_data=True, + entry_points={ + 'metadata.schemas_providers': [ + 'examples-catalog-schema = elyra_examples_connector.examples_schema_provider:ExamplesSchemasProvider' + ], + 'elyra.component.catalog_types': [ + 'examples-catalog = elyra_examples_connector.examples_connector:ExamplesCatalogConnector' + ], + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + ] +) + + +if __name__ == '__main__': + setup(**setup_args) diff --git a/component-catalog-connectors/example-components-connector/test_requirements.txt b/component-catalog-connectors/example-components-connector/test_requirements.txt new file mode 100644 index 0000000..28f28b5 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/test_requirements.txt @@ -0,0 +1 @@ +flake8>=3.5.0,<3.9.0 From 739ac9b8e6af774959c0294934107241e4fe1483 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Thu, 4 Nov 2021 13:55:45 -0700 Subject: [PATCH 02/18] Add airflow examples and fix bugs --- .../example-components-connector/.flake8 | 31 +++ .../example-components-connector/Makefile | 1 + .../example-components-connector/README.md | 44 ++-- .../bash_operator.py | 173 ++++++++++++++++ .../email_operator.py | 78 +++++++ .../http_operator.py | 99 +++++++++ .../slack_operator.py | 95 +++++++++ .../spark_sql_operator.py | 114 +++++++++++ .../spark_submit_operator.py | 190 ++++++++++++++++++ .../examples_connector.py | 72 ++++--- .../calculate_hash.yaml | 43 ++++ .../kfp_example_components/download_data.yaml | 26 +++ .../kfp_example_components/download_file.yaml | 24 --- .../filter_text_using_shell_and_grep.yaml | 42 ++++ .../run_notebook_using_papermill.yaml | 66 ++++++ .../example-components-connector/setup.py | 2 +- 16 files changed, 1015 insertions(+), 85 deletions(-) create mode 100644 component-catalog-connectors/example-components-connector/.flake8 create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/bash_operator.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/email_operator.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/http_operator.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/slack_operator.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_sql_operator.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_submit_operator.py create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/calculate_hash.yaml create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_data.yaml delete mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/filter_text_using_shell_and_grep.yaml create mode 100644 component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/run_notebook_using_papermill.yaml diff --git a/component-catalog-connectors/example-components-connector/.flake8 b/component-catalog-connectors/example-components-connector/.flake8 new file mode 100644 index 0000000..4237db1 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/.flake8 @@ -0,0 +1,31 @@ +[flake8] +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +exclude = + __init__.py + elyra_examples_connector/airflow_example_components/ +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # Allow breaks after binary operators + W504, + # Include name with TODOs as in # TODO(yourname) + H101, + # Do not import more than one module per line + H301, + # Alphabetically order imports by the full module path + H306, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405 +max-line-length = 120 + diff --git a/component-catalog-connectors/example-components-connector/Makefile b/component-catalog-connectors/example-components-connector/Makefile index ed0d0c3..ea50c62 100644 --- a/component-catalog-connectors/example-components-connector/Makefile +++ b/component-catalog-connectors/example-components-connector/Makefile @@ -26,6 +26,7 @@ clean: rm -rf dist/ rm -rf build/ rm -rf *.egg-info + - pip uninstall -y elyra-example-components-catalog test-dependencies: @pip install -q -r test_requirements.txt diff --git a/component-catalog-connectors/example-components-connector/README.md b/component-catalog-connectors/example-components-connector/README.md index 79320a4..f6208cc 100644 --- a/component-catalog-connectors/example-components-connector/README.md +++ b/component-catalog-connectors/example-components-connector/README.md @@ -1,28 +1,27 @@ -## Machine Learning Exchange catalog connector +## Elyra pipeline component examples catalog -This catalog connector enables Elyra to load Kubeflow Pipelines components from [Machine Learning Exchange](https://github.com/machine-learning-exchange) (MLX) deployments. +This catalog connector enables Elyra to load example pipeline components. ### Install the connector -You can install the MLX catalog connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. +You can install the example component connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. **Prerequisites** - [Install Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above). -- [Machine Learning Exchange deployment](https://github.com/machine-learning-exchange/mlx) ([quickstart guide](https://github.com/machine-learning-exchange/mlx/tree/main/quickstart)) **Install from PyPI** ``` - $ pip install mlx-component-catalog-connector + $ pip install elyra-example-components-catalog ``` **Install from source code** ``` $ git clone https://github.com/elyra-ai/examples.git - $ cd examples/component-catalog-connectors/mlx-connector/ - $ make source-install + $ cd examples/component-catalog-connectors/elyra-examples-connector/ + $ make clean source-install ``` ### Use the connector @@ -30,34 +29,23 @@ You can install the MLX catalog connector from PyPI or source code. Note that a 1. Launch JupyterLab. 1. [Open the `Manage Components` panel]( https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). -1. Add a new MLX component catalog ('`+`' > '`New Machine Learning Exchange Component Catalog`'). -1. Specify a catalog name, e.g. '`MLX dev catalog`'. -1. Select Kubeflow Pipelines as runtime. -1. (Optional) Specify a category under which the catalog's component will be organized in the palette. -1. Configure the `MLX API URL`, e.g. '`http://my-mlx-server.mydomain:8080/`'. - > Note: The Machine Learning Exchange API URL is different from the GUI URL! -1. Apply an optional filter expression to the component names. The `*` (zero or more characters) and `?` (zero or one character) wildcards are supported. +1. Add a new component examples catalog ('`+`' > '`New Elyra examples component catalog`'). +1. Specify a catalog name, e.g. '`Elyra example components`'. +1. Select a runtime from the list. +1. (Optional) Specify a category under which the example components will be organized in the palette. 1. Save the catalog entry. -1. Open the Kubeflow Pipelines Visual Pipeline Editor and expand the palette. The components that were loaded from the specified MLX URL are displayed. +1. Open the Visual Pipeline Editor for the chosen runtime and expand the palette. The example components are displayed. ### Uninstall the connector -1. Remove all MLX catalog entries from the '`Manage Components`' panel. +1. Remove all example catalog entries from the '`Manage Components`' panel. 1. Stop JupyterLab. -1. Uninstall the `mlx-component-catalog-connector` package. +1. Uninstall the `elyra-example-components-catalog` package. ``` - $ pip uninstall -y mlx-component-catalog-connector + $ pip uninstall -y elyra-example-components-catalog ``` ### Troubleshooting -**Problem: The palette does not diplay any components from the configured catalog.** - -**Solution:** If the the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. - -Example error message (The specified MLX URL is invalid): - -``` -Error fetching component list from MLX catalog http://localhost:8080: ... Failed to establish a new connection: [Errno 61] Connection refused' -``` - +**Q: No example components are displayed in the Visual Pipeline Editor palette.** +A: Verify that you imported the examples for the correct runtime environment and check the JupyterLab log file for error messages. \ No newline at end of file diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/bash_operator.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/bash_operator.py new file mode 100644 index 0000000..e27e746 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/bash_operator.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import os +import signal +from subprocess import Popen, STDOUT, PIPE +from tempfile import gettempdir, NamedTemporaryFile + +from builtins import bytes + +from airflow.exceptions import AirflowException +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from airflow.utils.file import TemporaryDirectory +from airflow.utils.operator_helpers import context_to_airflow_vars + + +class BashOperator(BaseOperator): + r""" + Execute a Bash script, command or set of commands. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:BashOperator` + + :param bash_command: The command, set of commands or reference to a + bash script (must be '.sh') to be executed. (templated) + :type bash_command: str + :param xcom_push: If xcom_push is True, the last line written to stdout + will also be pushed to an XCom when the bash command completes. + :type xcom_push: bool + :param env: If env is not None, it must be a mapping that defines the + environment variables for the new process; these are used instead + of inheriting the current process environment, which is the default + behavior. (templated) + :type env: dict + :param output_encoding: Output encoding of bash command + :type output_encoding: str + + .. warning:: + + Care should be taken with "user" input or when using Jinja templates in the + ``bash_command``, as this bash operator does not perform any escaping or + sanitization of the command. + + This applies mostly to using "dag_run" conf, as that can be submitted via + users in the Web UI. Most of the default template variables are not at + risk. + + For example, do **not** do this: + + .. code-block:: python + + bash_task = BashOperator( + task_id="bash_task", + bash_command='echo "Here is the message: \'{{ dag_run.conf["message"] if dag_run else "" }}\'"', + ) + + Instead, you should pass this via the ``env`` kwarg and use double-quotes + inside the bash_command, as below: + + .. code-block:: python + + bash_task = BashOperator( + task_id="bash_task", + bash_command='echo "here is the message: \'$message\'"', + env={'message': '{{ dag_run.conf["message"] if dag_run else "" }}'}, + ) + + """ + template_fields = ('bash_command', 'env') + template_ext = ('.sh', '.bash',) + ui_color = '#f0ede4' + + @apply_defaults + def __init__( + self, + bash_command, + xcom_push=False, + env=None, + output_encoding='utf-8', + *args, **kwargs): + + super(BashOperator, self).__init__(*args, **kwargs) + self.bash_command = bash_command + self.env = env + self.xcom_push_flag = xcom_push + self.output_encoding = output_encoding + self.sub_process = None + + def execute(self, context): + """ + Execute the bash command in a temporary directory + which will be cleaned afterwards + """ + self.log.info("Tmp dir root location: \n %s", gettempdir()) + + # Prepare env for child process. + env = self.env + if env is None: + env = os.environ.copy() + airflow_context_vars = context_to_airflow_vars(context, in_env_var_format=True) + self.log.debug('Exporting the following env vars:\n%s', + '\n'.join(["{}={}".format(k, v) + for k, v in airflow_context_vars.items()])) + env.update(airflow_context_vars) + + self.lineage_data = self.bash_command + + with TemporaryDirectory(prefix='airflowtmp') as tmp_dir: + with NamedTemporaryFile(dir=tmp_dir, prefix=self.task_id) as f: + + f.write(bytes(self.bash_command, 'utf_8')) + f.flush() + fname = f.name + script_location = os.path.abspath(fname) + self.log.info( + "Temporary script location: %s", + script_location + ) + + def pre_exec(): + # Restore default signal disposition and invoke setsid + for sig in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'): + if hasattr(signal, sig): + signal.signal(getattr(signal, sig), signal.SIG_DFL) + os.setsid() + + self.log.info("Running command: %s", self.bash_command) + self.sub_process = Popen( + ['bash', fname], + stdout=PIPE, stderr=STDOUT, + cwd=tmp_dir, env=env, + preexec_fn=pre_exec) + + self.log.info("Output:") + line = '' + for line in iter(self.sub_process.stdout.readline, b''): + line = line.decode(self.output_encoding).rstrip() + self.log.info(line) + self.sub_process.wait() + self.log.info( + "Command exited with return code %s", + self.sub_process.returncode + ) + + if self.sub_process.returncode: + raise AirflowException("Bash command failed") + + if self.xcom_push_flag: + return line + + def on_kill(self): + self.log.info('Sending SIGTERM signal to bash process group') + if self.sub_process and hasattr(self.sub_process, 'pid'): + os.killpg(os.getpgid(self.sub_process.pid), signal.SIGTERM) diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/email_operator.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/email_operator.py new file mode 100644 index 0000000..0ea7fc0 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/email_operator.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.models import BaseOperator +from airflow.utils.email import send_email +from airflow.utils.decorators import apply_defaults + + +class EmailOperator(BaseOperator): + """ + Sends an email. + + :param to: list of emails to send the email to. (templated) + :type to: list or string (comma or semicolon delimited) + :param subject: subject line for the email. (templated) + :type subject: str + :param html_content: content of the email, html markup + is allowed. (templated) + :type html_content: str + :param files: file names to attach in email + :type files: list + :param cc: list of recipients to be added in CC field + :type cc: list or string (comma or semicolon delimited) + :param bcc: list of recipients to be added in BCC field + :type bcc: list or string (comma or semicolon delimited) + :param mime_subtype: MIME sub content type + :type mime_subtype: str + :param mime_charset: character set parameter added to the Content-Type + header. + :type mime_charset: str + """ + + template_fields = ('to', 'subject', 'html_content') + template_ext = ('.html',) + ui_color = '#e6faf9' + + @apply_defaults + def __init__( + self, + to, + subject, + html_content, + files=None, + cc=None, + bcc=None, + mime_subtype='mixed', + mime_charset='us_ascii', + *args, **kwargs): + super(EmailOperator, self).__init__(*args, **kwargs) + self.to = to + self.subject = subject + self.html_content = html_content + self.files = files or [] + self.cc = cc + self.bcc = bcc + self.mime_subtype = mime_subtype + self.mime_charset = mime_charset + + def execute(self, context): + send_email(self.to, self.subject, self.html_content, + files=self.files, cc=self.cc, bcc=self.bcc, + mime_subtype=self.mime_subtype, mime_charset=self.mime_charset) diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/http_operator.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/http_operator.py new file mode 100644 index 0000000..8b8532b --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/http_operator.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from airflow.exceptions import AirflowException +from airflow.hooks.http_hook import HttpHook +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults + + +class SimpleHttpOperator(BaseOperator): + """ + Calls an endpoint on an HTTP system to execute an action + + :param http_conn_id: The connection to run the operator against + :type http_conn_id: str + :param endpoint: The relative part of the full url. (templated) + :type endpoint: str + :param method: The HTTP method to use, default = "POST" + :type method: str + :param data: The data to pass. POST-data in POST/PUT and params + in the URL for a GET request. (templated) + :type data: For POST/PUT, depends on the content-type parameter, + for GET a dictionary of key/value string pairs + :param headers: The HTTP headers to be added to the GET request + :type headers: a dictionary of string key/value pairs + :param response_check: A check against the 'requests' response object. + Returns True for 'pass' and False otherwise. + :type response_check: A lambda or defined function. + :param extra_options: Extra options for the 'requests' library, see the + 'requests' documentation (options to modify timeout, ssl, etc.) + :type extra_options: A dictionary of options, where key is string and value + depends on the option that's being modified. + :param xcom_push: Push the response to Xcom (default: False). + If xcom_push is True, response of an HTTP request will also + be pushed to an XCom. + :type xcom_push: bool + :param log_response: Log the response (default: False) + :type log_response: bool + """ + + template_fields = ['endpoint', 'data', 'headers', ] + template_ext = () + ui_color = '#f4a460' + + @apply_defaults + def __init__(self, + endpoint, + method='POST', + data=None, + headers=None, + response_check=None, + extra_options=None, + xcom_push=False, + http_conn_id='http_default', + log_response=False, + *args, **kwargs): + super(SimpleHttpOperator, self).__init__(*args, **kwargs) + self.http_conn_id = http_conn_id + self.method = method + self.endpoint = endpoint + self.headers = headers or {} + self.data = data or {} + self.response_check = response_check + self.extra_options = extra_options or {} + self.xcom_push_flag = xcom_push + self.log_response = log_response + + def execute(self, context): + http = HttpHook(self.method, http_conn_id=self.http_conn_id) + + self.log.info("Calling HTTP method") + + response = http.run(self.endpoint, + self.data, + self.headers, + self.extra_options) + if self.log_response: + self.log.info(response.text) + if self.response_check: + if not self.response_check(response): + raise AirflowException("Response check returned False.") + if self.xcom_push_flag: + return response.text diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/slack_operator.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/slack_operator.py new file mode 100644 index 0000000..cb40245 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/slack_operator.py @@ -0,0 +1,95 @@ +# flake8: noqa +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +from airflow.operators.slack_operator import SlackAPIOperator +from airflow.utils.decorators import apply_defaults + + +class SlackAPIPostOperator(SlackAPIOperator): + """ + Posts messages to a slack channel + + :param slack_conn_id: Slack connection ID which its password is Slack API token + :type slack_conn_id: str + :param token: Slack API token (https://api.slack.com/web) + :type token: str + :param method: The Slack API Method to Call (https://api.slack.com/methods) + :type method: str + :param api_params: API Method call parameters (https://api.slack.com/methods) + :type api_params: dict + :param channel: channel in which to post message on slack name (#general) or + ID (C12318391). (templated) + :type channel: str + :param username: Username that airflow will be posting to Slack as. (templated) + :type username: str + :param text: message to send to slack. (templated) + :type text: str + :param icon_url: url to icon used for this message + :type icon_url: str + :param attachments: extra formatting details. (templated) + - see https://api.slack.com/docs/attachments. + :type attachments: list of hashes + :param blocks: extra block layouts. (templated) + - see https://api.slack.com/reference/block-kit/blocks. + :type blocks: list of hashes + """ + + template_fields = ('username', 'text', 'attachments', 'blocks', 'channel') + ui_color = '#FFBA40' + + @apply_defaults + def __init__(self, + slack_conn_id=None, + token=None, + api_params=None, + channel='#general', + username='Airflow', + text='No message has been set.\n' + 'Here is a cat video instead\n' + 'https://www.youtube.com/watch?v=J---aiyznGQ', + icon_url='https://raw.githubusercontent.com/apache/' + 'airflow/master/airflow/www/static/pin_100.png', + attachments=None, + blocks=None, + *args, **kwargs): + self.method = 'chat.postMessage' + self.channel = channel + self.username = username + self.text = text + self.icon_url = icon_url + self.attachments = attachments or [] + self.blocks = blocks or [] + super(SlackAPIPostOperator, self).__init__(method=self.method, + token=token, + slack_conn_id=slack_conn_id, + api_params=api_params, + *args, **kwargs) + + def construct_api_call_params(self): + self.api_params = { + 'channel': self.channel, + 'username': self.username, + 'text': self.text, + 'icon_url': self.icon_url, + 'attachments': json.dumps(self.attachments), + 'blocks': json.dumps(self.blocks), + } diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_sql_operator.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_sql_operator.py new file mode 100644 index 0000000..06a51f7 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_sql_operator.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from airflow.models import BaseOperator +from airflow.utils.decorators import apply_defaults +from airflow.contrib.hooks.spark_sql_hook import SparkSqlHook + + +class SparkSqlOperator(BaseOperator): + """ + Execute Spark SQL query + + :param sql: The SQL query to execute. (templated) + :type sql: str + :param conf: arbitrary Spark configuration property + :type conf: str (format: PROP=VALUE) + :param conn_id: connection_id string + :type conn_id: str + :param total_executor_cores: (Standalone & Mesos only) Total cores for all + executors (Default: all the available cores on the worker) + :type total_executor_cores: int + :param executor_cores: (Standalone & YARN only) Number of cores per + executor (Default: 2) + :type executor_cores: int + :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G) + :type executor_memory: str + :param keytab: Full path to the file that contains the keytab + :type keytab: str + :param master: spark://host:port, mesos://host:port, yarn, or local + :type master: str + :param name: Name of the job + :type name: str + :param num_executors: Number of executors to launch + :type num_executors: int + :param verbose: Whether to pass the verbose flag to spark-sql + :type verbose: bool + :param yarn_queue: The YARN queue to submit to (Default: "default") + :type yarn_queue: str + """ + + template_fields = ["_sql"] + template_ext = [".sql", ".hql"] + + @apply_defaults + def __init__(self, + sql, + conf=None, + conn_id='spark_sql_default', + total_executor_cores=None, + executor_cores=None, + executor_memory=None, + keytab=None, + principal=None, + master='yarn', + name='default-name', + num_executors=None, + verbose=True, + yarn_queue='default', + *args, + **kwargs): + super(SparkSqlOperator, self).__init__(*args, **kwargs) + self._sql = sql + self._conf = conf + self._conn_id = conn_id + self._total_executor_cores = total_executor_cores + self._executor_cores = executor_cores + self._executor_memory = executor_memory + self._keytab = keytab + self._principal = principal + self._master = master + self._name = name + self._num_executors = num_executors + self._verbose = verbose + self._yarn_queue = yarn_queue + self._hook = None + + def execute(self, context): + """ + Call the SparkSqlHook to run the provided sql query + """ + self._hook = SparkSqlHook(sql=self._sql, + conf=self._conf, + conn_id=self._conn_id, + total_executor_cores=self._total_executor_cores, + executor_cores=self._executor_cores, + executor_memory=self._executor_memory, + keytab=self._keytab, + principal=self._principal, + name=self._name, + num_executors=self._num_executors, + master=self._master, + verbose=self._verbose, + yarn_queue=self._yarn_queue + ) + self._hook.run_query() + + def on_kill(self): + self._hook.kill() diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_submit_operator.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_submit_operator.py new file mode 100644 index 0000000..caee335 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_submit_operator.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook +from airflow.models import BaseOperator +from airflow.settings import WEB_COLORS +from airflow.utils.decorators import apply_defaults + + +class SparkSubmitOperator(BaseOperator): + """ + This hook is a wrapper around the spark-submit binary to kick off a spark-submit job. + It requires that the "spark-submit" binary is in the PATH or the spark-home is set + in the extra on the connection. + + :param application: The application that submitted as a job, either jar or py file. (templated) + :type application: str + :param conf: Arbitrary Spark configuration properties (templated) + :type conf: dict + :param conn_id: The connection id as configured in Airflow administration. When an + invalid connection_id is supplied, it will default to yarn. + :type conn_id: str + :param files: Upload additional files to the executor running the job, separated by a + comma. Files will be placed in the working directory of each executor. + For example, serialized objects. (templated) + :type files: str + :param py_files: Additional python files used by the job, can be .zip, .egg or .py. (templated) + :type py_files: str + :param jars: Submit additional jars to upload and place them in executor classpath. (templated) + :type jars: str + :param driver_class_path: Additional, driver-specific, classpath settings. (templated) + :type driver_class_path: str + :param java_class: the main class of the Java application + :type java_class: str + :param packages: Comma-separated list of maven coordinates of jars to include on the + driver and executor classpaths. (templated) + :type packages: str + :param exclude_packages: Comma-separated list of maven coordinates of jars to exclude + while resolving the dependencies provided in 'packages' (templated) + :type exclude_packages: str + :param repositories: Comma-separated list of additional remote repositories to search + for the maven coordinates given with 'packages' + :type repositories: str + :param total_executor_cores: (Standalone & Mesos only) Total cores for all executors + (Default: all the available cores on the worker) + :type total_executor_cores: int + :param executor_cores: (Standalone & YARN only) Number of cores per executor (Default: 2) + :type executor_cores: int + :param executor_memory: Memory per executor (e.g. 1000M, 2G) (Default: 1G) + :type executor_memory: str + :param driver_memory: Memory allocated to the driver (e.g. 1000M, 2G) (Default: 1G) + :type driver_memory: str + :param keytab: Full path to the file that contains the keytab (templated) + :type keytab: str + :param principal: The name of the kerberos principal used for keytab (templated) + :type principal: str + :param proxy_user: User to impersonate when submitting the application (templated) + :type proxy_user: str + :param name: Name of the job (default airflow-spark). (templated) + :type name: str + :param num_executors: Number of executors to launch + :type num_executors: int + :param status_poll_interval: Seconds to wait between polls of driver status in cluster + mode (Default: 1) + :type status_poll_interval: int + :param application_args: Arguments for the application being submitted (templated) + :type application_args: list + :param env_vars: Environment variables for spark-submit. It supports yarn and k8s mode too. (templated) + :type env_vars: dict + :param verbose: Whether to pass the verbose flag to spark-submit process for debugging + :type verbose: bool + :param spark_binary: The command to use for spark submit. + Some distros may use spark2-submit. + :type spark_binary: str + """ + template_fields = ('_application', '_conf', '_files', '_py_files', '_jars', '_driver_class_path', + '_packages', '_exclude_packages', '_keytab', '_principal', '_proxy_user', '_name', + '_application_args', '_env_vars') + ui_color = WEB_COLORS['LIGHTORANGE'] + + @apply_defaults + def __init__(self, + application='', + conf=None, + conn_id='spark_default', + files=None, + py_files=None, + archives=None, + driver_class_path=None, + jars=None, + java_class=None, + packages=None, + exclude_packages=None, + repositories=None, + total_executor_cores=None, + executor_cores=None, + executor_memory=None, + driver_memory=None, + keytab=None, + principal=None, + proxy_user=None, + name='airflow-spark', + num_executors=None, + status_poll_interval=1, + application_args=None, + env_vars=None, + verbose=False, + spark_binary=None, + *args, + **kwargs): + super(SparkSubmitOperator, self).__init__(*args, **kwargs) + self._application = application + self._conf = conf + self._files = files + self._py_files = py_files + self._archives = archives + self._driver_class_path = driver_class_path + self._jars = jars + self._java_class = java_class + self._packages = packages + self._exclude_packages = exclude_packages + self._repositories = repositories + self._total_executor_cores = total_executor_cores + self._executor_cores = executor_cores + self._executor_memory = executor_memory + self._driver_memory = driver_memory + self._keytab = keytab + self._principal = principal + self._proxy_user = proxy_user + self._name = name + self._num_executors = num_executors + self._status_poll_interval = status_poll_interval + self._application_args = application_args + self._env_vars = env_vars + self._verbose = verbose + self._spark_binary = spark_binary + self._hook = None + self._conn_id = conn_id + + def execute(self, context): + """ + Call the SparkSubmitHook to run the provided spark job + """ + self._hook = SparkSubmitHook( + conf=self._conf, + conn_id=self._conn_id, + files=self._files, + py_files=self._py_files, + archives=self._archives, + driver_class_path=self._driver_class_path, + jars=self._jars, + java_class=self._java_class, + packages=self._packages, + exclude_packages=self._exclude_packages, + repositories=self._repositories, + total_executor_cores=self._total_executor_cores, + executor_cores=self._executor_cores, + executor_memory=self._executor_memory, + driver_memory=self._driver_memory, + keytab=self._keytab, + principal=self._principal, + proxy_user=self._proxy_user, + name=self._name, + num_executors=self._num_executors, + status_poll_interval=self._status_poll_interval, + application_args=self._application_args, + env_vars=self._env_vars, + verbose=self._verbose, + spark_binary=self._spark_binary + ) + self._hook.submit(self._application) + + def on_kill(self): + self._hook.on_kill() diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py index d415c6f..c6ed38f 100644 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py @@ -14,7 +14,6 @@ # limitations under the License. # -from glob import glob from pathlib import Path from typing import Any from typing import Dict @@ -32,13 +31,16 @@ class ExamplesCatalogConnector(ComponentCatalogConnector): """ config = { 'kfp': { - 'root_dir': 'kfp_examples_components', + 'root_dir': 'kfp_example_components', 'file_filter': '*.yaml' + }, + 'airflow': { + 'root_dir': 'airflow_example_components', + 'file_filter': '*.py' } } def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str, Any]]: - """ Returns Elyra example custom components for the selected runtime (Kubeflow Pipelines and Apache Airflow only) @@ -46,25 +48,26 @@ def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str """ component_list = [] - if catalog_metadata.get('runtime') not in ExamplesCatalogConnector.config.keys(): - self.log.error('Example custom components are only available for Kubeflow Pipelines ' - 'and Apache Airflow.') + runtime_id = catalog_metadata.get('runtime') + + if runtime_id not in ExamplesCatalogConnector.config.keys(): + self.log.error(f'Cannot retrieve component list for runtime \'{runtime_id}\': ' + f'only {list(ExamplesCatalogConnector.config.keys())} are supported.') # return empty component specification list return component_list try: - root_dir = Path(__file__).parent / ExamplesCatalogConnector.config[catalog_metadata.get('runtime')] - self.log.info(f"Retrieving component list for runtime '{catalog_metadata.get('runtime')}' from " - f'{root_dir}') - file_spec = ExamplesCatalogConnector.config[catalog_metadata.get('runtime')].get('file_filter', '*') - for file in glob.iglob(root_dir / '**' / file_spec): - component_list.append({'component-id': file}) - - self.log.info(f'Component list: {component_list}') - + root_dir = Path(__file__).parent / ExamplesCatalogConnector.config[runtime_id]['root_dir'] + self.log.debug(f"Retrieving component list for runtime '{catalog_metadata.get('runtime')}' from " + f'{root_dir}') + pattern = ExamplesCatalogConnector.config[runtime_id].get('file_filter', '*') + self.log.debug(f'Pattern: {pattern}') + for file in root_dir.glob(f'**/{pattern}'): + component_list.append({'component-id': str(file)[len(str(root_dir)) + 1:]}) + self.log.debug(f'Component list: {component_list}') except Exception as ex: - self.log.warning(f"Error retrieving component list for runtime '{catalog_metadata.get('runtime')}'" - f" from {root_dir}: {ex}") + self.log.error(f"Error retrieving component list for runtime '{catalog_metadata.get('runtime')}'" + f" from {root_dir}: {ex}") return component_list @@ -72,29 +75,34 @@ def read_catalog_entry(self, catalog_entry_data: Dict[str, Any], catalog_metadata: Dict[str, Any]) -> Optional[str]: """ - Read a component definition for a single catalog entry using the its data (as returned from - get_catalog_entries()) and the catalog metadata, if needed - - :param catalog_entry_data: a dictionary that contains the information needed to read the content - of the component definition - :param catalog_metadata: the metadata associated with the catalog in which this catalog entry is - stored; in addition to catalog_entry_data, catalog_metadata may also be - needed to read the component definition for certain types of catalogs - - :returns: the requested component specification, if it exists + Retrieves a component from the catalog that is identified using + the information provided in catalog_entry_data. + + :param catalog_entry_data: an entry that was returned by get_catalog_entries + :type catalog_entry_data: Dict[str, Any] + :param catalog_metadata: the schema instance metadata, as defined in elyra-examples-catalog.json + :type catalog_metadata: Dict[str, Any] + :return: the component specification, if found + :rtype: Optional[str] """ - - component_id = catalog_entry_data['component-id'] + component_id = catalog_entry_data.get('component-id') if component_id is None: - self.log.error('Cannot retrieve example component specification: ' + self.log.error('Cannot retrieve component specification: ' 'A component id must be provided.') return None + runtime_id = catalog_metadata.get('runtime') + if runtime_id not in ExamplesCatalogConnector.config.keys(): + self.log.error(f'Cannot fetch component \'{component_id}\': ' + f'only {list(ExamplesCatalogConnector.config.keys())} are supported.') + return None + try: - with open(component_id, 'r') as fp: + root_dir = Path(__file__).parent / ExamplesCatalogConnector.config[runtime_id]['root_dir'] + with open(root_dir / component_id, 'r') as fp: return fp.read() except Exception as e: - self.log.error(f'Failed to download component specification {component_id} ' + self.log.error(f'Failed to fetch component \'{component_id}\' ' f': {str(e)}') return None diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/calculate_hash.yaml b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/calculate_hash.yaml new file mode 100644 index 0000000..16be33b --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/calculate_hash.yaml @@ -0,0 +1,43 @@ +name: Calculate data hash +inputs: +- {name: Data} +- {name: Hash algorithm, type: String, default: SHA256, description: "Hash algorithm to use. Supported values are MD5, SHA1, SHA256, SHA512, SHA3"} +outputs: +- {name: Hash} +metadata: + annotations: + author: Alexey Volkov +implementation: + container: + image: alpine + command: + - sh + - -exc + - | + data_path="$0" + hash_algorithm="$1" + hash_path="$2" + mkdir -p "$(dirname "$hash_path")" + + hash_algorithm=$(echo "$hash_algorithm" | tr '[:upper:]' '[:lower:]') + case "$hash_algorithm" in + md5|sha1|sha256|sha512|sha3) hash_program="${hash_algorithm}sum";; + *) echo "Unsupported hash algorithm $hash_algorithm"; exit 1;; + esac + + if [ -d "$data_path" ]; then + # Calculating hash for directory + cd "$data_path" + find . -type f -print0 | + sort -z | + xargs -0 "$hash_program" | + "$hash_program" | + cut -d ' ' -f 1 > "$hash_path" + else + # Calculating hash for file + "$hash_program" "$data_path" | + cut -d ' ' -f 1 > "$hash_path" + fi + - {inputPath: Data} + - {inputValue: Hash algorithm} + - {outputPath: Hash} \ No newline at end of file diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_data.yaml b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_data.yaml new file mode 100644 index 0000000..2ec0799 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_data.yaml @@ -0,0 +1,26 @@ +name: Download data +inputs: +- {name: Url, type: URI} +- {name: curl options, type: string, default: '--location', description: 'Additional options given to the curl program. See https://curl.haxx.se/docs/manpage.html'} +outputs: +- {name: Data} +metadata: + annotations: + author: Alexey Volkov +implementation: + container: + # image: curlimages/curl # Sets a non-root user which cannot write to mounted volumes. See https://github.com/curl/curl-docker/issues/22 + image: byrnedo/alpine-curl@sha256:548379d0a4a0c08b9e55d9d87a592b7d35d9ab3037f4936f5ccd09d0b625a342 + command: + - sh + - -exc + - | + url="$0" + output_path="$1" + curl_options="$2" + + mkdir -p "$(dirname "$output_path")" + curl --get "$url" --output "$output_path" $curl_options + - inputValue: Url + - outputPath: Data + - inputValue: curl options \ No newline at end of file diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml deleted file mode 100644 index 8ddd982..0000000 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_file.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: Download File -description: Downloads a file from a public HTTP/S URL using a GET request. - -inputs: -- {name: URL, type: String, optional: false, description: 'File URL'} - -outputs: -- {name: downloaded file, type: String, description: 'Content of the downloaded file.'} - -implementation: - container: - image: quay.io/elyra/kfp-tutorial-download-file-component@sha256:499c6e687ab94babb67664b99562f199289e081edb2f38f26a13bc198aa5cfb9 - # command is a list of strings (command-line arguments). - # The YAML language has two syntaxes for lists and you can use either of them. - # Here we use the "flow syntax" - comma-separated strings inside square brackets. - command: [ - python3, - # Path of the program inside the container - /pipelines/component/src/download-file.py, - --file-url, - {inputValue: URL}, - --downloaded-file-path, - {outputPath: downloaded file} - ] diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/filter_text_using_shell_and_grep.yaml b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/filter_text_using_shell_and_grep.yaml new file mode 100644 index 0000000..d4d124c --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/filter_text_using_shell_and_grep.yaml @@ -0,0 +1,42 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Component source location: https://raw.githubusercontent.com/kubeflow/pipelines/master/components/sample/Shell_script/component.yaml +# Component details: Takes a text file and a regex pattern filter to produce a filtered text file +name: Filter text +description: Filter input text according to the given regex pattern using shell and grep. +inputs: +- {name: Text, optional: false, description: 'Path to file to be filtered'} +- {name: Pattern, optional: true, default: '.*', description: 'Regex pattern'} +outputs: +- {name: Filtered text} +metadata: + annotations: + author: Alexey Volkov +implementation: + container: + image: alpine + command: + - sh + - -ec + - | + text_path=$0 + pattern=$1 + filtered_text_path=$2 + mkdir -p "$(dirname "$filtered_text_path")" + + grep "$pattern" < "$text_path" > "$filtered_text_path" + - {inputPath: Text} + - {inputValue: Pattern} + - {outputPath: Filtered text} diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/run_notebook_using_papermill.yaml b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/run_notebook_using_papermill.yaml new file mode 100644 index 0000000..29e9aa9 --- /dev/null +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/run_notebook_using_papermill.yaml @@ -0,0 +1,66 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Component source location: https://raw.githubusercontent.com/kubeflow/pipelines/1.4.1/components/notebooks/Run_notebook_using_papermill/component.yaml +# Component details: https://github.com/kubeflow/pipelines/tree/1.4.1/components/notebooks/samples +name: Run notebook using papermill +description: | + Run Jupyter notebook using papermill. + The notebook will receive the parameter values passed to it as well as the INPUT_DATA_PATH and OUTPUT_DATA_PATH variables that will be set to the input data path (if provided) and directory for the optional output data. +inputs: +- {name: Notebook, optional: false, type: JupyterNotebook, description: 'Required. Notebook to execute.'} +- {name: Parameters, optional: false, type: JsonObject, default: '{}', description: 'Map with notebook parameter values.'} +- {name: Packages to install, optional: false, type: JsonArray, default: '[]', description: 'Python packages to install'} +- {name: Input data, optional: true, description: 'Optional data that can be passed to notebook. In notebook, the INPUT_DATA_PATH variable will point to the data (if passed).'} +outputs: +- {name: Notebook, type: JupyterNotebook, description: 'Executed notebook.'} +- {name: Output data, description: 'Directory with any output data. In notebook, the OUTPUT_DATA_PATH variable will point to this directory, so that the notebook can write output data there.'} +metadata: + annotations: + author: Alexey Volkov +implementation: + container: + image: python:3.7 + command: + - sh + - -exc + - | + input_notebook_path="$0" + output_notebook_path="$1" + arguments="$2" + packages_to_install="$3" + input_data_path="$4" + output_data_path="$5" + mkdir -p "$(dirname "$output_notebook_path")" + mkdir -p "$output_data_path" + + # Converting packages_to_install from JSON to command-line arguments + packages_to_install=$(echo "$packages_to_install" | sed -E -e 's/^\[//' -e 's/]$//' -e 's/",/" /g' -e "s/\"/'/g") + # Installing packages + sh -c "python3 -m pip install --upgrade --quiet jupyter papermill==2.2.0 ${packages_to_install}" + # Running the notebook using papermill + papermill --parameters_yaml "$arguments" --parameters INPUT_DATA_PATH "$input_data_path" --parameters OUTPUT_DATA_PATH "$output_data_path" "$input_notebook_path" "$output_notebook_path" + + - {inputPath: Notebook} + - {outputPath: Notebook} + - {inputValue: Parameters} + - if: + cond: {isPresent: Packages to install} + then: [{inputValue: Packages to install}] + else: "{}" + - if: + cond: {isPresent: Input data} + then: [{inputPath: Input data}] + else: "" + - {outputPath: Output data} diff --git a/component-catalog-connectors/example-components-connector/setup.py b/component-catalog-connectors/example-components-connector/setup.py index 2aadcc7..cc2cec4 100644 --- a/component-catalog-connectors/example-components-connector/setup.py +++ b/component-catalog-connectors/example-components-connector/setup.py @@ -47,7 +47,7 @@ 'examples-catalog-schema = elyra_examples_connector.examples_schema_provider:ExamplesSchemasProvider' ], 'elyra.component.catalog_types': [ - 'examples-catalog = elyra_examples_connector.examples_connector:ExamplesCatalogConnector' + 'elyra-examples-catalog = elyra_examples_connector.examples_connector:ExamplesCatalogConnector' ], }, classifiers=[ From de8e0fd304017f91b307b515ce470c0a7a30ae3d Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Fri, 5 Nov 2021 12:09:26 -0700 Subject: [PATCH 03/18] Fix bugs and update instructions --- .flake8 | 4 +++- .../example-components-connector/MANIFEST.in | 2 +- .../example-components-connector/README.md | 13 +++++++------ .../elyra-examples-catalog.json | 17 +++++++++-------- .../examples_connector.py | 2 +- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.flake8 b/.flake8 index db17034..4237db1 100644 --- a/.flake8 +++ b/.flake8 @@ -3,7 +3,9 @@ # https://flake8.readthedocs.io/en/latest/user/configuration.html # https://flake8.readthedocs.io/en/latest/user/error-codes.html # https://docs.openstack.org/hacking/latest/user/hacking.html -exclude = __init__.py +exclude = + __init__.py + elyra_examples_connector/airflow_example_components/ ignore = # Import formatting E4, diff --git a/component-catalog-connectors/example-components-connector/MANIFEST.in b/component-catalog-connectors/example-components-connector/MANIFEST.in index 8660f5e..b5be0a3 100644 --- a/component-catalog-connectors/example-components-connector/MANIFEST.in +++ b/component-catalog-connectors/example-components-connector/MANIFEST.in @@ -36,4 +36,4 @@ recursive-exclude * *.py[co] include elyra_examples_connector/elyra-examples-catalog.json include elyra_examples_connector/kfp_example_components/*.yaml -recursive-include elyra_examples_connector +recursive-include elyra_examples_connector *.py diff --git a/component-catalog-connectors/example-components-connector/README.md b/component-catalog-connectors/example-components-connector/README.md index f6208cc..a8e4b83 100644 --- a/component-catalog-connectors/example-components-connector/README.md +++ b/component-catalog-connectors/example-components-connector/README.md @@ -1,10 +1,10 @@ ## Elyra pipeline component examples catalog -This catalog connector enables Elyra to load example pipeline components. +This catalog connector provides access to example pipeline components for Kubeflow Pipelines and Apache Airflow. -### Install the connector +### Install the component examples -You can install the example component connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. +You can install the component examples from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. **Prerequisites** @@ -27,11 +27,11 @@ You can install the example component connector from PyPI or source code. Note t ### Use the connector 1. Launch JupyterLab. -1. [Open the `Manage Components` panel]( +1. [Open the '`Manage Components`' panel]( https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). 1. Add a new component examples catalog ('`+`' > '`New Elyra examples component catalog`'). -1. Specify a catalog name, e.g. '`Elyra example components`'. -1. Select a runtime from the list. +1. Specify a catalog name, e.g. '`Elyra example components for Kubeflow`'. +1. Select the desired runtime from the list. 1. (Optional) Specify a category under which the example components will be organized in the palette. 1. Save the catalog entry. 1. Open the Visual Pipeline Editor for the chosen runtime and expand the palette. The example components are displayed. @@ -48,4 +48,5 @@ https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#manag ### Troubleshooting **Q: No example components are displayed in the Visual Pipeline Editor palette.** + A: Verify that you imported the examples for the correct runtime environment and check the JupyterLab log file for error messages. \ No newline at end of file diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json b/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json index d723390..2c4b1be 100644 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json @@ -1,16 +1,16 @@ { "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json", - "title": "Elyra examples component catalog", + "title": "Elyra component examples catalog", "name": "elyra-examples-catalog", - "display_name": "Elyra examples component catalog", + "display_name": "Elyra component examples catalog", "schemaspace": "component-registries", "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", "uihints": { - "title": "Elyra examples component catalog", + "title": "Elyra component examples catalog", "icon": "", - "reference_url": "https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html" + "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/example-components-connector" }, "properties": { "schema_name": { @@ -37,12 +37,13 @@ "properties": { "description": { "title": "Description", - "description": "Description of this Component Catalog", - "type": "string" + "description": "Description of this catalog entry.", + "type": "string", + "default": "Example pipeline components" }, "runtime": { "title": "Runtime", - "description": "The runtime associated with this Component Catalog", + "description": "The runtime for which to load the example components.", "type": "string", "enum": ["kfp", "airflow"], "default": "kfp", @@ -52,7 +53,7 @@ }, "categories": { "title": "Category Names", - "description": "Category names associated with this Component Catalog (the components defined in this registry will be organized in the component palette according to these categories)", + "description": "The example components will be added to the specified categories in the pipeline editor palette.", "type": "array", "items": { "type": "string", diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py index c6ed38f..f0aba3d 100644 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py +++ b/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py @@ -61,7 +61,7 @@ def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str self.log.debug(f"Retrieving component list for runtime '{catalog_metadata.get('runtime')}' from " f'{root_dir}') pattern = ExamplesCatalogConnector.config[runtime_id].get('file_filter', '*') - self.log.debug(f'Pattern: {pattern}') + self.log.debug(f'Component file pattern: {pattern}') for file in root_dir.glob(f'**/{pattern}'): component_list.append({'component-id': str(file)[len(str(root_dir)) + 1:]}) self.log.debug(f'Component list: {component_list}') From 4cc2958883c1d77b055b6f048cfee4b7552d6cb3 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Fri, 5 Nov 2021 12:13:51 -0700 Subject: [PATCH 04/18] Minor fixes --- .../example-components-connector/Makefile | 4 ++-- .../example-components-connector/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/component-catalog-connectors/example-components-connector/Makefile b/component-catalog-connectors/example-components-connector/Makefile index ea50c62..200e3dd 100644 --- a/component-catalog-connectors/example-components-connector/Makefile +++ b/component-catalog-connectors/example-components-connector/Makefile @@ -37,8 +37,8 @@ lint: test-dependencies dist: clean lint ## Build distribution python setup.py bdist_wheel sdist -source-install: dist ## Install example pipeline components package from source +source-install: dist ## Install pipeline components examples package from source pip install . -install: ## Install example pipeline components package from PyPI +install: ## Install pipeline components examples package from PyPI pip install elyra-example-components-catalog diff --git a/component-catalog-connectors/example-components-connector/README.md b/component-catalog-connectors/example-components-connector/README.md index a8e4b83..6b1371c 100644 --- a/component-catalog-connectors/example-components-connector/README.md +++ b/component-catalog-connectors/example-components-connector/README.md @@ -29,7 +29,7 @@ You can install the component examples from PyPI or source code. Note that a **r 1. Launch JupyterLab. 1. [Open the '`Manage Components`' panel]( https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). -1. Add a new component examples catalog ('`+`' > '`New Elyra examples component catalog`'). +1. Add a new component examples catalog ('`+`' > '`New Elyra component examples catalog`'). 1. Specify a catalog name, e.g. '`Elyra example components for Kubeflow`'. 1. Select the desired runtime from the list. 1. (Optional) Specify a category under which the example components will be organized in the palette. From b19fff5eedc9f563419daca9c28f840e31df8e73 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Mon, 8 Nov 2021 13:23:40 -0800 Subject: [PATCH 05/18] Split connector into two and update documentation --- component-catalog-connectors/README.md | 7 ++ .../.flake8 | 3 +- .../MANIFEST.in | 5 +- .../Makefile | 8 +- .../README.md | 60 ++++++++++ .../airflow_examples_connector}/__init__.py | 0 .../airflow_examples_connector}/_version.py | 0 .../elyra-airflow-catalog.json | 72 ++++++++++++ .../examples_connector.py | 51 ++++----- .../examples_schema_provider.py | 2 +- .../resources}/bash_operator.py | 0 .../resources}/email_operator.py | 0 .../resources}/http_operator.py | 0 .../resources}/slack_operator.py | 0 .../resources}/spark_sql_operator.py | 0 .../resources}/spark_submit_operator.py | 0 .../setup.cfg | 53 +++++++++ .../setup.py | 70 ++++++++++++ .../test_requirements.txt | 0 .../kfp-example-components-connector/.flake8 | 30 +++++ .../MANIFEST.in | 38 +++++++ .../kfp-example-components-connector/Makefile | 46 ++++++++ .../README.md | 24 ++-- .../kfp_examples_connector/__init__.py | 15 +++ .../kfp_examples_connector/_version.py | 16 +++ .../elyra-kfp-catalog.json} | 18 +-- .../examples_connector.py | 106 ++++++++++++++++++ .../examples_schema_provider.py | 49 ++++++++ .../resources}/calculate_hash.yaml | 0 .../resources}/download_data.yaml | 0 .../filter_text_using_shell_and_grep.yaml | 0 .../run_notebook_using_papermill.yaml | 0 .../setup.cfg | 4 +- .../setup.py | 15 ++- .../test_requirements.txt | 1 + 35 files changed, 627 insertions(+), 66 deletions(-) create mode 100644 component-catalog-connectors/README.md rename component-catalog-connectors/{example-components-connector => airflow-example-components-connector}/.flake8 (93%) rename component-catalog-connectors/{example-components-connector => airflow-example-components-connector}/MANIFEST.in (85%) rename component-catalog-connectors/{example-components-connector => airflow-example-components-connector}/Makefile (90%) create mode 100644 component-catalog-connectors/airflow-example-components-connector/README.md rename component-catalog-connectors/{example-components-connector/elyra_examples_connector => airflow-example-components-connector/airflow_examples_connector}/__init__.py (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector => airflow-example-components-connector/airflow_examples_connector}/_version.py (100%) create mode 100644 component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json rename component-catalog-connectors/{example-components-connector/elyra_examples_connector => airflow-example-components-connector/airflow_examples_connector}/examples_connector.py (65%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector => airflow-example-components-connector/airflow_examples_connector}/examples_schema_provider.py (97%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/airflow_example_components => airflow-example-components-connector/airflow_examples_connector/resources}/bash_operator.py (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/airflow_example_components => airflow-example-components-connector/airflow_examples_connector/resources}/email_operator.py (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/airflow_example_components => airflow-example-components-connector/airflow_examples_connector/resources}/http_operator.py (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/airflow_example_components => airflow-example-components-connector/airflow_examples_connector/resources}/slack_operator.py (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/airflow_example_components => airflow-example-components-connector/airflow_examples_connector/resources}/spark_sql_operator.py (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/airflow_example_components => airflow-example-components-connector/airflow_examples_connector/resources}/spark_submit_operator.py (100%) create mode 100644 component-catalog-connectors/airflow-example-components-connector/setup.cfg create mode 100644 component-catalog-connectors/airflow-example-components-connector/setup.py rename component-catalog-connectors/{example-components-connector => airflow-example-components-connector}/test_requirements.txt (100%) create mode 100644 component-catalog-connectors/kfp-example-components-connector/.flake8 create mode 100644 component-catalog-connectors/kfp-example-components-connector/MANIFEST.in create mode 100644 component-catalog-connectors/kfp-example-components-connector/Makefile rename component-catalog-connectors/{example-components-connector => kfp-example-components-connector}/README.md (58%) create mode 100644 component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/__init__.py create mode 100644 component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/elyra-examples-catalog.json => kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json} (80%) create mode 100644 component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py create mode 100644 component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_schema_provider.py rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/kfp_example_components => kfp-example-components-connector/kfp_examples_connector/resources}/calculate_hash.yaml (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/kfp_example_components => kfp-example-components-connector/kfp_examples_connector/resources}/download_data.yaml (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/kfp_example_components => kfp-example-components-connector/kfp_examples_connector/resources}/filter_text_using_shell_and_grep.yaml (100%) rename component-catalog-connectors/{example-components-connector/elyra_examples_connector/kfp_example_components => kfp-example-components-connector/kfp_examples_connector/resources}/run_notebook_using_papermill.yaml (100%) rename component-catalog-connectors/{example-components-connector => kfp-example-components-connector}/setup.cfg (93%) rename component-catalog-connectors/{example-components-connector => kfp-example-components-connector}/setup.py (78%) create mode 100644 component-catalog-connectors/kfp-example-components-connector/test_requirements.txt diff --git a/component-catalog-connectors/README.md b/component-catalog-connectors/README.md new file mode 100644 index 0000000..64e5aa5 --- /dev/null +++ b/component-catalog-connectors/README.md @@ -0,0 +1,7 @@ +## Component catalog connectors + +Component catalog connectors provide Elyra's Visual Pipeline Editor access to local and remote catalogs that store pipeline components. This directory contains the following component catalog connector implementations: +- [Kubeflow Pipelines example components](kfp-example-components-connector) +- [Apache Airflow example operators](airflow-example-components-connector) + +The connectors listed above are maintained by the Elyra community. You can find a complete list of available connectors on [this page](connector-directory.md). \ No newline at end of file diff --git a/component-catalog-connectors/example-components-connector/.flake8 b/component-catalog-connectors/airflow-example-components-connector/.flake8 similarity index 93% rename from component-catalog-connectors/example-components-connector/.flake8 rename to component-catalog-connectors/airflow-example-components-connector/.flake8 index 4237db1..d95624f 100644 --- a/component-catalog-connectors/example-components-connector/.flake8 +++ b/component-catalog-connectors/airflow-example-components-connector/.flake8 @@ -5,7 +5,8 @@ # https://docs.openstack.org/hacking/latest/user/hacking.html exclude = __init__.py - elyra_examples_connector/airflow_example_components/ + setup.py + airflow_examples_connector/resources ignore = # Import formatting E4, diff --git a/component-catalog-connectors/example-components-connector/MANIFEST.in b/component-catalog-connectors/airflow-example-components-connector/MANIFEST.in similarity index 85% rename from component-catalog-connectors/example-components-connector/MANIFEST.in rename to component-catalog-connectors/airflow-example-components-connector/MANIFEST.in index b5be0a3..4989028 100644 --- a/component-catalog-connectors/example-components-connector/MANIFEST.in +++ b/component-catalog-connectors/airflow-example-components-connector/MANIFEST.in @@ -34,6 +34,5 @@ include dist/*.tgz recursive-exclude * __pycache__ recursive-exclude * *.py[co] -include elyra_examples_connector/elyra-examples-catalog.json -include elyra_examples_connector/kfp_example_components/*.yaml -recursive-include elyra_examples_connector *.py +include airflow_examples_connector/elyra-airflow-catalog.json +recursive-include airflow_examples_connector/resources *.py diff --git a/component-catalog-connectors/example-components-connector/Makefile b/component-catalog-connectors/airflow-example-components-connector/Makefile similarity index 90% rename from component-catalog-connectors/example-components-connector/Makefile rename to component-catalog-connectors/airflow-example-components-connector/Makefile index 200e3dd..ada633d 100644 --- a/component-catalog-connectors/example-components-connector/Makefile +++ b/component-catalog-connectors/airflow-example-components-connector/Makefile @@ -18,6 +18,8 @@ SHELL:=/bin/bash +PACKAGE_NAME=elyra-airflow-example-components-catalog + help: # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -26,13 +28,13 @@ clean: rm -rf dist/ rm -rf build/ rm -rf *.egg-info - - pip uninstall -y elyra-example-components-catalog + - pip uninstall -y $(PACKAGE_NAME) test-dependencies: @pip install -q -r test_requirements.txt lint: test-dependencies - flake8 elyra_examples_connector + flake8 . dist: clean lint ## Build distribution python setup.py bdist_wheel sdist @@ -41,4 +43,4 @@ source-install: dist ## Install pipeline components examples package from sourc pip install . install: ## Install pipeline components examples package from PyPI - pip install elyra-example-components-catalog + pip install $(PACKAGE_NAME) diff --git a/component-catalog-connectors/airflow-example-components-connector/README.md b/component-catalog-connectors/airflow-example-components-connector/README.md new file mode 100644 index 0000000..df537a7 --- /dev/null +++ b/component-catalog-connectors/airflow-example-components-connector/README.md @@ -0,0 +1,60 @@ +## Elyra pipeline component examples catalog + +This catalog connector provides access to example pipeline components for Apache Airflow. + +### Install the component examples + +You can install the component examples from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. + +**Prerequisites** + +- [Install Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above). + +**Install from PyPI** + + ``` + $ pip install elyra-airflow-example-components-catalog + ``` + +**Install from source code** + + ``` + $ git clone https://github.com/elyra-ai/examples.git + $ cd examples/component-catalog-connectors/airflow-examples-connector/ + $ make clean source-install + ``` + +### Use the connector + +1. Launch JupyterLab. +1. [Open the '`Manage Components`' panel]( +https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). +1. Add a new component examples catalog ('`+`' > '`New Apache Airflow example components catalog`'). +1. Specify a catalog name, e.g. '`Elyra example components for Kubeflow`'. +1. (Optional) Specify a category under which the example components will be organized in the palette. +1. Save the catalog entry. +1. Open the Visual Pipeline Editor for Apache Airflow and expand the palette. The example components are displayed. + +### Customize the catalog + +This connector utilizes an embedded catalog as storage and is therefore a static catalog. To customize the catalog content according to your needs: + +1. Clone or fork the `https://github.com/elyra-ai/examples` repository. +1. Navigate to the [`airflow_examples_connector/resources`](airflow_examples_connector/resources) directory. This directory contains the Apache Airflow operators that this connector makes available to Elyra. Note that the operator packages must be installed on the Apache Airflow cluster, or DAG execution will fail. +1. Change the content of this directory as desired. +1. Install the customized connector from source. + +### Uninstall the connector + +1. Remove all example catalog entries from the '`Manage Components`' panel. +1. Stop JupyterLab. +1. Uninstall the `elyra-airflow-example-components-catalog` package. + ``` + $ pip uninstall -y elyra-airflow-example-components-catalog + ``` + +### Troubleshooting + +**Q: No example components are displayed in the Visual Pipeline Editor palette.** + +A: Verify that you imported the examples for the correct runtime environment and check the JupyterLab log file for error messages. \ No newline at end of file diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/__init__.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/__init__.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/__init__.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/__init__.py diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/_version.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/_version.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json new file mode 100644 index 0000000..9c5cae2 --- /dev/null +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json", + "title": "Apache Airflow example components catalog", + "name": "elyra-airflow-examples-catalog", + "display_name": "Apache Airflow example components catalog", + "schemaspace": "component-registries", + "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", + "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", + "uihints": { + "title": "Apache Airflow example components catalog", + "icon": "", + "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/airflow-example-components-connector" + }, + "properties": { + "schema_name": { + "title": "Schema Name", + "description": "The schema associated with this instance", + "type": "string", + "const": "elyra-airflow-examples-catalog" + }, + "display_name": { + "title": "Display Name", + "description": "Display name of this Component Catalog", + "type": "string", + "minLength": 1 + }, + "version": { + "title": "Version", + "description": "The version associated with this instance", + "type": "integer", + "const": 1 + }, + "metadata": { + "description": "Additional data specific to this metadata", + "type": "object", + "properties": { + "description": { + "title": "Description", + "description": "Curated collection of Apache Airflow components", + "type": "string", + "default": "Example pipeline components for Apache Airflow" + }, + "runtime": { + "title": "Runtime", + "description": "The runtime for which to load the example components.", + "type": "string", + "enum": ["airflow"], + "default": "airflow", + "uihints": { + "field_type": "dropdown" + } + }, + "categories": { + "title": "Category Names", + "description": "The example components will be added to the specified categories in the pipeline editor palette.", + "type": "array", + "items": { + "type": "string", + "maxLength": 18 + }, + "uihints": { + "field_type": "array", + "category": "Component Categories" + } + } + }, + "required": ["runtime"] + } + }, + "required": ["schema_name", "display_name", "version", "metadata"] +} diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py similarity index 65% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py index f0aba3d..01c75a5 100644 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_connector.py +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py @@ -26,47 +26,35 @@ class ExamplesCatalogConnector(ComponentCatalogConnector): """ - Makes example components for Kubeflow Pipelines and Apache Airflow - available to Elyra. + Makes a curated set of components for Apache Airflow available to Elyra. """ - config = { - 'kfp': { - 'root_dir': 'kfp_example_components', - 'file_filter': '*.yaml' - }, - 'airflow': { - 'root_dir': 'airflow_example_components', - 'file_filter': '*.py' - } - } def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str, Any]]: """ - Returns Elyra example custom components for the selected runtime (Kubeflow Pipelines - and Apache Airflow only) + Returns list of components that are stored in the '/resources' directory :param registry_metadata: the dictionary-form of the Metadata instance for a single registry """ component_list = [] - runtime_id = catalog_metadata.get('runtime') + runtime_type = catalog_metadata.get('runtime') - if runtime_id not in ExamplesCatalogConnector.config.keys(): - self.log.error(f'Cannot retrieve component list for runtime \'{runtime_id}\': ' - f'only {list(ExamplesCatalogConnector.config.keys())} are supported.') + if runtime_type != 'airflow': + self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' + 'Only Apache Airflow is supported.') # return empty component specification list return component_list try: - root_dir = Path(__file__).parent / ExamplesCatalogConnector.config[runtime_id]['root_dir'] - self.log.debug(f"Retrieving component list for runtime '{catalog_metadata.get('runtime')}' from " - f'{root_dir}') - pattern = ExamplesCatalogConnector.config[runtime_id].get('file_filter', '*') + root_dir = Path(__file__).parent / 'resources' + self.log.debug(f'Retrieving component list for runtime type \'{runtime_type}\' from ' + f'{root_dir}') + pattern = '**/*.py' self.log.debug(f'Component file pattern: {pattern}') - for file in root_dir.glob(f'**/{pattern}'): + for file in root_dir.glob(pattern): component_list.append({'component-id': str(file)[len(str(root_dir)) + 1:]}) self.log.debug(f'Component list: {component_list}') except Exception as ex: - self.log.error(f"Error retrieving component list for runtime '{catalog_metadata.get('runtime')}'" + self.log.error(f"Error retrieving component list for runtime type '{runtime_type}'" f" from {root_dir}: {ex}") return component_list @@ -87,23 +75,24 @@ def read_catalog_entry(self, """ component_id = catalog_entry_data.get('component-id') if component_id is None: - self.log.error('Cannot retrieve component specification: ' + self.log.error('Cannot retrieve component: ' 'A component id must be provided.') return None - runtime_id = catalog_metadata.get('runtime') - if runtime_id not in ExamplesCatalogConnector.config.keys(): - self.log.error(f'Cannot fetch component \'{component_id}\': ' - f'only {list(ExamplesCatalogConnector.config.keys())} are supported.') + runtime_type = catalog_metadata.get('runtime') + if runtime_type != 'airflow': + self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' + 'Only Apache Airflow is supported.') return None try: - root_dir = Path(__file__).parent / ExamplesCatalogConnector.config[runtime_id]['root_dir'] + # load component from resources directory + root_dir = Path(__file__).parent / 'resources' with open(root_dir / component_id, 'r') as fp: return fp.read() except Exception as e: self.log.error(f'Failed to fetch component \'{component_id}\' ' - f': {str(e)}') + f' from \'{root_dir}\': {str(e)}') return None def get_hash_keys(self) -> List[Any]: diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_schema_provider.py similarity index 97% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_schema_provider.py index 8a5a46e..d8bc687 100644 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/examples_schema_provider.py +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_schema_provider.py @@ -38,7 +38,7 @@ def get_schemas(self) -> List[Dict]: examples_catalog_schema_defs = [] try: # load examples schema definition - examples_catalog_connector_schema_file = Path(__file__).parent / 'elyra-examples-catalog.json' + examples_catalog_connector_schema_file = Path(__file__).parent / 'elyra-airflow-catalog.json' log.debug(f'Reading examples catalog connector schema from {examples_catalog_connector_schema_file}') with open(examples_catalog_connector_schema_file, 'r') as fp: examples_catalog_connector_schema = json.load(fp) diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/bash_operator.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/bash_operator.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/bash_operator.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/bash_operator.py diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/email_operator.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/email_operator.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/email_operator.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/email_operator.py diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/http_operator.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/http_operator.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/http_operator.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/http_operator.py diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/slack_operator.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/slack_operator.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/slack_operator.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/slack_operator.py diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_sql_operator.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/spark_sql_operator.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_sql_operator.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/spark_sql_operator.py diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_submit_operator.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/spark_submit_operator.py similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/airflow_example_components/spark_submit_operator.py rename to component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources/spark_submit_operator.py diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.cfg b/component-catalog-connectors/airflow-example-components-connector/setup.cfg new file mode 100644 index 0000000..569c65c --- /dev/null +++ b/component-catalog-connectors/airflow-example-components-connector/setup.cfg @@ -0,0 +1,53 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[bdist_wheel] +universal=0 + +[metadata] +description_file=README.md + +[flake8] +application-import-names = airflow_examples_connector +application-package-names = airflow_examples_connector +enable-extensions = G +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # File contains nothing but comments + H104, + # Include name with TODOs as in # TODO(yourname) + H101, + # Enable mocking + H216, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405, + # Allow breaks after binary operators + W504 +import-order-style = google +max-line-length = 120 diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.py b/component-catalog-connectors/airflow-example-components-connector/setup.py new file mode 100644 index 0000000..83a066d --- /dev/null +++ b/component-catalog-connectors/airflow-example-components-connector/setup.py @@ -0,0 +1,70 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from setuptools import find_packages, setup + +long_desc = """ + Curated Elyra component catalog for example + Airflow components. + """ + +here = os.path.abspath(os.path.dirname(__file__)) + +version_ns = {} +with open(os.path.join(here, 'airflow_examples_connector', '_version.py')) as f: + exec(f.read(), {}, version_ns) + +setup_args = dict( + name="elyra-airflow-example-components-catalog", + version=version_ns['__version__'], + url="https://github.com/elyra-ai/examples", + description="Curated Elyra component catalog for example Apache Airflow components.", + long_description=long_desc, + author="Elyra Maintainers", + license="Apache License Version 2.0", + packages=find_packages(), + install_requires=[ + 'elyra==3.3.0.dev0' + ], + setup_requires=['flake8'], + include_package_data=True, + entry_points={ + 'metadata.schemas_providers': [ + 'airflow-examples-catalog-schema = airflow_examples_connector.examples_schema_provider:ExamplesSchemasProvider' + ], + 'elyra.component.catalog_types': [ + 'elyra-airflow-examples-catalog = airflow_examples_connector.examples_connector:ExamplesCatalogConnector' + ], + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + ] +) + +if __name__ == '__main__': + setup(**setup_args) diff --git a/component-catalog-connectors/example-components-connector/test_requirements.txt b/component-catalog-connectors/airflow-example-components-connector/test_requirements.txt similarity index 100% rename from component-catalog-connectors/example-components-connector/test_requirements.txt rename to component-catalog-connectors/airflow-example-components-connector/test_requirements.txt diff --git a/component-catalog-connectors/kfp-example-components-connector/.flake8 b/component-catalog-connectors/kfp-example-components-connector/.flake8 new file mode 100644 index 0000000..05716f2 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/.flake8 @@ -0,0 +1,30 @@ +[flake8] +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +exclude = + __init__.py +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # Allow breaks after binary operators + W504, + # Include name with TODOs as in # TODO(yourname) + H101, + # Do not import more than one module per line + H301, + # Alphabetically order imports by the full module path + H306, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405 +max-line-length = 120 + diff --git a/component-catalog-connectors/kfp-example-components-connector/MANIFEST.in b/component-catalog-connectors/kfp-example-components-connector/MANIFEST.in new file mode 100644 index 0000000..7bcea57 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/MANIFEST.in @@ -0,0 +1,38 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# exclude from ANY directory +global-exclude *.ipynb +global-exclude *.py[cod] +global-exclude __pycache__ +global-exclude .git +global-exclude .ipynb_checkpoints +global-exclude .DS_Store +global-exclude *.sh +global-exclude docs +global-exclude tests + +# explicit includes +include CONTRIBUTING.md +include README.md +include LICENSE +include dist/*.tgz + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +include kfp_examples_connector/elyra-kfp-catalog.json +recursive-include kfp_examples_connector/resources *.yaml diff --git a/component-catalog-connectors/kfp-example-components-connector/Makefile b/component-catalog-connectors/kfp-example-components-connector/Makefile new file mode 100644 index 0000000..297d134 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/Makefile @@ -0,0 +1,46 @@ +# +# Copyright 2021-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: help clean lint test-dependencies source-install install dist + +SHELL:=/bin/bash + +PACKAGE_NAME=elyra-kfp-example-components-catalog + +help: +# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +clean: + rm -rf dist/ + rm -rf build/ + rm -rf *.egg-info + - pip uninstall -y $(PACKAGE_NAME) + +test-dependencies: + @pip install -q -r test_requirements.txt + +lint: test-dependencies + flake8 . + +dist: clean lint ## Build distribution + python setup.py bdist_wheel sdist + +source-install: dist ## Install pipeline components examples package from source + pip install . + +install: ## Install pipeline components examples package from PyPI + pip install $(PACKAGE_NAME) diff --git a/component-catalog-connectors/example-components-connector/README.md b/component-catalog-connectors/kfp-example-components-connector/README.md similarity index 58% rename from component-catalog-connectors/example-components-connector/README.md rename to component-catalog-connectors/kfp-example-components-connector/README.md index 6b1371c..a9d171d 100644 --- a/component-catalog-connectors/example-components-connector/README.md +++ b/component-catalog-connectors/kfp-example-components-connector/README.md @@ -1,6 +1,6 @@ ## Elyra pipeline component examples catalog -This catalog connector provides access to example pipeline components for Kubeflow Pipelines and Apache Airflow. +This catalog connector provides access to example pipeline components for Kubeflow Pipelines. ### Install the component examples @@ -13,14 +13,14 @@ You can install the component examples from PyPI or source code. Note that a **r **Install from PyPI** ``` - $ pip install elyra-example-components-catalog + $ pip install elyra-kfp-example-components-catalog ``` **Install from source code** ``` $ git clone https://github.com/elyra-ai/examples.git - $ cd examples/component-catalog-connectors/elyra-examples-connector/ + $ cd examples/component-catalog-connectors/kfp-examples-connector/ $ make clean source-install ``` @@ -29,20 +29,28 @@ You can install the component examples from PyPI or source code. Note that a **r 1. Launch JupyterLab. 1. [Open the '`Manage Components`' panel]( https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). -1. Add a new component examples catalog ('`+`' > '`New Elyra component examples catalog`'). +1. Add a new component examples catalog ('`+`' > '`New Kubeflow Pipelines example components catalog`'). 1. Specify a catalog name, e.g. '`Elyra example components for Kubeflow`'. -1. Select the desired runtime from the list. 1. (Optional) Specify a category under which the example components will be organized in the palette. 1. Save the catalog entry. -1. Open the Visual Pipeline Editor for the chosen runtime and expand the palette. The example components are displayed. +1. Open the Visual Pipeline Editor for Kubeflow Pipelines and expand the palette. The example components are displayed. + +### Customize the catalog + +This connector utilizes an embedded catalog as storage and is therefore a static catalog. To customize the catalog content according to your needs: + +1. Clone or fork the `https://github.com/elyra-ai/examples` repository. +1. Navigate to the [`kfp_examples_connector/resources`](kfp_examples_connector/resources) directory. This directory contains the Kubeflow Pipelines components that this connector makes available to Elyra. +1. Change the content of this directory as desired. +1. Install the connector from source. ### Uninstall the connector 1. Remove all example catalog entries from the '`Manage Components`' panel. 1. Stop JupyterLab. -1. Uninstall the `elyra-example-components-catalog` package. +1. Uninstall the `elyra-kfp-example-components-catalog` package. ``` - $ pip uninstall -y elyra-example-components-catalog + $ pip uninstall -y elyra-kfp-example-components-catalog ``` ### Troubleshooting diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/__init__.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/__init__.py new file mode 100644 index 0000000..febc0a2 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py new file mode 100644 index 0000000..4abcc16 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py @@ -0,0 +1,16 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = '0.0.1' diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json similarity index 80% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json rename to component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json index 2c4b1be..a97b0cd 100644 --- a/component-catalog-connectors/example-components-connector/elyra_examples_connector/elyra-examples-catalog.json +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json @@ -1,23 +1,23 @@ { "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json", - "title": "Elyra component examples catalog", - "name": "elyra-examples-catalog", - "display_name": "Elyra component examples catalog", + "title": "Kubeflow Pipelines example components catalog", + "name": "elyra-kfp-examples-catalog", + "display_name": "Kubeflow Pipelines example components catalog", "schemaspace": "component-registries", "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", "uihints": { - "title": "Elyra component examples catalog", + "title": "Kubeflow Pipelines example components catalog", "icon": "", - "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/example-components-connector" + "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/kfp-example-components-connector" }, "properties": { "schema_name": { "title": "Schema Name", "description": "The schema associated with this instance", "type": "string", - "const": "elyra-examples-catalog" + "const": "elyra-kfp-examples-catalog" }, "display_name": { "title": "Display Name", @@ -37,15 +37,15 @@ "properties": { "description": { "title": "Description", - "description": "Description of this catalog entry.", + "description": "Curated collection of Kubeflow Pipeline components", "type": "string", - "default": "Example pipeline components" + "default": "Example pipeline components for Kubeflow Pipelines" }, "runtime": { "title": "Runtime", "description": "The runtime for which to load the example components.", "type": "string", - "enum": ["kfp", "airflow"], + "enum": ["kfp"], "default": "kfp", "uihints": { "field_type": "dropdown" diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py new file mode 100644 index 0000000..d53ee85 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py @@ -0,0 +1,106 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + + +from elyra.pipeline.catalog_connector import ComponentCatalogConnector + + +class ExamplesCatalogConnector(ComponentCatalogConnector): + """ + Makes a curated set of components for Kubeflow Pipelines available to Elyra. + """ + + def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Returns list of components that are stored in the '/resources' directory + :param registry_metadata: the dictionary-form of the Metadata instance for a single registry + """ + component_list = [] + + runtime_type = catalog_metadata.get('runtime') + + if runtime_type != 'kfp': + self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' + 'Only Kubeflow Pipelines is supported.') + # return empty component specification list + return component_list + + try: + root_dir = Path(__file__).parent / 'resources' + self.log.debug(f'Retrieving component list for runtime type \'{runtime_type}\' from ' + f'{root_dir}') + pattern = '**/*.yaml' + self.log.debug(f'Component file pattern: {pattern}') + for file in root_dir.glob(pattern): + component_list.append({'component-id': str(file)[len(str(root_dir)) + 1:]}) + self.log.debug(f'Component list: {component_list}') + except Exception as ex: + self.log.error(f"Error retrieving component list for runtime type '{runtime_type}'" + f" from {root_dir}: {ex}") + + return component_list + + def read_catalog_entry(self, + catalog_entry_data: Dict[str, Any], + catalog_metadata: Dict[str, Any]) -> Optional[str]: + """ + Retrieves a component from the catalog that is identified using + the information provided in catalog_entry_data. + + :param catalog_entry_data: an entry that was returned by get_catalog_entries + :type catalog_entry_data: Dict[str, Any] + :param catalog_metadata: the schema instance metadata, as defined in elyra-examples-catalog.json + :type catalog_metadata: Dict[str, Any] + :return: the component specification, if found + :rtype: Optional[str] + """ + component_id = catalog_entry_data.get('component-id') + if component_id is None: + self.log.error('Cannot retrieve component: ' + 'A component id must be provided.') + return None + + runtime_type = catalog_metadata.get('runtime') + if runtime_type != 'kfp': + self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' + 'Only Kubeflow Pipelines is supported.') + return None + + try: + # load component from resources directory + root_dir = Path(__file__).parent / 'resources' + with open(root_dir / component_id, 'r') as fp: + return fp.read() + except Exception as e: + self.log.error(f'Failed to fetch component \'{component_id}\' ' + f' from \'{root_dir}\': {str(e)}') + return None + + def get_hash_keys(self) -> List[Any]: + """ + Identifies the key(s) that read_catalog_entry method + requires to be present in the catalog_entry_data parameter + to allow for retrieval of a component. + + :returns: a list of keys + """ + return ['component-id'] diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_schema_provider.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_schema_provider.py new file mode 100644 index 0000000..7d7d330 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_schema_provider.py @@ -0,0 +1,49 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +from pathlib import Path +from typing import Dict +from typing import List + +from elyra.metadata.schema import SchemasProvider + + +class ExamplesSchemasProvider(SchemasProvider): + """ + Provides access to example custom components + for Kubeflow Pipelines and Apache Airflow. + """ + + def get_schemas(self) -> List[Dict]: + """ + Return the example schema + """ + # use Elyra logger + log = logging.getLogger('ElyraApp') + examples_catalog_schema_defs = [] + try: + # load examples schema definition + examples_catalog_connector_schema_file = Path(__file__).parent / 'elyra-kfp-catalog.json' + log.debug(f'Reading examples catalog connector schema from {examples_catalog_connector_schema_file}') + with open(examples_catalog_connector_schema_file, 'r') as fp: + examples_catalog_connector_schema = json.load(fp) + examples_catalog_schema_defs.append(examples_catalog_connector_schema) + except Exception as ex: + log.error(f'Error reading examples catalog connector schema {examples_catalog_connector_schema_file}: {ex}') + + return examples_catalog_schema_defs diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/calculate_hash.yaml b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/calculate_hash.yaml similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/calculate_hash.yaml rename to component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/calculate_hash.yaml diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_data.yaml b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/download_data.yaml similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/download_data.yaml rename to component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/download_data.yaml diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/filter_text_using_shell_and_grep.yaml b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/filter_text_using_shell_and_grep.yaml similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/filter_text_using_shell_and_grep.yaml rename to component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/filter_text_using_shell_and_grep.yaml diff --git a/component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/run_notebook_using_papermill.yaml b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/run_notebook_using_papermill.yaml similarity index 100% rename from component-catalog-connectors/example-components-connector/elyra_examples_connector/kfp_example_components/run_notebook_using_papermill.yaml rename to component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources/run_notebook_using_papermill.yaml diff --git a/component-catalog-connectors/example-components-connector/setup.cfg b/component-catalog-connectors/kfp-example-components-connector/setup.cfg similarity index 93% rename from component-catalog-connectors/example-components-connector/setup.cfg rename to component-catalog-connectors/kfp-example-components-connector/setup.cfg index e35c831..400a619 100644 --- a/component-catalog-connectors/example-components-connector/setup.cfg +++ b/component-catalog-connectors/kfp-example-components-connector/setup.cfg @@ -21,8 +21,8 @@ universal=0 description_file=README.md [flake8] -application-import-names = catalog_connector -application-package-names = catalog_connector +application-import-names = kfp_examples_connector +application-package-names = kfp_examples_connector enable-extensions = G # References: # https://flake8.readthedocs.io/en/latest/user/configuration.html diff --git a/component-catalog-connectors/example-components-connector/setup.py b/component-catalog-connectors/kfp-example-components-connector/setup.py similarity index 78% rename from component-catalog-connectors/example-components-connector/setup.py rename to component-catalog-connectors/kfp-example-components-connector/setup.py index cc2cec4..441fb10 100644 --- a/component-catalog-connectors/example-components-connector/setup.py +++ b/component-catalog-connectors/kfp-example-components-connector/setup.py @@ -18,21 +18,21 @@ from setuptools import find_packages, setup long_desc = """ - Elyra component catalog for example Kubeflow Pipelines - and Apache Airflow components. + Curated Elyra component catalog for example + Kubeflow Pipelines components. """ here = os.path.abspath(os.path.dirname(__file__)) version_ns = {} -with open(os.path.join(here, 'elyra_examples_connector', '_version.py')) as f: +with open(os.path.join(here, 'kfp_examples_connector', '_version.py')) as f: exec(f.read(), {}, version_ns) setup_args = dict( - name="elyra-example-components-catalog", + name="elyra-kfp-example-components-catalog", version=version_ns['__version__'], url="https://github.com/elyra-ai/examples", - description="Kubeflow Pipelines and Airflow example components for Elyra", + description="Curated Elyra component catalog for example Kubeflow Pipelines components.", long_description=long_desc, author="Elyra Maintainers", license="Apache License Version 2.0", @@ -44,10 +44,10 @@ include_package_data=True, entry_points={ 'metadata.schemas_providers': [ - 'examples-catalog-schema = elyra_examples_connector.examples_schema_provider:ExamplesSchemasProvider' + 'kfp-examples-catalog-schema = kfp_examples_connector.examples_schema_provider:ExamplesSchemasProvider' ], 'elyra.component.catalog_types': [ - 'elyra-examples-catalog = elyra_examples_connector.examples_connector:ExamplesCatalogConnector' + 'elyra-kfp-examples-catalog = kfp_examples_connector.examples_connector:ExamplesCatalogConnector' ], }, classifiers=[ @@ -66,6 +66,5 @@ ] ) - if __name__ == '__main__': setup(**setup_args) diff --git a/component-catalog-connectors/kfp-example-components-connector/test_requirements.txt b/component-catalog-connectors/kfp-example-components-connector/test_requirements.txt new file mode 100644 index 0000000..28f28b5 --- /dev/null +++ b/component-catalog-connectors/kfp-example-components-connector/test_requirements.txt @@ -0,0 +1 @@ +flake8>=3.5.0,<3.9.0 From f662285680744e68b83cc326c13cd8153034ae10 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Mon, 8 Nov 2021 13:29:30 -0800 Subject: [PATCH 06/18] Fix readme issues --- .../airflow-example-components-connector/README.md | 2 +- .../kfp-example-components-connector/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/README.md b/component-catalog-connectors/airflow-example-components-connector/README.md index df537a7..7e4877e 100644 --- a/component-catalog-connectors/airflow-example-components-connector/README.md +++ b/component-catalog-connectors/airflow-example-components-connector/README.md @@ -20,7 +20,7 @@ You can install the component examples from PyPI or source code. Note that a **r ``` $ git clone https://github.com/elyra-ai/examples.git - $ cd examples/component-catalog-connectors/airflow-examples-connector/ + $ cd examples/component-catalog-connectors/airflow-example-components-connector/ $ make clean source-install ``` diff --git a/component-catalog-connectors/kfp-example-components-connector/README.md b/component-catalog-connectors/kfp-example-components-connector/README.md index a9d171d..7323db4 100644 --- a/component-catalog-connectors/kfp-example-components-connector/README.md +++ b/component-catalog-connectors/kfp-example-components-connector/README.md @@ -20,7 +20,7 @@ You can install the component examples from PyPI or source code. Note that a **r ``` $ git clone https://github.com/elyra-ai/examples.git - $ cd examples/component-catalog-connectors/kfp-examples-connector/ + $ cd examples/component-catalog-connectors/kfp-example-components-connector/ $ make clean source-install ``` From 4f882b1db674f82a4401bc44dff0b8f81ccd5df0 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Mon, 8 Nov 2021 16:32:15 -0800 Subject: [PATCH 07/18] Add support for runtime_type --- .../elyra-airflow-catalog.json | 10 +++--- .../examples_connector.py | 33 +++++++++++------- .../elyra-kfp-catalog.json | 10 +++--- .../examples_connector.py | 34 ++++++++++++------- 4 files changed, 52 insertions(+), 35 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json index 9c5cae2..b5a54c7 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json @@ -41,12 +41,12 @@ "type": "string", "default": "Example pipeline components for Apache Airflow" }, - "runtime": { - "title": "Runtime", + "runtime_type": { + "title": "Runtime type", "description": "The runtime for which to load the example components.", "type": "string", - "enum": ["airflow"], - "default": "airflow", + "enum": ["APACHE_AIRFLOW"], + "default": "APACHE_AIRFLOW", "uihints": { "field_type": "dropdown" } @@ -65,7 +65,7 @@ } } }, - "required": ["runtime"] + "required": ["runtime_type"] } }, "required": ["schema_name", "display_name", "version", "metadata"] diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py index 01c75a5..ebc9140 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py @@ -22,6 +22,7 @@ from elyra.pipeline.catalog_connector import ComponentCatalogConnector +from elyra.pipeline.runtime_type import RuntimeProcessorType class ExamplesCatalogConnector(ComponentCatalogConnector): @@ -36,25 +37,28 @@ def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str """ component_list = [] - runtime_type = catalog_metadata.get('runtime') + runtime_type_name = catalog_metadata.get('runtime_type') + runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name) + # The runtime type's user-friendly display name + runtime_type_display_name = runtime_type.value - if runtime_type != 'airflow': - self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' - 'Only Apache Airflow is supported.') + if runtime_type != RuntimeProcessorType.APACHE_AIRFLOW: + self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type_display_name}\': ' + f'Only \'{RuntimeProcessorType.APACHE_AIRFLOW.value}\' is supported.') # return empty component specification list return component_list try: root_dir = Path(__file__).parent / 'resources' - self.log.debug(f'Retrieving component list for runtime type \'{runtime_type}\' from ' - f'{root_dir}') + self.log.debug(f'Retrieving component list for runtime type \'{runtime_type_display_name}\' from ' + f'{root_dir}') pattern = '**/*.py' self.log.debug(f'Component file pattern: {pattern}') for file in root_dir.glob(pattern): component_list.append({'component-id': str(file)[len(str(root_dir)) + 1:]}) self.log.debug(f'Component list: {component_list}') except Exception as ex: - self.log.error(f"Error retrieving component list for runtime type '{runtime_type}'" + self.log.error(f"Error retrieving component list for runtime type '{runtime_type_display_name}'" f" from {root_dir}: {ex}") return component_list @@ -79,15 +83,20 @@ def read_catalog_entry(self, 'A component id must be provided.') return None - runtime_type = catalog_metadata.get('runtime') - if runtime_type != 'airflow': - self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' - 'Only Apache Airflow is supported.') - return None + runtime_type_name = catalog_metadata.get('runtime_type') + runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name) + # The runtime type's user-friendly display name + runtime_type_display_name = runtime_type.value + + if runtime_type != RuntimeProcessorType.APACHE_AIRFLOW: + self.log.error(f'Cannot retrieve component for runtime type \'{runtime_type_name}\': ' + f'Only \'{RuntimeProcessorType.APACHE_AIRFLOW.value}\' is supported.') try: # load component from resources directory root_dir = Path(__file__).parent / 'resources' + self.log.debug(f'Retrieving component of runtime type \'{runtime_type_display_name}\' from ' + f'{root_dir}') with open(root_dir / component_id, 'r') as fp: return fp.read() except Exception as e: diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json index a97b0cd..53d3f6b 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json @@ -41,12 +41,12 @@ "type": "string", "default": "Example pipeline components for Kubeflow Pipelines" }, - "runtime": { - "title": "Runtime", + "runtime_type": { + "title": "Runtime type", "description": "The runtime for which to load the example components.", "type": "string", - "enum": ["kfp"], - "default": "kfp", + "enum": ["KUBEFLOW_PIPELINES"], + "default": "KUBEFLOW_PIPELINES", "uihints": { "field_type": "dropdown" } @@ -65,7 +65,7 @@ } } }, - "required": ["runtime"] + "required": ["runtime_type"] } }, "required": ["schema_name", "display_name", "version", "metadata"] diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py index d53ee85..cefe1ab 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py @@ -20,8 +20,8 @@ from typing import List from typing import Optional - from elyra.pipeline.catalog_connector import ComponentCatalogConnector +from elyra.pipeline.runtime_type import RuntimeProcessorType class ExamplesCatalogConnector(ComponentCatalogConnector): @@ -36,25 +36,28 @@ def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str """ component_list = [] - runtime_type = catalog_metadata.get('runtime') + runtime_type_name = catalog_metadata.get('runtime_type') + runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name) + # The runtime type's user-friendly display name + runtime_type_display_name = runtime_type.value - if runtime_type != 'kfp': - self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' - 'Only Kubeflow Pipelines is supported.') + if runtime_type != RuntimeProcessorType.KUBEFLOW_PIPELINES: + self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type_display_name}\': ' + f'Only \'{RuntimeProcessorType.KUBEFLOW_PIPELINES.value}\' is supported.') # return empty component specification list return component_list try: root_dir = Path(__file__).parent / 'resources' - self.log.debug(f'Retrieving component list for runtime type \'{runtime_type}\' from ' - f'{root_dir}') + self.log.debug(f'Retrieving component list for runtime type \'{runtime_type_display_name}\' from ' + f'{root_dir}') pattern = '**/*.yaml' self.log.debug(f'Component file pattern: {pattern}') for file in root_dir.glob(pattern): component_list.append({'component-id': str(file)[len(str(root_dir)) + 1:]}) self.log.debug(f'Component list: {component_list}') except Exception as ex: - self.log.error(f"Error retrieving component list for runtime type '{runtime_type}'" + self.log.error(f"Error retrieving component list for runtime type '{runtime_type_display_name}'" f" from {root_dir}: {ex}") return component_list @@ -79,15 +82,20 @@ def read_catalog_entry(self, 'A component id must be provided.') return None - runtime_type = catalog_metadata.get('runtime') - if runtime_type != 'kfp': - self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type}\': ' - 'Only Kubeflow Pipelines is supported.') - return None + runtime_type_name = catalog_metadata.get('runtime_type') + runtime_type = RuntimeProcessorType.get_instance_by_name(runtime_type_name) + # The runtime type's user-friendly display name + runtime_type_display_name = runtime_type.value + + if runtime_type != RuntimeProcessorType.KUBEFLOW_PIPELINES: + self.log.error(f'Cannot retrieve component for runtime type \'{runtime_type_display_name}\': ' + f'Only \'{RuntimeProcessorType.KUBEFLOW_PIPELINES.value}\' is supported.') try: # load component from resources directory root_dir = Path(__file__).parent / 'resources' + self.log.debug(f'Retrieving component of runtime type \'{runtime_type_display_name}\' from ' + f'{root_dir}') with open(root_dir / component_id, 'r') as fp: return fp.read() except Exception as e: From c63e962d45e7515c3e26d99ce63104163a7b0215 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Mon, 8 Nov 2021 16:38:46 -0800 Subject: [PATCH 08/18] fix linting error --- .../airflow-example-components-connector/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.py b/component-catalog-connectors/airflow-example-components-connector/setup.py index 83a066d..ad4c4ed 100644 --- a/component-catalog-connectors/airflow-example-components-connector/setup.py +++ b/component-catalog-connectors/airflow-example-components-connector/setup.py @@ -44,10 +44,12 @@ include_package_data=True, entry_points={ 'metadata.schemas_providers': [ - 'airflow-examples-catalog-schema = airflow_examples_connector.examples_schema_provider:ExamplesSchemasProvider' + 'airflow-examples-catalog-schema =\ + airflow_examples_connector.examples_schema_provider:ExamplesSchemasProvider' ], 'elyra.component.catalog_types': [ - 'elyra-airflow-examples-catalog = airflow_examples_connector.examples_connector:ExamplesCatalogConnector' + 'elyra-airflow-examples-catalog =\ + airflow_examples_connector.examples_connector:ExamplesCatalogConnector' ], }, classifiers=[ From 164dfba8f61b87a6dee6f5e624cc6b0eb5523722 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Tue, 9 Nov 2021 09:46:38 -0800 Subject: [PATCH 09/18] Address review comments --- .../airflow-example-components-connector/Makefile | 2 +- .../airflow-example-components-connector/README.md | 11 +++++------ .../airflow_examples_connector/_version.py | 2 +- .../elyra-airflow-catalog.json | 1 + .../airflow_examples_connector/examples_connector.py | 10 ---------- .../airflow-example-components-connector/setup.py | 4 ++-- .../kfp-example-components-connector/Makefile | 2 +- .../kfp-example-components-connector/README.md | 11 +++++------ .../kfp_examples_connector/_version.py | 2 +- .../kfp_examples_connector/elyra-kfp-catalog.json | 1 + .../kfp_examples_connector/examples_connector.py | 10 ---------- .../kfp-example-components-connector/setup.py | 4 ++-- 12 files changed, 20 insertions(+), 40 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/Makefile b/component-catalog-connectors/airflow-example-components-connector/Makefile index ada633d..de92b09 100644 --- a/component-catalog-connectors/airflow-example-components-connector/Makefile +++ b/component-catalog-connectors/airflow-example-components-connector/Makefile @@ -18,7 +18,7 @@ SHELL:=/bin/bash -PACKAGE_NAME=elyra-airflow-example-components-catalog +PACKAGE_NAME=elyra-examples-airflow-catalog help: # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html diff --git a/component-catalog-connectors/airflow-example-components-connector/README.md b/component-catalog-connectors/airflow-example-components-connector/README.md index 7e4877e..b6398e3 100644 --- a/component-catalog-connectors/airflow-example-components-connector/README.md +++ b/component-catalog-connectors/airflow-example-components-connector/README.md @@ -13,7 +13,7 @@ You can install the component examples from PyPI or source code. Note that a **r **Install from PyPI** ``` - $ pip install elyra-airflow-example-components-catalog + $ pip install elyra-examples-airflow-catalog ``` **Install from source code** @@ -30,8 +30,7 @@ You can install the component examples from PyPI or source code. Note that a **r 1. [Open the '`Manage Components`' panel]( https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). 1. Add a new component examples catalog ('`+`' > '`New Apache Airflow example components catalog`'). -1. Specify a catalog name, e.g. '`Elyra example components for Kubeflow`'. -1. (Optional) Specify a category under which the example components will be organized in the palette. +1. Specify a catalog name, e.g. '`Example components for Apache Airflow`'. 1. Save the catalog entry. 1. Open the Visual Pipeline Editor for Apache Airflow and expand the palette. The example components are displayed. @@ -48,13 +47,13 @@ This connector utilizes an embedded catalog as storage and is therefore a static 1. Remove all example catalog entries from the '`Manage Components`' panel. 1. Stop JupyterLab. -1. Uninstall the `elyra-airflow-example-components-catalog` package. +1. Uninstall the `elyra-examples-airflow-catalog` package. ``` - $ pip uninstall -y elyra-airflow-example-components-catalog + $ pip uninstall -y elyra-examples-airflow-catalog ``` ### Troubleshooting **Q: No example components are displayed in the Visual Pipeline Editor palette.** -A: Verify that you imported the examples for the correct runtime environment and check the JupyterLab log file for error messages. \ No newline at end of file +A: Check the JupyterLab log file for error messages. \ No newline at end of file diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py index 4abcc16..8c548d6 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = '0.0.1' +__version__ = '0.1.0' diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json index b5a54c7..46568b1 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json @@ -59,6 +59,7 @@ "type": "string", "maxLength": 18 }, + "default": ["examples"], "uihints": { "field_type": "array", "category": "Component Categories" diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py index ebc9140..a83d408 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/examples_connector.py @@ -42,12 +42,6 @@ def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str # The runtime type's user-friendly display name runtime_type_display_name = runtime_type.value - if runtime_type != RuntimeProcessorType.APACHE_AIRFLOW: - self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type_display_name}\': ' - f'Only \'{RuntimeProcessorType.APACHE_AIRFLOW.value}\' is supported.') - # return empty component specification list - return component_list - try: root_dir = Path(__file__).parent / 'resources' self.log.debug(f'Retrieving component list for runtime type \'{runtime_type_display_name}\' from ' @@ -88,10 +82,6 @@ def read_catalog_entry(self, # The runtime type's user-friendly display name runtime_type_display_name = runtime_type.value - if runtime_type != RuntimeProcessorType.APACHE_AIRFLOW: - self.log.error(f'Cannot retrieve component for runtime type \'{runtime_type_name}\': ' - f'Only \'{RuntimeProcessorType.APACHE_AIRFLOW.value}\' is supported.') - try: # load component from resources directory root_dir = Path(__file__).parent / 'resources' diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.py b/component-catalog-connectors/airflow-example-components-connector/setup.py index ad4c4ed..9afe830 100644 --- a/component-catalog-connectors/airflow-example-components-connector/setup.py +++ b/component-catalog-connectors/airflow-example-components-connector/setup.py @@ -29,7 +29,7 @@ exec(f.read(), {}, version_ns) setup_args = dict( - name="elyra-airflow-example-components-catalog", + name="elyra-examples-airflow-catalog", version=version_ns['__version__'], url="https://github.com/elyra-ai/examples", description="Curated Elyra component catalog for example Apache Airflow components.", @@ -38,7 +38,7 @@ license="Apache License Version 2.0", packages=find_packages(), install_requires=[ - 'elyra==3.3.0.dev0' + 'elyra>3.2' ], setup_requires=['flake8'], include_package_data=True, diff --git a/component-catalog-connectors/kfp-example-components-connector/Makefile b/component-catalog-connectors/kfp-example-components-connector/Makefile index 297d134..b3598c3 100644 --- a/component-catalog-connectors/kfp-example-components-connector/Makefile +++ b/component-catalog-connectors/kfp-example-components-connector/Makefile @@ -18,7 +18,7 @@ SHELL:=/bin/bash -PACKAGE_NAME=elyra-kfp-example-components-catalog +PACKAGE_NAME=elyra-examples-kfp-catalog help: # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html diff --git a/component-catalog-connectors/kfp-example-components-connector/README.md b/component-catalog-connectors/kfp-example-components-connector/README.md index 7323db4..149a516 100644 --- a/component-catalog-connectors/kfp-example-components-connector/README.md +++ b/component-catalog-connectors/kfp-example-components-connector/README.md @@ -13,7 +13,7 @@ You can install the component examples from PyPI or source code. Note that a **r **Install from PyPI** ``` - $ pip install elyra-kfp-example-components-catalog + $ pip install elyra-examples-kfp-catalog ``` **Install from source code** @@ -30,8 +30,7 @@ You can install the component examples from PyPI or source code. Note that a **r 1. [Open the '`Manage Components`' panel]( https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). 1. Add a new component examples catalog ('`+`' > '`New Kubeflow Pipelines example components catalog`'). -1. Specify a catalog name, e.g. '`Elyra example components for Kubeflow`'. -1. (Optional) Specify a category under which the example components will be organized in the palette. +1. Specify a catalog name, e.g. '`Example components for Kubeflow Pipelines`'. 1. Save the catalog entry. 1. Open the Visual Pipeline Editor for Kubeflow Pipelines and expand the palette. The example components are displayed. @@ -48,13 +47,13 @@ This connector utilizes an embedded catalog as storage and is therefore a static 1. Remove all example catalog entries from the '`Manage Components`' panel. 1. Stop JupyterLab. -1. Uninstall the `elyra-kfp-example-components-catalog` package. +1. Uninstall the `elyra-examples-kfp-catalog` package. ``` - $ pip uninstall -y elyra-kfp-example-components-catalog + $ pip uninstall -y elyra-examples-kfp-catalog ``` ### Troubleshooting **Q: No example components are displayed in the Visual Pipeline Editor palette.** -A: Verify that you imported the examples for the correct runtime environment and check the JupyterLab log file for error messages. \ No newline at end of file +A: Check the JupyterLab log file for error messages. \ No newline at end of file diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py index 4abcc16..8c548d6 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = '0.0.1' +__version__ = '0.1.0' diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json index 53d3f6b..7aad919 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json @@ -59,6 +59,7 @@ "type": "string", "maxLength": 18 }, + "default": ["examples"], "uihints": { "field_type": "array", "category": "Component Categories" diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py index cefe1ab..0235497 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/examples_connector.py @@ -41,12 +41,6 @@ def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str # The runtime type's user-friendly display name runtime_type_display_name = runtime_type.value - if runtime_type != RuntimeProcessorType.KUBEFLOW_PIPELINES: - self.log.error(f'Cannot retrieve component list for runtime type \'{runtime_type_display_name}\': ' - f'Only \'{RuntimeProcessorType.KUBEFLOW_PIPELINES.value}\' is supported.') - # return empty component specification list - return component_list - try: root_dir = Path(__file__).parent / 'resources' self.log.debug(f'Retrieving component list for runtime type \'{runtime_type_display_name}\' from ' @@ -87,10 +81,6 @@ def read_catalog_entry(self, # The runtime type's user-friendly display name runtime_type_display_name = runtime_type.value - if runtime_type != RuntimeProcessorType.KUBEFLOW_PIPELINES: - self.log.error(f'Cannot retrieve component for runtime type \'{runtime_type_display_name}\': ' - f'Only \'{RuntimeProcessorType.KUBEFLOW_PIPELINES.value}\' is supported.') - try: # load component from resources directory root_dir = Path(__file__).parent / 'resources' diff --git a/component-catalog-connectors/kfp-example-components-connector/setup.py b/component-catalog-connectors/kfp-example-components-connector/setup.py index 441fb10..97c905a 100644 --- a/component-catalog-connectors/kfp-example-components-connector/setup.py +++ b/component-catalog-connectors/kfp-example-components-connector/setup.py @@ -29,7 +29,7 @@ exec(f.read(), {}, version_ns) setup_args = dict( - name="elyra-kfp-example-components-catalog", + name="elyra-examples-kfp-catalog", version=version_ns['__version__'], url="https://github.com/elyra-ai/examples", description="Curated Elyra component catalog for example Kubeflow Pipelines components.", @@ -38,7 +38,7 @@ license="Apache License Version 2.0", packages=find_packages(), install_requires=[ - 'elyra==3.3.0.dev0' + 'elyra>=3.2' ], setup_requires=['flake8'], include_package_data=True, From 75bd4d3c544f39192ec9c64e4f75fed23873f335 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Tue, 9 Nov 2021 09:54:44 -0800 Subject: [PATCH 10/18] Improve README --- .../airflow-example-components-connector/README.md | 2 +- .../kfp-example-components-connector/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/README.md b/component-catalog-connectors/airflow-example-components-connector/README.md index b6398e3..b3a197a 100644 --- a/component-catalog-connectors/airflow-example-components-connector/README.md +++ b/component-catalog-connectors/airflow-example-components-connector/README.md @@ -39,7 +39,7 @@ https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#manag This connector utilizes an embedded catalog as storage and is therefore a static catalog. To customize the catalog content according to your needs: 1. Clone or fork the `https://github.com/elyra-ai/examples` repository. -1. Navigate to the [`airflow_examples_connector/resources`](airflow_examples_connector/resources) directory. This directory contains the Apache Airflow operators that this connector makes available to Elyra. Note that the operator packages must be installed on the Apache Airflow cluster, or DAG execution will fail. +1. Navigate to the [`examples/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/resources`](airflow_examples_connector/resources) directory. This directory contains the Apache Airflow operators that this connector makes available to Elyra. Note that the operator packages must be installed on the Apache Airflow cluster, or DAG execution will fail. 1. Change the content of this directory as desired. 1. Install the customized connector from source. diff --git a/component-catalog-connectors/kfp-example-components-connector/README.md b/component-catalog-connectors/kfp-example-components-connector/README.md index 149a516..6f350a2 100644 --- a/component-catalog-connectors/kfp-example-components-connector/README.md +++ b/component-catalog-connectors/kfp-example-components-connector/README.md @@ -39,7 +39,7 @@ https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#manag This connector utilizes an embedded catalog as storage and is therefore a static catalog. To customize the catalog content according to your needs: 1. Clone or fork the `https://github.com/elyra-ai/examples` repository. -1. Navigate to the [`kfp_examples_connector/resources`](kfp_examples_connector/resources) directory. This directory contains the Kubeflow Pipelines components that this connector makes available to Elyra. +1. Navigate to the [`examples/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/resources`](kfp_examples_connector/resources) directory. This directory contains the Kubeflow Pipelines components that this connector makes available to Elyra. 1. Change the content of this directory as desired. 1. Install the connector from source. From 796fc44ace40afce277737f2663bd3b15f29dd14 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Tue, 9 Nov 2021 11:10:11 -0800 Subject: [PATCH 11/18] Update schemas per review feedback --- .../airflow_examples_connector/elyra-airflow-catalog.json | 3 +-- .../kfp_examples_connector/elyra-kfp-catalog.json | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json index 46568b1..eeeac8e 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json @@ -1,9 +1,8 @@ { "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", - "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json", "title": "Apache Airflow example components catalog", "name": "elyra-airflow-examples-catalog", - "display_name": "Apache Airflow example components catalog", "schemaspace": "component-registries", "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json index 7aad919..92d7aed 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json @@ -1,9 +1,8 @@ { "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", - "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json", "title": "Kubeflow Pipelines example components catalog", "name": "elyra-kfp-examples-catalog", - "display_name": "Kubeflow Pipelines example components catalog", "schemaspace": "component-registries", "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", From 0b093cc0d745878e54b4a8260e483c92ee718f1c Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Tue, 9 Nov 2021 15:14:46 -0800 Subject: [PATCH 12/18] Add MLX connector and documentation --- README.md | 6 + .../elyra-airflow-catalog.json | 8 +- .../build-a-custom-connector.md | 3 + .../connector-directory.md | 13 ++ .../elyra-kfp-catalog.json | 8 +- .../mlx-connector/MANIFEST.in | 37 ++++ .../mlx-connector/Makefile | 46 +++++ .../mlx-connector/README.md | 61 ++++++ .../mlx_catalog_connector/__init__.py | 15 ++ .../mlx_catalog_connector/_version.py | 16 ++ .../mlx_catalog_connector/mlx-catalog.json | 82 ++++++++ .../mlx_component_catalog_connector.py | 177 ++++++++++++++++++ .../mlx_schema_provider.py | 48 +++++ .../mlx-connector/setup.cfg | 53 ++++++ .../mlx-connector/setup.py | 71 +++++++ .../mlx-connector/test_requirements.txt | 1 + 16 files changed, 631 insertions(+), 14 deletions(-) create mode 100644 component-catalog-connectors/build-a-custom-connector.md create mode 100644 component-catalog-connectors/connector-directory.md create mode 100644 component-catalog-connectors/mlx-connector/MANIFEST.in create mode 100644 component-catalog-connectors/mlx-connector/Makefile create mode 100644 component-catalog-connectors/mlx-connector/README.md create mode 100644 component-catalog-connectors/mlx-connector/mlx_catalog_connector/__init__.py create mode 100644 component-catalog-connectors/mlx-connector/mlx_catalog_connector/_version.py create mode 100644 component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json create mode 100644 component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_component_catalog_connector.py create mode 100644 component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_schema_provider.py create mode 100644 component-catalog-connectors/mlx-connector/setup.cfg create mode 100644 component-catalog-connectors/mlx-connector/setup.py create mode 100644 component-catalog-connectors/mlx-connector/test_requirements.txt diff --git a/README.md b/README.md index becccd8..9964a79 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,12 @@ Pipeline nodes are implemented using - [Custom components for Kubeflow Pipelines](pipelines/kubeflow_pipelines_component_examples) - [Custom components for Apache Airflow](pipelines/airflow_component_examples) +### Pipeline component catalog connectors + +Elyra loads [custom components](https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html) from component catalogs: +- [Find catalog connectors](component-catalog-connectors/connector-directory.md) +- [Build a custom catalog connector](component-catalog-connectors/build-a-custom-connector.md) + ### Example pipelines This repository includes the following example pipelines: - [Analyzing NOAA weather data](pipelines/dax_noaa_weather_data) diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json index eeeac8e..a1daa8e 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json @@ -24,12 +24,6 @@ "type": "string", "minLength": 1 }, - "version": { - "title": "Version", - "description": "The version associated with this instance", - "type": "integer", - "const": 1 - }, "metadata": { "description": "Additional data specific to this metadata", "type": "object", @@ -68,5 +62,5 @@ "required": ["runtime_type"] } }, - "required": ["schema_name", "display_name", "version", "metadata"] + "required": ["schema_name", "display_name", "metadata"] } diff --git a/component-catalog-connectors/build-a-custom-connector.md b/component-catalog-connectors/build-a-custom-connector.md new file mode 100644 index 0000000..e816b62 --- /dev/null +++ b/component-catalog-connectors/build-a-custom-connector.md @@ -0,0 +1,3 @@ +## How to build a component catalog connector + +This content is under development. \ No newline at end of file diff --git a/component-catalog-connectors/connector-directory.md b/component-catalog-connectors/connector-directory.md new file mode 100644 index 0000000..b11b418 --- /dev/null +++ b/component-catalog-connectors/connector-directory.md @@ -0,0 +1,13 @@ +## Component catalog connectors + +The following catalog connectors should work with Elyra version 3.3 and above. Connectors are provided as-is and, unless specified otherwise, are not maintained by the Elyra core committers. + +- To add your connector to the list [create a pull request](https://github.com/elyra-ai/examples/pulls). +- Learn [how to build your own catalog connector](build-a-custom-connector.md). + +| Connector | Description | +| ----------- | ----------- | +| [Apache Airflow example catalog](airflow-example-components-connector) | Provides access to a small set of curated Apache Airflow operators that you can use to get started with the Visual Pipeline Editor. | +| [Kubeflow Pipelines example catalog](kfp-example-components-connector) | Provides access to a small set of curated Kubeflow Pipelines components that you can use to get started with the Visual Pipeline Editor. | +| [Machine Learning Exchange](mlx-connector/) | This LFAI project provides an open source Data and AI assets catalog and execution engine for Kubeflow Pipelines. | + diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json index 92d7aed..812fc41 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json @@ -24,12 +24,6 @@ "type": "string", "minLength": 1 }, - "version": { - "title": "Version", - "description": "The version associated with this instance", - "type": "integer", - "const": 1 - }, "metadata": { "description": "Additional data specific to this metadata", "type": "object", @@ -68,5 +62,5 @@ "required": ["runtime_type"] } }, - "required": ["schema_name", "display_name", "version", "metadata"] + "required": ["schema_name", "display_name", "metadata"] } diff --git a/component-catalog-connectors/mlx-connector/MANIFEST.in b/component-catalog-connectors/mlx-connector/MANIFEST.in new file mode 100644 index 0000000..4bd1206 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/MANIFEST.in @@ -0,0 +1,37 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# exclude from ANY directory +global-exclude *.ipynb +global-exclude *.py[cod] +global-exclude __pycache__ +global-exclude .git +global-exclude .ipynb_checkpoints +global-exclude .DS_Store +global-exclude *.sh +global-exclude docs +global-exclude tests + +# explicit includes +include CONTRIBUTING.md +include README.md +include LICENSE +include dist/*.tgz + +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include mlx_catalog_connector mlx-catalog.json diff --git a/component-catalog-connectors/mlx-connector/Makefile b/component-catalog-connectors/mlx-connector/Makefile new file mode 100644 index 0000000..9164a61 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/Makefile @@ -0,0 +1,46 @@ +# +# Copyright 2021-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +.PHONY: help clean lint test-dependencies source-install install dist + +SHELL:=/bin/bash + +PACKAGE_NAME=mlx-component-catalog-connector + +help: +# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +clean: + rm -rf dist/ + rm -rf build/ + rm -rf *.egg-info + - pip uninstall -y $(PACKAGE_NAME) + +test-dependencies: + @pip install -q -r test_requirements.txt + +lint: test-dependencies + flake8 mlx_catalog_connector + +dist: clean lint ## Build distribution + python setup.py bdist_wheel sdist + +source-install: dist ## Install MLX component connector package from source + pip install . + +install: ## Install MLX component connector package from PyPI + pip install $(PACKAGE_NAME) diff --git a/component-catalog-connectors/mlx-connector/README.md b/component-catalog-connectors/mlx-connector/README.md new file mode 100644 index 0000000..954f6ee --- /dev/null +++ b/component-catalog-connectors/mlx-connector/README.md @@ -0,0 +1,61 @@ +## Machine Learning Exchange catalog connector + +This catalog connector enables Elyra to load Kubeflow Pipelines components from [Machine Learning Exchange](https://github.com/machine-learning-exchange) (MLX) deployments. + +### Install the connector + +You can install the MLX catalog connector from PyPI or source code. Note that a **rebuild of JupyterLab is not required**. + +**Prerequisites** + +- [Install Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above). +- [Machine Learning Exchange deployment](https://github.com/machine-learning-exchange/mlx) ([quickstart guide](https://github.com/machine-learning-exchange/mlx/tree/main/quickstart)) + +**Install from PyPI** + + ``` + $ pip install mlx-component-catalog-connector + ``` + +**Install from source code** + + ``` + $ git clone https://github.com/elyra-ai/examples.git + $ cd examples/component-catalog-connectors/mlx-connector/ + $ make source-install + ``` + +### Use the connector + +1. Launch JupyterLab. +1. [Open the '`Manage Components`' panel]( +https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui). +1. Add a new MLX component catalog ('`+`' > '`New Machine Learning Exchange Component Catalog`'). +1. Specify a catalog name, e.g. '`MLX dev catalog`'. +1. (Optional) Specify a category under which the catalog's components will be organized in the palette. +1. Configure the `MLX API URL`, e.g. '`http://my-mlx-server.mydomain:8080/`'. + > Note: The Machine Learning Exchange API URL is different from the GUI URL! +1. Apply an optional filter expression to the component names. The `*` (zero or more characters) and `?` (zero or one character) wildcards are supported. +1. Save the catalog entry. +1. Open the Kubeflow Pipelines Visual Pipeline Editor and expand the palette. The catalog components are displayed. + +### Uninstall the connector + +1. Remove all MLX catalog entries from the '`Manage Components`' panel. +1. Stop JupyterLab. +1. Uninstall the `mlx-component-catalog-connector` package. + ``` + $ pip uninstall -y mlx-component-catalog-connector + ``` + +### Troubleshooting + +**Problem: The palette does not display any components from the configured catalog.** + +**Solution:** If the the Elyra GUI does not display any error message indicating that a problem was encountered, inspect the JupyterLab log file. + +Example error message (The specified MLX URL is invalid): + +``` +Error fetching component list from MLX catalog http://localhost:8080: ... Failed to establish a new connection: [Errno 61] Connection refused' +``` diff --git a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/__init__.py b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/__init__.py new file mode 100644 index 0000000..febc0a2 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/_version.py b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/_version.py new file mode 100644 index 0000000..8c548d6 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/_version.py @@ -0,0 +1,16 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = '0.1.0' diff --git a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json new file mode 100644 index 0000000..9ddfe97 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json", + "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json", + "title": "Machine Learning Exchange Component Catalog", + "name": "mlx-catalog", + "schemaspace": "component-registries", + "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", + "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", + "uihints": { + "title": "Machine Learning Exchange Component Catalog", + "icon": "", + "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/mlx-connector" + }, + "properties": { + "schema_name": { + "title": "Schema Name", + "description": "The schema associated with this instance", + "type": "string", + "const": "mlx-catalog" + }, + "display_name": { + "title": "Display Name", + "description": "Display name of this Component Catalog", + "type": "string", + "minLength": 1 + }, + "metadata": { + "description": "Additional data specific to this metadata", + "type": "object", + "properties": { + "description": { + "title": "Description", + "description": "Description of this Component Catalog", + "type": "string", + "default": "MLX component catalog" + }, + "runtime_type": { + "title": "Runtime", + "description": "The Machine Learning Exchange only supports Kubeflow Pipeline components.", + "type": "string", + "enum": ["KUBEFLOW_PIPELINES"], + "default": "KUBEFLOW_PIPELINES", + "uihints": { + "field_type": "dropdown" + } + }, + "categories": { + "title": "Category Names", + "description": "Assign the components in the catalog to one or more categories, to group them in the visual pipeline editor palette.", + "type": "array", + "items": { + "type": "string", + "maxLength": 18 + }, + "uihints": { + "field_type": "array", + "category": "Component Categories" + } + }, + "mlx_api_url": { + "title": "MLX API URL", + "description": "API endpoint URL for the Machine Learning Exchange server", + "type": "string", + "format": "uri", + "uihints": { + "category": "Source" + } + }, + "filter": { + "title": "Component name filter", + "description": "Only return components that match the specified name filter expression. * and ? are valid wildcards.", + "type": "string", + "uihints": { + "category": "Source" + } + } + }, + "required": ["runtime_type", "mlx_api_url"] + } + }, + "required": ["schema_name", "display_name", "metadata"] +} diff --git a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_component_catalog_connector.py b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_component_catalog_connector.py new file mode 100644 index 0000000..cd96512 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_component_catalog_connector.py @@ -0,0 +1,177 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from http import HTTPStatus +from io import BytesIO +import re +import tarfile +from tempfile import TemporaryFile +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from urllib.parse import urlparse + +from elyra.pipeline.catalog_connector import ComponentCatalogConnector +import requests + + +class MLXComponentCatalogConnector(ComponentCatalogConnector): + """ + Read component definitions from a Machine Learning Exchange catalog + https://github.com/machine-learning-exchange + """ + + def get_catalog_entries(self, catalog_metadata: Dict[str, Any]) -> List[Dict[str, Any]]: + + """ + Returns a list of component_metadata instances, one per component found in the given registry. + The form that component_metadata takes is determined by requirements of the reader class. + + :param registry_metadata: the dictionary-form of the Metadata instance for a single registry + """ + component_list = [] + + # verify that the required inputs were provided + mlx_api_url = catalog_metadata.get('mlx_api_url') + + if mlx_api_url is None: + self.log.error('Cannot connect to MLX catalog: An API endpoint URL must be provided.') + # return empty component list + return component_list + + try: + # invoke endpoint to retrieve component list from the catalog + self.log.debug(f'Retrieving component list from MLX catalog \'{mlx_api_url}\'.') + u = urlparse(mlx_api_url) + # assemble MLX component list endpoint URL + endpoint = u._replace(path='/apis/v1alpha1/components').geturl() + + # Query MLX catalog components endpoint + res = requests.get(endpoint) + if res.status_code != HTTPStatus.OK: + self.log.warning(f'Error fetching component list from MLX catalog {mlx_api_url}: ' + f'Request: {endpoint} HTTP code: {res.status_code}.') + return component_list + + if res.headers['Content-Type'] != 'application/json': + self.log.warning(f'Error fetching component list from MLX catalog {mlx_api_url}: ' + f'Unexpected content type: {res.headers["Content-Type"]}.' + f'Content: {res.content}') + return component_list + + # the response is JSON formatted: + # "components": [ + # { + # "id": "component-id-used-for-retrieval", + # "name": "user friendly component name" + # ... + # } + # ] + + # create component filter regex if a filter condition was + # specified by the user + filter_expression = catalog_metadata.get('filter', '').strip() + regex = None + if len(filter_expression) > 0: + regex = filter_expression.replace('*', '.*').replace('?', '.?') + + # post-process the component list by applying the filter regex, if + # one was specified + for component in res.json().get('components', []): + if regex: + if re.fullmatch(regex, component.get('name', ''), flags=re.IGNORECASE): + component_list.append({'mlx_component_id': component.get('id')}) + else: + component_list.append({'mlx_component_id': component.get('id')}) + + except Exception as ex: + self.log.warning(f'Error fetching component list from MLX catalog {mlx_api_url}: {ex}') + + return component_list + + def read_catalog_entry(self, + catalog_entry_data: Dict[str, Any], + catalog_metadata: Dict[str, Any]) -> Optional[str]: + """ + Fetch the component that is identified by catalog_entry_data from + the MLX catalog. + + :param catalog_entry_data: a dictionary that contains the information needed to read the content + of the component definition + :param catalog_metadata: the metadata associated with the catalog in which this catalog entry is + stored; in addition to catalog_entry_data, catalog_metadata may also be + needed to read the component definition for certain types of catalogs + + :returns: the content of the given catalog entry's definition in string form + """ + + # verify that the required inputs were provided + mlx_api_url = catalog_metadata.get('mlx_api_url') + if mlx_api_url is None: + self.log.error('Cannot connect to MLX catalog: An API endpoint URL must be provided.') + return None + + mlx_component_id = catalog_entry_data['mlx_component_id'] + if mlx_component_id is None: + self.log.error(f'Cannot retrieve component specification from MLX catalog {mlx_api_url}: ' + 'A component id must be provided.') + return None + + try: + u = urlparse(mlx_api_url) + # assemble MLX component list endpoint URL + endpoint = u._replace(path=f'apis/v1alpha1/components/{mlx_component_id}/download').geturl() + res = requests.get(endpoint) + except Exception as e: + self.log.error(f'Failed to download component specification {mlx_component_id} ' + f'from {mlx_api_url}: {str(e)}') + return None + + if res.status_code != HTTPStatus.OK: + self.log.error(f'Error fetching component specification {mlx_component_id} ' + f'from MLX catalog {mlx_api_url}: ' + f'Request: {endpoint} HTTP code: {res.status_code}.') + return None + + # response type should be 'application/gzip' + # Content-Disposition: attachment; filename=model-fairness-check.tgz + with TemporaryFile() as fp: + fp.write(res.content) + fp.seek(0) + try: + tar = tarfile.open(fileobj=BytesIO(fp.read()), + mode='r:gz') + if len(tar.getnames()) > 1: + self.log.error(f'The response archive contains more than one member: {tar.getnames()}') + + return tar.extractfile(tar.getnames()[0]).read() + except Exception as ex: + # the response is not a tgz file + self.log.error(f'The MLX catalog response could not be processed: {ex}') + + return None + + def get_hash_keys(self) -> List[Any]: + """ + Identifies the unique MLX catalog key that read_catalog_entry can use + to fetch an entry from the catalog. Method get_catalog_entries retrieves + the list of available key values. + + :returns: a list of keys, which is for MLX the component id + """ + return ['mlx_component_id'] diff --git a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_schema_provider.py b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_schema_provider.py new file mode 100644 index 0000000..043ac34 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx_schema_provider.py @@ -0,0 +1,48 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +from pathlib import Path +from typing import Dict +from typing import List + +from elyra.metadata.schema import SchemasProvider + + +class MLXSchemasProvider(SchemasProvider): + """ + Enables BYO catalog connector for the Machine Learning Exchange + """ + + def get_schemas(self) -> List[Dict]: + """ + Return the MLX catalog connector schema + """ + # use Elyra logger + log = logging.getLogger('ElyraApp') + mlx_catalog_schema_defs = [] + try: + # load MLX catalog schema definition + mlx_catalog_connector_schema_file = Path(__file__).parent / 'mlx-catalog.json' + log.debug(f'Reading MLX catalog connector schema from {mlx_catalog_connector_schema_file}') + with open(mlx_catalog_connector_schema_file, 'r') as fp: + mlx_catalog_connector_schema = json.load(fp) + mlx_catalog_schema_defs.append(mlx_catalog_connector_schema) + except Exception as ex: + log.error(f'Error reading MLX catalog connector schema {mlx_catalog_connector_schema_file}: {ex}') + + return mlx_catalog_schema_defs diff --git a/component-catalog-connectors/mlx-connector/setup.cfg b/component-catalog-connectors/mlx-connector/setup.cfg new file mode 100644 index 0000000..e35c831 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/setup.cfg @@ -0,0 +1,53 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[bdist_wheel] +universal=0 + +[metadata] +description_file=README.md + +[flake8] +application-import-names = catalog_connector +application-package-names = catalog_connector +enable-extensions = G +# References: +# https://flake8.readthedocs.io/en/latest/user/configuration.html +# https://flake8.readthedocs.io/en/latest/user/error-codes.html +# https://docs.openstack.org/hacking/latest/user/hacking.html +ignore = + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # File contains nothing but comments + H104, + # Include name with TODOs as in # TODO(yourname) + H101, + # Enable mocking + H216, + # Multi line docstrings should start without a leading new line + H404, + # Multi line docstrings should start with a one line summary followed by an empty line + H405, + # Allow breaks after binary operators + W504 +import-order-style = google +max-line-length = 120 diff --git a/component-catalog-connectors/mlx-connector/setup.py b/component-catalog-connectors/mlx-connector/setup.py new file mode 100644 index 0000000..2d2aa29 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/setup.py @@ -0,0 +1,71 @@ +# +# Copyright 2018-2021 Elyra Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from setuptools import find_packages, setup + +long_desc = """ + Elyra component catalog connector for the Machine Learning Exchange + """ + +here = os.path.abspath(os.path.dirname(__file__)) + +version_ns = {} +with open(os.path.join(here, 'mlx_catalog_connector', '_version.py')) as f: + exec(f.read(), {}, version_ns) + +setup_args = dict( + name="mlx-component-catalog-connector", + version=version_ns['__version__'], + url="https://github.com/elyra-ai/examples", + description="Elyra component catalog connector for the Machine Learning Exchange", + long_description=long_desc, + author="Elyra Maintainers", + license="Apache License Version 2.0", + packages=find_packages(), + install_requires=[ + 'elyra>=3.2', + 'requests' + ], + setup_requires=['flake8'], + include_package_data=True, + entry_points={ + 'metadata.schemas_providers': [ + 'mlx-catalog-schema = mlx_catalog_connector.mlx_schema_provider:MLXSchemasProvider' + ], + 'elyra.component.catalog_types': [ + 'mlx-catalog = mlx_catalog_connector.mlx_component_catalog_connector:MLXComponentCatalogConnector' + ], + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + ] +) + + +if __name__ == '__main__': + setup(**setup_args) diff --git a/component-catalog-connectors/mlx-connector/test_requirements.txt b/component-catalog-connectors/mlx-connector/test_requirements.txt new file mode 100644 index 0000000..28f28b5 --- /dev/null +++ b/component-catalog-connectors/mlx-connector/test_requirements.txt @@ -0,0 +1 @@ +flake8>=3.5.0,<3.9.0 From 047929b5645339e57afa0c44554555715fd0c813 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Thu, 11 Nov 2021 11:57:10 -0800 Subject: [PATCH 13/18] Add PyPI upload targets to Makefiles --- .../airflow-example-components-connector/Makefile | 9 ++++++++- .../test_requirements.txt | 1 + .../kfp-example-components-connector/Makefile | 7 +++++++ component-catalog-connectors/mlx-connector/Makefile | 7 +++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/Makefile b/component-catalog-connectors/airflow-example-components-connector/Makefile index de92b09..12af335 100644 --- a/component-catalog-connectors/airflow-example-components-connector/Makefile +++ b/component-catalog-connectors/airflow-example-components-connector/Makefile @@ -14,7 +14,8 @@ # limitations under the License. # -.PHONY: help clean lint test-dependencies source-install install dist +.PHONY: help clean lint test-dependencies source-install install distributed +.PHONY: publish test-publish SHELL:=/bin/bash @@ -44,3 +45,9 @@ source-install: dist ## Install pipeline components examples package from sourc install: ## Install pipeline components examples package from PyPI pip install $(PACKAGE_NAME) + +test-publish: dist ## Upload package to test PyPI + twine upload --repository testpypi dist/* + +publish: dist ## Upload package to PyPI + twine upload dist/* diff --git a/component-catalog-connectors/airflow-example-components-connector/test_requirements.txt b/component-catalog-connectors/airflow-example-components-connector/test_requirements.txt index 28f28b5..10ec18c 100644 --- a/component-catalog-connectors/airflow-example-components-connector/test_requirements.txt +++ b/component-catalog-connectors/airflow-example-components-connector/test_requirements.txt @@ -1 +1,2 @@ flake8>=3.5.0,<3.9.0 +twine diff --git a/component-catalog-connectors/kfp-example-components-connector/Makefile b/component-catalog-connectors/kfp-example-components-connector/Makefile index b3598c3..80b31cd 100644 --- a/component-catalog-connectors/kfp-example-components-connector/Makefile +++ b/component-catalog-connectors/kfp-example-components-connector/Makefile @@ -15,6 +15,7 @@ # .PHONY: help clean lint test-dependencies source-install install dist +.PHONY: publish test-publish SHELL:=/bin/bash @@ -44,3 +45,9 @@ source-install: dist ## Install pipeline components examples package from sourc install: ## Install pipeline components examples package from PyPI pip install $(PACKAGE_NAME) + +test-publish: dist ## Upload package to test PyPI + twine upload --repository testpypi dist/* + +publish: dist ## Upload package to PyPI + twine upload dist/* \ No newline at end of file diff --git a/component-catalog-connectors/mlx-connector/Makefile b/component-catalog-connectors/mlx-connector/Makefile index 9164a61..2d439c1 100644 --- a/component-catalog-connectors/mlx-connector/Makefile +++ b/component-catalog-connectors/mlx-connector/Makefile @@ -15,6 +15,7 @@ # .PHONY: help clean lint test-dependencies source-install install dist +.PHONY: publish test-publish SHELL:=/bin/bash @@ -44,3 +45,9 @@ source-install: dist ## Install MLX component connector package from source install: ## Install MLX component connector package from PyPI pip install $(PACKAGE_NAME) + +test-publish: dist ## Upload package to test PyPI + twine upload --repository testpypi dist/* + +publish: dist ## Upload package to PyPI + twine upload dist/* From 661ffb90e5694df6886aea4ef70227f3409e1dce Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Thu, 11 Nov 2021 12:42:30 -0800 Subject: [PATCH 14/18] Remove elyra dependency in setup.py --- .../airflow-example-components-connector/setup.py | 1 - .../kfp-example-components-connector/setup.py | 1 - component-catalog-connectors/mlx-connector/setup.py | 1 - 3 files changed, 3 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.py b/component-catalog-connectors/airflow-example-components-connector/setup.py index 9afe830..a572e25 100644 --- a/component-catalog-connectors/airflow-example-components-connector/setup.py +++ b/component-catalog-connectors/airflow-example-components-connector/setup.py @@ -38,7 +38,6 @@ license="Apache License Version 2.0", packages=find_packages(), install_requires=[ - 'elyra>3.2' ], setup_requires=['flake8'], include_package_data=True, diff --git a/component-catalog-connectors/kfp-example-components-connector/setup.py b/component-catalog-connectors/kfp-example-components-connector/setup.py index 97c905a..934aefe 100644 --- a/component-catalog-connectors/kfp-example-components-connector/setup.py +++ b/component-catalog-connectors/kfp-example-components-connector/setup.py @@ -38,7 +38,6 @@ license="Apache License Version 2.0", packages=find_packages(), install_requires=[ - 'elyra>=3.2' ], setup_requires=['flake8'], include_package_data=True, diff --git a/component-catalog-connectors/mlx-connector/setup.py b/component-catalog-connectors/mlx-connector/setup.py index 2d2aa29..282ecff 100644 --- a/component-catalog-connectors/mlx-connector/setup.py +++ b/component-catalog-connectors/mlx-connector/setup.py @@ -37,7 +37,6 @@ license="Apache License Version 2.0", packages=find_packages(), install_requires=[ - 'elyra>=3.2', 'requests' ], setup_requires=['flake8'], From aa9ebbdedd2a07b96b4d1fb31dda8956a6e27b80 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Thu, 11 Nov 2021 17:44:51 -0800 Subject: [PATCH 15/18] Update schemaspace information --- .../airflow_examples_connector/elyra-airflow-catalog.json | 4 ++-- .../kfp_examples_connector/elyra-kfp-catalog.json | 4 ++-- .../mlx-connector/mlx_catalog_connector/mlx-catalog.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json index a1daa8e..89d580a 100644 --- a/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json +++ b/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json @@ -3,8 +3,8 @@ "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json", "title": "Apache Airflow example components catalog", "name": "elyra-airflow-examples-catalog", - "schemaspace": "component-registries", - "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", + "schemaspace": "component-catalogs", + "schemaspace_id": "8dc89ca3-4b90-41fd-adb9-9510ad346620", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", "uihints": { "title": "Apache Airflow example components catalog", diff --git a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json index 812fc41..e4c7e69 100644 --- a/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json +++ b/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json @@ -3,8 +3,8 @@ "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/kfp-example-components-connector/kfp_examples_connector/elyra-kfp-catalog.json", "title": "Kubeflow Pipelines example components catalog", "name": "elyra-kfp-examples-catalog", - "schemaspace": "component-registries", - "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", + "schemaspace": "component-catalogs", + "schemaspace_id": "8dc89ca3-4b90-41fd-adb9-9510ad346620", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", "uihints": { "title": "Kubeflow Pipelines example components catalog", diff --git a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json index 9ddfe97..56e99ea 100644 --- a/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json +++ b/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json @@ -3,8 +3,8 @@ "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/mlx-connector/mlx_catalog_connector/mlx-catalog.json", "title": "Machine Learning Exchange Component Catalog", "name": "mlx-catalog", - "schemaspace": "component-registries", - "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70", + "schemaspace": "component-catalogs", + "schemaspace_id": "8dc89ca3-4b90-41fd-adb9-9510ad346620", "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata", "uihints": { "title": "Machine Learning Exchange Component Catalog", From 25eda019a5c9cac92b16c41fcf279359658fb280 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Fri, 12 Nov 2021 07:23:57 -0800 Subject: [PATCH 16/18] Add badges and update homepage links --- .../airflow-example-components-connector/README.md | 2 ++ .../airflow-example-components-connector/setup.py | 2 +- .../kfp-example-components-connector/README.md | 2 ++ .../kfp-example-components-connector/setup.py | 3 ++- component-catalog-connectors/mlx-connector/README.md | 2 ++ component-catalog-connectors/mlx-connector/setup.py | 3 ++- 6 files changed, 11 insertions(+), 3 deletions(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/README.md b/component-catalog-connectors/airflow-example-components-connector/README.md index b3a197a..4775ea1 100644 --- a/component-catalog-connectors/airflow-example-components-connector/README.md +++ b/component-catalog-connectors/airflow-example-components-connector/README.md @@ -1,3 +1,5 @@ +[![PyPI version](https://badge.fury.io/py/elyra-examples-airflow-catalog.svg)](https://badge.fury.io/py/elyra-examples-airflow-catalog) + ## Elyra pipeline component examples catalog This catalog connector provides access to example pipeline components for Apache Airflow. diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.py b/component-catalog-connectors/airflow-example-components-connector/setup.py index a572e25..e87695e 100644 --- a/component-catalog-connectors/airflow-example-components-connector/setup.py +++ b/component-catalog-connectors/airflow-example-components-connector/setup.py @@ -31,7 +31,7 @@ setup_args = dict( name="elyra-examples-airflow-catalog", version=version_ns['__version__'], - url="https://github.com/elyra-ai/examples", + url="https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/airflow-example-components-connector", description="Curated Elyra component catalog for example Apache Airflow components.", long_description=long_desc, author="Elyra Maintainers", diff --git a/component-catalog-connectors/kfp-example-components-connector/README.md b/component-catalog-connectors/kfp-example-components-connector/README.md index 6f350a2..60db727 100644 --- a/component-catalog-connectors/kfp-example-components-connector/README.md +++ b/component-catalog-connectors/kfp-example-components-connector/README.md @@ -1,3 +1,5 @@ +[![PyPI version](https://badge.fury.io/py/elyra-examples-kfp-catalog.svg)](https://badge.fury.io/py/elyra-examples-kfp-catalog) + ## Elyra pipeline component examples catalog This catalog connector provides access to example pipeline components for Kubeflow Pipelines. diff --git a/component-catalog-connectors/kfp-example-components-connector/setup.py b/component-catalog-connectors/kfp-example-components-connector/setup.py index 934aefe..a26e34c 100644 --- a/component-catalog-connectors/kfp-example-components-connector/setup.py +++ b/component-catalog-connectors/kfp-example-components-connector/setup.py @@ -31,7 +31,8 @@ setup_args = dict( name="elyra-examples-kfp-catalog", version=version_ns['__version__'], - url="https://github.com/elyra-ai/examples", + url="https://github.com/elyra-ai/examples/" + "tree/master/component-catalog-connectors/kfp-example-components-connector", description="Curated Elyra component catalog for example Kubeflow Pipelines components.", long_description=long_desc, author="Elyra Maintainers", diff --git a/component-catalog-connectors/mlx-connector/README.md b/component-catalog-connectors/mlx-connector/README.md index 954f6ee..cbbf38e 100644 --- a/component-catalog-connectors/mlx-connector/README.md +++ b/component-catalog-connectors/mlx-connector/README.md @@ -1,3 +1,5 @@ +[![PyPI version](https://badge.fury.io/py/mlx-component-catalog-connector.svg)](https://badge.fury.io/py/mlx-component-catalog-connector) + ## Machine Learning Exchange catalog connector This catalog connector enables Elyra to load Kubeflow Pipelines components from [Machine Learning Exchange](https://github.com/machine-learning-exchange) (MLX) deployments. diff --git a/component-catalog-connectors/mlx-connector/setup.py b/component-catalog-connectors/mlx-connector/setup.py index 282ecff..c242321 100644 --- a/component-catalog-connectors/mlx-connector/setup.py +++ b/component-catalog-connectors/mlx-connector/setup.py @@ -30,7 +30,8 @@ setup_args = dict( name="mlx-component-catalog-connector", version=version_ns['__version__'], - url="https://github.com/elyra-ai/examples", + url="https://github.com/elyra-ai/examples/" + "tree/master/component-catalog-connectors/mlx-connector", description="Elyra component catalog connector for the Machine Learning Exchange", long_description=long_desc, author="Elyra Maintainers", From deeb83ca8f02ab61aff65f617bd0a4bd5c2e5f8e Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Fri, 12 Nov 2021 07:30:39 -0800 Subject: [PATCH 17/18] Improve READMEs --- component-catalog-connectors/README.md | 3 ++- .../airflow-example-components-connector/README.md | 2 +- .../kfp-example-components-connector/README.md | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/component-catalog-connectors/README.md b/component-catalog-connectors/README.md index 64e5aa5..50cf7f7 100644 --- a/component-catalog-connectors/README.md +++ b/component-catalog-connectors/README.md @@ -3,5 +3,6 @@ Component catalog connectors provide Elyra's Visual Pipeline Editor access to local and remote catalogs that store pipeline components. This directory contains the following component catalog connector implementations: - [Kubeflow Pipelines example components](kfp-example-components-connector) - [Apache Airflow example operators](airflow-example-components-connector) +- [Machine Learning Exchange catalog connector](mlx-connector) -The connectors listed above are maintained by the Elyra community. You can find a complete list of available connectors on [this page](connector-directory.md). \ No newline at end of file +The connectors listed above are maintained by the Elyra community. You can find a complete list of available connectors on [this page](connector-directory.md). diff --git a/component-catalog-connectors/airflow-example-components-connector/README.md b/component-catalog-connectors/airflow-example-components-connector/README.md index 4775ea1..7ebee96 100644 --- a/component-catalog-connectors/airflow-example-components-connector/README.md +++ b/component-catalog-connectors/airflow-example-components-connector/README.md @@ -1,6 +1,6 @@ [![PyPI version](https://badge.fury.io/py/elyra-examples-airflow-catalog.svg)](https://badge.fury.io/py/elyra-examples-airflow-catalog) -## Elyra pipeline component examples catalog +## Apache Airflow component examples catalog This catalog connector provides access to example pipeline components for Apache Airflow. diff --git a/component-catalog-connectors/kfp-example-components-connector/README.md b/component-catalog-connectors/kfp-example-components-connector/README.md index 60db727..ee0cc6f 100644 --- a/component-catalog-connectors/kfp-example-components-connector/README.md +++ b/component-catalog-connectors/kfp-example-components-connector/README.md @@ -1,6 +1,6 @@ [![PyPI version](https://badge.fury.io/py/elyra-examples-kfp-catalog.svg)](https://badge.fury.io/py/elyra-examples-kfp-catalog) -## Elyra pipeline component examples catalog +## Kubeflow Pipelines component examples catalog This catalog connector provides access to example pipeline components for Kubeflow Pipelines. From f62af484565dc63488659c05b94e4b230f3b4a5d Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Fri, 12 Nov 2021 07:40:50 -0800 Subject: [PATCH 18/18] Fix linting issue --- .../airflow-example-components-connector/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/component-catalog-connectors/airflow-example-components-connector/setup.py b/component-catalog-connectors/airflow-example-components-connector/setup.py index e87695e..ea39d5c 100644 --- a/component-catalog-connectors/airflow-example-components-connector/setup.py +++ b/component-catalog-connectors/airflow-example-components-connector/setup.py @@ -31,7 +31,8 @@ setup_args = dict( name="elyra-examples-airflow-catalog", version=version_ns['__version__'], - url="https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/airflow-example-components-connector", + url="https://github.com/elyra-ai/examples/" + "tree/master/component-catalog-connectors/airflow-example-components-connector", description="Curated Elyra component catalog for example Apache Airflow components.", long_description=long_desc, author="Elyra Maintainers",