Add pipeline component examples catalogs for KFP and AA and MLX connector

.flake8

@@ -3,7 +3,9 @@
 # https://flake8.readthedocs.io/en/latest/user/configuration.html
 # https://flake8.readthedocs.io/en/latest/user/error-codes.html
 # https://docs.openstack.org/hacking/latest/user/hacking.html
-exclude = __init__.py
+exclude = 
+    __init__.py
+    elyra_examples_connector/airflow_example_components/  
 ignore =
     # Import formatting
     E4,

component-catalog-connectors/README.md

@@ -0,0 +1,7 @@
+## Component catalog connectors
+
+Component catalog connectors provide Elyra's Visual Pipeline Editor access to local and remote catalogs that store pipeline components. This directory contains the following component catalog connector implementations:
+- [Kubeflow Pipelines example components](kfp-example-components-connector)
+- [Apache Airflow example operators](airflow-example-components-connector)
+
+The connectors listed above are maintained by the Elyra community. You can find a complete list of available connectors on [this page](connector-directory.md).

component-catalog-connectors/airflow-example-components-connector/.flake8

@@ -0,0 +1,32 @@
+[flake8]
+# References:
+# https://flake8.readthedocs.io/en/latest/user/configuration.html
+# https://flake8.readthedocs.io/en/latest/user/error-codes.html
+# https://docs.openstack.org/hacking/latest/user/hacking.html
+exclude = 
+    __init__.py
+    setup.py
+    airflow_examples_connector/resources
+ignore =
+    # Import formatting
+    E4,
+    # Comparing types instead of isinstance
+    E721,
+    # Assigning lambda expression
+    E731,
+    # Ambiguous variable names
+    E741,
+    # Allow breaks after binary operators
+    W504,
+    # Include name with TODOs as in # TODO(yourname)
+    H101,
+    # Do not import more than one module per line
+    H301,
+    # Alphabetically order imports by the full module path
+    H306,
+    # Multi line docstrings should start without a leading new line
+    H404,
+    # Multi line docstrings should start with a one line summary followed by an empty line
+    H405
+max-line-length = 120
+

component-catalog-connectors/airflow-example-components-connector/MANIFEST.in

@@ -0,0 +1,38 @@
+#
+# Copyright 2018-2021 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# exclude from ANY directory
+global-exclude *.ipynb
+global-exclude *.py[cod]
+global-exclude __pycache__
+global-exclude .git
+global-exclude .ipynb_checkpoints
+global-exclude .DS_Store
+global-exclude *.sh
+global-exclude docs
+global-exclude tests
+
+# explicit includes
+include CONTRIBUTING.md
+include README.md
+include LICENSE
+include dist/*.tgz
+
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+
+include airflow_examples_connector/elyra-airflow-catalog.json
+recursive-include airflow_examples_connector/resources *.py

component-catalog-connectors/airflow-example-components-connector/Makefile

@@ -0,0 +1,46 @@
+#
+# Copyright 2021-2021 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.PHONY: help clean lint test-dependencies source-install install dist
+
+SHELL:=/bin/bash
+
+PACKAGE_NAME=elyra-examples-airflow-catalog
+
+help:
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+clean:
+	rm -rf dist/
+	rm -rf build/
+	rm -rf *.egg-info
+	- pip uninstall -y $(PACKAGE_NAME)
+
+test-dependencies:
+	@pip install -q -r test_requirements.txt
+
+lint: test-dependencies
+	flake8 .
+
+dist: clean lint  ## Build distribution
+	python setup.py bdist_wheel sdist	
+
+source-install: dist  ## Install pipeline components examples package from source 
+	pip install .
+
+install:  ## Install pipeline components examples package from PyPI
+	pip install $(PACKAGE_NAME)

component-catalog-connectors/airflow-example-components-connector/README.md

@@ -0,0 +1,59 @@
+## Elyra pipeline component examples catalog
+
+This catalog connector provides access to example pipeline components for Apache Airflow.
+
+### Install the component examples
+
+You can install the component examples from PyPI or source code. Note that a **rebuild of JupyterLab is not required**.
+
+**Prerequisites**
+
+- [Install Elyra](https://elyra.readthedocs.io/en/stable/getting_started/installation.html) (version 3.3 and above).
+
+**Install from PyPI**
+
+  ```
+  $ pip install elyra-examples-airflow-catalog
+  ```
+
+**Install from source code**
+
+   ```
+   $ git clone https://github.com/elyra-ai/examples.git
+   $ cd examples/component-catalog-connectors/airflow-example-components-connector/
+   $ make clean source-install
+   ```
+
+### Use the connector
+
+1. Launch JupyterLab.
+1. [Open the '`Manage Components`' panel](
+https://elyra.readthedocs.io/en/stable/user_guide/pipeline-components.html#managing-custom-components-using-the-jupyterlab-ui).
+1. Add a new component examples catalog ('`+`' > '`New Apache Airflow example components catalog`').
+1. Specify a catalog name, e.g. '`Example components for Apache Airflow`'.
+1. Save the catalog entry.
+1. Open the Visual Pipeline Editor for Apache Airflow and expand the palette. The example components are displayed.
+
+### Customize the catalog
+
+This connector utilizes an embedded catalog as storage and is therefore a static catalog. To customize the catalog content according to your needs:
+
+1. Clone or fork the `https://github.com/elyra-ai/examples` repository.
+1. Navigate to the [`airflow_examples_connector/resources`](airflow_examples_connector/resources) directory. This directory contains the Apache Airflow operators that this connector makes available to Elyra. Note that the operator packages must be installed on the Apache Airflow cluster, or DAG execution will fail.
+1. Change the content of this directory as desired.
+1. Install the customized connector from source.
+
+### Uninstall the connector
+
+1. Remove all example catalog entries from the '`Manage Components`' panel.
+1. Stop JupyterLab.
+1. Uninstall the `elyra-examples-airflow-catalog` package.
+   ```
+   $ pip uninstall -y elyra-examples-airflow-catalog
+   ```
+
+### Troubleshooting
+
+**Q: No example components are displayed in the Visual Pipeline Editor palette.**
+
+A: Check the JupyterLab log file for error messages.

component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2018-2021 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/_version.py

@@ -0,0 +1,16 @@
+#
+# Copyright 2018-2021 Elyra Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+__version__ = '0.1.0'

component-catalog-connectors/airflow-example-components-connector/airflow_examples_connector/elyra-airflow-catalog.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "https://raw.githubusercontent.com/elyra-ai/elyra/master/elyra/metadata/schemas/meta-schema.json",
+  "$id": "https://raw.githubusercontent.com/elyra-ai/examples/master/component-catalog-connectors/elyra_examples_connector/elyra-examples-catalog.json",
+  "title": "Apache Airflow example components catalog",
+  "name": "elyra-airflow-examples-catalog",
+  "display_name": "Apache Airflow example components catalog",
+  "schemaspace": "component-registries",
+  "schemaspace_id": "ae79159a-489d-4656-83a6-1adfbc567c70",
+  "metadata_class_name": "elyra.pipeline.component_metadata.ComponentCatalogMetadata",
+  "uihints": {
+    "title": "Apache Airflow example components catalog",
+    "icon": "",
+    "reference_url": "https://github.com/elyra-ai/examples/tree/master/component-catalog-connectors/airflow-example-components-connector"
+  },
+  "properties": {
+    "schema_name": {
+      "title": "Schema Name",
+      "description": "The schema associated with this instance",
+      "type": "string",
+      "const": "elyra-airflow-examples-catalog"
+    },
+    "display_name": {
+      "title": "Display Name",
+      "description": "Display name of this Component Catalog",
+      "type": "string",
+      "minLength": 1
+    },
+    "version": {
+      "title": "Version",
+      "description": "The version associated with this instance",
+      "type": "integer",
+      "const": 1
+    },
+    "metadata": {
+      "description": "Additional data specific to this metadata",
+      "type": "object",
+      "properties": {
+        "description": {
+          "title": "Description",
+          "description": "Curated collection of Apache Airflow components",
+          "type": "string",
+          "default": "Example pipeline components for Apache Airflow"
+        },
+        "runtime_type": {
+          "title": "Runtime type",
+          "description": "The runtime for which to load the example components.",
+          "type": "string",
+          "enum": ["APACHE_AIRFLOW"],
+          "default": "APACHE_AIRFLOW",
+          "uihints": {
+            "field_type": "dropdown"
+          }
+        },
+        "categories": {
+          "title": "Category Names",
+          "description": "The example components will be added to the specified categories in the pipeline editor palette.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "maxLength": 18
+          },
+          "default": ["examples"],
+          "uihints": {
+            "field_type": "array",
+            "category": "Component Categories"
+          }
+        }
+      },
+      "required": ["runtime_type"]
+    }
+  },
+  "required": ["schema_name", "display_name", "version", "metadata"]
+}