Update pipeline version to 6 and pin versions (#2321)

Fixes #2262 Sister PR to elyra-ai/pipeline-editor#172 Update the pipeline version and migration code to handle the changes in Elyra 3.3 Pin package versions to prevent future backwards compatibility errors Co-authored-by: Alan Chin <[email protected]>
elyra-ai · Nov 30, 2021 · cd0aea5 · cd0aea5
1 parent af6e7a8
commit cd0aea5
Show file tree

Hide file tree

Showing 40 changed files with 167 additions and 127 deletions.
diff --git a/elyra/pipeline/pipeline.py b/elyra/pipeline/pipeline.py
@@ -23,7 +23,7 @@
 
 # TODO: Make pipeline version available more widely
 # as today is only available on the pipeline editor
-PIPELINE_CURRENT_VERSION = 5
+PIPELINE_CURRENT_VERSION = 6
 PIPELINE_CURRENT_SCHEMA = 3.0
 
 

diff --git a/elyra/tests/cli/resources/kfp_3_node_custom.pipeline b/elyra/tests/cli/resources/kfp_3_node_custom.pipeline
@@ -175,8 +175,7 @@
         "ui_data": {
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "runtime_type": "KUBEFLOW_PIPELINES",
         "properties": {
           "name": "kfp_custom",
@@ -187,4 +186,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/airflow/test_processor_airflow.py b/elyra/tests/pipeline/airflow/test_processor_airflow.py
@@ -524,7 +524,7 @@ def test_process_dictionary_value_function(processor):
 
 
 @pytest.mark.parametrize('parsed_pipeline',
-                         ['resources/validation_pipelines/aa_operator_same_name.pipeline'],
+                         ['resources/validation_pipelines/aa_operator_same_name.json'],
                          indirect=True)
 @pytest.mark.parametrize('component_cache_instance', [AIRFLOW_COMPONENT_CACHE_INSTANCE], indirect=True)
 def test_same_name_operator_in_pipeline(monkeypatch, processor, parsed_pipeline, sample_metadata):

diff --git a/elyra/tests/pipeline/resources/validation_pipelines/aa_invalid_node_op.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/aa_invalid_node_op.pipeline
@@ -65,15 +65,15 @@
           },
           "comments": []
         },
-        "version": 5,
-        "runtime": "airflow",
+        "version": 6,
         "properties": {
           "name": "aa_invalid_node_op",
           "runtime": "Apache Airflow"
-        }
+        },
+        "runtime_type": "APACHE_AIRFLOW"
       },
       "runtime_ref": ""
     }
   ],
   "schemas": []
-}
+}
diff --git a/..._pipelines/aa_operator_same_name.pipeline → ...tion_pipelines/aa_operator_same_name.json b/..._pipelines/aa_operator_same_name.pipeline → ...tion_pipelines/aa_operator_same_name.json
@@ -324,16 +324,13 @@
         "ui_data": {
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "runtime": "airflow",
-        "runtime_type": "APACHE_AIRFLOW",
         "runtime_config": "kfp-yukked1",
-        "properties": {
-          "name": "untitled4"
-        }
+        "source": "aa_operator_same_name.json"
       },
       "runtime_ref": ""
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/generic_basic_filepath_check.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/generic_basic_filepath_check.pipeline
@@ -66,7 +66,7 @@
         "ui_data": {
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_basic_filepath_check",
           "runtime": "Generic"
@@ -76,4 +76,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/...sts/pipeline/resources/validation_pipelines/generic_basic_pipeline_only_notebook.pipeline b/...sts/pipeline/resources/validation_pipelines/generic_basic_pipeline_only_notebook.pipeline
@@ -359,7 +359,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_basic_pipeline_only_notebook",
           "runtime": "Generic"
@@ -369,4 +369,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/...ests/pipeline/resources/validation_pipelines/generic_basic_pipeline_with_scripts.pipeline b/...ests/pipeline/resources/validation_pipelines/generic_basic_pipeline_with_scripts.pipeline
@@ -248,7 +248,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_basic_pipeline_with_scripts",
           "runtime": "Generic"
@@ -258,4 +258,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/generic_double_cycle.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/generic_double_cycle.pipeline
@@ -312,7 +312,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_double_cycle",
           "runtime": "Generic"
@@ -322,4 +322,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/...esources/validation_pipelines/generic_invalid_node_property_dependency_file_path.pipeline b/...esources/validation_pipelines/generic_invalid_node_property_dependency_file_path.pipeline
@@ -361,7 +361,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_invalid_node_property_dependency_file_path",
           "runtime": "Generic"
@@ -371,4 +371,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/.../resources/validation_pipelines/generic_invalid_node_property_hardware_resources.pipeline b/.../resources/validation_pipelines/generic_invalid_node_property_hardware_resources.pipeline
@@ -360,7 +360,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_invalid_node_property_hardware_resources",
           "runtime": "Generic"
@@ -370,4 +370,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/...pipeline/resources/validation_pipelines/generic_invalid_node_property_image_name.pipeline b/...pipeline/resources/validation_pipelines/generic_invalid_node_property_image_name.pipeline
@@ -379,7 +379,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_invalid_node_property_image_name",
           "runtime": "Generic"
@@ -389,4 +389,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/.../pipeline/resources/validation_pipelines/generic_invalid_node_property_structure.pipeline b/.../pipeline/resources/validation_pipelines/generic_invalid_node_property_structure.pipeline
@@ -368,8 +368,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
-        "runtime_type": "GENERIC",
+        "version": 6,
         "properties": {
           "name": "generic_invalid_node_property_structure"
         }
@@ -378,4 +377,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/generic_single_cycle.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/generic_single_cycle.pipeline
@@ -307,7 +307,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_single_cycle",
           "runtime": "Generic"
@@ -317,4 +317,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/generic_singleton.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/generic_singleton.pipeline
@@ -177,7 +177,7 @@
           "runtime": null,
           "comments": []
         },
-        "version": 5,
+        "version": 6,
         "properties": {
           "name": "generic_singleton",
           "runtime": "Generic"
@@ -187,4 +187,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_inputpath_parameter.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_inputpath_parameter.pipeline
@@ -181,8 +181,7 @@
         "ui_data": {
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "runtime_type": "KUBEFLOW_PIPELINES",
         "properties": {
           "name": "kf_inputpath_parameter"
@@ -192,4 +191,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/.../pipeline/resources/validation_pipelines/kf_invalid_inputpath_missing_connection.pipeline b/.../pipeline/resources/validation_pipelines/kf_invalid_inputpath_missing_connection.pipeline
@@ -172,8 +172,7 @@
         "ui_data": {
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "runtime_type": "KUBEFLOW_PIPELINES",
         "properties": {
           "name": "kf_invalid_inputpath_missing_connection"
@@ -183,4 +182,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_parameter.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_parameter.pipeline
@@ -180,8 +180,7 @@
         "ui_data": {
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "runtime_type": "KUBEFLOW_PIPELINES",
         "properties": {
           "name": "kf_invalid_inputpath_parameter"
@@ -191,4 +190,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_node_op.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_node_op.pipeline
@@ -120,15 +120,15 @@
           },
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "properties": {
           "name": "kf_invalid_node_op",
           "runtime": "Kubeflow Pipelines"
-        }
+        },
+        "runtime_type": "KUBEFLOW_PIPELINES"
       },
       "runtime_ref": ""
     }
   ],
   "schemas": []
-}
+}
diff --git a/.../tests/pipeline/resources/validation_pipelines/kf_invalid_node_op_with_supernode.pipeline b/.../tests/pipeline/resources/validation_pipelines/kf_invalid_node_op_with_supernode.pipeline
@@ -18,7 +18,7 @@
               "parameters": "{}",
               "packages_to_install": "",
               "input_data": ""
-              },
+            },
             "label": "Run notebook using papermill",
             "component_source": "/Users/akchin/opt/anaconda3/envs/test-package/lib/python3.8/site-packages/elyra/pipeline/resources/kfp/run_notebook_using_papermill/component.yaml",
             "ui_data": {
@@ -76,7 +76,7 @@
             "component_parameters": {
               "text": "../B.txt",
               "pattern": ".*"
-              },
+            },
             "label": "Filter text",
             "component_source": "/Users/akchin/opt/anaconda3/envs/test-package/lib/python3.8/site-packages/elyra/pipeline/resources/kfp/filter_text_using_shell_and_grep/component.yaml",
             "ui_data": {
@@ -174,12 +174,12 @@
           },
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "properties": {
           "name": "kf_invalid_node_op_with_supernode",
           "runtime": "Kubeflow Pipelines"
-        }
+        },
+        "runtime_type": "KUBEFLOW_PIPELINES"
       },
       "runtime_ref": ""
     },
@@ -192,9 +192,9 @@
           "op": "invalid_node_op",
           "app_data": {
             "component_parameters": {
-                "text": "B.txt",
-                "pattern": ".*"
-              },
+              "text": "B.txt",
+              "pattern": ".*"
+            },
             "label": "Filter text",
             "component_source": "/Users/akchin/opt/anaconda3/envs/test-package/lib/python3.8/site-packages/elyra/pipeline/resources/kfp/filter_text_using_shell_and_grep/component.yaml",
             "ui_data": {
@@ -269,4 +269,4 @@
     }
   ],
   "schemas": []
-}
+}
diff --git a/...ts/pipeline/resources/validation_pipelines/kf_invalid_node_property_in_component.pipeline b/...ts/pipeline/resources/validation_pipelines/kf_invalid_node_property_in_component.pipeline
@@ -144,15 +144,15 @@
           },
           "comments": []
         },
-        "version": 5,
-        "runtime": "kfp",
+        "version": 6,
         "properties": {
           "name": "kf_invalid_node_property_in_component",
           "runtime": "Kubeflow Pipelines"
-        }
+        },
+        "runtime_type": "KUBEFLOW_PIPELINES"
       },
       "runtime_ref": ""
     }
   ],
   "schemas": []
-}
+}