From 1a83e16d67576ffc4570b0046e6bfc4b497c4c9f Mon Sep 17 00:00:00 2001 From: Gayatri Aniruddha <103038659+gayatri-projects97@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:52:10 +0800 Subject: [PATCH] initial commit - combining reading & processing --- dlg-graphs/A_read_process_catalogue.graph | 528 ++++++++++++++++++++++ 1 file changed, 528 insertions(+) create mode 100644 dlg-graphs/A_read_process_catalogue.graph diff --git a/dlg-graphs/A_read_process_catalogue.graph b/dlg-graphs/A_read_process_catalogue.graph new file mode 100644 index 00000000..32a6d7e9 --- /dev/null +++ b/dlg-graphs/A_read_process_catalogue.graph @@ -0,0 +1,528 @@ +{ + "modelData": { + "filePath": "dlg-graphs/A_read_process_catalogue.graph", + "fileType": "Graph", + "shortDescription": "", + "detailedDescription": "", + "repoService": "GitHub", + "repoBranch": "main", + "repo": "ICRAR/wallaby-hires", + "eagleVersion": "v6.2.0:html-graph-renderer", + "eagleCommitHash": "a1d68d87110d9cf6614ceb5e9f6b807ffad4498f", + "schemaVersion": "OJS", + "readonly": true, + "repositoryUrl": "", + "commitHash": "", + "downloadUrl": "", + "signature": "", + "lastModifiedName": "", + "lastModifiedEmail": "", + "lastModifiedDatetime": 0, + "numLGNodes": 3 + }, + "nodeDataArray": [ + { + "category": "PythonApp", + "categoryType": "Application", + "isGroup": false, + "color": "#0059a5", + "drawOrderHint": 0, + "key": -1, + "name": "A_read_and_process_csv_file", + "description": "Reads a CSV file and returns its contents as a list of lists.\n ", + "x": 731.4000096877262, + "y": 329.6300581263572, + "radius": 25, + "collapsed": false, + "subject": null, + "expanded": true, + "repositoryUrl": "code", + "commitHash": "0.1", + "paletteDownloadUrl": "", + "dataHash": "f391799b99ebb9dbc7fb746cf21c5a7a53c2cb177b0ea6b8a887b1ec1c61af86", + "fields": [ + { + "name": "filename", + "value": "", + "defaultValue": "", + "description": "The name of the CSV file to be read. Returns: ------- ", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "f000741a-a868-4c41-aaa2-d64aba544266", + "parameterType": "ApplicationArgument", + "usage": "InputPort" + }, + { + "name": "input_parser", + "value": "path", + "defaultValue": "pickle", + "description": "Input port parsing technique", + "readonly": false, + "type": "Select", + "precious": false, + "options": [ + "pickle", + "eval", + "npy", + "path", + "dataurl" + ], + "positional": false, + "keyAttribute": false, + "id": "ea3c0c27-2e86-49c9-8c21-0621fdde2f94", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "output_parser", + "value": "pickle", + "defaultValue": "pickle", + "description": "Output port parsing technique", + "readonly": false, + "type": "Select", + "precious": false, + "options": [ + "pickle", + "eval", + "npy", + "path", + "dataurl" + ], + "positional": false, + "keyAttribute": false, + "id": "37f6ce98-e25e-4821-a8ca-96d29098adb9", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "execution_time", + "value": 5, + "defaultValue": "5", + "description": "Estimate of execution time (in seconds) for this application.", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "ffb425ba-7c9b-43e3-8f6c-d41d6907470a", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "num_cpus", + "value": 1, + "defaultValue": "1", + "description": "Number of cores used.", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "5384b04d-9371-43e8-8f23-1715f2c41f96", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "group_start", + "value": false, + "defaultValue": "false", + "description": "Is this node the start of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "2263db26-ee0f-4ff0-9516-155d3a351cc5", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dropclass", + "value": "dlg.apps.pyfunc.PyFuncApp", + "defaultValue": "dlg.apps.pyfunc.PyFuncApp", + "description": "The python class that implements this application", + "readonly": true, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "65f1e484-0184-4dab-8539-197f73915488", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "func_name", + "value": "A_read_catalogue.read_csv_file_a", + "defaultValue": "A_read_catalogue.read_csv_file_as_list", + "description": "Complete import path of function", + "readonly": true, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "e4ad4cab-cb24-4098-afe1-612c7d3d0d34", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "source_list", + "value": "", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "ad0f51f2-5d42-4bd5-88b0-1a228e0a1668", + "parameterType": "ComponentParameter", + "usage": "OutputPort" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + }, + { + "category": "File", + "categoryType": "Data", + "isGroup": false, + "color": "#2c2c2c", + "drawOrderHint": 0, + "key": -2, + "name": "test_catalogue.csv", + "description": "A standard file on a filesystem mounted to the deployment machine", + "x": 492, + "y": 334.23, + "radius": 25, + "collapsed": false, + "subject": null, + "expanded": true, + "repositoryUrl": "https://github.com/ICRAR/daliuge", + "commitHash": "44edda3", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", + "dataHash": "f40f919804d78c6b8e2674e3089e84fcb9a43bd732aec6bd8e29ddffb724de4a", + "fields": [ + { + "name": "filepath", + "value": "$DLG_ROOT/testdata/test_catalogue.csv", + "defaultValue": "", + "description": "File path for this file. In many cases this does not need to be specified. If it has a \\/ at the end it will be treated as a directory name and the filename will be generated. If it does not have a \\/, the last part will be treated as a filename. If filepath does not start with \\/ (relative path) then the session directory will be prepended to make the path absolute.\"", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "44822cf3-eb05-47f0-b426-848f99a614fc", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + }, + { + "name": "check_filepath_exists", + "value": false, + "defaultValue": "False", + "description": "Perform a check to make sure the file path exists before proceeding with the application", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "1c533c8d-f509-4bb6-96d0-4598b0c4ae13", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dropclass", + "value": "dlg.data.drops.file.FileDROP", + "defaultValue": "dlg.data.drops.file.FileDROP", + "description": "Drop class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "88234b4a-ecba-4188-81cb-d902620fab40", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "streaming", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component streams input and output data", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "4c5807c6-4d55-48be-a576-9bbdf61071d3", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "persist", + "value": true, + "defaultValue": "True", + "description": "Specifies whether this data component contains data that should not be deleted after execution", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "68e759f5-f1d2-4084-8546-e8d9d32f0ebe", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "data_volume", + "value": 5, + "defaultValue": "5", + "description": "Estimated size of the data contained in this node", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "65a568d0-d97e-4d2c-b88a-1108962ef742", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "group_end", + "value": false, + "defaultValue": "False", + "description": "Is this node the end of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "33f73c1c-94ac-4b9f-b4d3-b5405e6d2882", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dummy", + "value": "", + "defaultValue": "", + "description": "Dummy port", + "readonly": false, + "type": "Object", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "7aa0b981-f4e8-4782-b04e-2b5d9d1080b1", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + }, + { + "category": "File", + "categoryType": "Data", + "isGroup": false, + "color": "#2c2c2c", + "drawOrderHint": 0, + "key": -3, + "name": "sources", + "description": "A standard file on a filesystem mounted to the deployment machine", + "x": 985.5999580198531, + "y": 337.23006458484133, + "radius": 25, + "collapsed": false, + "subject": null, + "expanded": true, + "repositoryUrl": "https://github.com/ICRAR/daliuge", + "commitHash": "44edda3", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", + "dataHash": "f40f919804d78c6b8e2674e3089e84fcb9a43bd732aec6bd8e29ddffb724de4a", + "fields": [ + { + "name": "filepath", + "value": "", + "defaultValue": "", + "description": "File path for this file. In many cases this does not need to be specified. If it has a \\/ at the end it will be treated as a directory name and the filename will be generated. If it does not have a \\/, the last part will be treated as a filename. If filepath does not start with \\/ (relative path) then the session directory will be prepended to make the path absolute.\"", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "f5c27be6-1936-407c-bf21-e728897bf0ba", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + }, + { + "name": "check_filepath_exists", + "value": false, + "defaultValue": "False", + "description": "Perform a check to make sure the file path exists before proceeding with the application", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "915bafec-ceb7-4385-96bc-85b4a81307b6", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dropclass", + "value": "dlg.data.drops.file.FileDROP", + "defaultValue": "dlg.data.drops.file.FileDROP", + "description": "Drop class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "be8e87ee-a85f-421d-8b3e-79ebe49818bf", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "streaming", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component streams input and output data", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "f44be006-b129-401f-bb33-a5881ff1e8d1", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "persist", + "value": true, + "defaultValue": "True", + "description": "Specifies whether this data component contains data that should not be deleted after execution", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "6dffb25f-6313-4310-8592-1c34b0f36924", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "data_volume", + "value": 5, + "defaultValue": "5", + "description": "Estimated size of the data contained in this node", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "79b56d40-3ec8-40a2-8f73-5a220db32b2a", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "group_end", + "value": false, + "defaultValue": "False", + "description": "Is this node the end of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "8e85a3b9-5eea-4eb9-8e2d-f01bba1472b3", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dummy", + "value": "", + "defaultValue": "", + "description": "Dummy port", + "readonly": false, + "type": "Object", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "e74b59bf-2bdc-495a-afbe-e85ac0d6cd8e", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + } + ], + "linkDataArray": [ + { + "from": -2, + "fromPort": "7aa0b981-f4e8-4782-b04e-2b5d9d1080b1", + "to": -1, + "toPort": "f000741a-a868-4c41-aaa2-d64aba544266", + "dataType": "Object", + "loop_aware": "0", + "closesLoop": false + }, + { + "from": -1, + "fromPort": "ad0f51f2-5d42-4bd5-88b0-1a228e0a1668", + "to": -3, + "toPort": "e74b59bf-2bdc-495a-afbe-e85ac0d6cd8e", + "dataType": "String", + "loop_aware": "0", + "closesLoop": false + } + ] +} \ No newline at end of file