From 616d5a1d97c32613ef4fef5ab4b039ddca8b0f30 Mon Sep 17 00:00:00 2001 From: "ajin.thampi" Date: Thu, 3 Oct 2024 14:29:20 +0530 Subject: [PATCH 1/3] Feature:Python inline task added --- core/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/build.gradle b/core/build.gradle index beef8f7a2..ec8e78dcf 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,6 +43,9 @@ dependencies { implementation "org.openjdk.nashorn:nashorn-core:15.4" + //jython dependency + implementation "org.python:jython-standalone:2.7.4" + // JAXB is not bundled with Java 11, dependencies added explicitly // These are needed by Apache BVAL implementation "jakarta.xml.bind:jakarta.xml.bind-api:${revJAXB}" From 58e806f307464dab79d38a86a5bc7c5422c9c882 Mon Sep 17 00:00:00 2001 From: "ajin.thampi" Date: Thu, 3 Oct 2024 14:47:30 +0530 Subject: [PATCH 2/3] python evaluator and python task for java sdk --- .../execution/evaluators/PythonEvaluator.java | 89 +++++++++++++++++++ .../sdk/workflow/def/tasks/Python.java | 26 ++++++ 2 files changed, 115 insertions(+) create mode 100644 core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java create mode 100644 java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Python.java diff --git a/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java b/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java new file mode 100644 index 000000000..74f84e4c1 --- /dev/null +++ b/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java @@ -0,0 +1,89 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.core.execution.evaluators; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.Map; +import java.util.Properties; + +import org.python.core.PyObject; +import org.python.util.PythonInterpreter; +import org.springframework.stereotype.Component; + +@Component(PythonEvaluator.NAME) +public class PythonEvaluator implements Evaluator { + public static final String NAME = "python"; + + @Override + public Object evaluate(String expression, Object inputs) { + + PythonInterpreter interpreter = null; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + try { + Properties props = new Properties(); + props.setProperty("python.import.site", "false"); + PythonInterpreter.initialize(System.getProperties(), props, new String[] {}); + + interpreter = new PythonInterpreter(); + interpreter.setOut(new PrintStream(outputStream)); + + if (inputs instanceof Map) { + Map input = (Map) inputs; + + // Set variables in the interpreter + for (Map.Entry entry : input.entrySet()) { + interpreter.set(entry.getKey(), entry.getValue()); + } + // Build the global declaration dynamically + StringBuilder globalDeclaration = new StringBuilder("def evaluate():\n global "); + + for (Map.Entry entry : input.entrySet()) { + globalDeclaration.append(entry.getKey()).append(", "); + } + + // Remove the trailing comma and space, and add a newline + if (globalDeclaration.length() > 0) { + globalDeclaration.setLength(globalDeclaration.length() - 2); + } + globalDeclaration.append("\n"); + + // Wrap the expression in a function to handle multi-line statements + String wrappedExpression = + globalDeclaration.toString() // Add global declaration line + + " " + + expression.replace( + "\n", "\n ") // Indent the original expression + + "\n" + + "result = evaluate()"; + + // Execute the wrapped expression + interpreter.exec(wrappedExpression); + + // Get the result + PyObject result = interpreter.get("result"); + return result.__tojava__(Object.class); + } else { + return null; + } + } catch (Exception e) { + e.printStackTrace(); + return null; + } finally { + if (interpreter != null) { + interpreter.close(); + } + } + } +} diff --git a/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Python.java b/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Python.java new file mode 100644 index 000000000..ae7158c05 --- /dev/null +++ b/java-sdk/src/main/java/com/netflix/conductor/sdk/workflow/def/tasks/Python.java @@ -0,0 +1,26 @@ +/* + * Copyright 2024 Conductor Authors. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package com.netflix.conductor.sdk.workflow.def.tasks; + +import com.netflix.conductor.common.metadata.tasks.TaskType; + +public class Python extends Task { + public Python(String taskReferenceName, String script) { + super(taskReferenceName, TaskType.INLINE); + if (script == null || script.isEmpty()) { + throw new IllegalArgumentException("Python script cannot be null or empty"); + } + super.input("evaluatorType", "python"); + super.input("expression", script); + } +} From 0e05543e612ca498180c0cd48634c2c0c892efb7 Mon Sep 17 00:00:00 2001 From: "ajin.thampi" Date: Wed, 9 Oct 2024 12:04:03 +0530 Subject: [PATCH 3/3] python evaluator using gralVm --- core/build.gradle | 6 +- .../execution/evaluators/PythonEvaluator.java | 71 ++++++++----------- docker/docker-compose.yaml | 1 + 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index ec8e78dcf..4b4330ade 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,8 +43,10 @@ dependencies { implementation "org.openjdk.nashorn:nashorn-core:15.4" - //jython dependency - implementation "org.python:jython-standalone:2.7.4" + //gralVm dependencies for executing python + implementation("org.graalvm.polyglot:polyglot:24.1.0") + implementation("org.graalvm.polyglot:python:24.1.0") + implementation "org.graalvm.sdk:graal-sdk:24.1.0" // JAXB is not bundled with Java 11, dependencies added explicitly // These are needed by Apache BVAL diff --git a/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java b/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java index 74f84e4c1..1a71ee9b8 100644 --- a/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java +++ b/core/src/main/java/com/netflix/conductor/core/execution/evaluators/PythonEvaluator.java @@ -12,44 +12,35 @@ */ package com.netflix.conductor.core.execution.evaluators; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; import java.util.Map; -import java.util.Properties; -import org.python.core.PyObject; -import org.python.util.PythonInterpreter; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import com.netflix.conductor.core.exception.TerminateWorkflowException; + @Component(PythonEvaluator.NAME) public class PythonEvaluator implements Evaluator { public static final String NAME = "python"; + private static final Logger LOGGER = LoggerFactory.getLogger(PythonEvaluator.class); @Override - public Object evaluate(String expression, Object inputs) { - - PythonInterpreter interpreter = null; - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - - try { - Properties props = new Properties(); - props.setProperty("python.import.site", "false"); - PythonInterpreter.initialize(System.getProperties(), props, new String[] {}); + public Object evaluate(String expression, Object input) { + try (Context context = Context.newBuilder("python").allowAllAccess(true).build()) { + if (input instanceof Map) { + Map inputMap = (Map) input; - interpreter = new PythonInterpreter(); - interpreter.setOut(new PrintStream(outputStream)); - - if (inputs instanceof Map) { - Map input = (Map) inputs; - - // Set variables in the interpreter - for (Map.Entry entry : input.entrySet()) { - interpreter.set(entry.getKey(), entry.getValue()); + // Set inputs as variables in the GraalVM context + for (Map.Entry entry : inputMap.entrySet()) { + context.getBindings("python").putMember(entry.getKey(), entry.getValue()); } + // Build the global declaration dynamically StringBuilder globalDeclaration = new StringBuilder("def evaluate():\n global "); - - for (Map.Entry entry : input.entrySet()) { + for (Map.Entry entry : inputMap.entrySet()) { globalDeclaration.append(entry.getKey()).append(", "); } @@ -60,30 +51,28 @@ public Object evaluate(String expression, Object inputs) { globalDeclaration.append("\n"); // Wrap the expression in a function to handle multi-line statements - String wrappedExpression = - globalDeclaration.toString() // Add global declaration line - + " " - + expression.replace( - "\n", "\n ") // Indent the original expression - + "\n" - + "result = evaluate()"; + StringBuilder wrappedExpression = new StringBuilder(globalDeclaration); + for (String line : expression.split("\n")) { + wrappedExpression.append(" ").append(line).append("\n"); + } + + // Add the call to the function and capture the result + wrappedExpression.append("\nresult = evaluate()"); // Execute the wrapped expression - interpreter.exec(wrappedExpression); + context.eval("python", wrappedExpression.toString()); // Get the result - PyObject result = interpreter.get("result"); - return result.__tojava__(Object.class); + Value result = context.getBindings("python").getMember("result"); + + // Convert the result to a Java object and return it + return result.as(Object.class); } else { return null; } } catch (Exception e) { - e.printStackTrace(); - return null; - } finally { - if (interpreter != null) { - interpreter.close(); - } + LOGGER.error("Error evaluating expression: {}", e.getMessage(), e); + throw new TerminateWorkflowException(e.getMessage()); } } } diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 9425f735a..e812ac61b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -4,6 +4,7 @@ services: conductor-server: environment: - CONFIG_PROP=config-redis.properties + - JAVA_OPTS=-Dpolyglot.engine.WarnInterpreterOnly=false image: conductor:server container_name: conductor-server build: