From 80ff871c37861b0ae4c72bc8aaadd7903eec2698 Mon Sep 17 00:00:00 2001 From: Christoph Deppisch Date: Fri, 11 Oct 2024 09:33:28 +0200 Subject: [PATCH] CAMEL-21336: Allow Kamelets configured by EnvVars only - Consider environment variables when validating Kamelet parameter configuration - Avoids errors due to missing required Kamelet parameter validation when parameter is configured via environment variables --- .../camel/component/kamelet/Kamelet.java | 50 +++++++++++++++++++ .../component/kamelet/KameletComponent.java | 9 ++++ .../apache/camel/RouteTemplateContext.java | 22 ++++++++ .../org/apache/camel/impl/DefaultModel.java | 17 ++++--- .../model/DefaultRouteTemplateContext.java | 27 ++++++++++ .../java/org/apache/camel/util/IOHelper.java | 35 +++++++------ 6 files changed, 134 insertions(+), 26 deletions(-) diff --git a/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/Kamelet.java b/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/Kamelet.java index 94d94e229cc12..613f134aff87a 100644 --- a/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/Kamelet.java +++ b/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/Kamelet.java @@ -17,6 +17,7 @@ package org.apache.camel.component.kamelet; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.function.Predicate; @@ -30,6 +31,7 @@ import org.apache.camel.spi.UuidGenerator; import org.apache.camel.support.CamelContextHelper; import org.apache.camel.support.SimpleUuidGenerator; +import org.apache.camel.util.IOHelper; import org.apache.camel.util.ObjectHelper; import org.apache.camel.util.StringHelper; @@ -37,6 +39,7 @@ public final class Kamelet { public static final String PROPERTIES_PREFIX = "camel.kamelet."; + public static final String ENV_VAR_PREFIX = "CAMEL_KAMELET_"; public static final String SCHEME = "kamelet"; public static final String SOURCE_ID = "source"; public static final String SINK_ID = "sink"; @@ -128,6 +131,53 @@ public static void extractKameletProperties(CamelContext context, Map properties, String... elements) { + StringBuilder prefixBuffer = new StringBuilder(Kamelet.ENV_VAR_PREFIX); + + // Map contains parameter name as key and full environment variable as value + Map propertyMappings = new HashMap<>(); + for (String element : elements) { + if (element == null) { + continue; + } + prefixBuffer.append(IOHelper.normalizeEnvironmentVariable(element)).append('_'); + + String prefix = prefixBuffer.toString(); + System.getenv().keySet().stream() + .filter(Kamelet.startsWith(prefix)) + .forEach(name -> propertyMappings.put(name.substring(prefix.length()), name)); + } + + prefixBuffer = new StringBuilder(Kamelet.ENV_VAR_PREFIX); + + for (String element : elements) { + if (element == null) { + continue; + } + prefixBuffer.append(IOHelper.normalizeEnvironmentVariable(StringHelper.camelCaseToDash(element))).append('_'); + + String prefix = prefixBuffer.toString(); + System.getenv().keySet().stream() + .filter(Kamelet.startsWith(prefix)) + .forEach(name -> propertyMappings.put(name.substring(prefix.length()), name)); + } + + for (Map.Entry mapping : propertyMappings.entrySet()) { + String value = System.getenv(mapping.getValue()); + if (value != null) { + properties.put(mapping.getKey(), value); + } + } + } + public static RouteDefinition templateToRoute(RouteTemplateDefinition in, Map parameters) { final String rid = (String) parameters.get(PARAM_ROUTE_ID); final boolean noErrorHandler = (boolean) parameters.get(NO_ERROR_HANDLER); diff --git a/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/KameletComponent.java b/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/KameletComponent.java index ace60eda2fefc..15c926c3c5e53 100644 --- a/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/KameletComponent.java +++ b/components/camel-kamelet/src/main/java/org/apache/camel/component/kamelet/KameletComponent.java @@ -218,6 +218,15 @@ protected void doInit() throws Exception { Kamelet.extractKameletProperties(getCamelContext(), kameletProperties, templateId, routeId); } + // + // Look for OS environment variables that match the Kamelet properties + // Environment variables are loaded in the following order: + // + // CAMEL_KAMELET_" + templateId + // CAMEL_KAMELET_" + templateId + "_" routeId + // + Kamelet.extractKameletEnvVars(kameletProperties, templateId, routeId); + // // Uri params have the highest precedence // diff --git a/core/camel-api/src/main/java/org/apache/camel/RouteTemplateContext.java b/core/camel-api/src/main/java/org/apache/camel/RouteTemplateContext.java index 35cba9e23c8dc..877655a235f29 100644 --- a/core/camel-api/src/main/java/org/apache/camel/RouteTemplateContext.java +++ b/core/camel-api/src/main/java/org/apache/camel/RouteTemplateContext.java @@ -108,6 +108,17 @@ default void bind(String id, Object bean) { */ Object getProperty(String name); + /** + * Gets the environment variable parameter that matches the given property name. The match is performed by + * normalizing the given property name as an OS environment variable name. The environment variable name may use + * pure uppercase or camelCase converted to underscore property names. As an example bucketNameOrArn property + * matches BUCKETNAMEORARN and BUCKET_NAME_OR_ARN environment variables. + * + * @param name name of property + * @return the property value or null if no property exists + */ + Object getEnvVar(String name); + /** * Gets the property with the given name * @@ -139,6 +150,17 @@ default void bind(String id, Object bean) { */ boolean hasParameter(String name); + /** + * Whether the route template has an environment variable parameter that matches the given parameter name. The match + * is performed by normalizing the given name as an OS environment variable name. The environment variable name may + * use pure uppercase or camelCase converted to underscore property names. As an example bucketNameOrArn property + * matches BUCKETNAMEORARN and BUCKET_NAME_OR_ARN environment variables. + * + * @param name the parameter name + * @return true if exists + */ + boolean hasEnvVar(String name); + /** * Gets the local bean repository for the route template when creating the new route */ diff --git a/core/camel-core-engine/src/main/java/org/apache/camel/impl/DefaultModel.java b/core/camel-core-engine/src/main/java/org/apache/camel/impl/DefaultModel.java index 8651458a9d313..d27e4f7fa84c7 100644 --- a/core/camel-core-engine/src/main/java/org/apache/camel/impl/DefaultModel.java +++ b/core/camel-core-engine/src/main/java/org/apache/camel/impl/DefaultModel.java @@ -460,23 +460,24 @@ public String addRouteFromTemplate( final Map propDefaultValues = new HashMap<>(); // include default values first from the template (and validate that we have inputs for all required parameters) if (target.getTemplateParameters() != null) { - StringJoiner templatesBuilder = new StringJoiner(", "); + StringJoiner missingParameters = new StringJoiner(", "); for (RouteTemplateParameterDefinition temp : target.getTemplateParameters()) { if (temp.getDefaultValue() != null) { addProperty(prop, temp.getName(), temp.getDefaultValue()); addProperty(propDefaultValues, temp.getName(), temp.getDefaultValue()); - } else { - if (temp.isRequired() && !routeTemplateContext.hasParameter(temp.getName())) { - // this is a required parameter which is missing - templatesBuilder.add(temp.getName()); - } + } else if (routeTemplateContext.hasEnvVar(temp.getName())) { + // property is configured via environment variables + addProperty(prop, temp.getName(), routeTemplateContext.getEnvVar(temp.getName())); + } else if (temp.isRequired() && !routeTemplateContext.hasParameter(temp.getName())) { + // this is a required parameter which is missing + missingParameters.add(temp.getName()); } } - if (templatesBuilder.length() > 0) { + if (missingParameters.length() > 0) { throw new IllegalArgumentException( "Route template " + routeTemplateId + " the following mandatory parameters must be provided: " - + templatesBuilder); + + missingParameters); } } diff --git a/core/camel-core-model/src/main/java/org/apache/camel/model/DefaultRouteTemplateContext.java b/core/camel-core-model/src/main/java/org/apache/camel/model/DefaultRouteTemplateContext.java index 68f85c0395c2a..7cc877eefecce 100644 --- a/core/camel-core-model/src/main/java/org/apache/camel/model/DefaultRouteTemplateContext.java +++ b/core/camel-core-model/src/main/java/org/apache/camel/model/DefaultRouteTemplateContext.java @@ -27,6 +27,7 @@ import org.apache.camel.RouteTemplateContext; import org.apache.camel.spi.BeanRepository; import org.apache.camel.support.LocalBeanRegistry; +import org.apache.camel.util.IOHelper; import org.apache.camel.util.StringHelper; /** @@ -129,6 +130,32 @@ public boolean hasParameter(String name) { return false; } + @Override + public boolean hasEnvVar(String name) { + String normalizedKey = IOHelper.normalizeEnvironmentVariable(name); + if (parameters.containsKey(normalizedKey)) { + return true; + } + normalizedKey = IOHelper.normalizeEnvironmentVariable(StringHelper.camelCaseToDash(name)); + if (parameters.containsKey(normalizedKey)) { + return true; + } + return false; + } + + @Override + public Object getEnvVar(String name) { + String normalizedKey = IOHelper.normalizeEnvironmentVariable(name); + if (parameters.containsKey(normalizedKey)) { + return parameters.get(normalizedKey); + } + normalizedKey = IOHelper.normalizeEnvironmentVariable(StringHelper.camelCaseToDash(name)); + if (parameters.containsKey(normalizedKey)) { + return parameters.get(normalizedKey); + } + return null; + } + @Override public BeanRepository getLocalBeanRepository() { return registry; diff --git a/core/camel-util/src/main/java/org/apache/camel/util/IOHelper.java b/core/camel-util/src/main/java/org/apache/camel/util/IOHelper.java index 7cd201a1bbca7..de347a4561259 100644 --- a/core/camel-util/src/main/java/org/apache/camel/util/IOHelper.java +++ b/core/camel-util/src/main/java/org/apache/camel/util/IOHelper.java @@ -674,7 +674,7 @@ public static String normalizeCharset(String charset) { * For example given an ENV variable in either format: - CAMEL_KAMELET_AWS_S3_SOURCE_BUCKETNAMEORARN=myArn - * CAMEL_KAMELET_AWS_S3_SOURCE_BUCKET_NAME_OR_ARN=myArn * - * Then the following keys can lookup both ENV formats above: - camel.kamelet.awsS3Source.bucketNameOrArn - + * Then the following keys can look up both ENV formats above: - camel.kamelet.awsS3Source.bucketNameOrArn - * camel.kamelet.aws-s3-source.bucketNameOrArn - camel.kamelet.aws-s3-source.bucket-name-or-arn */ public static String lookupEnvironmentVariable(String key) { @@ -683,31 +683,30 @@ public static String lookupEnvironmentVariable(String key) { String value = System.getenv(upperKey); if (value == null) { - // some OS do not support dashes in keys, so replace with underscore - String normalizedKey = upperKey.replace('-', '_'); - - // and replace dots with underscores so keys like my.key are - // translated to MY_KEY - normalizedKey = normalizedKey.replace('.', '_'); - - value = System.getenv(normalizedKey); + value = System.getenv(normalizeEnvironmentVariable(upperKey)); } if (value == null) { // camelCase keys should use underscore as separator String caseKey = StringHelper.camelCaseToDash(key); - caseKey = caseKey.toUpperCase(); - // some OS do not support dashes in keys, so replace with underscore - String normalizedKey = caseKey.replace('-', '_'); - - // and replace dots with underscores so keys like my.key are - // translated to MY_KEY - normalizedKey = normalizedKey.replace('.', '_'); - - value = System.getenv(normalizedKey); + value = System.getenv(normalizeEnvironmentVariable(caseKey)); } return value; } + /** + * Convert given key into an OS environment variable. Uses uppercase keys and converts dashes and dots to + * underscores. + */ + public static String normalizeEnvironmentVariable(String key) { + String upperKey = key.toUpperCase(); + // some OS do not support dashes in keys, so replace with underscore + String normalizedKey = upperKey.replace('-', '_'); + + // and replace dots with underscores so keys like my.key are + // translated to MY_KEY + return normalizedKey.replace('.', '_'); + } + /** * Encoding-aware input stream. */