From c4f9cb000eda671aaef0f9269c03fee76d994aff Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 30 Nov 2022 17:19:39 +0100 Subject: [PATCH 01/16] test(validation): create UT to reflect payload validation with conditions --- .../JsonSchemaValidatorFactoryExt.java | 13 +- .../server/service/ValidationServiceTest.java | 318 ++++++++++++++++++ .../server/test/custom/CustomProcessor.java | 99 +++++- 3 files changed, 420 insertions(+), 10 deletions(-) create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java diff --git a/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java b/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java index 9f27941061b4e..752dc0ac9721f 100644 --- a/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java +++ b/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java @@ -23,10 +23,8 @@ import org.apache.johnzon.jsonschema.regex.JavaRegex; import org.apache.johnzon.jsonschema.spi.builtin.PatternValidation; -import org.talend.sdk.component.form.internal.validation.spi.ext.EnumValidationWithDefaultValue; import org.talend.sdk.component.form.internal.validation.spi.ext.MaximumValidation; import org.talend.sdk.component.form.internal.validation.spi.ext.MinimumValidation; -import org.talend.sdk.component.form.internal.validation.spi.ext.RequiredValidation; import org.talend.sdk.component.form.internal.validation.spi.ext.TypeValidation; public class JsonSchemaValidatorFactoryExt extends org.apache.johnzon.jsonschema.JsonSchemaValidatorFactory { @@ -43,17 +41,18 @@ public List createDefault List validations = super.createDefaultValidations() .stream() .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.TypeValidation.class.isInstance(v)) - .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.EnumValidation.class.isInstance(v)) + // .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.EnumValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.MinimumValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.MaximumValidation.class.isInstance(v)) - .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.RequiredValidation.class.isInstance(v)) + // .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.RequiredValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.PatternValidation.class.isInstance(v)) - .collect(Collectors.toList()); + .collect(Collectors.toList()) + ; validations.add(new TypeValidation()); - validations.add(new EnumValidationWithDefaultValue()); + // validations.add(new EnumValidationWithDefaultValue()); validations.add(new MinimumValidation()); validations.add(new MaximumValidation()); - validations.add(new RequiredValidation()); + // validations.add(new RequiredValidation()); return validations; } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java new file mode 100644 index 0000000000000..ed1ccd6881072 --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -0,0 +1,318 @@ +/* + * Copyright (C) 2006-2022 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + */ +package org.talend.sdk.component.server.service; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toList; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + +import javax.inject.Inject; +import javax.json.Json; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonValue; +import javax.json.bind.Jsonb; +import javax.json.bind.JsonbBuilder; +import javax.ws.rs.client.WebTarget; + +import org.apache.johnzon.jsonschema.JsonSchemaValidator; +import org.apache.johnzon.jsonschema.JsonSchemaValidatorFactory; +import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.talend.sdk.component.form.api.Client; +import org.talend.sdk.component.form.api.UiSpecService; +import org.talend.sdk.component.form.internal.converter.PropertyContext; +import org.talend.sdk.component.form.internal.converter.PropertyContext.Configuration; +import org.talend.sdk.component.form.internal.converter.impl.JsonSchemaConverter; +import org.talend.sdk.component.form.internal.lang.CompletionStages; +import org.talend.sdk.component.form.internal.validation.JsonSchemaValidatorFactoryExt; +import org.talend.sdk.component.form.model.Ui; +import org.talend.sdk.component.form.model.jsonschema.JsonSchema; +import org.talend.sdk.component.form.model.uischema.UiSchema; +import org.talend.sdk.component.server.front.model.ConfigTypeNode; +import org.talend.sdk.component.server.front.model.ConfigTypeNodes; +import org.talend.sdk.component.server.test.ComponentClient; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@MonoMeecrowaveConfig +public class ValidationServiceTest { + @Inject + private WebTarget base; + + @Inject + private ComponentClient client; + + + private Jsonb jsonb = JsonbBuilder.create(); + + + final String lang = "en"; + final String family = "custom"; + final String component = "noop"; + ConfigTypeNodes details; + ConfigTypeNode connection; + JsonSchema jsonSchema; + private Collection uiSchemas; + + private final UiSpecService uiSpecService = new UiSpecService<>(new Client() { + + @Override // for dynamic_values, just return an empty schema + public CompletionStage> action(final String family, final String type, + final String action, final String lang, final Map params, final Object context) { + final Map result = new HashMap<>(); + result.put("items", emptyList()); + return CompletableFuture.completedFuture(result); + } + + @Override + public void close() { + // no-op + } + }); + + private final JsonSchemaValidatorFactory factory = new JsonSchemaValidatorFactoryExt(); + + @Data + @NoArgsConstructor + @AllArgsConstructor + class Result { + + private Collection errors; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class ValidationError { + + private String field; + + private String message; + + } + + + public CompletionStage validate(final ConfigTypeNode config, final JsonObject properties) throws Exception { + return getValidator(config) + .thenApply(validator -> validator.apply(properties)) + .thenApply(vr -> new Result(vr + .getErrors() + .stream() + .map(e -> new ValidationError(e.getField(), e.getMessage())) + .collect(toList()))); + } + + public CompletionStage getValidator(final ConfigTypeNode config) throws Exception { + + JsonSchema xschema = new UiSpecService<>(null, jsonb) + .convert("test", "en", config, null) + .toCompletableFuture() + .get().getJsonSchema(); + JsonObject xjson = jsonb.fromJson(jsonb.toJson(xschema), JsonObject.class); + // xschema should be same as jsonSchema + final JsonSchema jschema = new JsonSchema(); + final JsonSchemaConverter converter = new JsonSchemaConverter(jsonb, jschema, config.getProperties()); + return CompletableFuture + .allOf(config + .getProperties() + .stream() + .filter(Objects::nonNull) + .filter(p -> p.getName().equals(p.getPath())) + .map(it -> new PropertyContext<>(it, this, new Configuration(false))) + .map(CompletionStages::toStage) + .map(converter::convert) + .toArray(CompletableFuture[]::new)) + .thenApply(r -> jschema) + .thenApply(schema -> jsonb.fromJson(jsonb.toJson(schema), JsonObject.class)) + .thenApply(factory::newInstance); + } + + @BeforeEach + void setup() throws Exception { + details = base + .path("configurationtype/details") + .queryParam("identifiers", "Y29tcG9uZW50LXdpdGgtdXNlci1qYXJzI2N1c3RvbSNkYXRhc2V0I2RhdGFzZXQ") + .request(APPLICATION_JSON_TYPE) + .header("Accept-Encoding", "gzip") + .get(ConfigTypeNodes.class); + connection = details.getNodes().values().iterator().next(); + uiSpecService.setConfiguration(new Configuration(true)); + Ui ui = uiSpecService.convert(family, lang, connection, null).toCompletableFuture().get(); + jsonSchema = ui.getJsonSchema(); + uiSchemas = ui.getUiSchema(); + } + + private void checkAsserts(JsonObject payload, List expected) throws Exception { + Result errors = validate(connection, payload).toCompletableFuture().get(); + + System.out.println("==="); + long ecnt = errors.getErrors().stream() + .peek(e -> log.warn("[checkAsserts] {}", e)) + .filter(e -> !expected.contains(e)) + .peek(e -> log.error("[checkAsserts]Validation uncaught: {}", e)) + // .map(e-> fail(e)) + .count(); + expected.stream().forEach(e -> assertTrue(errors.getErrors().contains(e))); + } + + @Test + void testValidation () throws Exception { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + // + final List uniques = Arrays.asList("one", "two", "three"); + final List notUniques = Arrays.asList("one", "two", "three", "one"); + // + ValidationError conn = new ValidationError("/configuration", "connection is required and is not present"); + ValidationError min = new ValidationError("/configuration/limit", "1.0 is less than 100.0"); + ValidationError max = new ValidationError("/configuration/limit", "1000.0 is more than 150.0"); + ValidationError url0 = new ValidationError("/configuration/connection/url0", "\"mailto://toto@titi.org\" doesn't match JavascriptRegex{/^https?://.*/}"); + ValidationError url1Patten = new ValidationError("/configuration/connection/url1", "\"mailto://toto@titi.org\" doesn't match JavascriptRegex{/^https?://.*/}"); + ValidationError url1Required = new ValidationError("/configuration/connection", "url1 is required and is not present"); + ValidationError username = new ValidationError("/configuration/connection", "username is required and is not present"); + ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", "Invalid value, got null, expected: [\"VALUE_1\",\"VALUE_2\",\"VALUE_3\"]"); + ValidationError password = new ValidationError("/configuration/connection", "password is required and is not present"); + ValidationError activedIfs = new ValidationError("/configuration/connection", "activedIfs is required and is not present"); + ValidationError uniqVals = new ValidationError("/configuration/connection/uniqVals", "duplicated items: []"); + + + JsonObject payload; + /** + * connection is required! + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connectn", "").build()).build(); + checkAsserts(payload, Arrays.asList(conn, valueEval)); + /** + * min/max + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", JsonValue.NULL) + .add("limit", 1)) + .build(); + checkAsserts(payload, Arrays.asList(min, valueEval, conn)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", JsonValue.NULL) + .add("limit", 1000)) + .build(); + checkAsserts(payload, Arrays.asList(max, valueEval, conn)); + /* + * url0 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "mailto://toto@titi.org").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url0, valueEval, username, password, url1Required, activedIfs)); + /* + * url1 + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url1", "mailto://toto@titi.org").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(valueEval, username, url1Patten, password, activedIfs)); + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url101", "mailto://toto@titi.org").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(valueEval, username, url1Required, password, activedIfs)); + + /* + * uniques + */ + final JsonObject try3 = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url", "http://t") + .add("username", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .build()) + .add("limit", 100)) + .build(); + checkAsserts(payload, Arrays.asList(valueEval, username, url1Required, password, activedIfs)); + + /* + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + */ + + /* + * valueEval @ActiveIf(target = "checkbox1", value = "true") + * + * TODO Fails + * + */ + + + /* + * uniqVals + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + + checkAsserts(payload, Arrays.asList(uniqVals, valueEval, username, password, activedIfs, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(uniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + + checkAsserts(payload, Arrays.asList(valueEval, username, password, activedIfs, url1Required)); + } + + + } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java index 935e5d46920bf..da17f09a3e6c2 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2022 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,21 @@ package org.talend.sdk.component.server.test.custom; import java.io.Serializable; +import java.util.List; +import java.util.Map; +import org.talend.sdk.component.api.component.MigrationHandler; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.condition.ActiveIf; +import org.talend.sdk.component.api.configuration.condition.ActiveIfs; +import org.talend.sdk.component.api.configuration.constraint.Max; +import org.talend.sdk.component.api.configuration.constraint.Min; +import org.talend.sdk.component.api.configuration.constraint.Pattern; +import org.talend.sdk.component.api.configuration.constraint.Required; +import org.talend.sdk.component.api.configuration.constraint.Uniques; +import org.talend.sdk.component.api.configuration.type.DataSet; +import org.talend.sdk.component.api.configuration.type.DataStore; import org.talend.sdk.component.api.processor.ElementListener; import org.talend.sdk.component.api.processor.Processor; import org.talend.sdk.component.api.record.Record; @@ -24,8 +38,87 @@ @Processor(family = "custom", name = "noop") public class CustomProcessor implements Serializable { + @Option + DataSetCustom dataset; + + public CustomProcessor (@Option("configuration") final DataSetCustom dataset) { + this.dataset = dataset; + } + @ElementListener public Record onElement(final Record data) { return data; } + + @DataSet("dataset") + static class DataSetCustom { + + @Option + @Required + private Connection connection; + + @Option + @Min(100) + @Max(150) + private int limit; + } + + @Version(migrationHandler = Connection.ConnectionMigration.class) + @DataStore("Connection") + static class Connection { + + @Option + @Pattern("^https?://.*") + private String url0; + + @Option + @Required + @Pattern("^https?://.*") + private String url1; + + @Option + @Required + private String username; + + @Option + @Required + @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + private String password; + + @Option + @Uniques + private List uniqVals; + + @Option + private boolean checkbox1; + @Option + private boolean checkbox2; + + @Option + @ActiveIf(target = "checkbox1", value = "true") + private ValueEval valueEval= ValueEval.VALUE_2; + + @Option + @Required + @ActiveIfs(operator = ActiveIfs.Operator.AND, value = { + @ActiveIf(target = "checkbox1", value = "true"), + @ActiveIf(target = "checkbox2", value = "true") + }) + private String activedIfs; + + enum ValueEval { + VALUE_1, + VALUE_2, + VALUE_3; + } + + static class ConnectionMigration implements MigrationHandler { + + @Override + public Map migrate(final int incomingVersion, final Map incomingData) { + incomingData.put("url", "http://migrated"); + return incomingData; + } + } + } } From 461a7f20c6cee540bd1cef47e7b024b4d7d1c9fa Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Fri, 9 Dec 2022 15:37:05 +0800 Subject: [PATCH 02/16] Fix[TCOMP-2260]: check for activeIf --- .../server/service/ValidationServiceTest.java | 163 +++++++++++++++--- 1 file changed, 141 insertions(+), 22 deletions(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java index ed1ccd6881072..043c4e1cb34d7 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -27,6 +27,8 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import javax.inject.Inject; import javax.json.Json; @@ -54,6 +56,7 @@ import org.talend.sdk.component.form.model.uischema.UiSchema; import org.talend.sdk.component.server.front.model.ConfigTypeNode; import org.talend.sdk.component.server.front.model.ConfigTypeNodes; +import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; import org.talend.sdk.component.server.test.ComponentClient; import lombok.AllArgsConstructor; @@ -121,6 +124,8 @@ static class ValidationError { public CompletionStage validate(final ConfigTypeNode config, final JsonObject properties) throws Exception { + findActiveIf(config, properties); + return getValidator(config) .thenApply(validator -> validator.apply(properties)) .thenApply(vr -> new Result(vr @@ -155,6 +160,46 @@ public CompletionStage getValidator(final ConfigTypeNode co .thenApply(factory::newInstance); } + private void findActiveIf(final ConfigTypeNode config, JsonObject properties) { + //TCOMP-2260: find out all activeIf's target , and its value in jsonObject; if it is not active, clear its validation + config + .getProperties() + .stream() + .filter(p -> p.getMetadata().get("condition::if::target") != null) + .forEach(simplePropertyDefinition -> { + if (isNotActive(simplePropertyDefinition, properties)) + simplePropertyDefinition.setValidation(null); + }); + } + + //should own the same first parent. + // 1) A.B.C depends on A.B.D, A.B -> A.C :support + // 2) A.B.C depends on A.D.E : TO check + private boolean isNotActive(final SimplePropertyDefinition simplePropertyDefinition, final JsonObject properties) { + String[] targetPath = simplePropertyDefinition.getMetadata().get("condition::if::target").split("\\."); + String[] selfPath = simplePropertyDefinition.getPath().split("\\."); + String targetValue = simplePropertyDefinition.getMetadata().get("condition::if::value"); + + AtomicBoolean isNotActive = new AtomicBoolean(false); + AtomicReference object = new AtomicReference<>(properties); + Arrays.stream(selfPath).forEach( onePath -> { + Arrays.stream(targetPath).forEach( targetP -> { + if (object.get().get(onePath) != null) { + if(object.get().get(onePath).getValueType().equals(JsonValue.ValueType.OBJECT)) { + object.set(object.get().get(onePath).asJsonObject()); + } else { + if (object.get().get(targetP) != null) { + String str = object.get().get(targetP).toString(); + isNotActive.set(str.contains(targetValue)); + } + } + } + }); + }); + + return !isNotActive.get(); + } + @BeforeEach void setup() throws Exception { details = base @@ -173,13 +218,14 @@ void setup() throws Exception { private void checkAsserts(JsonObject payload, List expected) throws Exception { Result errors = validate(connection, payload).toCompletableFuture().get(); - System.out.println("==="); + System.err.println(" ====== "+ expected.size()); long ecnt = errors.getErrors().stream() .peek(e -> log.warn("[checkAsserts] {}", e)) .filter(e -> !expected.contains(e)) .peek(e -> log.error("[checkAsserts]Validation uncaught: {}", e)) // .map(e-> fail(e)) .count(); +// System.out.println("Actual errors: "+ errors.getErrors().size() + ", uncaught errors: " + ecnt); expected.stream().forEach(e -> assertTrue(errors.getErrors().contains(e))); } @@ -210,7 +256,7 @@ void testValidation () throws Exception { payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connectn", "").build()).build(); - checkAsserts(payload, Arrays.asList(conn, valueEval)); + checkAsserts(payload, Arrays.asList(conn)); /** * min/max **/ @@ -219,14 +265,14 @@ void testValidation () throws Exception { .add("connection", JsonValue.NULL) .add("limit", 1)) .build(); - checkAsserts(payload, Arrays.asList(min, valueEval, conn)); + checkAsserts(payload, Arrays.asList(min, conn)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", JsonValue.NULL) .add("limit", 1000)) .build(); - checkAsserts(payload, Arrays.asList(max, valueEval, conn)); + checkAsserts(payload, Arrays.asList(max, conn)); /* * url0 pattern */ @@ -235,7 +281,7 @@ void testValidation () throws Exception { .add("connection", factory.createObjectBuilder() .add("url0", "mailto://toto@titi.org").build()) .build()).build(); - checkAsserts(payload, Arrays.asList(url0, valueEval, username, password, url1Required, activedIfs)); + checkAsserts(payload, Arrays.asList(url0, username, url1Required, activedIfs)); /* * url1 */ @@ -244,13 +290,14 @@ void testValidation () throws Exception { .add("connection", factory.createObjectBuilder() .add("url1", "mailto://toto@titi.org").build()) .build()).build(); - checkAsserts(payload, Arrays.asList(valueEval, username, url1Patten, password, activedIfs)); + checkAsserts(payload, Arrays.asList(username, url1Patten, activedIfs)); + payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("url101", "mailto://toto@titi.org").build()) .build()).build(); - checkAsserts(payload, Arrays.asList(valueEval, username, url1Required, password, activedIfs)); + checkAsserts(payload, Arrays.asList(username, url1Required, activedIfs)); /* * uniques @@ -264,19 +311,7 @@ void testValidation () throws Exception { .build()) .add("limit", 100)) .build(); - checkAsserts(payload, Arrays.asList(valueEval, username, url1Required, password, activedIfs)); - - /* - * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") - */ - - /* - * valueEval @ActiveIf(target = "checkbox1", value = "true") - * - * TODO Fails - * - */ - + checkAsserts(payload, Arrays.asList(username, url1Required, activedIfs)); /* * uniqVals @@ -295,7 +330,7 @@ void testValidation () throws Exception { .add("limit", 100)) .build(); - checkAsserts(payload, Arrays.asList(uniqVals, valueEval, username, password, activedIfs, url1Required)); + checkAsserts(payload, Arrays.asList(uniqVals, username, activedIfs, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -311,8 +346,92 @@ void testValidation () throws Exception { .add("limit", 100)) .build(); - checkAsserts(payload, Arrays.asList(valueEval, username, password, activedIfs, url1Required)); + checkAsserts(payload, Arrays.asList(username, activedIfs, url1Required)); } + @Test + void testValidation_activeIf () throws Exception { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + // + ValidationError url1Required = new ValidationError("/configuration/connection", "url1 is required and is not present"); + ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", "Invalid value, got null, expected: [\"VALUE_1\",\"VALUE_2\",\"VALUE_3\"]"); + ValidationError password = new ValidationError("/configuration/connection", "password is required and is not present"); + ValidationError activedIfs = new ValidationError("/configuration/connection", "activedIfs is required and is not present"); + + JsonObject payload; + + /* + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", JsonValue.NULL).build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url1Required, activedIfs, password)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("password", JsonValue.NULL).build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", "abc").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + } + + @Test + void testValidation_activeIf_2 () throws Exception { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + // + ValidationError url1Required = new ValidationError("/configuration/connection", "url1 is required and is not present"); + ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", "Invalid value, got null, expected: [\"VALUE_1\",\"VALUE_2\",\"VALUE_3\"]"); + ValidationError password = new ValidationError("/configuration/connection", "password is required and is not present"); + ValidationError activedIfs = new ValidationError("/configuration/connection", "activedIfs is required and is not present"); + + JsonObject payload; + + + /* + * valueEval @ActiveIf(target = "checkbox1", value = "true") + * + * TODO Fails + * + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", JsonValue.NULL).build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url1Required, activedIfs, valueEval)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("checkbox1", JsonValue.FALSE) + .add("valueEval", JsonValue.NULL).build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", "VALUE_2").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + + } + } From f8d5327d5ca9cd33205a1e692f3b71ea6a9c0aa0 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 15 Dec 2022 10:11:56 +0800 Subject: [PATCH 03/16] Fix[TCOMP-2260]:extract a method --- .../manager/reflect/ReflectionService.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index a81f48e22bdf7..2f0618e23dc0b 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -208,19 +208,23 @@ public Function, Object[]> parameterFactory(final Executable return config -> { final Map notNullConfig = ofNullable(config).orElseGet(Collections::emptyMap); - final PayloadValidator visitor = new PayloadValidator(); - if (!visitor.skip) { - visitor.globalPayload = new PayloadMapper((a, b) -> { - }).visitAndMap(metas, notNullConfig); - final PayloadMapper payloadMapper = new PayloadMapper(visitor); - payloadMapper.setGlobalPayload(visitor.globalPayload); - payloadMapper.visitAndMap(metas, notNullConfig); - visitor.throwIfFailed(); - } + checkPayload(metas, notNullConfig); return factories.stream().map(f -> f.apply(notNullConfig)).toArray(Object[]::new); }; } + public static void checkPayload(List metas, Map notNullConfig) { + final PayloadValidator visitor = new PayloadValidator(); + if (!visitor.skip) { + visitor.globalPayload = new PayloadMapper((a, b) -> { + }).visitAndMap(metas, notNullConfig); + final PayloadMapper payloadMapper = new PayloadMapper(visitor); + payloadMapper.setGlobalPayload(visitor.globalPayload); + payloadMapper.visitAndMap(metas, notNullConfig); + visitor.throwIfFailed(); + } + } + public Function, Object> createContextualSupplier(final ClassLoader loader) { return supplier -> { final Thread thread = Thread.currentThread(); From 71c4c962de982bbaa0ff0e2368462cb1b5643497 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 15 Dec 2022 10:17:11 +0800 Subject: [PATCH 04/16] Fix[TCOMP-2260]:add junits --- .../server/service/ValidationServiceTest.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java index 043c4e1cb34d7..dbb6eef733bf8 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -15,6 +15,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toList; import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; import static org.junit.Assert.assertTrue; @@ -29,6 +30,7 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import javax.inject.Inject; import javax.json.Json; @@ -42,6 +44,7 @@ import org.apache.johnzon.jsonschema.JsonSchemaValidator; import org.apache.johnzon.jsonschema.JsonSchemaValidatorFactory; import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; +import org.apache.xbean.propertyeditor.PropertyEditorRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.talend.sdk.component.form.api.Client; @@ -54,6 +57,10 @@ import org.talend.sdk.component.form.model.Ui; import org.talend.sdk.component.form.model.jsonschema.JsonSchema; import org.talend.sdk.component.form.model.uischema.UiSchema; +import org.talend.sdk.component.runtime.manager.ParameterMeta; +import org.talend.sdk.component.runtime.manager.reflect.ParameterModelService; +import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.BaseParameterEnricher; +import org.talend.sdk.component.runtime.manager.service.LocalConfigurationService; import org.talend.sdk.component.server.front.model.ConfigTypeNode; import org.talend.sdk.component.server.front.model.ConfigTypeNodes; import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; @@ -160,7 +167,9 @@ public CompletionStage getValidator(final ConfigTypeNode co .thenApply(factory::newInstance); } - private void findActiveIf(final ConfigTypeNode config, JsonObject properties) { + private void findActiveIf(final ConfigTypeNode config, JsonObject properties) throws NoSuchMethodException { + extracted(config.getProperties().get(0)); + extracted(config.getProperties().get(1)); //TCOMP-2260: find out all activeIf's target , and its value in jsonObject; if it is not active, clear its validation config .getProperties() @@ -172,6 +181,18 @@ private void findActiveIf(final ConfigTypeNode config, JsonObject properties) { }); } + private static void extracted(final SimplePropertyDefinition properties) throws NoSuchMethodException { + final ParameterModelService service = new ParameterModelService(new PropertyEditorRegistry()); +// final List metas = service +// .buildParameterMetas(SimplePropertyDefinition.class.getMethod("getMetadata"), +// "", new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "server"))); + final List metas2 = service.buildParameterMetas(Stream.of(new ParameterModelService.Param(SimplePropertyDefinition.class, SimplePropertyDefinition.class.getAnnotations(), properties.getName())), + SimplePropertyDefinition.class, + ofNullable(SimplePropertyDefinition.class.getPackage()).map(Package::getName).orElse(""), true, + new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "server-model"))); + System.err.println("size: " + metas2.size()); + } + //should own the same first parent. // 1) A.B.C depends on A.B.D, A.B -> A.C :support // 2) A.B.C depends on A.D.E : TO check @@ -213,8 +234,19 @@ void setup() throws Exception { Ui ui = uiSpecService.convert(family, lang, connection, null).toCompletableFuture().get(); jsonSchema = ui.getJsonSchema(); uiSchemas = ui.getUiSchema(); + int i = 0; } + @Test + public void test1(){ + final ParameterModelService service = new ParameterModelService(new PropertyEditorRegistry()); + final List metas2 = service.buildParameterMetas(Stream.of(new ParameterModelService.Param(JsonSchema.class, + JsonSchema.class.getAnnotations(), jsonSchema.getTitle())), + JsonSchema.class, + ofNullable(JsonSchema.class.getPackage()).map(Package::getName).orElse(""), true, + new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "server-model"))); + System.err.println("size: " + metas2.size()); + } private void checkAsserts(JsonObject payload, List expected) throws Exception { Result errors = validate(connection, payload).toCompletableFuture().get(); From 44ebabaca61e257e059e30bf5776fc6d9554e308 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 11 Jan 2023 16:48:11 +0800 Subject: [PATCH 05/16] Fix[TCOMP-2260]: extract a common method for 2 ways. --- .../manager/reflect/ReflectionService.java | 9 +- .../server/front/model/ConfigTypeNode.java | 3 + .../server/service/PropertiesService.java | 96 +++++++++++-- .../service/PropertyValidationService.java | 56 ++++++-- .../server/service/PropertiesServiceTest.java | 136 +++++++++++++++--- .../server/service/ValidationServiceTest.java | 8 +- 6 files changed, 262 insertions(+), 46 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index 2f0618e23dc0b..1930a689aef4c 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -225,6 +225,14 @@ public static void checkPayload(List metas, Map n } } +// public static void checkPayload(List metas, Map notNullConfig, JsonObject payload) { +// final PayloadValidator visitor = new PayloadValidator(); +// final PayloadMapper payloadMapper = new PayloadMapper(visitor); +// payloadMapper.setGlobalPayload(payload); +// payloadMapper.visitAndMap(metas, notNullConfig); +// visitor.throwIfFailed(); +// } + public Function, Object> createContextualSupplier(final ClassLoader loader) { return supplier -> { final Thread thread = Thread.currentThread(); @@ -912,7 +920,6 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) { if (!VISIBILITY_SERVICE.build(meta).isVisible(globalPayload)) { return; } - if (Boolean.parseBoolean(meta.getMetadata().get("tcomp::validation::required")) && value == JsonValue.NULL) { errors.add(MESSAGES.required(meta.getPath())); diff --git a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java index 3c5b4c5a30d89..a6b9cae44a33b 100644 --- a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java +++ b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java @@ -48,4 +48,7 @@ public class ConfigTypeNode { private Collection actions; + public void visibility(final SimplePropertyDefinition spd){ + //no-op + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 8baf37d00e683..d18bdde5647ba 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -15,31 +15,35 @@ */ package org.talend.sdk.component.server.service; +import static java.util.Collections.emptyList; import static java.util.Optional.ofNullable; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toSet; import static org.talend.sdk.component.server.lang.CustomCollectors.toLinkedMap; -import java.util.AbstractMap; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonPointer; +import javax.json.JsonValue; import javax.json.bind.Jsonb; +import javax.json.spi.JsonProvider; +import org.apache.cxf.common.util.StringUtils; import org.talend.sdk.component.runtime.internationalization.ParameterBundle; import org.talend.sdk.component.runtime.manager.ParameterMeta; +import org.talend.sdk.component.runtime.manager.reflect.ReflectionService; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.ValidationParameterEnricher; import org.talend.sdk.component.runtime.manager.util.DefaultValueInspector; import org.talend.sdk.component.server.configuration.ComponentServerConfiguration; +import org.talend.sdk.component.server.front.model.ConfigTypeNode; import org.talend.sdk.component.server.front.model.PropertyValidation; import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; import org.talend.sdk.component.server.service.qualifier.ComponentServer; @@ -67,6 +71,59 @@ public Stream buildProperties(final List from JsonObject(payload). + private Map extractConfig(ConfigTypeNode configNode, JsonObject payload) { + Map payloadConfig = new HashMap<>(); + for(SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { + JsonValue value = toPointer(propertyDefinition.getPath()).getValue(payload); + if (value != null) { + payloadConfig.put(propertyDefinition.getPath(), value.toString()); + } + } + + return payloadConfig; + } + + private List buildParameterMetas(List properties){ + List parameterMetaList = new ArrayList<>(); + for(SimplePropertyDefinition propertyDefinition : properties) { + final String path = sanitizePropertyName(propertyDefinition.getPath()); + final String name = sanitizePropertyName(propertyDefinition.getName()); + final ParameterMeta.Type type = findType(propertyDefinition.getType()); + //translate PropertyValidation + final Map sanitizedMetadata = ofNullable(getSanitizedMetadata(propertyDefinition.getMetadata())) + .orElse(new LinkedHashMap<>()); + if (propertyDefinition.getValidation() != null) { + sanitizedMetadata.putAll(propertyValidationService.mapMeta(propertyDefinition.getValidation())); + } + + parameterMetaList.add(new ParameterMeta(null, null, type, path, name,null, + emptyList(), null, sanitizedMetadata, false)); + } + return parameterMetaList; + } + + private static LinkedHashMap getSanitizedMetadata(Map p) { + return ofNullable(p) + .map(m -> m + .entrySet() + .stream() + .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) + .collect(toLinkedMap(e -> e.getKey().replace("tcomp::", ""), Map.Entry::getValue))) + .orElse(null); + } + + private Stream buildProperties(final List meta, final ClassLoader loader, final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { return meta.stream().flatMap(p -> { @@ -81,13 +138,7 @@ private Stream buildProperties(final List sanitizedMetadata = ofNullable(p.getMetadata()) - .map(m -> m - .entrySet() - .stream() - .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) - .collect(toLinkedMap(e -> e.getKey().replace("tcomp::", ""), Map.Entry::getValue))) - .orElse(null); + final Map sanitizedMetadata = getSanitizedMetadata(p.getMetadata()); final Map metadata; if (parent != null) { metadata = sanitizedMetadata; @@ -182,4 +233,21 @@ private String toDefault(final DefaultValueInspector.Instance instance, final Pa private String sanitizePropertyName(final String path) { return path.replace("${index}", ""); } + + private ParameterMeta.Type findType(final String type) { + if (ParameterMeta.Type.STRING.name().equals(type)) { + return ParameterMeta.Type.STRING; + } else if (ParameterMeta.Type.BOOLEAN.name().equals(type)) { + return ParameterMeta.Type.BOOLEAN; + } else if (ParameterMeta.Type.NUMBER.name().equals(type)) { + return ParameterMeta.Type.NUMBER; + } else if (ParameterMeta.Type.ENUM.name().equals(type)) { + return ParameterMeta.Type.ENUM; + } else if (ParameterMeta.Type.ARRAY.name().equals(type)) { + return ParameterMeta.Type.ARRAY; + } else if (ParameterMeta.Type.OBJECT.name().equals(type)) { + return ParameterMeta.Type.OBJECT; + } + return ParameterMeta.Type.OBJECT; + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java index 74d7e6516ecec..1a6bb2bb159e0 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2022 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ import static java.util.stream.Collectors.toList; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; @@ -35,6 +36,8 @@ public class PropertyValidationService { private Function, PropertyValidation> propertyValidationCreator; + private Function> propertyMetaCreator; + @PostConstruct private void initMapper() { // precompute the mapping of validations to centralize the convention - note: can be moved to impl for setters @@ -56,28 +59,61 @@ private void initMapper() { f.setAccessible(true); } return (BiFunction, Boolean>) (instance, - meta) -> ofNullable(meta.get(ValidationParameterEnricher.META_PREFIX + f.getName())) - .map(valueConverter) + meta) -> ofNullable(meta.get(ValidationParameterEnricher.META_PREFIX + f.getName())) + .map(valueConverter) + .map(val -> { + try { + f.set(instance, val); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + return true; + }) + .orElse(false); + }).collect(toList()); + propertyValidationCreator = config -> { + final PropertyValidation validation = new PropertyValidation(); + if (validationSetters.stream().filter(s -> s.apply(validation, config)).count() == 0) { + return null; + } + return validation; + }; + + final Collection, Boolean>> validationGetters = + Stream.of(PropertyValidation.class.getDeclaredFields()).map(f -> { + f.setAccessible(true); + return (BiFunction, Boolean>) (instance, + meta) -> { + try { + return ofNullable(f.get(instance)) .map(val -> { try { - f.set(instance, val); + meta.put(ValidationParameterEnricher.META_PREFIX + f.getName(), f.get(instance).toString()); } catch (IllegalAccessException e) { throw new IllegalStateException(e); } return true; }) .orElse(false); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + }; }).collect(toList()); - propertyValidationCreator = config -> { - final PropertyValidation validation = new PropertyValidation(); - if (validationSetters.stream().filter(s -> s.apply(validation, config)).count() == 0) { + propertyMetaCreator = config -> { + final Map meta = new HashMap<>(); + if (validationGetters.stream().filter(s -> s.apply(config, meta)).count() == 0) { return null; } - return validation; + return meta; }; } public PropertyValidation map(final Map meta) { return propertyValidationCreator.apply(meta); } + + public Map mapMeta(final PropertyValidation propertyValidation) { + return propertyMetaCreator.apply(propertyValidation); + } } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index cdd7bf8c54c50..dab1ddc2591c3 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -15,24 +15,7 @@ */ package org.talend.sdk.component.server.service; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - -import javax.inject.Inject; - +import lombok.Data; import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; import org.apache.xbean.propertyeditor.PropertyEditorRegistry; import org.junit.jupiter.api.Test; @@ -42,9 +25,26 @@ import org.talend.sdk.component.runtime.manager.reflect.ParameterModelService; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.BaseParameterEnricher; import org.talend.sdk.component.runtime.manager.service.LocalConfigurationService; +import org.talend.sdk.component.server.front.model.ConfigTypeNode; +import org.talend.sdk.component.server.front.model.ConfigTypeNodes; import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; -import lombok.Data; +import javax.inject.Inject; +import javax.json.Json; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonValue; +import javax.ws.rs.client.WebTarget; +import java.util.*; + +import static java.util.Arrays.asList; +import static java.util.Collections.*; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @MonoMeecrowaveConfig class PropertiesServiceTest { @@ -52,6 +52,11 @@ class PropertiesServiceTest { @Inject private PropertiesService propertiesService; + @Inject + private WebTarget base; + + private ConfigTypeNode connection; + private static void gridLayout(@Option final WithLayout layout) { // no-op } @@ -172,6 +177,99 @@ void validateProp() { }); } + @Test // the class BaseConfig don't contains attribute + void validateConfigNode() { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + final List uniques = Arrays.asList("one", "two", "three"); + final List notUniques = Arrays.asList("one", "two", "three", "one"); + + String activeIfsError = "- Property 'configuration.connection.activedIfs' is required."; + String connectionError = "- Property 'configuration.connection' is required."; + String passwordError = "- Property 'configuration.connection.password' is required."; + String url1Error = "- Property 'configuration.connection.url1' is required."; + String usernameError = "- Property 'configuration.connection.username' is required."; + String minError = "Property 'limit' min"; + String maxError = "Property 'limit' max"; + + connection = base + .path("configurationtype/details") + .queryParam("identifiers", "Y29tcG9uZW50LXdpdGgtdXNlci1qYXJzI2N1c3RvbSNkYXRhc2V0I2RhdGFzZXQ") + .request(APPLICATION_JSON_TYPE) + .header("Accept-Encoding", "gzip") + .get(ConfigTypeNodes.class) + .getNodes().values().iterator().next(); + + JsonObject payload; + + /** + * min/max + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url", "http://t") + .add("username", "abc") + .add("password", "aaa") + .build()) + .add("limit", 100)) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error, minError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("limit", 1000)).build()) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error, maxError)); + + /* + * url0 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "mailto://toto@titi.org").build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Error, usernameError)); + + /* + * uniqVals + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Error, usernameError)); + + } + + private void checkErrors(JsonObject payload, List expected) { + try { + propertiesService.validate(connection, payload); + if (expected != null && expected.size() >0) { + fail("There should be errors: "+expected); + } + } catch (Exception errors) { + StringBuffer expectedBuffer = new StringBuffer(); + expected.stream().forEach(e -> { + expectedBuffer.append(e).append("\n"); + }); + expectedBuffer.delete(expectedBuffer.lastIndexOf("\n"), expectedBuffer.length()); + assertEquals(expectedBuffer.toString(), errors.getMessage()); + } + } + @Data public static class BoolBool { diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java index dbb6eef733bf8..449709da7d223 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -238,9 +238,13 @@ void setup() throws Exception { } @Test - public void test1(){ + public void test1() throws NoSuchMethodException { final ParameterModelService service = new ParameterModelService(new PropertyEditorRegistry()); - final List metas2 = service.buildParameterMetas(Stream.of(new ParameterModelService.Param(JsonSchema.class, + final List metas = service + .buildParameterMetas(ConfigTypeNodes.class.getMethod("visibility", SimplePropertyDefinition.class), + "def", new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "test"))); + + final List metas2 = service.buildParameterMetas(Stream.of(new ParameterModelService.Param(JsonSchema.class, JsonSchema.class.getAnnotations(), jsonSchema.getTitle())), JsonSchema.class, ofNullable(JsonSchema.class.getPackage()).map(Package::getName).orElse(""), true, From aba50562c62433fc9e2af06e3f1099609e1e9416 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Fri, 3 Mar 2023 16:54:42 +0800 Subject: [PATCH 06/16] Feat[TCOMP-2260]: combine 2 validations into 1, and make common checks passed. --- .../reflect/visibility/PayloadMapper.java | 10 +- .../server/service/PropertiesService.java | 30 +++- .../service/PropertyValidationService.java | 51 +++--- .../server/service/PropertiesServiceTest.java | 149 ++++++++++++++++-- 4 files changed, 187 insertions(+), 53 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java index 00c2e70079e36..5216da8f4b6c5 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java @@ -84,7 +84,7 @@ private void onProperty(final String contextualPrefix, final Collection json.add(name, Boolean.parseBoolean(v))); break; case NUMBER: - final String numberValue = config.get(newPath); + final String numberValue = getValue(config, newPath, definition); if (numberValue == null || numberValue.isEmpty()) { parameterVisitor.onParameter(definition, JsonValue.NULL); } else { @@ -114,7 +114,7 @@ private void onProperty(final String contextualPrefix, final Collection json.add(name, v)); break; @@ -123,6 +123,10 @@ private void onProperty(final String contextualPrefix, final Collection config, String newPath, ParameterMeta definition) { + return config.get(newPath) == null ? config.get(definition.getPath()) : config.get(newPath); + } + private void onObject(final Collection definitions, final ParameterMeta meta, final Map config, final JsonObjectBuilder json, final String name, final String currentPath) { diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index d18bdde5647ba..847646fc32c32 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -29,10 +29,7 @@ import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; -import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; -import javax.json.JsonPointer; -import javax.json.JsonValue; +import javax.json.*; import javax.json.bind.Jsonb; import javax.json.spi.JsonProvider; @@ -85,15 +82,32 @@ private JsonPointer toPointer(final String absoluteTargetPath) { private Map extractConfig(ConfigTypeNode configNode, JsonObject payload) { Map payloadConfig = new HashMap<>(); for(SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { - JsonValue value = toPointer(propertyDefinition.getPath()).getValue(payload); - if (value != null) { - payloadConfig.put(propertyDefinition.getPath(), value.toString()); - } + try { + JsonValue value = toPointer(propertyDefinition.getPath()).getValue(payload); + if (value != null && !JsonObject.class.isInstance(value)) { + payloadConfig.put(propertyDefinition.getPath(), getValue(value)); + } + } catch (JsonException e) { + continue; + } } return payloadConfig; } + private String getValue(JsonValue value) { + switch (value.getValueType()) { + case TRUE: + case FALSE: + case NUMBER: + return String.valueOf(value); + case STRING: + return JsonString.class.cast(value).getString(); + default: + return null; + } + } + private List buildParameterMetas(List properties){ List parameterMetaList = new ArrayList<>(); for(SimplePropertyDefinition propertyDefinition : properties) { diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java index 1a6bb2bb159e0..7ac5000f4a4c3 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java @@ -79,34 +79,6 @@ private void initMapper() { return validation; }; - final Collection, Boolean>> validationGetters = - Stream.of(PropertyValidation.class.getDeclaredFields()).map(f -> { - f.setAccessible(true); - return (BiFunction, Boolean>) (instance, - meta) -> { - try { - return ofNullable(f.get(instance)) - .map(val -> { - try { - meta.put(ValidationParameterEnricher.META_PREFIX + f.getName(), f.get(instance).toString()); - } catch (IllegalAccessException e) { - throw new IllegalStateException(e); - } - return true; - }) - .orElse(false); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - }; - }).collect(toList()); - propertyMetaCreator = config -> { - final Map meta = new HashMap<>(); - if (validationGetters.stream().filter(s -> s.apply(config, meta)).count() == 0) { - return null; - } - return meta; - }; } public PropertyValidation map(final Map meta) { @@ -114,6 +86,27 @@ public PropertyValidation map(final Map meta) { } public Map mapMeta(final PropertyValidation propertyValidation) { - return propertyMetaCreator.apply(propertyValidation); + final Map metaMap = new HashMap<>(); + + Stream.of(PropertyValidation.class.getDeclaredFields()).filter(field -> { + try { + field.setAccessible(true); + return field.get(propertyValidation) != null; + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + }) + .forEach(field -> { + field.setAccessible(true); + try { + Object value = field.get(propertyValidation); + if (value != null) { + metaMap.put(ValidationParameterEnricher.META_PREFIX + field.getName(), String.valueOf(value));//f.get(instance).toString()); + } + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + return metaMap; } } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index dab1ddc2591c3..0ccb8ed3f0f07 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -186,10 +186,12 @@ void validateConfigNode() { String activeIfsError = "- Property 'configuration.connection.activedIfs' is required."; String connectionError = "- Property 'configuration.connection' is required."; String passwordError = "- Property 'configuration.connection.password' is required."; - String url1Error = "- Property 'configuration.connection.url1' is required."; + String url1Required = "- Property 'configuration.connection.url1' is required."; + String url0Error = "- 'configuration.connection.url0' does not match '^https?://.*'."; + String url1Error = "- 'configuration.connection.url1' does not match '^https?://.*'."; String usernameError = "- Property 'configuration.connection.username' is required."; - String minError = "Property 'limit' min"; - String maxError = "Property 'limit' max"; + String minError = "- Property 'configuration.limit' should be > 100, got 99."; + String maxError = "- Property 'configuration.limit' should be < 150, got 200."; connection = base .path("configurationtype/details") @@ -201,28 +203,52 @@ void validateConfigNode() { JsonObject payload; + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("password", JsonValue.NULL).build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + + /** * min/max **/ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() - .add("url", "http://t") + .add("url0", "http://t") .add("username", "abc") .add("password", "aaa") .build()) - .add("limit", 100)) + .add("limit", 110)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error, minError)); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "https://t") + .add("url1", "https://t") + .add("username", "abc") + .add("password", "aaa") + .build()) + .add("limit", 99)) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, minError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "abc") .add("password", "abc") - .add("limit", 1000)).build()) + .add("url0", "https://t") + .add("url1", "https://t") + .build()) + .add("limit", 200)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error, maxError)); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, maxError)); /* * url0 pattern @@ -230,9 +256,36 @@ void validateConfigNode() { payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() - .add("url0", "mailto://toto@titi.org").build()) - .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Error, usernameError)); + .add("username", "abc") + .add("password", "abc") + .add("url0", "https://www.talend.com") + .build())) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url0", "mailto://toto@titi.org") + .build())) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url0Error, url1Required)); + + /* + * url1 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url1", "mailto://toto@titi.org") + .build())) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + /* * uniqVals @@ -242,7 +295,7 @@ void validateConfigNode() { .add("connection", factory.createObjectBuilder() .add("url0", JsonValue.NULL) .add("url1", JsonValue.NULL) - .add("username", JsonValue.NULL) + .add("username", "abc") .add("password", JsonValue.NULL) .add("uniqVals", factory.createArrayBuilder(notUniques).build()) .add("checkbox1", JsonValue.TRUE) @@ -250,8 +303,78 @@ void validateConfigNode() { .build()) .add("limit", 100)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Error, usernameError)); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Required)); + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(uniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Required, usernameError)); + + /* + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", JsonValue.NULL).build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("password", JsonValue.NULL).build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", "abc").build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + + /* + * valueEval @ActiveIf(target = "checkbox1", value = "true") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", JsonValue.NULL).build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("checkbox1", JsonValue.FALSE) + .add("valueEval", JsonValue.NULL).build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", "VALUE_2").build()) + .build()).build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); } private void checkErrors(JsonObject payload, List expected) { From a465acb9cfa93d5842399890719bf314c7f68639 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Mon, 6 Mar 2023 18:09:28 +0800 Subject: [PATCH 07/16] Feat{TCOMP-2260]: refactor: make password check passed, --- .../server/service/PropertiesService.java | 12 +++++- .../server/service/PropertiesServiceTest.java | 37 +++++++------------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 847646fc32c32..7a06efb7fcfce 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -115,7 +115,7 @@ private List buildParameterMetas(List p final String name = sanitizePropertyName(propertyDefinition.getName()); final ParameterMeta.Type type = findType(propertyDefinition.getType()); //translate PropertyValidation - final Map sanitizedMetadata = ofNullable(getSanitizedMetadata(propertyDefinition.getMetadata())) + final Map sanitizedMetadata = ofNullable(getParamMetadata(propertyDefinition.getMetadata())) .orElse(new LinkedHashMap<>()); if (propertyDefinition.getValidation() != null) { sanitizedMetadata.putAll(propertyValidationService.mapMeta(propertyDefinition.getValidation())); @@ -127,6 +127,16 @@ private List buildParameterMetas(List p return parameterMetaList; } + private static LinkedHashMap getParamMetadata(Map p) { + return ofNullable(p) + .map(m -> m + .entrySet() + .stream() + .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) + .collect(toLinkedMap(e -> e.getKey().replace("condition::", "tcomp::condition::"), Map.Entry::getValue))) + .orElse(null); + } + private static LinkedHashMap getSanitizedMetadata(Map p) { return ofNullable(p) .map(m -> m diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index 0ccb8ed3f0f07..9f8470a7b354b 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -203,15 +203,6 @@ void validateConfigNode() { JsonObject payload; - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("username", "abcd") - .add("password", JsonValue.NULL).build()) - .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); - - /** * min/max **/ @@ -224,7 +215,7 @@ void validateConfigNode() { .build()) .add("limit", 110)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -236,7 +227,7 @@ void validateConfigNode() { .build()) .add("limit", 99)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, minError)); + checkErrors(payload, Arrays.asList(connectionError, minError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -248,7 +239,7 @@ void validateConfigNode() { .build()) .add("limit", 200)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, maxError)); + checkErrors(payload, Arrays.asList(connectionError, maxError)); /* * url0 pattern @@ -261,7 +252,7 @@ void validateConfigNode() { .add("url0", "https://www.talend.com") .build())) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -271,7 +262,7 @@ void validateConfigNode() { .add("url0", "mailto://toto@titi.org") .build())) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url0Error, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, url0Error, url1Required)); /* * url1 pattern @@ -284,7 +275,7 @@ void validateConfigNode() { .add("url1", "mailto://toto@titi.org") .build())) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + checkErrors(payload, Arrays.asList(connectionError, url1Error)); /* @@ -303,7 +294,7 @@ void validateConfigNode() { .build()) .add("limit", 100)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -319,7 +310,7 @@ void validateConfigNode() { .add("limit", 100)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Required, usernameError)); + checkErrors(payload, Arrays.asList(connectionError, url1Required, usernameError)); /* * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") @@ -330,7 +321,7 @@ void validateConfigNode() { .add("username", "undx") .add("password", JsonValue.NULL).build()) .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, passwordError, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, passwordError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -338,7 +329,7 @@ void validateConfigNode() { .add("username", "abcd") .add("password", JsonValue.NULL).build()) .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() @@ -347,7 +338,7 @@ void validateConfigNode() { .add("username", "undx") .add("password", "abc").build()) .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); /* * valueEval @ActiveIf(target = "checkbox1", value = "true") @@ -358,7 +349,7 @@ void validateConfigNode() { .add("checkbox1", JsonValue.TRUE) .add("valueEval", JsonValue.NULL).build()) .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + checkErrors(payload, Arrays.asList(connectionError, url1Required, usernameError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -366,7 +357,7 @@ void validateConfigNode() { .add("checkbox1", JsonValue.FALSE) .add("valueEval", JsonValue.NULL).build()) .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + checkErrors(payload, Arrays.asList(connectionError, url1Error)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -374,7 +365,7 @@ void validateConfigNode() { .add("checkbox1", JsonValue.TRUE) .add("valueEval", "VALUE_2").build()) .build()).build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Error)); + checkErrors(payload, Arrays.asList(connectionError, url1Error)); } private void checkErrors(JsonObject payload, List expected) { From 50cd4ea4611ade8e61a8454e80c5c5fdee89fff6 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Thu, 9 Mar 2023 16:08:00 +0800 Subject: [PATCH 08/16] except array, others passed --- .../runtime/manager/reflect/ReflectionService.java | 11 ++++++++--- .../component/server/service/PropertiesService.java | 4 ++-- .../server/service/PropertiesServiceTest.java | 11 +++++++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index 1930a689aef4c..48b57d13b6115 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -214,12 +214,17 @@ public Function, Object[]> parameterFactory(final Executable } public static void checkPayload(List metas, Map notNullConfig) { + JsonObject globalPayload = new PayloadMapper((a, b) -> { + }).visitAndMap(metas, notNullConfig); + checkWithPayload(metas, notNullConfig, globalPayload); + } + + public static void checkWithPayload(List metas, Map notNullConfig, final JsonObject payload) { final PayloadValidator visitor = new PayloadValidator(); if (!visitor.skip) { - visitor.globalPayload = new PayloadMapper((a, b) -> { - }).visitAndMap(metas, notNullConfig); + visitor.globalPayload = payload; final PayloadMapper payloadMapper = new PayloadMapper(visitor); - payloadMapper.setGlobalPayload(visitor.globalPayload); + payloadMapper.setGlobalPayload(payload); payloadMapper.visitAndMap(metas, notNullConfig); visitor.throwIfFailed(); } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 7a06efb7fcfce..6d3c752a54054 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -69,8 +69,8 @@ public Stream buildProperties(final List Date: Wed, 15 Mar 2023 14:49:58 +0800 Subject: [PATCH 09/16] extract into one way and all junit cases passed --- .../manager/reflect/ReflectionService.java | 111 +++++++++--------- .../manager/reflect/Messages.properties | 1 + .../server/service/PropertiesService.java | 42 +++---- .../server/service/PropertiesServiceTest.java | 41 ++++--- 4 files changed, 100 insertions(+), 95 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index 48b57d13b6115..14666cf8c42f6 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2022 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,18 +33,7 @@ import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.BiFunction; @@ -101,7 +90,7 @@ public class ReflectionService { // IMPORTANT: ensure to be able to read all data (including collection) from a // map to support system properties override public Function, Object[]> parameterFactory(final Executable executable, - final Map, Object> precomputed, final List metas) { + final Map, Object> precomputed, final List metas) { final ClassLoader loader = Thread.currentThread().getContextClassLoader(); final Function, Object> contextualSupplier = createContextualSupplier(loader); final Collection, Object>> factories = @@ -215,7 +204,7 @@ public Function, Object[]> parameterFactory(final Executable public static void checkPayload(List metas, Map notNullConfig) { JsonObject globalPayload = new PayloadMapper((a, b) -> { - }).visitAndMap(metas, notNullConfig); + }).visitAndMap(metas, notNullConfig); checkWithPayload(metas, notNullConfig, globalPayload); } @@ -252,8 +241,8 @@ public Function, Object> createContextualSupplier(final ClassLo } public Function, Object> createConfigFactory(final Map, Object> precomputed, - final ClassLoader loader, final Function, Object> contextualSupplier, final String name, - final Configuration configuration, final Annotation[] allAnnotations, final Class configClass) + final ClassLoader loader, final Function, Object> contextualSupplier, final String name, + final Configuration configuration, final Annotation[] allAnnotations, final Class configClass) throws NoSuchMethodException { final Constructor constructor = configClass.getConstructor(); final LocalConfiguration config = LocalConfiguration.class.cast(precomputed.get(LocalConfiguration.class)); @@ -305,7 +294,7 @@ private List translate(final List metas, final Str } private Collector createMapCollector(final Class mapType, final Class keyItemClass, - final Class valueItemClass, final Map, Object> precomputed) { + final Class valueItemClass, final Map, Object> precomputed) { final Function, Object> keyMapper = o -> doConvert(keyItemClass, o.getKey(), precomputed); final Function, Object> valueMapper = o -> doConvert(valueItemClass, o.getValue(), precomputed); return ConcurrentMap.class.isAssignableFrom(mapType) ? toConcurrentMap(keyMapper, valueMapper) @@ -313,9 +302,9 @@ private Collector createMapCollector(final Class mapType, final Class keyI } private Object createList(final ClassLoader loader, final Function, Object> contextualSupplier, - final String name, final Class collectionType, final Class itemClass, final Collector collector, - final BiFunction, Object> itemFactory, final Map config, - final List metas, final Map, Object> precomputed) { + final String name, final Class collectionType, final Class itemClass, final Collector collector, + final BiFunction, Object> itemFactory, final Map config, + final List metas, final Map, Object> precomputed) { final Object obj = config.get(name); if (collectionType.isInstance(obj)) { return Collection.class @@ -336,7 +325,7 @@ private Object createList(final ClassLoader loader, final Function k.startsWith(configName + "."))) { // object - // mapping + // mapping if (paramIdx == 0) { args = findArgsName(itemClass); } @@ -363,9 +352,9 @@ private Integer getArrayMaxLength(final String prefix, final Map } private Object createMap(final String name, final Class mapType, - final BiFunction, Object> keyItemFactory, - final BiFunction, Object> valueItemFactory, final Collector collector, - final Map config) { + final BiFunction, Object> keyItemFactory, + final BiFunction, Object> valueItemFactory, final Collector collector, + final Map config) { final Object obj = config.get(name); if (mapType.isInstance(obj)) { return Map.class.cast(obj).entrySet().stream().collect(collector); @@ -395,8 +384,8 @@ private Object createMap(final String name, final Class mapType, } private BiFunction, Object> createObjectFactory(final ClassLoader loader, - final Function, Object> contextualSupplier, final Type type, - final List metas, final Map, Object> precomputed) { + final Function, Object> contextualSupplier, final Type type, + final List metas, final Map, Object> precomputed) { final Class clazz = Class.class.cast(type); if (clazz.isPrimitive() || Primitives.unwrap(clazz) != clazz || String.class == clazz) { return (name, config) -> doConvert(clazz, config.get(name), precomputed); @@ -425,7 +414,7 @@ private String[] findArgsName(final Class clazz) { } private JsonValue createJsonValue(final Object value, final Map, Object> precomputed, - final Function fallbackReaderCreator) { + final Function fallbackReaderCreator) { final StringReader sr = new StringReader(String.valueOf(value).trim()); try (final JsonReader reader = ofNullable(precomputed.get(JsonReaderFactory.class)) .map(JsonReaderFactory.class::cast) @@ -436,8 +425,8 @@ private JsonValue createJsonValue(final Object value, final Map, Object } private Object createObject(final ClassLoader loader, final Function, Object> contextualSupplier, - final Class clazz, final String[] args, final String name, final Map config, - final List metas, final Map, Object> precomputed) { + final Class clazz, final String[] args, final String name, final Map config, + final List metas, final Map, Object> precomputed) { final Object potentialJsonValue = config.get(name); if (JsonObject.class == clazz && String.class.isInstance(potentialJsonValue)) { return createJsonValue(potentialJsonValue, precomputed, Json::createReader).asJsonObject(); @@ -468,11 +457,11 @@ private Object createObject(final ClassLoader loader, final Function mapEntries = specificMapping.entrySet().stream().filter(e -> { - final String key = e.getKey(); - final int idxStart = key.indexOf('[', prefix.length()); - return idxStart > 0 && ((idxStart > ".key".length() && key.startsWith(".key", idxStart - ".key".length())) - || (idxStart > ".value".length() && key.startsWith(".value", idxStart - ".value".length()))); - }) + final String key = e.getKey(); + final int idxStart = key.indexOf('[', prefix.length()); + return idxStart > 0 && ((idxStart > ".key".length() && key.startsWith(".key", idxStart - ".key".length())) + || (idxStart > ".value".length() && key.startsWith(".value", idxStart - ".value".length()))); + }) .sorted(this::sortIndexEntry) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, noMerge(), LinkedHashMap::new)); mapEntries.keySet().forEach(specificMapping::remove); @@ -512,12 +501,12 @@ private Object createObject(final ClassLoader loader, final Function listEntries = specificMapping.entrySet().stream().filter(e -> { - final String key = e.getKey(); - final int idxStart = key.indexOf('[', prefix.length()); - final int idxEnd = key.indexOf(']', prefix.length()); - final int sep = key.indexOf('.', prefix.length() + 1); - return idxStart > 0 && key.endsWith("]") && (sep > idxEnd || sep < 0); - }) + final String key = e.getKey(); + final int idxStart = key.indexOf('[', prefix.length()); + final int idxEnd = key.indexOf(']', prefix.length()); + final int sep = key.indexOf('.', prefix.length() + 1); + return idxStart > 0 && key.endsWith("]") && (sep > idxEnd || sep < 0); + }) .sorted(this::sortIndexEntry) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, noMerge(), LinkedHashMap::new)); listEntries.keySet().forEach(specificMapping::remove); @@ -679,7 +668,7 @@ private Object createObject(final ClassLoader loader, final Function, Object> contextualSupplier, - final Map config, final String prefix, final Map preparedObjects, - final String nestedName, final String listName, final ParameterizedType pt, final Supplier init, - final List metas, final Map, Object> precomputed) { + final Map config, final String prefix, final Map preparedObjects, + final String nestedName, final String listName, final ParameterizedType pt, final Supplier init, + final List metas, final Map, Object> precomputed) { // CHECKSTYLE:ON final Collection aggregator = Collection.class.cast(preparedObjects.computeIfAbsent(listName, k -> init.get())); @@ -904,6 +893,8 @@ public interface Messages { String uniqueItems(String property); String pattern(String property, String pattern); + + String enumValues(String property, String enums, String val); } @RequiredArgsConstructor @@ -1012,6 +1003,20 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) { } } } + { + final String enumValues = metadata.get("tcomp::validation::enumValues"); + if (enumValues != null) {//value = null or not in enumValues: add error + if (value == null) { + errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, null)); + } else { + final String val = JsonValue.class.cast(value).toString().replace("\"", ""); + String[] enums = enumValues.substring(1, enumValues.length() - 1).split(","); + if (Arrays.stream(enums).noneMatch(s -> s.trim().equalsIgnoreCase(val))) { + errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, val)); + } + } + } + } } private void throwIfFailed() { @@ -1023,7 +1028,7 @@ private void throwIfFailed() { /** * Helper function for creating an instance from a configuration map. - * + * * @param clazz Class of the wanted instance. * @param Type managed * @return function that generate the wanted instance when calling @@ -1037,11 +1042,11 @@ public BiFunction, T> createObjectFactory(final if (clazz.isEnum()) { return (name, config) -> (T) ofNullable(config.get(name)) - .map(String.class::cast) - .map(String::trim) - .filter(it -> !it.isEmpty()) - .map(v -> Enum.valueOf((Class) clazz, v)) - .orElse(null); + .map(String.class::cast) + .map(String::trim) + .filter(it -> !it.isEmpty()) + .map(v -> Enum.valueOf((Class) clazz, v)) + .orElse(null); } final ClassLoader loader = Thread.currentThread().getContextClassLoader(); final Function, Object> contextualSupplier = createContextualSupplier(loader); diff --git a/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties b/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties index 08995f3faed65..2fc0b38ed634b 100644 --- a/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties +++ b/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties @@ -20,3 +20,4 @@ org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.minI org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.maxItems = Length of property ''{0}'' should be < {1}, got {2}. org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.uniqueItems = ''{0}'' has duplicated items. org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.pattern = ''{0}'' does not match ''{1}''. +org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.enumValues = Invalid value for Property ''{0}'' expected: ''{1}'', got {2}. \ No newline at end of file diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 6d3c752a54054..8ef06be3704b3 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2022 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -64,11 +64,11 @@ public class PropertiesService { private Jsonb defaultMapper; public Stream buildProperties(final List meta, final ClassLoader loader, - final Locale locale, final DefaultValueInspector.Instance rootInstance) { + final Locale locale, final DefaultValueInspector.Instance rootInstance) { return buildProperties(meta, loader, locale, rootInstance, null); } - public void validate(final ConfigTypeNode configNode, final JsonObject payload){ + public void validate(final ConfigTypeNode configNode, final JsonObject payload) { ReflectionService.checkWithPayload(buildParameterMetas(configNode.getProperties()), extractConfig(configNode, payload), payload); } @@ -81,12 +81,12 @@ private JsonPointer toPointer(final String absoluteTargetPath) { //Map from JsonObject(payload). private Map extractConfig(ConfigTypeNode configNode, JsonObject payload) { Map payloadConfig = new HashMap<>(); - for(SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { + for (SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { try { JsonValue value = toPointer(propertyDefinition.getPath()).getValue(payload); if (value != null && !JsonObject.class.isInstance(value)) { - payloadConfig.put(propertyDefinition.getPath(), getValue(value)); - } + payloadConfig.put(propertyDefinition.getPath(), getValue(value)); + } } catch (JsonException e) { continue; } @@ -108,9 +108,9 @@ private String getValue(JsonValue value) { } } - private List buildParameterMetas(List properties){ + private List buildParameterMetas(List properties) { List parameterMetaList = new ArrayList<>(); - for(SimplePropertyDefinition propertyDefinition : properties) { + for (SimplePropertyDefinition propertyDefinition : properties) { final String path = sanitizePropertyName(propertyDefinition.getPath()); final String name = sanitizePropertyName(propertyDefinition.getName()); final ParameterMeta.Type type = findType(propertyDefinition.getType()); @@ -121,7 +121,7 @@ private List buildParameterMetas(List p sanitizedMetadata.putAll(propertyValidationService.mapMeta(propertyDefinition.getValidation())); } - parameterMetaList.add(new ParameterMeta(null, null, type, path, name,null, + parameterMetaList.add(new ParameterMeta(null, null, type, path, name, null, emptyList(), null, sanitizedMetadata, false)); } return parameterMetaList; @@ -149,7 +149,7 @@ private static LinkedHashMap getSanitizedMetadata(Map buildProperties(final List meta, final ClassLoader loader, - final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { + final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { return meta.stream().flatMap(p -> { final String path = sanitizePropertyName(p.getPath()); final String name = sanitizePropertyName(p.getName()); @@ -177,12 +177,12 @@ private Stream buildProperties(final List rewriteMetadataForLocale(final Map metadata, - final ParameterBundle parentBundle, final ParameterBundle bundle) { + final ParameterBundle parentBundle, final ParameterBundle bundle) { return rewriteLayoutMetadata(rewriteDocMetadata(metadata, parentBundle, bundle), parentBundle, bundle); } private Map rewriteDocMetadata(final Map metadata, - final ParameterBundle parentBundle, final ParameterBundle bundle) { + final ParameterBundle parentBundle, final ParameterBundle bundle) { final String defaultDoc = metadata.get("documentation::value"); final String bundleDoc = bundle.documentation(parentBundle).orElse(null); if (bundleDoc == null || bundleDoc.equals(defaultDoc)) { @@ -212,7 +212,7 @@ private Map rewriteDocMetadata(final Map metadat } private Map rewriteLayoutMetadata(final Map metadata, - final ParameterBundle parentBundle, final ParameterBundle bundle) { + final ParameterBundle parentBundle, final ParameterBundle bundle) { if (!componentServerConfiguration.getTranslateGridLayoutTabNames()) { return metadata; } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index 083e080d77845..20881263195d8 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -1,12 +1,12 @@ /** * Copyright (C) 2006-2022 Talend Inc. - www.talend.com - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -66,7 +66,7 @@ private static void boolWrapper(@Option final BoolBool wrapper) { } private static void multipleParams(@Option("aa") final String first, @Option("b") final BoolWrapper config, - @Option("a") final String last) { + @Option("a") final String last) { // no-op } @@ -119,7 +119,7 @@ void booleanDefault() throws NoSuchMethodException { } private List getProperties(final String locale) { - final String[] i18nPackages = { Config.class.getPackage().getName() }; + final String[] i18nPackages = {Config.class.getPackage().getName()}; final ParameterMeta host = new ParameterMeta(null, Config.class, ParameterMeta.Type.STRING, "configuration.host", "host", i18nPackages, emptyList(), null, emptyMap(), false); @@ -162,7 +162,8 @@ void buildPropertiesFr() { assertEquals("User Name FR", props.get(4).getDisplayName()); } - @Test // the class BaseConfig don't contains attribute + @Test + // the class BaseConfig don't contains attribute void validateProp() { assertThrows(IllegalArgumentException.class, () -> { ParameterMeta attribute = new ParameterMeta(null, BadConfig.class, ParameterMeta.Type.STRING, @@ -177,7 +178,8 @@ void validateProp() { }); } - @Test // the class BaseConfig don't contains attribute + @Test + // the class BaseConfig don't contains attribute void validateConfigNode() { final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); final List uniques = Arrays.asList("one", "two", "three"); @@ -192,6 +194,7 @@ void validateConfigNode() { String usernameError = "- Property 'configuration.connection.username' is required."; String minError = "- Property 'configuration.limit' should be > 100, got 99."; String maxError = "- Property 'configuration.limit' should be < 150, got 200."; + String ValueEvalError = "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; connection = base .path("configurationtype/details") @@ -203,13 +206,6 @@ void validateConfigNode() { JsonObject payload; - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("checkbox1", JsonValue.TRUE) - .add("valueEval", JsonValue.NULL).build()) - .build()).build(); - checkErrors(payload, Arrays.asList(connectionError, url1Required, usernameError)); /** * min/max **/ @@ -301,7 +297,7 @@ void validateConfigNode() { .build()) .add("limit", 100)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required)); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required, ValueEvalError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -317,7 +313,7 @@ void validateConfigNode() { .add("limit", 100)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError)); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); /* * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") @@ -353,33 +349,36 @@ void validateConfigNode() { payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() + .add("username", "abcd") .add("checkbox1", JsonValue.TRUE) .add("valueEval", JsonValue.NULL).build()) .build()).build(); - checkErrors(payload, Arrays.asList(connectionError, url1Required, usernameError)); + checkErrors(payload, Arrays.asList(connectionError, url1Required, ValueEvalError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() + .add("username", "abcd") .add("checkbox1", JsonValue.FALSE) .add("valueEval", JsonValue.NULL).build()) .build()).build(); - checkErrors(payload, Arrays.asList(connectionError, url1Error)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() + .add("username", "abcd") .add("checkbox1", JsonValue.TRUE) .add("valueEval", "VALUE_2").build()) .build()).build(); - checkErrors(payload, Arrays.asList(connectionError, url1Error)); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); } private void checkErrors(JsonObject payload, List expected) { try { propertiesService.validate(connection, payload); - if (expected != null && expected.size() >0) { - fail("There should be errors: "+expected); + if (expected != null && expected.size() > 0) { + fail("There should be errors: " + expected); } } catch (Exception errors) { StringBuffer expectedBuffer = new StringBuffer(); From 6e81aa560a10c4d8b0fb195d67bef36057674579 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 29 Mar 2023 14:42:21 +0800 Subject: [PATCH 10/16] [TCOMP-2260]: fix style --- .../runtime/manager/reflect/ReflectionService.java | 12 ++---------- .../manager/reflect/visibility/PayloadMapper.java | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index 70eec541f2387..87b76acecddfe 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -202,13 +202,13 @@ public Function, Object[]> parameterFactory(final Executable }; } - public static void checkPayload(List metas, Map notNullConfig) { + public static void checkPayload(final List metas, final Map notNullConfig) { JsonObject globalPayload = new PayloadMapper((a, b) -> { }).visitAndMap(metas, notNullConfig); checkWithPayload(metas, notNullConfig, globalPayload); } - public static void checkWithPayload(List metas, Map notNullConfig, final JsonObject payload) { + public static void checkWithPayload(final List metas, final Map notNullConfig, final JsonObject payload) { final PayloadValidator visitor = new PayloadValidator(); if (!visitor.skip) { visitor.globalPayload = payload; @@ -219,14 +219,6 @@ public static void checkWithPayload(List metas, Map metas, Map notNullConfig, JsonObject payload) { -// final PayloadValidator visitor = new PayloadValidator(); -// final PayloadMapper payloadMapper = new PayloadMapper(visitor); -// payloadMapper.setGlobalPayload(payload); -// payloadMapper.visitAndMap(metas, notNullConfig); -// visitor.throwIfFailed(); -// } - public Function, Object> createContextualSupplier(final ClassLoader loader) { return supplier -> { final Thread thread = Thread.currentThread(); diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java index ebb9270dfd5f1..bb42abbdef327 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java @@ -123,7 +123,7 @@ private void onProperty(final String contextualPrefix, final Collection config, String newPath, ParameterMeta definition) { + private static String getValue(final Map config, final String newPath, final ParameterMeta definition) { return config.get(newPath) == null ? config.get(definition.getPath()) : config.get(newPath); } From f813eb8c07727dd5ab5e23efa7867bd5c2d4ff13 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 29 Mar 2023 16:14:54 +0800 Subject: [PATCH 11/16] [TCOMP-2260]:fix junit cases --- .../JsonSchemaValidatorFactoryExt.java | 10 +- .../server/service/ValidationServiceTest.java | 430 +++++++----------- 2 files changed, 159 insertions(+), 281 deletions(-) diff --git a/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java b/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java index ae064093dfd16..547c713347d7f 100644 --- a/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java +++ b/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java @@ -23,8 +23,10 @@ import org.apache.johnzon.jsonschema.regex.JavaRegex; import org.apache.johnzon.jsonschema.spi.builtin.PatternValidation; +import org.talend.sdk.component.form.internal.validation.spi.ext.EnumValidationWithDefaultValue; import org.talend.sdk.component.form.internal.validation.spi.ext.MaximumValidation; import org.talend.sdk.component.form.internal.validation.spi.ext.MinimumValidation; +import org.talend.sdk.component.form.internal.validation.spi.ext.RequiredValidation; import org.talend.sdk.component.form.internal.validation.spi.ext.TypeValidation; public class JsonSchemaValidatorFactoryExt extends org.apache.johnzon.jsonschema.JsonSchemaValidatorFactory { @@ -41,18 +43,18 @@ public List createDefault List validations = super.createDefaultValidations() .stream() .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.TypeValidation.class.isInstance(v)) - // .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.EnumValidation.class.isInstance(v)) + .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.EnumValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.MinimumValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.MaximumValidation.class.isInstance(v)) - // .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.RequiredValidation.class.isInstance(v)) + .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.RequiredValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.PatternValidation.class.isInstance(v)) .collect(Collectors.toList()) ; validations.add(new TypeValidation()); - // validations.add(new EnumValidationWithDefaultValue()); + validations.add(new EnumValidationWithDefaultValue()); validations.add(new MinimumValidation()); validations.add(new MaximumValidation()); - // validations.add(new RequiredValidation()); + validations.add(new RequiredValidation()); return validations; } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java index 449709da7d223..d19529ca02e8a 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -15,22 +15,17 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; -import static java.util.Optional.ofNullable; -import static java.util.stream.Collectors.toList; import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; import javax.inject.Inject; import javax.json.Json; @@ -41,29 +36,19 @@ import javax.json.bind.JsonbBuilder; import javax.ws.rs.client.WebTarget; -import org.apache.johnzon.jsonschema.JsonSchemaValidator; import org.apache.johnzon.jsonschema.JsonSchemaValidatorFactory; import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; -import org.apache.xbean.propertyeditor.PropertyEditorRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.talend.sdk.component.form.api.Client; import org.talend.sdk.component.form.api.UiSpecService; -import org.talend.sdk.component.form.internal.converter.PropertyContext; import org.talend.sdk.component.form.internal.converter.PropertyContext.Configuration; -import org.talend.sdk.component.form.internal.converter.impl.JsonSchemaConverter; -import org.talend.sdk.component.form.internal.lang.CompletionStages; import org.talend.sdk.component.form.internal.validation.JsonSchemaValidatorFactoryExt; import org.talend.sdk.component.form.model.Ui; import org.talend.sdk.component.form.model.jsonschema.JsonSchema; import org.talend.sdk.component.form.model.uischema.UiSchema; -import org.talend.sdk.component.runtime.manager.ParameterMeta; -import org.talend.sdk.component.runtime.manager.reflect.ParameterModelService; -import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.BaseParameterEnricher; -import org.talend.sdk.component.runtime.manager.service.LocalConfigurationService; import org.talend.sdk.component.server.front.model.ConfigTypeNode; import org.talend.sdk.component.server.front.model.ConfigTypeNodes; -import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; import org.talend.sdk.component.server.test.ComponentClient; import lombok.AllArgsConstructor; @@ -80,10 +65,11 @@ public class ValidationServiceTest { @Inject private ComponentClient client; + @Inject + private PropertiesService propertiesService; private Jsonb jsonb = JsonbBuilder.create(); - final String lang = "en"; final String family = "custom"; final String component = "noop"; @@ -129,98 +115,6 @@ static class ValidationError { } - - public CompletionStage validate(final ConfigTypeNode config, final JsonObject properties) throws Exception { - findActiveIf(config, properties); - - return getValidator(config) - .thenApply(validator -> validator.apply(properties)) - .thenApply(vr -> new Result(vr - .getErrors() - .stream() - .map(e -> new ValidationError(e.getField(), e.getMessage())) - .collect(toList()))); - } - - public CompletionStage getValidator(final ConfigTypeNode config) throws Exception { - - JsonSchema xschema = new UiSpecService<>(null, jsonb) - .convert("test", "en", config, null) - .toCompletableFuture() - .get().getJsonSchema(); - JsonObject xjson = jsonb.fromJson(jsonb.toJson(xschema), JsonObject.class); - // xschema should be same as jsonSchema - final JsonSchema jschema = new JsonSchema(); - final JsonSchemaConverter converter = new JsonSchemaConverter(jsonb, jschema, config.getProperties()); - return CompletableFuture - .allOf(config - .getProperties() - .stream() - .filter(Objects::nonNull) - .filter(p -> p.getName().equals(p.getPath())) - .map(it -> new PropertyContext<>(it, this, new Configuration(false))) - .map(CompletionStages::toStage) - .map(converter::convert) - .toArray(CompletableFuture[]::new)) - .thenApply(r -> jschema) - .thenApply(schema -> jsonb.fromJson(jsonb.toJson(schema), JsonObject.class)) - .thenApply(factory::newInstance); - } - - private void findActiveIf(final ConfigTypeNode config, JsonObject properties) throws NoSuchMethodException { - extracted(config.getProperties().get(0)); - extracted(config.getProperties().get(1)); - //TCOMP-2260: find out all activeIf's target , and its value in jsonObject; if it is not active, clear its validation - config - .getProperties() - .stream() - .filter(p -> p.getMetadata().get("condition::if::target") != null) - .forEach(simplePropertyDefinition -> { - if (isNotActive(simplePropertyDefinition, properties)) - simplePropertyDefinition.setValidation(null); - }); - } - - private static void extracted(final SimplePropertyDefinition properties) throws NoSuchMethodException { - final ParameterModelService service = new ParameterModelService(new PropertyEditorRegistry()); -// final List metas = service -// .buildParameterMetas(SimplePropertyDefinition.class.getMethod("getMetadata"), -// "", new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "server"))); - final List metas2 = service.buildParameterMetas(Stream.of(new ParameterModelService.Param(SimplePropertyDefinition.class, SimplePropertyDefinition.class.getAnnotations(), properties.getName())), - SimplePropertyDefinition.class, - ofNullable(SimplePropertyDefinition.class.getPackage()).map(Package::getName).orElse(""), true, - new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "server-model"))); - System.err.println("size: " + metas2.size()); - } - - //should own the same first parent. - // 1) A.B.C depends on A.B.D, A.B -> A.C :support - // 2) A.B.C depends on A.D.E : TO check - private boolean isNotActive(final SimplePropertyDefinition simplePropertyDefinition, final JsonObject properties) { - String[] targetPath = simplePropertyDefinition.getMetadata().get("condition::if::target").split("\\."); - String[] selfPath = simplePropertyDefinition.getPath().split("\\."); - String targetValue = simplePropertyDefinition.getMetadata().get("condition::if::value"); - - AtomicBoolean isNotActive = new AtomicBoolean(false); - AtomicReference object = new AtomicReference<>(properties); - Arrays.stream(selfPath).forEach( onePath -> { - Arrays.stream(targetPath).forEach( targetP -> { - if (object.get().get(onePath) != null) { - if(object.get().get(onePath).getValueType().equals(JsonValue.ValueType.OBJECT)) { - object.set(object.get().get(onePath).asJsonObject()); - } else { - if (object.get().get(targetP) != null) { - String str = object.get().get(targetP).toString(); - isNotActive.set(str.contains(targetValue)); - } - } - } - }); - }); - - return !isNotActive.get(); - } - @BeforeEach void setup() throws Exception { details = base @@ -237,162 +131,147 @@ void setup() throws Exception { int i = 0; } - @Test - public void test1() throws NoSuchMethodException { - final ParameterModelService service = new ParameterModelService(new PropertyEditorRegistry()); - final List metas = service - .buildParameterMetas(ConfigTypeNodes.class.getMethod("visibility", SimplePropertyDefinition.class), - "def", new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "test"))); - - final List metas2 = service.buildParameterMetas(Stream.of(new ParameterModelService.Param(JsonSchema.class, - JsonSchema.class.getAnnotations(), jsonSchema.getTitle())), - JsonSchema.class, - ofNullable(JsonSchema.class.getPackage()).map(Package::getName).orElse(""), true, - new BaseParameterEnricher.Context(new LocalConfigurationService(emptyList(), "server-model"))); - System.err.println("size: " + metas2.size()); - } - private void checkAsserts(JsonObject payload, List expected) throws Exception { - Result errors = validate(connection, payload).toCompletableFuture().get(); - - System.err.println(" ====== "+ expected.size()); - long ecnt = errors.getErrors().stream() - .peek(e -> log.warn("[checkAsserts] {}", e)) - .filter(e -> !expected.contains(e)) - .peek(e -> log.error("[checkAsserts]Validation uncaught: {}", e)) - // .map(e-> fail(e)) - .count(); -// System.out.println("Actual errors: "+ errors.getErrors().size() + ", uncaught errors: " + ecnt); - expected.stream().forEach(e -> assertTrue(errors.getErrors().contains(e))); + private void checkAsserts(JsonObject payload, List expected) throws Exception { + try { + propertiesService.validate(connection, payload); + if (expected != null && expected.size() > 0) { + fail("There should be errors: " + expected); + } + } catch (Exception errors) { + StringBuffer expectedBuffer = new StringBuffer(); + expected.stream().forEach(e -> { + expectedBuffer.append(e).append("\n"); + }); + expectedBuffer.delete(expectedBuffer.lastIndexOf("\n"), expectedBuffer.length()); + assertEquals(expectedBuffer.toString(), errors.getMessage()); + } } - @Test - void testValidation () throws Exception { - final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); - // - final List uniques = Arrays.asList("one", "two", "three"); - final List notUniques = Arrays.asList("one", "two", "three", "one"); - // - ValidationError conn = new ValidationError("/configuration", "connection is required and is not present"); - ValidationError min = new ValidationError("/configuration/limit", "1.0 is less than 100.0"); - ValidationError max = new ValidationError("/configuration/limit", "1000.0 is more than 150.0"); - ValidationError url0 = new ValidationError("/configuration/connection/url0", "\"mailto://toto@titi.org\" doesn't match JavascriptRegex{/^https?://.*/}"); - ValidationError url1Patten = new ValidationError("/configuration/connection/url1", "\"mailto://toto@titi.org\" doesn't match JavascriptRegex{/^https?://.*/}"); - ValidationError url1Required = new ValidationError("/configuration/connection", "url1 is required and is not present"); - ValidationError username = new ValidationError("/configuration/connection", "username is required and is not present"); - ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", "Invalid value, got null, expected: [\"VALUE_1\",\"VALUE_2\",\"VALUE_3\"]"); - ValidationError password = new ValidationError("/configuration/connection", "password is required and is not present"); - ValidationError activedIfs = new ValidationError("/configuration/connection", "activedIfs is required and is not present"); - ValidationError uniqVals = new ValidationError("/configuration/connection/uniqVals", "duplicated items: []"); - - - JsonObject payload; - /** - * connection is required! - **/ - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connectn", "").build()).build(); - checkAsserts(payload, Arrays.asList(conn)); - /** - * min/max - **/ - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", JsonValue.NULL) - .add("limit", 1)) - .build(); - checkAsserts(payload, Arrays.asList(min, conn)); - - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", JsonValue.NULL) - .add("limit", 1000)) - .build(); - checkAsserts(payload, Arrays.asList(max, conn)); - /* - * url0 pattern - */ - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("url0", "mailto://toto@titi.org").build()) - .build()).build(); - checkAsserts(payload, Arrays.asList(url0, username, url1Required, activedIfs)); - /* - * url1 - */ - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("url1", "mailto://toto@titi.org").build()) - .build()).build(); - checkAsserts(payload, Arrays.asList(username, url1Patten, activedIfs)); - - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("url101", "mailto://toto@titi.org").build()) - .build()).build(); - checkAsserts(payload, Arrays.asList(username, url1Required, activedIfs)); - - /* - * uniques - */ - final JsonObject try3 = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("url", "http://t") - .add("username", JsonValue.NULL) - .add("uniqVals", factory.createArrayBuilder(notUniques).build()) - .build()) - .add("limit", 100)) - .build(); - checkAsserts(payload, Arrays.asList(username, url1Required, activedIfs)); - - /* - * uniqVals - */ - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("url0", JsonValue.NULL) - .add("url1", JsonValue.NULL) - .add("username", JsonValue.NULL) - .add("password", JsonValue.NULL) - .add("uniqVals", factory.createArrayBuilder(notUniques).build()) - .add("checkbox1", JsonValue.TRUE) - .add("checkbox2", JsonValue.TRUE) - .build()) - .add("limit", 100)) - .build(); - - checkAsserts(payload, Arrays.asList(uniqVals, username, activedIfs, url1Required)); - - payload = factory.createObjectBuilder() - .add("configuration", factory.createObjectBuilder() - .add("connection", factory.createObjectBuilder() - .add("url0", JsonValue.NULL) - .add("url1", JsonValue.NULL) - .add("username", JsonValue.NULL) - .add("password", JsonValue.NULL) - .add("uniqVals", factory.createArrayBuilder(uniques).build()) - .add("checkbox1", JsonValue.TRUE) - .add("checkbox2", JsonValue.TRUE) - .build()) - .add("limit", 100)) - .build(); - - checkAsserts(payload, Arrays.asList(username, activedIfs, url1Required)); - } + @Test + void testValidation() throws Exception { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + // + final List uniques = Arrays.asList("one", "two", "three"); + final List notUniques = Arrays.asList("one", "two", "three", "one"); + // + String activeIfsError = "- Property 'configuration.connection.activedIfs' is required."; + String connectionError = "- Property 'configuration.connection' is required."; + String url1Required = "- Property 'configuration.connection.url1' is required."; + String url0Error = "- 'configuration.connection.url0' does not match '^https?://.*'."; + String url1Error = "- 'configuration.connection.url1' does not match '^https?://.*'."; + String usernameError = "- Property 'configuration.connection.username' is required."; + String minError = "- Property 'configuration.limit' should be > 100, got 1."; + String maxError = "- Property 'configuration.limit' should be < 150, got 1,000."; + String ValueEvalError = "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; + + JsonObject payload; + /** + * connection is required! + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connectn", "").build()).build(); + checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); + /** + * min/max + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", JsonValue.NULL) + .add("limit", 1)) + .build(); + checkAsserts(payload, Arrays.asList(connectionError, minError, url1Required, usernameError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", JsonValue.NULL) + .add("limit", 1000)) + .build(); + checkAsserts(payload, Arrays.asList(connectionError, maxError, url1Required, usernameError)); + /* + * url0 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "mailto://toto@titi.org").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(connectionError, url0Error, url1Required, usernameError)); + /* + * url1 + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url1", "mailto://toto@titi.org").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(connectionError, url1Error, usernameError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url101", "mailto://toto@titi.org").build()) + .build()).build(); + checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); + + /* + * uniques + */ + final JsonObject try3 = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url", "http://t") + .add("username", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .build()) + .add("limit", 100)) + .build(); + checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); + + /* + * uniqVals + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + + checkAsserts(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(uniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + + checkAsserts(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); + } @Test - void testValidation_activeIf () throws Exception { + void testValidation_activeIf() throws Exception { final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); // - ValidationError url1Required = new ValidationError("/configuration/connection", "url1 is required and is not present"); - ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", "Invalid value, got null, expected: [\"VALUE_1\",\"VALUE_2\",\"VALUE_3\"]"); - ValidationError password = new ValidationError("/configuration/connection", "password is required and is not present"); - ValidationError activedIfs = new ValidationError("/configuration/connection", "activedIfs is required and is not present"); + String activeIfsError = "- Property 'configuration.connection.activedIfs' is required."; + String connectionError = "- Property 'configuration.connection' is required."; + String passwordError = "- Property 'configuration.connection.password' is required."; + String url1Required = "- Property 'configuration.connection.url1' is required."; JsonObject payload; @@ -405,7 +284,7 @@ void testValidation_activeIf () throws Exception { .add("username", "undx") .add("password", JsonValue.NULL).build()) .build()).build(); - checkAsserts(payload, Arrays.asList(url1Required, activedIfs, password)); + checkAsserts(payload, Arrays.asList(connectionError, passwordError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -413,7 +292,7 @@ void testValidation_activeIf () throws Exception { .add("username", "abcd") .add("password", JsonValue.NULL).build()) .build()).build(); - checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + checkAsserts(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() @@ -422,26 +301,23 @@ void testValidation_activeIf () throws Exception { .add("username", "undx") .add("password", "abc").build()) .build()).build(); - checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + checkAsserts(payload, Arrays.asList(connectionError, url1Required)); } @Test - void testValidation_activeIf_2 () throws Exception { + void testValidation_activeIf_2() throws Exception { final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); // - ValidationError url1Required = new ValidationError("/configuration/connection", "url1 is required and is not present"); - ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", "Invalid value, got null, expected: [\"VALUE_1\",\"VALUE_2\",\"VALUE_3\"]"); - ValidationError password = new ValidationError("/configuration/connection", "password is required and is not present"); - ValidationError activedIfs = new ValidationError("/configuration/connection", "activedIfs is required and is not present"); + String activeIfsError = "- Property 'configuration.connection.activedIfs' is required."; + String connectionError = "- Property 'configuration.connection' is required."; + String usernameError = "- Property 'configuration.connection.username' is required."; + String url1Required = "- Property 'configuration.connection.url1' is required."; + String ValueEvalError = "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; JsonObject payload; - - /* * valueEval @ActiveIf(target = "checkbox1", value = "true") * - * TODO Fails - * */ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -449,7 +325,7 @@ void testValidation_activeIf_2 () throws Exception { .add("checkbox1", JsonValue.TRUE) .add("valueEval", JsonValue.NULL).build()) .build()).build(); - checkAsserts(payload, Arrays.asList(url1Required, activedIfs, valueEval)); + checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError, ValueEvalError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -457,17 +333,17 @@ void testValidation_activeIf_2 () throws Exception { .add("checkbox1", JsonValue.FALSE) .add("valueEval", JsonValue.NULL).build()) .build()).build(); - checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("checkbox1", JsonValue.TRUE) .add("valueEval", "VALUE_2").build()) - .build()).build(); - checkAsserts(payload, Arrays.asList(url1Required, activedIfs)); + .build()).build(); + checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); } - } +} From ebf7f50d3ab8631c5a2f5c9986ce1cdb0dd91cfd Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 29 Mar 2023 16:58:30 +0800 Subject: [PATCH 12/16] [TCOMP-2260]: fix errors --- .../manager/reflect/ReflectionService.java | 79 ++++++++++--------- .../reflect/visibility/PayloadMapper.java | 3 +- .../server/front/model/ConfigTypeNode.java | 4 +- .../server/service/PropertiesService.java | 29 ++++--- 4 files changed, 64 insertions(+), 51 deletions(-) diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index 87b76acecddfe..df2de5d73d167 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -90,7 +90,7 @@ public class ReflectionService { // IMPORTANT: ensure to be able to read all data (including collection) from a // map to support system properties override public Function, Object[]> parameterFactory(final Executable executable, - final Map, Object> precomputed, final List metas) { + final Map, Object> precomputed, final List metas) { final ClassLoader loader = Thread.currentThread().getContextClassLoader(); final Function, Object> contextualSupplier = createContextualSupplier(loader); final Collection, Object>> factories = @@ -208,7 +208,8 @@ public static void checkPayload(final List metas, final Map metas, final Map notNullConfig, final JsonObject payload) { + public static void checkWithPayload(final List metas, final Map notNullConfig, + final JsonObject payload) { final PayloadValidator visitor = new PayloadValidator(); if (!visitor.skip) { visitor.globalPayload = payload; @@ -233,8 +234,8 @@ public Function, Object> createContextualSupplier(final ClassLo } public Function, Object> createConfigFactory(final Map, Object> precomputed, - final ClassLoader loader, final Function, Object> contextualSupplier, final String name, - final Configuration configuration, final Annotation[] allAnnotations, final Class configClass) + final ClassLoader loader, final Function, Object> contextualSupplier, final String name, + final Configuration configuration, final Annotation[] allAnnotations, final Class configClass) throws NoSuchMethodException { final Constructor constructor = configClass.getConstructor(); final LocalConfiguration config = LocalConfiguration.class.cast(precomputed.get(LocalConfiguration.class)); @@ -286,7 +287,7 @@ private List translate(final List metas, final Str } private Collector createMapCollector(final Class mapType, final Class keyItemClass, - final Class valueItemClass, final Map, Object> precomputed) { + final Class valueItemClass, final Map, Object> precomputed) { final Function, Object> keyMapper = o -> doConvert(keyItemClass, o.getKey(), precomputed); final Function, Object> valueMapper = o -> doConvert(valueItemClass, o.getValue(), precomputed); return ConcurrentMap.class.isAssignableFrom(mapType) ? toConcurrentMap(keyMapper, valueMapper) @@ -294,9 +295,9 @@ private Collector createMapCollector(final Class mapType, final Class keyI } private Object createList(final ClassLoader loader, final Function, Object> contextualSupplier, - final String name, final Class collectionType, final Class itemClass, final Collector collector, - final BiFunction, Object> itemFactory, final Map config, - final List metas, final Map, Object> precomputed) { + final String name, final Class collectionType, final Class itemClass, final Collector collector, + final BiFunction, Object> itemFactory, final Map config, + final List metas, final Map, Object> precomputed) { final Object obj = config.get(name); if (collectionType.isInstance(obj)) { return Collection.class @@ -344,9 +345,9 @@ private Integer getArrayMaxLength(final String prefix, final Map } private Object createMap(final String name, final Class mapType, - final BiFunction, Object> keyItemFactory, - final BiFunction, Object> valueItemFactory, final Collector collector, - final Map config) { + final BiFunction, Object> keyItemFactory, + final BiFunction, Object> valueItemFactory, final Collector collector, + final Map config) { final Object obj = config.get(name); if (mapType.isInstance(obj)) { return Map.class.cast(obj).entrySet().stream().collect(collector); @@ -376,8 +377,8 @@ private Object createMap(final String name, final Class mapType, } private BiFunction, Object> createObjectFactory(final ClassLoader loader, - final Function, Object> contextualSupplier, final Type type, - final List metas, final Map, Object> precomputed) { + final Function, Object> contextualSupplier, final Type type, + final List metas, final Map, Object> precomputed) { final Class clazz = Class.class.cast(type); if (clazz.isPrimitive() || Primitives.unwrap(clazz) != clazz || String.class == clazz) { return (name, config) -> doConvert(clazz, config.get(name), precomputed); @@ -406,7 +407,7 @@ private String[] findArgsName(final Class clazz) { } private JsonValue createJsonValue(final Object value, final Map, Object> precomputed, - final Function fallbackReaderCreator) { + final Function fallbackReaderCreator) { final StringReader sr = new StringReader(String.valueOf(value).trim()); try (final JsonReader reader = ofNullable(precomputed.get(JsonReaderFactory.class)) .map(JsonReaderFactory.class::cast) @@ -417,8 +418,8 @@ private JsonValue createJsonValue(final Object value, final Map, Object } private Object createObject(final ClassLoader loader, final Function, Object> contextualSupplier, - final Class clazz, final String[] args, final String name, final Map config, - final List metas, final Map, Object> precomputed) { + final Class clazz, final String[] args, final String name, final Map config, + final List metas, final Map, Object> precomputed) { final Object potentialJsonValue = config.get(name); if (JsonObject.class == clazz && String.class.isInstance(potentialJsonValue)) { return createJsonValue(potentialJsonValue, precomputed, Json::createReader).asJsonObject(); @@ -449,11 +450,11 @@ private Object createObject(final ClassLoader loader, final Function mapEntries = specificMapping.entrySet().stream().filter(e -> { - final String key = e.getKey(); - final int idxStart = key.indexOf('[', prefix.length()); - return idxStart > 0 && ((idxStart > ".key".length() && key.startsWith(".key", idxStart - ".key".length())) - || (idxStart > ".value".length() && key.startsWith(".value", idxStart - ".value".length()))); - }) + final String key = e.getKey(); + final int idxStart = key.indexOf('[', prefix.length()); + return idxStart > 0 && ((idxStart > ".key".length() && key.startsWith(".key", idxStart - ".key".length())) + || (idxStart > ".value".length() && key.startsWith(".value", idxStart - ".value".length()))); + }) .sorted(this::sortIndexEntry) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, noMerge(), LinkedHashMap::new)); mapEntries.keySet().forEach(specificMapping::remove); @@ -493,12 +494,12 @@ private Object createObject(final ClassLoader loader, final Function listEntries = specificMapping.entrySet().stream().filter(e -> { - final String key = e.getKey(); - final int idxStart = key.indexOf('[', prefix.length()); - final int idxEnd = key.indexOf(']', prefix.length()); - final int sep = key.indexOf('.', prefix.length() + 1); - return idxStart > 0 && key.endsWith("]") && (sep > idxEnd || sep < 0); - }) + final String key = e.getKey(); + final int idxStart = key.indexOf('[', prefix.length()); + final int idxEnd = key.indexOf(']', prefix.length()); + final int sep = key.indexOf('.', prefix.length() + 1); + return idxStart > 0 && key.endsWith("]") && (sep > idxEnd || sep < 0); + }) .sorted(this::sortIndexEntry) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, noMerge(), LinkedHashMap::new)); listEntries.keySet().forEach(specificMapping::remove); @@ -660,7 +661,7 @@ private Object createObject(final ClassLoader loader, final Function, Object> contextualSupplier, - final Map config, final String prefix, final Map preparedObjects, - final String nestedName, final String listName, final ParameterizedType pt, final Supplier init, - final List metas, final Map, Object> precomputed) { + final Map config, final String prefix, final Map preparedObjects, + final String nestedName, final String listName, final ParameterizedType pt, final Supplier init, + final List metas, final Map, Object> precomputed) { // CHECKSTYLE:ON final Collection aggregator = Collection.class.cast(preparedObjects.computeIfAbsent(listName, k -> init.get())); @@ -997,7 +998,7 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) { } { final String enumValues = metadata.get("tcomp::validation::enumValues"); - if (enumValues != null) {//value = null or not in enumValues: add error + if (enumValues != null) {// value = null or not in enumValues: add error if (value == null) { errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, null)); } else { @@ -1034,11 +1035,11 @@ public BiFunction, T> createObjectFactory(final if (clazz.isEnum()) { return (name, config) -> (T) ofNullable(config.get(name)) - .map(String.class::cast) - .map(String::trim) - .filter(it -> !it.isEmpty()) - .map(v -> Enum.valueOf((Class) clazz, v)) - .orElse(null); + .map(String.class::cast) + .map(String::trim) + .filter(it -> !it.isEmpty()) + .map(v -> Enum.valueOf((Class) clazz, v)) + .orElse(null); } final ClassLoader loader = Thread.currentThread().getContextClassLoader(); final Function, Object> contextualSupplier = createContextualSupplier(loader); diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java index bb42abbdef327..20562c5c82463 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java @@ -123,7 +123,8 @@ private void onProperty(final String contextualPrefix, final Collection config, final String newPath, final ParameterMeta definition) { + private static String getValue(final Map config, final String newPath, + final ParameterMeta definition) { return config.get(newPath) == null ? config.get(definition.getPath()) : config.get(newPath); } diff --git a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java index 2a42c727d53aa..80fda57a8d571 100644 --- a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java +++ b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java @@ -48,7 +48,7 @@ public class ConfigTypeNode { private Collection actions; - public void visibility(final SimplePropertyDefinition spd){ - //no-op + public void visibility(final SimplePropertyDefinition spd) { + // no-op } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 7c70e24a9d1d3..c73dd91227ebe 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -21,19 +21,30 @@ import static java.util.stream.Collectors.toSet; import static org.talend.sdk.component.server.lang.CustomCollectors.toLinkedMap; -import java.util.*; -import java.util.concurrent.atomic.AtomicReference; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; -import javax.json.*; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonPointer; +import javax.json.JsonString; +import javax.json.JsonValue; import javax.json.bind.Jsonb; import javax.json.spi.JsonProvider; -import org.apache.cxf.common.util.StringUtils; import org.talend.sdk.component.runtime.internationalization.ParameterBundle; import org.talend.sdk.component.runtime.manager.ParameterMeta; import org.talend.sdk.component.runtime.manager.reflect.ReflectionService; @@ -79,7 +90,7 @@ private JsonPointer toPointer(final String absoluteTargetPath) { //Map from JsonObject(payload). - private Map extractConfig(ConfigTypeNode configNode, JsonObject payload) { + private Map extractConfig(final ConfigTypeNode configNode, final JsonObject payload) { Map payloadConfig = new HashMap<>(); for (SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { try { @@ -95,7 +106,7 @@ private Map extractConfig(ConfigTypeNode configNode, JsonObject return payloadConfig; } - private String getValue(JsonValue value) { + private String getValue(final JsonValue value) { switch (value.getValueType()) { case TRUE: case FALSE: @@ -108,7 +119,7 @@ private String getValue(JsonValue value) { } } - private List buildParameterMetas(List properties) { + private List buildParameterMetas(final List properties) { List parameterMetaList = new ArrayList<>(); for (SimplePropertyDefinition propertyDefinition : properties) { final String path = sanitizePropertyName(propertyDefinition.getPath()); @@ -127,7 +138,7 @@ private List buildParameterMetas(List p return parameterMetaList; } - private static LinkedHashMap getParamMetadata(Map p) { + private static LinkedHashMap getParamMetadata(final Map p) { return ofNullable(p) .map(m -> m .entrySet() @@ -137,7 +148,7 @@ private static LinkedHashMap getParamMetadata(Map getSanitizedMetadata(Map p) { + private static LinkedHashMap getSanitizedMetadata(final Map p) { return ofNullable(p) .map(m -> m .entrySet() From 788d9d311bc3c848e0b6faa5b5714130f982720d Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 29 Mar 2023 17:33:55 +0800 Subject: [PATCH 13/16] [TCOMP-2260]:order imports --- .../talend/sdk/component/server/service/PropertiesService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index c73dd91227ebe..ba83d8677ba90 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -26,11 +26,10 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; import java.util.Map; - import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; From 17cff6d6d11e57a0692c07a8848dd9978a700720 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Fri, 31 Mar 2023 16:57:45 +0800 Subject: [PATCH 14/16] fix{TCOMP-2260): fix junit --- .../server/service/PropertiesService.java | 51 +++++---- .../service/PropertyValidationService.java | 43 ++++---- .../ConfigurationTypeResourceImplTest.java | 2 +- .../server/service/PropertiesServiceTest.java | 66 ++++++----- .../server/service/ValidationServiceTest.java | 104 ++++++++++++------ .../server/test/custom/CustomProcessor.java | 9 +- 6 files changed, 162 insertions(+), 113 deletions(-) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index ba83d8677ba90..d7e5969310bd3 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,7 +74,7 @@ public class PropertiesService { private Jsonb defaultMapper; public Stream buildProperties(final List meta, final ClassLoader loader, - final Locale locale, final DefaultValueInspector.Instance rootInstance) { + final Locale locale, final DefaultValueInspector.Instance rootInstance) { return buildProperties(meta, loader, locale, rootInstance, null); } @@ -87,8 +87,7 @@ private JsonPointer toPointer(final String absoluteTargetPath) { return JsonProvider.provider().createPointer('/' + absoluteTargetPath.replace('.', '/')); } - - //Map from JsonObject(payload). + // Map from JsonObject(payload). private Map extractConfig(final ConfigTypeNode configNode, final JsonObject payload) { Map payloadConfig = new HashMap<>(); for (SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { @@ -107,14 +106,14 @@ private Map extractConfig(final ConfigTypeNode configNode, final private String getValue(final JsonValue value) { switch (value.getValueType()) { - case TRUE: - case FALSE: - case NUMBER: - return String.valueOf(value); - case STRING: - return JsonString.class.cast(value).getString(); - default: - return null; + case TRUE: + case FALSE: + case NUMBER: + return String.valueOf(value); + case STRING: + return JsonString.class.cast(value).getString(); + default: + return null; } } @@ -124,7 +123,7 @@ private List buildParameterMetas(final List sanitizedMetadata = ofNullable(getParamMetadata(propertyDefinition.getMetadata())) .orElse(new LinkedHashMap<>()); if (propertyDefinition.getValidation() != null) { @@ -143,7 +142,8 @@ private static LinkedHashMap getParamMetadata(final Map !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) - .collect(toLinkedMap(e -> e.getKey().replace("condition::", "tcomp::condition::"), Map.Entry::getValue))) + .collect(toLinkedMap(e -> e.getKey().replace("condition::", "tcomp::condition::"), + Map.Entry::getValue))) .orElse(null); } @@ -157,9 +157,8 @@ private static LinkedHashMap getSanitizedMetadata(final Map buildProperties(final List meta, final ClassLoader loader, - final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { + final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { return meta.stream().flatMap(p -> { final String path = sanitizePropertyName(p.getPath()); final String name = sanitizePropertyName(p.getName()); @@ -187,12 +186,12 @@ private Stream buildProperties(final List rewriteMetadataForLocale(final Map metadata, - final ParameterBundle parentBundle, final ParameterBundle bundle) { + final ParameterBundle parentBundle, final ParameterBundle bundle) { return rewriteLayoutMetadata(rewriteDocMetadata(metadata, parentBundle, bundle), parentBundle, bundle); } private Map rewriteDocMetadata(final Map metadata, - final ParameterBundle parentBundle, final ParameterBundle bundle) { + final ParameterBundle parentBundle, final ParameterBundle bundle) { final String defaultDoc = metadata.get("documentation::value"); final String bundleDoc = bundle.documentation(parentBundle).orElse(null); if (bundleDoc == null || bundleDoc.equals(defaultDoc)) { @@ -222,7 +221,7 @@ private Map rewriteDocMetadata(final Map metadat } private Map rewriteLayoutMetadata(final Map metadata, - final ParameterBundle parentBundle, final ParameterBundle bundle) { + final ParameterBundle parentBundle, final ParameterBundle bundle) { if (!componentServerConfiguration.getTranslateGridLayoutTabNames()) { return metadata; } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java index 50a63b4594b4d..6e80d83254b53 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,17 +59,17 @@ private void initMapper() { f.setAccessible(true); } return (BiFunction, Boolean>) (instance, - meta) -> ofNullable(meta.get(ValidationParameterEnricher.META_PREFIX + f.getName())) - .map(valueConverter) - .map(val -> { - try { - f.set(instance, val); - } catch (IllegalAccessException e) { - throw new IllegalStateException(e); - } - return true; - }) - .orElse(false); + meta) -> ofNullable(meta.get(ValidationParameterEnricher.META_PREFIX + f.getName())) + .map(valueConverter) + .map(val -> { + try { + f.set(instance, val); + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } + return true; + }) + .orElse(false); }).collect(toList()); propertyValidationCreator = config -> { final PropertyValidation validation = new PropertyValidation(); @@ -89,19 +89,20 @@ public Map mapMeta(final PropertyValidation propertyValidation) final Map metaMap = new HashMap<>(); Stream.of(PropertyValidation.class.getDeclaredFields()).filter(field -> { - try { - field.setAccessible(true); - return field.get(propertyValidation) != null; - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - }) + try { + field.setAccessible(true); + return field.get(propertyValidation) != null; + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + }) .forEach(field -> { field.setAccessible(true); try { Object value = field.get(propertyValidation); if (value != null) { - metaMap.put(ValidationParameterEnricher.META_PREFIX + field.getName(), String.valueOf(value));//f.get(instance).toString()); + metaMap.put(ValidationParameterEnricher.META_PREFIX + field.getName(), + String.valueOf(value));// f.get(instance).toString()); } } catch (IllegalAccessException e) { throw new RuntimeException(e); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java index 00c0c53b64c35..b4da2e2e05545 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java @@ -182,7 +182,7 @@ void migrateOver8196DefaultByteBuffer() { } private void assertIndex(final ConfigTypeNodes index) { - assertEquals(5, index.getNodes().size()); + assertEquals(8, index.getNodes().size()); index.getNodes().keySet().forEach(Assertions::assertNotNull); // assert no null ids // assert there is at least one parent node assertTrue(index.getNodes().values().stream().anyMatch(n -> n.getParentId() == null)); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index 3f093e73be74e..7d6120430a4e6 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -66,7 +66,7 @@ private static void boolWrapper(@Option final BoolBool wrapper) { } private static void multipleParams(@Option("aa") final String first, @Option("b") final BoolWrapper config, - @Option("a") final String last) { + @Option("a") final String last) { // no-op } @@ -119,7 +119,7 @@ void booleanDefault() throws NoSuchMethodException { } private List getProperties(final String locale) { - final String[] i18nPackages = {Config.class.getPackage().getName()}; + final String[] i18nPackages = { Config.class.getPackage().getName() }; final ParameterMeta host = new ParameterMeta(null, Config.class, ParameterMeta.Type.STRING, "configuration.host", "host", i18nPackages, emptyList(), null, emptyMap(), false); @@ -163,7 +163,7 @@ void buildPropertiesFr() { } @Test - // the class BaseConfig don't contains attribute + // the class BaseConfig don't contains attribute void validateProp() { assertThrows(IllegalArgumentException.class, () -> { ParameterMeta attribute = new ParameterMeta(null, BadConfig.class, ParameterMeta.Type.STRING, @@ -179,7 +179,7 @@ void validateProp() { } @Test - // the class BaseConfig don't contains attribute + // the class BaseConfig don't contains attribute void validateConfigNode() { final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); final List uniques = Arrays.asList("one", "two", "three"); @@ -194,7 +194,8 @@ void validateConfigNode() { String usernameError = "- Property 'configuration.connection.username' is required."; String minError = "- Property 'configuration.limit' should be > 100, got 99."; String maxError = "- Property 'configuration.limit' should be < 150, got 200."; - String ValueEvalError = "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; + String ValueEvalError = + "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; connection = base .path("configurationtype/details") @@ -202,7 +203,10 @@ void validateConfigNode() { .request(APPLICATION_JSON_TYPE) .header("Accept-Encoding", "gzip") .get(ConfigTypeNodes.class) - .getNodes().values().iterator().next(); + .getNodes() + .values() + .iterator() + .next(); JsonObject payload; @@ -280,7 +284,6 @@ void validateConfigNode() { .build(); checkErrors(payload, Arrays.asList(connectionError, url1Error)); - /* * uniqVals */ @@ -313,46 +316,55 @@ void validateConfigNode() { .add("limit", 100)) .build(); - checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); + checkErrors(payload, + Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); /* - * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = + * "undx") */ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "undx") - .add("password", JsonValue.NULL).build()) - .build()).build(); + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); checkErrors(payload, Arrays.asList(connectionError, passwordError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "abcd") - .add("password", JsonValue.NULL).build()) - .build()).build(); + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); checkErrors(payload, Arrays.asList(connectionError, url1Required)); - payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "undx") - .add("password", "abc").build()) - .build()).build(); + .add("password", "abc") + .build()) + .build()) + .build(); checkErrors(payload, Arrays.asList(connectionError, url1Required)); /* - * valueEval @ActiveIf(target = "checkbox1", value = "true") + * valueEval @ActiveIf(target = "checkbox1", value = "true") */ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "abcd") .add("checkbox1", JsonValue.TRUE) - .add("valueEval", JsonValue.NULL).build()) - .build()).build(); + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); checkErrors(payload, Arrays.asList(connectionError, url1Required, ValueEvalError)); payload = factory.createObjectBuilder() @@ -360,8 +372,10 @@ void validateConfigNode() { .add("connection", factory.createObjectBuilder() .add("username", "abcd") .add("checkbox1", JsonValue.FALSE) - .add("valueEval", JsonValue.NULL).build()) - .build()).build(); + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); checkErrors(payload, Arrays.asList(connectionError, url1Required)); payload = factory.createObjectBuilder() @@ -369,8 +383,10 @@ void validateConfigNode() { .add("connection", factory.createObjectBuilder() .add("username", "abcd") .add("checkbox1", JsonValue.TRUE) - .add("valueEval", "VALUE_2").build()) - .build()).build(); + .add("valueEval", "VALUE_2") + .build()) + .build()) + .build(); checkErrors(payload, Arrays.asList(connectionError, url1Required)); } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java index d19529ca02e8a..066d3a6e3aa50 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -1,15 +1,17 @@ -/* - * Copyright (C) 2006-2022 Talend Inc. - www.talend.com +/** + * Copyright (C) 2006-2023 Talend Inc. - www.talend.com * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.talend.sdk.component.server.service; @@ -59,6 +61,7 @@ @Slf4j @MonoMeecrowaveConfig public class ValidationServiceTest { + @Inject private WebTarget base; @@ -71,18 +74,24 @@ public class ValidationServiceTest { private Jsonb jsonb = JsonbBuilder.create(); final String lang = "en"; + final String family = "custom"; + final String component = "noop"; + ConfigTypeNodes details; + ConfigTypeNode connection; + JsonSchema jsonSchema; + private Collection uiSchemas; private final UiSpecService uiSpecService = new UiSpecService<>(new Client() { @Override // for dynamic_values, just return an empty schema public CompletionStage> action(final String family, final String type, - final String action, final String lang, final Map params, final Object context) { + final String action, final String lang, final Map params, final Object context) { final Map result = new HashMap<>(); result.put("items", emptyList()); return CompletableFuture.completedFuture(result); @@ -162,7 +171,8 @@ void testValidation() throws Exception { String usernameError = "- Property 'configuration.connection.username' is required."; String minError = "- Property 'configuration.limit' should be > 100, got 1."; String maxError = "- Property 'configuration.limit' should be < 150, got 1,000."; - String ValueEvalError = "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; + String ValueEvalError = + "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; JsonObject payload; /** @@ -170,7 +180,9 @@ void testValidation() throws Exception { **/ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() - .add("connectn", "").build()).build(); + .add("connectn", "") + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); /** * min/max @@ -194,8 +206,10 @@ void testValidation() throws Exception { payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() - .add("url0", "mailto://toto@titi.org").build()) - .build()).build(); + .add("url0", "mailto://toto@titi.org") + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url0Error, url1Required, usernameError)); /* * url1 @@ -203,15 +217,19 @@ void testValidation() throws Exception { payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() - .add("url1", "mailto://toto@titi.org").build()) - .build()).build(); + .add("url1", "mailto://toto@titi.org") + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Error, usernameError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() - .add("url101", "mailto://toto@titi.org").build()) - .build()).build(); + .add("url101", "mailto://toto@titi.org") + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); /* @@ -245,7 +263,8 @@ void testValidation() throws Exception { .add("limit", 100)) .build(); - checkAsserts(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); + checkAsserts(payload, + Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() @@ -261,7 +280,8 @@ void testValidation() throws Exception { .add("limit", 100)) .build(); - checkAsserts(payload, Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); + checkAsserts(payload, + Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); } @Test @@ -276,31 +296,37 @@ void testValidation_activeIf() throws Exception { JsonObject payload; /* - * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = + * "undx") */ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "undx") - .add("password", JsonValue.NULL).build()) - .build()).build(); + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, passwordError, url1Required)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "abcd") - .add("password", JsonValue.NULL).build()) - .build()).build(); + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required)); - payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("username", "undx") - .add("password", "abc").build()) - .build()).build(); + .add("password", "abc") + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required)); } @@ -312,38 +338,44 @@ void testValidation_activeIf_2() throws Exception { String connectionError = "- Property 'configuration.connection' is required."; String usernameError = "- Property 'configuration.connection.username' is required."; String url1Required = "- Property 'configuration.connection.url1' is required."; - String ValueEvalError = "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; + String ValueEvalError = + "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; JsonObject payload; /* - * valueEval @ActiveIf(target = "checkbox1", value = "true") + * valueEval @ActiveIf(target = "checkbox1", value = "true") * */ payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("checkbox1", JsonValue.TRUE) - .add("valueEval", JsonValue.NULL).build()) - .build()).build(); + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError, ValueEvalError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("checkbox1", JsonValue.FALSE) - .add("valueEval", JsonValue.NULL).build()) - .build()).build(); + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); payload = factory.createObjectBuilder() .add("configuration", factory.createObjectBuilder() .add("connection", factory.createObjectBuilder() .add("checkbox1", JsonValue.TRUE) - .add("valueEval", "VALUE_2").build()) - .build()).build(); + .add("valueEval", "VALUE_2") + .build()) + .build()) + .build(); checkAsserts(payload, Arrays.asList(connectionError, url1Required, usernameError)); } - } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java index d315118b310c3..d635734f5b3e0 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java @@ -4,9 +4,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,7 +41,7 @@ public class CustomProcessor implements Serializable { @Option DataSetCustom dataset; - public CustomProcessor (@Option("configuration") final DataSetCustom dataset) { + public CustomProcessor(@Option("configuration") final DataSetCustom dataset) { this.dataset = dataset; } @@ -91,12 +91,13 @@ static class Connection { @Option private boolean checkbox1; + @Option private boolean checkbox2; @Option @ActiveIf(target = "checkbox1", value = "true") - private ValueEval valueEval= ValueEval.VALUE_2; + private ValueEval valueEval = ValueEval.VALUE_2; @Option @Required From 64054801e0c4f84872df38017874ebec40ee9df7 Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Wed, 12 Apr 2023 10:49:09 +0800 Subject: [PATCH 15/16] fix(TCOMP-2260): fix imports --- .../validation/JsonSchemaValidatorFactoryExt.java | 3 +-- .../runtime/manager/reflect/ReflectionService.java | 14 +++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java b/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java index 547c713347d7f..766c1b23a4704 100644 --- a/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java +++ b/component-form/component-form-core/src/main/java/org/talend/sdk/component/form/internal/validation/JsonSchemaValidatorFactoryExt.java @@ -48,8 +48,7 @@ public List createDefault .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.MaximumValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.RequiredValidation.class.isInstance(v)) .filter(v -> !org.apache.johnzon.jsonschema.spi.builtin.PatternValidation.class.isInstance(v)) - .collect(Collectors.toList()) - ; + .collect(Collectors.toList()); validations.add(new TypeValidation()); validations.add(new EnumValidationWithDefaultValue()); validations.add(new MinimumValidation()); diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index df2de5d73d167..a5bb62d152859 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -33,7 +33,19 @@ import java.lang.reflect.Field; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import java.util.*; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.BiFunction; From c07b3079b26ce72769f6a359b4187a26506e158c Mon Sep 17 00:00:00 2001 From: yyin-talend Date: Tue, 18 Apr 2023 16:56:54 +0800 Subject: [PATCH 16/16] fix(TCOMP-2260): merge TCOMP-2407 --- .gitignore | 5 +- .jenkins/scripts/docker_build.sh | 34 +- Jenkinsfile | 584 ++++++++++++++---- ci/Jenkinsfile-scan | 2 +- .../service/MavenRepositoryResolverTest.java | 14 +- .../pages/_partials/generated_changelog.adoc | 32 +- .../_partials/generated_contributors.adoc | 18 +- .../src/main/antora/site-template.yml | 5 + images/component-server-image/pom.xml | 2 +- pom.xml | 13 + reporting/pom.xml | 36 +- 11 files changed, 599 insertions(+), 146 deletions(-) diff --git a/.gitignore b/.gitignore index fb027cfeb4965..4dde0cee36f75 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,7 @@ documentation/rebuildpdf.bat # release process .build/ -.build_source/ \ No newline at end of file +.build_source/ +# direnv +.envrc +.env \ No newline at end of file diff --git a/.jenkins/scripts/docker_build.sh b/.jenkins/scripts/docker_build.sh index f1e105ce70a7d..9e5d1a22ff879 100755 --- a/.jenkins/scripts/docker_build.sh +++ b/.jenkins/scripts/docker_build.sh @@ -20,20 +20,32 @@ set -xe # Parameters: # $1: docker tag version # $2: should tag as latest +# $3: requested image, if not given, all will be pushed -tag="${1?Missing tag}" -latest="${2:-false}" +_TAG="${1?Missing tag}" +_IS_LATEST="${2-'false'}" +_ONLY_ONE_IMAGE="${3-''}" dockerBuild() { - echo ">> Building and pushing $1:${tag}" - cd "images/${1}-image" - mvn package jib:build@build -Ddocker.talend.image.tag=${tag} - if [[ ${latest} == 'true' ]]; then - mvn package jib:build@build -Ddocker.talend.image.tag=latest + _IMAGE="${1}" + echo ">> Building and push $_IMAGE:${_TAG}" + + mvn package jib:build@build \ + --file "images/${_IMAGE}-image/pom.xml" \ + --define docker.talend.image.tag="${_TAG}" + + if [[ ${_IS_LATEST} == 'true' ]]; then + mvn package jib:build@build \ + --file "images/${_IMAGE}-image/pom.xml" \ + --define docker.talend.image.tag=latest fi - cd ../.. } -dockerBuild "component-server" -dockerBuild "component-starter-server" -dockerBuild "remote-engine-customizer" +if [[ -n "${_ONLY_ONE_IMAGE}" ]]; then + dockerBuild "${_ONLY_ONE_IMAGE}" +else + dockerBuild "component-server" + dockerBuild "component-starter-server" + dockerBuild "remote-engine-customizer" +fi + diff --git a/Jenkinsfile b/Jenkinsfile index cb05e158972f2..f9c38199dc5aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,8 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +// Imports +import java.time.LocalDateTime +import java.util.regex.Matcher + // Credentials final def ossrhCredentials = usernamePassword(credentialsId: 'ossrh-credentials', usernameVariable: 'OSSRH_USER', passwordVariable: 'OSSRH_PASS') +final def nexusCredentials = usernamePassword(credentialsId: 'nexus-artifact-zl-credentials', usernameVariable: 'NEXUS_USER', passwordVariable: 'NEXUS_PASS') final def jetbrainsCredentials = usernamePassword(credentialsId: 'jetbrains-credentials', usernameVariable: 'JETBRAINS_USER', passwordVariable: 'JETBRAINS_PASS') final def jiraCredentials = usernamePassword(credentialsId: 'jira-credentials', usernameVariable: 'JIRA_USER', passwordVariable: 'JIRA_PASS') final def gitCredentials = usernamePassword(credentialsId: 'github-credentials', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_PASS') @@ -28,14 +33,24 @@ final String slackChannel = 'components-ci' final Boolean isMasterBranch = env.BRANCH_NAME == "master" final Boolean isStdBranch = (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("maintenance/")) final Boolean hasPostLoginScript = params.POST_LOGIN_SCRIPT != "" -final Boolean hasExtraBuildArgs = params.EXTRA_BUILD_ARGS != "" -final String buildTimestamp = String.format('-%tY% + dev branches are deploying on ''') + booleanParam( + name: 'DOCKER_PUSH', + defaultValue: false, + description: ''' + Force DOCKER push stage for development branches. No effect on master and maintenance. + INFO: master/maintenance and dev branches are deploying on ''') + string( + name: 'VERSION_QUALIFIER', + defaultValue: 'DEFAULT', + description: ''' + Deploy jars with the given version qualifier. No effect on master and maintenance. + - DEFAULT means the qualifier will be the Jira id extracted from the branch name. + From "user/JIRA-12345_some_information" the qualifier will be JIRA-12345.''') string( - name: 'EXTRA_BUILD_ARGS', + name: 'EXTRA_BUILD_PARAMS', defaultValue: '', description: 'Add some extra parameters to maven commands. Applies to all maven calls.') string( @@ -80,7 +109,15 @@ pipeline { defaultValue: '', description: 'Execute a shell command after login. Useful for maintenance.') booleanParam( - name: 'DEBUG_BEFORE_EXITING', + name: 'FORCE_SONAR', + defaultValue: false, + description: 'Force Sonar analysis') + booleanParam( + name: 'FORCE_DOC', + defaultValue: false, + description: 'Force documentation stage for development branches. No effect on master and maintenance.') + booleanParam( + name: 'JENKINS_DEBUG', defaultValue: false, description: 'Add an extra step to the pipeline allowing to keep the pod alive for debug purposes.') } @@ -88,6 +125,9 @@ pipeline { stages { stage('Preliminary steps') { steps { + /////////////////////////////////////////// + // Login tasks + /////////////////////////////////////////// script { withCredentials([gitCredentials]) { sh """ bash .jenkins/scripts/git_login.sh "\${GITHUB_USER}" "\${GITHUB_PASS}" """ @@ -96,16 +136,107 @@ pipeline { sh """ bash .jenkins/scripts/docker_login.sh "${ARTIFACTORY_REGISTRY}" "\${DOCKER_USER}" "\${DOCKER_PASS}" """ } withCredentials([keyImportCredentials]) { - sh """ bash .jenkins/scripts/setup_gpg.sh""" + sh """ bash .jenkins/scripts/setup_gpg.sh """ + } + } + /////////////////////////////////////////// + // asdf install + /////////////////////////////////////////// + script { + println "asdf install the content of repository .tool-versions'\n" + sh 'bash .jenkins/scripts/asdf_install.sh' + } + /////////////////////////////////////////// + // Variables init + /////////////////////////////////////////// + script { + stdBranch_buildOnly = isStdBranch && params.Action != 'RELEASE' + devBranch_mavenDeploy = !isStdBranch && params.MAVEN_DEPLOY + devBranch_dockerPush = !isStdBranch && params.DOCKER_PUSH + + needQualify = devBranch_mavenDeploy || devBranch_dockerPush + + if (needQualify) { + // Qualified version have to be released on talend_repository + // Overwrite the deployOptions + deployOptions = "$skipOptions --activate-profiles private_repository -Denforcer.skip=true" + } + + // By default the doc is skipped for standards branches + Boolean skip_documentation = !( params.FORCE_DOC || isStdBranch ) + extraBuildParams = assemblyExtraBuildParams(skip_documentation) + + } + /////////////////////////////////////////// + // Pom version and Qualifier management + /////////////////////////////////////////// + script{ + final def pom = readMavenPom file: 'pom.xml' + pomVersion = pom.version + + echo 'Manage the version qualifier' + if (!needQualify) { + println """ + No need to add qualifier in followings cases: + - We are on Master or Maintenance branch + - We do not want to deploy on dev branch + """.stripIndent() + finalVersion = pomVersion } + else { + branch_user = "" + branch_ticket = "" + branch_description = "" + if (params.VERSION_QUALIFIER != ("DEFAULT")) { + // If the qualifier is given, use it + println """ + No need to add qualifier, use the given one: "$params.VERSION_QUALIFIER" + """.stripIndent() + } + else { + echo "Validate the branch name" + (branch_user, + branch_ticket, + branch_description) = extract_branch_info("$env.BRANCH_NAME") - def pom = readMavenPom file: 'pom.xml' - env.PROJECT_VERSION = pom.version - try { - EXTRA_BUILD_ARGS = params.EXTRA_BUILD_ARGS - } catch (ignored) { - EXTRA_BUILD_ARGS = "" + // Check only branch_user, because if there is an error all three params are empty. + if (branch_user == ("")) { + println """ + ERROR: The branch name doesn't comply with the format: user/JIRA-1234-Description + It is MANDATORY for artifact management. + You have few options: + - You do not need to deploy, uncheck MAVEN_DEPLOY checkbox + - Change the VERSION_QUALIFIER text box to a personal qualifier, BUT you need to do it on ALL se/ee and cloud-components build + - Rename your branch + """.stripIndent() + currentBuild.description = ("ERROR: The branch name is not correct") + sh """exit 1""" + } + } + + echo "Insert a qualifier in pom version..." + finalVersion = add_qualifier_to_version( + pomVersion, + branch_ticket, + "$params.VERSION_QUALIFIER" as String) + + echo """ + Configure the version qualifier for the curent branche: $env.BRANCH_NAME + requested qualifier: $params.VERSION_QUALIFIER + with User = $branch_user, Ticket = $branch_ticket, Description = $branch_description + Qualified Version = $finalVersion""" + + // On development branches the connectors version shall be edited for deployment + // Maven documentation about maven_version: + // https://docs.oracle.com/middleware/1212/core/MAVEN/maven_version.htm + println "Edit version on dev branches, new version is ${finalVersion}" + sh """\ + #!/usr/bin/env bash + mvn versions:set --define newVersion=${finalVersion} + mvn versions:set --file bom/pom.xml --define newVersion=${finalVersion} + """.stripIndent() } + } /////////////////////////////////////////// // Updating build displayName and description @@ -114,29 +245,42 @@ pipeline { String user_name = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause').userId[0] if ( user_name == null) { user_name = "auto" } + String deploy_info = '' + if (stdBranch_buildOnly || devBranch_mavenDeploy){ + deploy_info = deploy_info + '+DEPLOY' + } + if (devBranch_dockerPush){ + deploy_info = deploy_info + '+DOCKER' + } + currentBuild.displayName = ( - "#$currentBuild.number-$params.Action: $user_name" + "#$currentBuild.number-$params.Action" + deploy_info + ": $user_name" ) // updating build description - currentBuild.description = (""" - User: $user_name - $params.Action Build - Sonar: $params.FORCE_SONAR - Script: $hasPostLoginScript - Extra args: $hasExtraBuildArgs - Debug: $params.DEBUG_BEFORE_EXITING""".stripIndent() - ) + String description = """ + Version = $finalVersion - $params.Action Build + Sonar forced: $params.FORCE_SONAR - Script: $hasPostLoginScript + Debug: $params.JENKINS_DEBUG + Extra build args: $extraBuildParams""".stripIndent() + job_description_append(description) } - /////////////////////////////////////////// - // asdf install - /////////////////////////////////////////// - script { - println "asdf install the content of repository .tool-versions'\n" - sh 'bash .jenkins/scripts/asdf_install.sh' + } + post { + always { + println "Artifact Poms files for analysis if needed" + archiveArtifacts artifacts: '**/*pom.*', allowEmptyArchive: false, onlyIfSuccessful: false } } } stage('Post login') { steps { - withCredentials([gitCredentials, dockerCredentials, ossrhCredentials, jetbrainsCredentials, jiraCredentials, gpgCredentials]) { + withCredentials([gitCredentials, + dockerCredentials, + ossrhCredentials, + jetbrainsCredentials, + jiraCredentials, + gpgCredentials]) { script { try { sh """\ @@ -151,68 +295,130 @@ pipeline { } } } - stage('Standard maven build') { + stage('Maven validate to install') { when { expression { params.Action != 'RELEASE' } } steps { withCredentials([ossrhCredentials]) { sh """\ #!/usr/bin/env bash - mvn clean install $BUILD_ARGS $EXTRA_BUILD_ARGS -s .jenkins/settings.xml + set -xe + mvn clean install --file bom/pom.xml + mvn clean install $BUILD_ARGS \ + $extraBuildParams \ + --settings .jenkins/settings.xml """.stripIndent() } } + post { + always { + recordIssues( + enabledForFailure: true, + tools: [ + junitParser( + id: 'unit-test', + name: 'Unit Test', + pattern: '**/target/surefire-reports/*.xml' + ) + ] + ) + } + } } - stage('Deploy artifacts') { + stage('Maven deploy') { when { - allOf { - expression { params.Action != 'RELEASE' } - expression { isStdBranch } + anyOf { + expression { stdBranch_buildOnly } + expression { devBranch_mavenDeploy } } } steps { - withCredentials([ossrhCredentials, gpgCredentials]) { - sh """\ + script { + withCredentials([ossrhCredentials, + gpgCredentials, + nexusCredentials]) { + sh """\ #!/usr/bin/env bash - bash mvn deploy $DEPLOY_OPTS $EXTRA_BUILD_ARGS -s .jenkins/settings.xml + set -xe + bash mvn deploy $deployOptions \ + $extraBuildParams \ + --settings .jenkins/settings.xml """.stripIndent() + } + } + // Add description to job + script { + def repo + if (devBranch_mavenDeploy) { + repo = ['artifacts-zl.talend.com', + 'https://artifacts-zl.talend.com/nexus/content/repositories/snapshots/org/talend/sdk/component'] + } else { + repo = ['oss.sonatype.org', + 'https://oss.sonatype.org/content/repositories/snapshots/org/talend/sdk/component/'] + } + + job_description_append("Maven artefact deployed as ${finalVersion} on [${repo[0]}](${repo[1]})") } } } - stage('Docker images') { + stage('Docker build/push') { when { - allOf { - expression { params.Action != 'RELEASE' } - expression { isStdBranch } + anyOf { + expression { stdBranch_buildOnly } + expression { devBranch_dockerPush } } } steps { script { configFileProvider([configFile(fileId: 'maven-settings-nexus-zl', variable: 'MAVEN_SETTINGS')]) { - sh """\ - #!/usr/bin/env bash - bash .jenkins/scripts/docker_build.sh ${env.PROJECT_VERSION}${buildTimestamp} - """.stripIndent() + + String images_options = '' + if (isStdBranch){ + // Build and push all images + job_description_append("Docker images deployed: component-server, component-starter-server and remote-engine-customizer") + } + else{ + images_options = 'false component-server' + job_description_append("Docker images deployed: component-server") + job_description_append("As ${finalVersion}${buildTimestamp} on [artifactory.datapwn.com](https://artifactory.datapwn.com/tlnd-docker-dev/talend/common/tacokit)" as String) + + } + + // Build and push specific image + sh """ + bash .jenkins/scripts/docker_build.sh \ + ${finalVersion}${buildTimestamp} \ + ${images_options} + """ + + job_description_append("docker pull artifactory.datapwn.com/tlnd-docker-dev/talend/common/tacokit/component-server:${finalVersion}${buildTimestamp}") } + } } } stage('Documentation') { when { - allOf { - expression { params.Action != 'RELEASE' } - expression { isMasterBranch } + expression { + params.FORCE_DOC || (params.Action != 'RELEASE' && isMasterBranch) } } steps { withCredentials([ossrhCredentials, gitCredentials]) { sh """\ #!/usr/bin/env bash - cd documentation && mvn verify pre-site -Pgh-pages -Dgpg.skip=true $SKIP_OPTS $EXTRA_BUILD_ARGS -s ../.jenkins/settings.xml && cd - + set -xe + mvn verify pre-site --file documentation/pom.xml \ + --settings .jenkins/settings.xml \ + --activate-profiles gh-pages \ + --define gpg.skip=true \ + $skipOptions \ + $extraBuildParams + """.stripIndent() } } } - stage('Master Post Build Tasks') { + stage('OSS security analysis') { when { expression { params.Action != 'RELEASE' } branch 'master' @@ -222,59 +428,100 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh """\ #!/usr/bin/env bash - mvn ossindex:audit-aggregate -pl '!bom' -Dossindex.fail=false -Dossindex.reportFile=target/audit.txt -s .jenkins/settings.xml - mvn versions:dependency-updates-report versions:plugin-updates-report versions:property-updates-report -pl '!bom' + set -xe + mvn ossindex:audit-aggregate -pl '!bom' \ + --define ossindex.fail=false \ + --define ossindex.reportFile=target/audit.txt \ + --settings .jenkins/settings.xml """.stripIndent() } } - withCredentials([sonarCredentials]) { + } + post { + always { + publishHTML( + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/', + reportFiles : 'audit.txt', + reportName : "security::audit" + ]) + } + } + } + stage('Deps report') { + when { + expression { params.Action != 'RELEASE' } + branch 'master' + } + steps { + withCredentials([ossrhCredentials]) { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - // TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache) sh """\ #!/usr/bin/env bash - _JAVA_OPTIONS='--add-opens=java.base/java.lang=ALL-UNNAMED' mvn -Dsonar.host.url=https://sonar-eks.datapwn.com -Dsonar.login='$SONAR_USER' -Dsonar.password='$SONAR_PASS' -Dsonar.branch.name=${env.BRANCH_NAME} -Dsonar.analysisCache.enabled=false sonar:sonar - """.stripIndent() + set -xe + mvn versions:dependency-updates-report versions:plugin-updates-report \ + versions:property-updates-report \ + -pl '!bom' + """.stripIndent() } } } post { always { publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/', - reportFiles : 'audit.txt', - reportName : "security::audit" - ]) - publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/site/', - reportFiles : 'property-updates-report.html', - reportName : "outdated::property" - ]) + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/site/', + reportFiles : 'property-updates-report.html', + reportName : "outdated::property" + ]) publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/site/', - reportFiles : 'dependency-updates-report.html', - reportName : "outdated::dependency" - ]) + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/site/', + reportFiles : 'dependency-updates-report.html', + reportName : "outdated::dependency" + ]) publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/site/', - reportFiles : 'plugin-updates-report.html', - reportName : "outdated::plugins" - ]) + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/site/', + reportFiles : 'plugin-updates-report.html', + reportName : "outdated::plugins" + ]) + } + } + } + stage('Sonar') { + when { + expression { params.Action != 'RELEASE' } + branch 'master' + } + steps { + withCredentials([sonarCredentials]) { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + // TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache) + sh """\ + #!/usr/bin/env bash + set -xe + _JAVA_OPTIONS='--add-opens=java.base/java.lang=ALL-UNNAMED' + mvn sonar:sonar \ + --define sonar.host.url=https://sonar-eks.datapwn.com \ + --define sonar.login='$SONAR_USER' \ + --define sonar.password='$SONAR_PASS' \ + --define sonar.branch.name=${env.BRANCH_NAME} \ + --define sonar.analysisCache.enabled=false + """.stripIndent() + } } } } @@ -291,17 +538,13 @@ pipeline { configFileProvider([configFile(fileId: 'maven-settings-nexus-zl', variable: 'MAVEN_SETTINGS')]) { sh """\ #!/usr/bin/env bash - bash .jenkins/scripts/release.sh ${env.BRANCH_NAME} ${env.PROJECT_VERSION} + bash .jenkins/scripts/release.sh ${env.BRANCH_NAME} ${finalVersion} """.stripIndent() } } } } } - stage('Debug') { - when { expression { return params.DEBUG_BEFORE_EXITING } } - steps { script { input message: 'Finish the job?', ok: 'Yes' } } - } } post { success { @@ -337,7 +580,7 @@ pipeline { //Only post results to Slack for Master and Maintenance branches if (isStdBranch) { //if previous build was a success, ping channel in the Slack message - if ("SUCCESS".equals(currentBuild.previousBuild.result)) { + if ("SUCCESS" == currentBuild.previousBuild.result) { slackSend( color: '#FF0000', message: "@here : NEW FAILURE: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})", @@ -358,11 +601,6 @@ pipeline { recordIssues( enabledForFailure: true, tools: [ - junitParser( - id: 'unit-test', - name: 'Unit Test', - pattern: '**/target/surefire-reports/*.xml' - ), taskScanner( id: 'disabled', name: '@Disabled', @@ -381,10 +619,142 @@ pipeline { ] ) script { - println '====== Archive artifacts' - println "Artifact 1: ${_ARTIFACT_COVERAGE}" - archiveArtifacts artifacts: "${_ARTIFACT_COVERAGE}", allowEmptyArchive: true, onlyIfSuccessful: false + println '====== Archive jacoco reports artifacts' + archiveArtifacts artifacts: "${'**/jacoco-aggregate/**/*.*'}", allowEmptyArchive: true, onlyIfSuccessful: false + } + + script { + if (params.JENKINS_DEBUG) { + jenkinsBreakpoint() + } } } } } + + +/** + * Append a new line to job description + * This is MARKDOWN, do not forget double space at the end of line + * + * @param new line + * @return void + */ +private void job_description_append(String new_line) { + if (currentBuild.description == null) { + println "Create the job description with: \n$new_line" + currentBuild.description = new_line + } else { + println "Edit the job description adding: $new_line" + currentBuild.description = currentBuild.description + '\n' + new_line + } +} + +/** + * Assembly all needed items to put inside extraBuildParams + * + * @param Boolean skip_doc, if set to true documentation build will be skipped + * + * @return extraBuildParams as a string ready for mvn cmd + */ +private String assemblyExtraBuildParams(Boolean skip_doc) { + String extraBuildParams + + println 'Processing extraBuildParams' + final List buildParamsAsArray = [] + + println 'Manage user params' + if ( params.EXTRA_BUILD_PARAMS ) { + buildParamsAsArray.add(params.EXTRA_BUILD_PARAMS as String) + } + + println 'Manage the skip_doc option' + if (skip_doc) { + buildParamsAsArray.add('--projects !documentation') + buildParamsAsArray.add('--define documentation.skip=true') + } + + println 'Construct final params content' + extraBuildParams = buildParamsAsArray.join(' ') + println "extraBuildParams: $extraBuildParams" + + return extraBuildParams +} + +/** + * Implement a simple breakpoint to stop actual job + * Use the method anywhere you need to stop + * The first usage is to keep the pod alive on post stage. + * Change and restore the job description to be more visible + * + * @param none + * @return void + */ +private void jenkinsBreakpoint() { + // Backup the description + String job_description_backup = currentBuild.description + // updating build description + currentBuild.description = "ACTION NEEDED TO CONTINUE \n ${job_description_backup}" + // Request user action + input message: 'Finish the job?', ok: 'Yes' + // updating build description + currentBuild.description = "$job_description_backup" +} + +/** + * create a new version from actual one and given jira ticket or user qualifier + * Priority to user qualifier + * + * The branch name has comply with the format: user/JIRA-1234-Description + * It is MANDATORY for artifact management. + * + * @param String version actual version to edit + * @param GString ticket + * @param GString user_qualifier to be checked as priority qualifier + * + * @return String new_version with added qualifier + */ +private static String add_qualifier_to_version(String version, String ticket, String user_qualifier) { + String new_version + + if (user_qualifier.contains("DEFAULT")) { + if (version.contains("-SNAPSHOT")) { + new_version = version.replace("-SNAPSHOT", "-$ticket-SNAPSHOT" as String) + } else { + new_version = "$version-$ticket".toString() + } + } else { + new_version = version.replace("-SNAPSHOT", "-$user_qualifier-SNAPSHOT" as String) + } + return new_version +} + +/** + * extract given branch information + * + * The branch name has comply with the format: user/JIRA-1234-Description + * It is MANDATORY for artifact management. + * + * @param branch_name row name of the branch + * + * @return A list containing the extracted: [user, ticket, description] + * The method also raise an assert exception in case of wrong branch name + */ +private static ArrayList extract_branch_info(GString branch_name) { + + String branchRegex = /^(?.*)\/(?[A-Z]{2,8}-\d{1,6})[_-](?.*)/ + Matcher branchMatcher = branch_name =~ branchRegex + + try { + assert branchMatcher.matches() + } + catch (AssertionError ignored) { + return ["", "", ""] + } + + String user = branchMatcher.group("user") + String ticket = branchMatcher.group("ticket") + String description = branchMatcher.group("description") + + return [user, ticket, description] +} \ No newline at end of file diff --git a/ci/Jenkinsfile-scan b/ci/Jenkinsfile-scan index 2a36a5f96539c..fee7cd439c37e 100644 --- a/ci/Jenkinsfile-scan +++ b/ci/Jenkinsfile-scan @@ -21,7 +21,7 @@ final def veracodeCredentials = usernamePassword( final def nexusCredentials = usernamePassword( credentialsId: 'nexus-artifact-zl-credentials', usernameVariable: 'NEXUS_USER', - passwordVariable: 'NEXUS_PASSWORD') + passwordVariable: 'NEXUS_PASS') String git_branch_name = "" diff --git a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java index c98012157639a..9691d1d2c77a5 100644 --- a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java +++ b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java @@ -17,7 +17,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.M2_REPOSITORY; import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.STUDIO_MVN_REPOSITORY; import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.TALEND_COMPONENT_MANAGER_M2_REPOSITORY; import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.TALEND_COMPONENT_MANAGER_M2_SETTINGS; @@ -39,6 +38,8 @@ class MavenRepositoryResolverTest { + static String M2_REPOSITORY = ".m2" + File.separator + "repository"; + private final MavenRepositoryResolver resolver = new MavenRepositoryDefaultResolver(); final MavenRepositoryDefaultResolver mavenSettingsOnlyResolver = new MavenRepositoryDefaultResolver() { @@ -63,7 +64,7 @@ public Path get(final String path) { } }; - private final String fallback = System.getProperty("user.home") + "/" + M2_REPOSITORY; + private final String fallback = System.getProperty("user.home") + File.separator + M2_REPOSITORY; private final Path repository = Paths.get(new File("target/test-classes").getAbsolutePath()); @@ -164,7 +165,8 @@ void discoverFromSettingsWindowsPath() { mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck); final Path m2 = mavenSettingsOnlyResolver.discover(); assertNotNull(m2); - assertEquals("C:/Users/maven/repository", m2.toString()); + assertEquals("C:" + File.separator + "Users" + File.separator + "maven" + File.separator + "repository", + m2.toString()); System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS); } @@ -174,7 +176,8 @@ void discoverFromSettingsTildePath() { mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck); final Path m2 = mavenSettingsOnlyResolver.discover(); assertNotNull(m2); - assertEquals(System.getProperty("user.home") + "/mvn_home_dev/repository", m2.toString()); + assertEquals(System.getProperty("user.home") + File.separator + "mvn_home_dev" + File.separator + "repository", + m2.toString()); System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS); } @@ -184,7 +187,8 @@ void discoverFromSettingsUserHomeProperty() { mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck); final Path m2 = mavenSettingsOnlyResolver.discover(); assertNotNull(m2); - assertEquals(System.getProperty("user.home") + "/mvn_home_dev/repository", m2.toString()); + assertEquals(System.getProperty("user.home") + File.separator + "mvn_home_dev" + File.separator + "repository", + m2.toString()); System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS); } diff --git a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc index 799181716dc32..d0c8afc737ca5 100644 --- a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc +++ b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc @@ -1,5 +1,17 @@ +== Version 1.56.0 + +=== Bug + +- link:https://jira.talendforge.org/browse/TCOMP-2355[TCOMP-2355^]: Error on language support for xx_YY language files link:search.html?query=component-server[component-server^,role='dockey'] + + + +=== Work Item + +- link:https://jira.talendforge.org/browse/TCOMP-2405[TCOMP-2405^]: Upgrade snakeyaml to 2.0 link:search.html?query=build[build^,role='dockey'] + == Version 1.55.0 === Bug @@ -702,6 +714,14 @@ + + + +== Version 1.38.9 + +=== Work Item + +- link:https://jira.talendforge.org/browse/TCOMP-2412[TCOMP-2412^]: Upgrade tomcat to 9.0.69 link:search.html?query=component-server[component-server^,role='dockey'] == Version 1.38.8 @@ -1481,12 +1501,6 @@ - -== Version 1.1.15.2 - -=== Work Item - -- link:https://jira.talendforge.org/browse/TCOMP-1752[TCOMP-1752^]: Make component-runtime class loader find classes in RemoteEngine JobServer == Version 1.1.15 @@ -1521,6 +1535,12 @@ - link:https://jira.talendforge.org/browse/TCOMP-1578[TCOMP-1578^]: Upgrade asciidoctor-pdf to v1.5.0-beta.7 - link:https://jira.talendforge.org/browse/TCOMP-1581[TCOMP-1581^]: Support JUnit5 meta annotations for our extensions +== Version 1.1.15.2 + +=== Work Item + +- link:https://jira.talendforge.org/browse/TCOMP-1752[TCOMP-1752^]: Make component-runtime class loader find classes in RemoteEngine JobServer + == Version 1.1.15.1 === New Feature diff --git a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc index 5677d2360061e..17355d93380c9 100644 --- a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc +++ b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc @@ -8,7 +8,7 @@ "name":"Romain Manni-Bucau" }, { - "commits":689, + "commits":695, "description":"Software engineer @Talend.\r\n\r\nComponents team member.\n\nBlog: undx.github.io", "gravatar":"https://avatars.githubusercontent.com/u/265575?v=4", "id":"undx", @@ -50,7 +50,7 @@ "name":"ypiel" }, { - "commits":27, + "commits":28, "description":"", "gravatar":"https://avatars.githubusercontent.com/u/7742508?v=4", "id":"yyin-talend", @@ -70,6 +70,13 @@ "id":"wwang-talend", "name":"wang wei" }, + { + "commits":15, + "description":"QA Automation @ Talend Nantes", + "gravatar":"https://avatars.githubusercontent.com/u/1255625?v=4", + "id":"acatoire", + "name":"Axel CATOIRE" + }, { "commits":10, "description":"", @@ -98,13 +105,6 @@ "id":"jsomsanith-tlnd", "name":"Jimmy Somsanith" }, - { - "commits":8, - "description":"QA Automation @ Talend Nantes", - "gravatar":"https://avatars.githubusercontent.com/u/1255625?v=4", - "id":"acatoire", - "name":"Axel CATOIRE" - }, { "commits":8, "description":"", diff --git a/documentation/src/main/antora/site-template.yml b/documentation/src/main/antora/site-template.yml index 222b35cc4f14f..57e35f1eab826 100644 --- a/documentation/src/main/antora/site-template.yml +++ b/documentation/src/main/antora/site-template.yml @@ -50,6 +50,7 @@ content: - '!component-runtime-1.38.5' - '!component-runtime-1.38.6' - '!component-runtime-1.38.7' + - '!component-runtime-1.38.8' - '!component-runtime-1.39.0' - '!component-runtime-1.39.1' - '!component-runtime-1.39.2' @@ -70,6 +71,10 @@ content: - '!component-runtime-1.50.3' - '!component-runtime-1.51.0' - '!component-runtime-1.51.1' + - '!component-runtime-1.52.0' + - '!component-runtime-1.52.1' + - '!component-runtime-1.53.0' + - '!component-runtime-1.54.0' branches: - HEAD start_path: documentation/src/main/antora diff --git a/images/component-server-image/pom.xml b/images/component-server-image/pom.xml index f801281ce15b1..4d757c8f7e968 100644 --- a/images/component-server-image/pom.xml +++ b/images/component-server-image/pom.xml @@ -231,7 +231,7 @@ ${TALEND_HOME}/connectors v1/tenants-keyrings/decrypt/{x-talend-tenant-id} ${docker.talend.dir.base}component-kit/custom/*:${docker.talend.dir.base}custom/*:${TALEND_HOME}/extensions/*:${docker.talend.dir.app}resources:${docker.talend.dir.app}classes - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED -Dlog4j.configurationFile=${docker.talend.dir.app}conf/log4j2-component-server-TEXT.xml + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.net.nio=ALL-UNNAMED -Dlog4j.configurationFile=${docker.talend.dir.app}conf/log4j2-component-server-TEXT.xml docker run -d -p ${docker.image.port}:${docker.image.port} ${docker.image.to} diff --git a/pom.xml b/pom.xml index a00086202a795..7d0c91ef5dcc2 100644 --- a/pom.xml +++ b/pom.xml @@ -1746,6 +1746,19 @@ + + private_repository + + + + talend.snapshots + https://artifacts-zl.talend.com/nexus/content/repositories/snapshots + + + travis diff --git a/reporting/pom.xml b/reporting/pom.xml index ebbf841c0e3ed..94f518117cfd6 100644 --- a/reporting/pom.xml +++ b/reporting/pom.xml @@ -29,6 +29,13 @@ true + + false @@ -187,11 +194,6 @@ remote-engine-customizer-image ${project.version} - - org.talend.sdk.component - documentation - ${project.version} - org.talend.sdk.component talend-component-kit-intellij-plugin @@ -241,4 +243,28 @@ + + + + documentation + + + + documentation.skip + false + + + + + org.talend.sdk.component + documentation + ${project.version} + ${documentation.skip} + + + +