diff --git a/examples/https/pom.xml b/examples/https/pom.xml index 2bd5ae131..d1b4e8e70 100644 --- a/examples/https/pom.xml +++ b/examples/https/pom.xml @@ -12,7 +12,11 @@ io.quarkus - quarkus-resteasy + quarkus-rest-jackson + + + io.quarkus + quarkus-rest-client-jackson io.quarkus diff --git a/examples/https/src/test/java/io/quarkus/qe/OpenShiftServingCertificatesIT.java b/examples/https/src/test/java/io/quarkus/qe/OpenShiftServingCertificatesIT.java new file mode 100644 index 000000000..d47a7d946 --- /dev/null +++ b/examples/https/src/test/java/io/quarkus/qe/OpenShiftServingCertificatesIT.java @@ -0,0 +1,62 @@ +package io.quarkus.qe; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import io.quarkus.qe.hero.Hero; +import io.quarkus.qe.hero.HeroClient; +import io.quarkus.qe.hero.HeroClientResource; +import io.quarkus.qe.hero.HeroResource; +import io.quarkus.test.bootstrap.Protocol; +import io.quarkus.test.bootstrap.RestService; +import io.quarkus.test.scenarios.OpenShiftScenario; +import io.quarkus.test.scenarios.annotations.DisabledOnNative; +import io.quarkus.test.services.Certificate; +import io.quarkus.test.services.QuarkusApplication; +import io.quarkus.test.utils.AwaitilityUtils; + +/** + * Test OpenShift serving certificate support provided by our framework. + */ +@DisabledOnNative // building 2 apps is costly and point here is to test FW support not Quarkus +@OpenShiftScenario +public class OpenShiftServingCertificatesIT { + + private static final String CLIENT_TLS_CONFIG_NAME = "cert-serving-test-client"; + private static final String SERVER_TLS_CONFIG_NAME = "cert-serving-test-server"; + + @QuarkusApplication(ssl = true, certificates = @Certificate(tlsConfigName = SERVER_TLS_CONFIG_NAME, servingCertificates = @Certificate.ServingCertificates(addServiceCertificate = true)), classes = { + HeroResource.class, Hero.class }) + static final RestService server = new RestService() + .withProperty("quarkus.http.ssl.client-auth", "request") + .withProperty("quarkus.http.insecure-requests", "DISABLED"); + + @QuarkusApplication(certificates = @Certificate(tlsConfigName = CLIENT_TLS_CONFIG_NAME, servingCertificates = @Certificate.ServingCertificates(injectCABundle = true)), classes = { + HeroClient.class, Hero.class, HeroClientResource.class }) + static final RestService client = new RestService() + .withProperty("quarkus.rest-client.hero.tls-configuration-name", CLIENT_TLS_CONFIG_NAME) + .withProperty("quarkus.rest-client.hero.uri", () -> server.getURI(Protocol.HTTPS).getRestAssuredStyleUri()); + + @Test + public void testSecuredCommunicationBetweenClientAndServer() { + // REST client use OpenShift internal CA + // server is configured with OpenShift serving certificates + // ad "untilAsserted": hopefully it's not necessary, but once I experienced unknown SAN, + // so to avoid flakiness I am adding here retry: + AwaitilityUtils.untilAsserted(() -> { + var hero = client.given() + .get("hero-client-resource") + .then() + .statusCode(200) + .extract() + .as(Hero.class); + assertNotNull(hero); + assertNotNull(hero.name()); + assertTrue(hero.name().startsWith("Name-")); + assertTrue(hero.otherName().startsWith("Other-")); + }); + } + +} diff --git a/examples/https/src/test/java/io/quarkus/qe/hero/Hero.java b/examples/https/src/test/java/io/quarkus/qe/hero/Hero.java new file mode 100644 index 000000000..dbfae87c3 --- /dev/null +++ b/examples/https/src/test/java/io/quarkus/qe/hero/Hero.java @@ -0,0 +1,4 @@ +package io.quarkus.qe.hero; + +public record Hero(Long id, String name, String otherName, int level, String picture, String powers) { +} diff --git a/examples/https/src/test/java/io/quarkus/qe/hero/HeroClient.java b/examples/https/src/test/java/io/quarkus/qe/hero/HeroClient.java new file mode 100644 index 000000000..4deee5a04 --- /dev/null +++ b/examples/https/src/test/java/io/quarkus/qe/hero/HeroClient.java @@ -0,0 +1,15 @@ +package io.quarkus.qe.hero; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; + +import org.eclipse.microprofile.rest.client.inject.RegisterRestClient; + +@RegisterRestClient(configKey = "hero") +public interface HeroClient { + + @GET + @Path("/api/heroes/random") + Hero getRandomHero(); + +} diff --git a/examples/https/src/test/java/io/quarkus/qe/hero/HeroClientResource.java b/examples/https/src/test/java/io/quarkus/qe/hero/HeroClientResource.java new file mode 100644 index 000000000..54d532015 --- /dev/null +++ b/examples/https/src/test/java/io/quarkus/qe/hero/HeroClientResource.java @@ -0,0 +1,19 @@ +package io.quarkus.qe.hero; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; + +import org.eclipse.microprofile.rest.client.inject.RestClient; + +@Path("hero-client-resource") +public class HeroClientResource { + + @RestClient + HeroClient heroClient; + + @GET + public Hero triggerClientToServerCommunication() { + return heroClient.getRandomHero(); + } + +} diff --git a/examples/https/src/test/java/io/quarkus/qe/hero/HeroResource.java b/examples/https/src/test/java/io/quarkus/qe/hero/HeroResource.java new file mode 100644 index 000000000..312db76ac --- /dev/null +++ b/examples/https/src/test/java/io/quarkus/qe/hero/HeroResource.java @@ -0,0 +1,17 @@ +package io.quarkus.qe.hero; + +import java.util.random.RandomGenerator; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; + +@Path("/api/heroes/random") +public class HeroResource { + + @GET + public Hero getRandomHero() { + long random = RandomGenerator.getDefault().nextLong(); + return new Hero(random, "Name-" + random, "Other-" + random, 1, "placeholder", "root"); + } + +} diff --git a/examples/https/src/test/resources/test.properties b/examples/https/src/test/resources/test.properties index eec132864..56fc29b87 100644 --- a/examples/https/src/test/resources/test.properties +++ b/examples/https/src/test/resources/test.properties @@ -1 +1,4 @@ ts.app.log.enable=true + +# serving certs only works for internal DNS +ts.server.openshift.use-internal-service-as-url=true diff --git a/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilder.java b/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilder.java index c17dfcdda..a24a6a489 100644 --- a/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilder.java +++ b/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilder.java @@ -1,10 +1,14 @@ package io.quarkus.test.security.certificate; import static io.quarkus.test.security.certificate.Certificate.createCertsTempDir; +import static io.quarkus.test.services.Certificate.DEFAULT_CONFIG; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import io.quarkus.test.utils.TestExecutionProperties; + public interface CertificateBuilder { /** @@ -16,6 +20,8 @@ public interface CertificateBuilder { Certificate findCertificateByPrefix(String prefix); + ServingCertificateConfig servingCertificateConfig(); + /** * Regenerates certificate with {@code prefix}. * The new certificate will be stored at the same location as the original one. @@ -24,7 +30,7 @@ public interface CertificateBuilder { Certificate regenerateCertificate(String prefix, CertificateRequestCustomizer... customizers); static CertificateBuilder of(Certificate certificate) { - return new CertificateBuilderImp(List.of(certificate)); + return new CertificateBuilderImpl(List.of(certificate), null); } static CertificateBuilder of(io.quarkus.test.services.Certificate[] certificates) { @@ -35,18 +41,45 @@ static CertificateBuilder of(io.quarkus.test.services.Certificate[] certificates } private static CertificateBuilder createBuilder(io.quarkus.test.services.Certificate[] certificates) { - Certificate[] generatedCerts = new Certificate[certificates.length]; + var svcCertConfigBuilder = ServingCertificateConfig.builder(); + List generatedCerts = new ArrayList<>(); for (int i = 0; i < certificates.length; i++) { var cert = certificates[i]; + configureServingCertificates(cert, svcCertConfigBuilder); + boolean generateCerts = cert.configureHttpServer() || cert.configureManagementInterface() + || cert.configureKeystore() || cert.configureTruststore() || cert.clientCertificates().length > 0; + if (!generateCerts) { + continue; + } var clientCertReqs = Arrays.stream(cert.clientCertificates()) .map(cc -> new ClientCertificateRequest(cc.cnAttribute(), cc.unknownToServer())) .toArray(ClientCertificateRequest[]::new); - generatedCerts[i] = Certificate.ofInterchangeable(new CertificateOptions(cert.prefix(), cert.format(), + generatedCerts.add(Certificate.ofInterchangeable(new CertificateOptions(cert.prefix(), cert.format(), cert.password(), cert.configureKeystore(), cert.configureTruststore(), cert.configureManagementInterface(), clientCertReqs, createCertsTempDir(cert.prefix()), new DefaultContainerMountStrategy(cert.prefix()), - false, null, null, null, null, cert.useTlsRegistry(), cert.tlsConfigName(), cert.configureHttpServer())); + false, null, null, null, null, cert.useTlsRegistry(), cert.tlsConfigName(), cert.configureHttpServer()))); + } + return new CertificateBuilderImpl(List.copyOf(generatedCerts), svcCertConfigBuilder.build()); + } + + private static void configureServingCertificates(io.quarkus.test.services.Certificate cert, + ServingCertificateConfig.ServingCertificateConfigBuilder svcCertConfigBuilder) { + if (TestExecutionProperties.isOpenshiftPlatform() && cert.useTlsRegistry()) { + boolean servingCertificatesEnabled = cert.servingCertificates().length > 0; + if (servingCertificatesEnabled) { + for (var servingCertificate : cert.servingCertificates()) { + if (servingCertificate.addServiceCertificate()) { + svcCertConfigBuilder.withAddServiceCertificate(true); + } + if (servingCertificate.injectCABundle()) { + svcCertConfigBuilder.withInjectCABundle(true); + } + } + if (!DEFAULT_CONFIG.equals(cert.tlsConfigName())) { + svcCertConfigBuilder.withTlsConfigName(cert.tlsConfigName()); + } + } } - return new CertificateBuilderImp(List.of(generatedCerts)); } } diff --git a/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilderImp.java b/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilderImpl.java similarity index 87% rename from quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilderImp.java rename to quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilderImpl.java index 8042325dc..42846e1a2 100644 --- a/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilderImp.java +++ b/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/CertificateBuilderImpl.java @@ -3,7 +3,8 @@ import java.util.List; import java.util.Objects; -record CertificateBuilderImp(List certificates) implements CertificateBuilder { +record CertificateBuilderImpl(List certificates, + ServingCertificateConfig servingCertificateConfig) implements CertificateBuilder { @Override public Certificate findCertificateByPrefix(String prefix) { Objects.requireNonNull(prefix); diff --git a/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/ServingCertificateConfig.java b/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/ServingCertificateConfig.java new file mode 100644 index 000000000..3e42bd282 --- /dev/null +++ b/quarkus-test-core/src/main/java/io/quarkus/test/security/certificate/ServingCertificateConfig.java @@ -0,0 +1,55 @@ +package io.quarkus.test.security.certificate; + +import io.quarkus.test.bootstrap.ServiceContext; + +public record ServingCertificateConfig(boolean injectCABundle, boolean addServiceCertificate, String tlsConfigName) { + + public static final String SERVING_CERTIFICATE_KEY = "serving-certificate-config-key"; + + public static boolean isServingCertificateScenario(ServiceContext serviceContext) { + return get(serviceContext) != null; + } + + public static ServingCertificateConfig get(ServiceContext serviceContext) { + if (serviceContext.get(SERVING_CERTIFICATE_KEY) instanceof ServingCertificateConfig config) { + return config; + } + return null; + } + + static ServingCertificateConfigBuilder builder() { + return new ServingCertificateConfigBuilder(); + } + + static final class ServingCertificateConfigBuilder { + + private boolean injectCABundle = false; + private boolean addServiceCertificate = false; + private String tlsConfigName = null; + + private ServingCertificateConfigBuilder() { + } + + ServingCertificateConfigBuilder withInjectCABundle(boolean injectCABundle) { + this.injectCABundle = injectCABundle; + return this; + } + + ServingCertificateConfigBuilder withAddServiceCertificate(boolean addServiceCertificate) { + this.addServiceCertificate = addServiceCertificate; + return this; + } + + ServingCertificateConfigBuilder withTlsConfigName(String tlsConfigName) { + this.tlsConfigName = tlsConfigName; + return this; + } + + ServingCertificateConfig build() { + if (injectCABundle || addServiceCertificate) { + return new ServingCertificateConfig(injectCABundle, addServiceCertificate, tlsConfigName); + } + return null; + } + } +} diff --git a/quarkus-test-core/src/main/java/io/quarkus/test/services/Certificate.java b/quarkus-test-core/src/main/java/io/quarkus/test/services/Certificate.java index 90817cb0c..2490dcf52 100644 --- a/quarkus-test-core/src/main/java/io/quarkus/test/services/Certificate.java +++ b/quarkus-test-core/src/main/java/io/quarkus/test/services/Certificate.java @@ -112,6 +112,11 @@ enum Format { */ String tlsConfigName() default DEFAULT_CONFIG; + /** + * Facilitates support for OpenShift serving certificates. + */ + ServingCertificates[] servingCertificates() default {}; + /** * Specify client certificates that should be generated. * Generation of more than one client certificate is only implemented for {@link Format#PKCS12}. @@ -133,4 +138,26 @@ enum Format { boolean unknownToServer() default false; } + /** + * OpenShift serving certificates configuration. This only works when internal service is used as URL. + */ + @interface ServingCertificates { + + /** + * Whether service CA bundle should be mounted to Quarkus pod. + * This CA bundle can be used by a REST client to communicate with pod using service certificate. + * To put it blunt, this is client side, while {@link #addServiceCertificate()} is server side. + */ + boolean injectCABundle() default false; + + /** + * Whether certificate generated by OpenShift should be mounted to Quarkus pod and the TLS registry + * extension should be configured with the certificate. These certificates are only valid for the internal + * service DNS and won't work outside the OpenShift cluster. That is, use internal service DNS name + * to communicate with other pods. + */ + boolean addServiceCertificate() default false; + + } + } diff --git a/quarkus-test-core/src/main/java/io/quarkus/test/services/quarkus/QuarkusApplicationManagedResourceBuilder.java b/quarkus-test-core/src/main/java/io/quarkus/test/services/quarkus/QuarkusApplicationManagedResourceBuilder.java index 0170484ab..f001fe6eb 100644 --- a/quarkus-test-core/src/main/java/io/quarkus/test/services/quarkus/QuarkusApplicationManagedResourceBuilder.java +++ b/quarkus-test-core/src/main/java/io/quarkus/test/services/quarkus/QuarkusApplicationManagedResourceBuilder.java @@ -1,5 +1,6 @@ package io.quarkus.test.services.quarkus; +import static io.quarkus.test.security.certificate.ServingCertificateConfig.SERVING_CERTIFICATE_KEY; import static io.quarkus.test.utils.PropertiesUtils.resolveProperty; import static java.util.stream.Collectors.toSet; @@ -227,6 +228,9 @@ protected void configureCertificates() { .forEach(certificate -> certificate .configProperties() .forEach((k, v) -> getContext().withTestScopeConfigProperty(k, v))); + if (certificateBuilder.servingCertificateConfig() != null) { + getContext().put(SERVING_CERTIFICATE_KEY, certificateBuilder.servingCertificateConfig()); + } } } diff --git a/quarkus-test-openshift/src/main/java/io/quarkus/test/bootstrap/inject/OpenShiftClient.java b/quarkus-test-openshift/src/main/java/io/quarkus/test/bootstrap/inject/OpenShiftClient.java index c1e94b078..8563cc4c9 100644 --- a/quarkus-test-openshift/src/main/java/io/quarkus/test/bootstrap/inject/OpenShiftClient.java +++ b/quarkus-test-openshift/src/main/java/io/quarkus/test/bootstrap/inject/OpenShiftClient.java @@ -2,6 +2,12 @@ import static io.quarkus.test.model.CustomVolume.VolumeType.CONFIG_MAP; import static io.quarkus.test.model.CustomVolume.VolumeType.SECRET; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.getAnnotatedConfigMap; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.getMountConfigMap; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.getMountSecret; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.isAnnotatedConfigMap; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.isMountConfigMap; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.isMountSecret; import static io.quarkus.test.utils.AwaitilityUtils.AwaitilitySettings; import static io.quarkus.test.utils.AwaitilityUtils.untilIsNotNull; import static io.quarkus.test.utils.AwaitilityUtils.untilIsTrue; @@ -59,6 +65,7 @@ import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; import io.fabric8.kubernetes.client.dsl.PodResource; +import io.fabric8.kubernetes.client.dsl.Resource; import io.fabric8.kubernetes.client.utils.Serialization; import io.fabric8.openshift.api.model.ImageStream; import io.fabric8.openshift.api.model.Route; @@ -75,6 +82,7 @@ import io.quarkus.test.configuration.PropertyLookup; import io.quarkus.test.logging.Log; import io.quarkus.test.model.CustomVolume; +import io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils; import io.quarkus.test.services.URILike; import io.quarkus.test.services.operator.model.CustomResourceStatus; import io.quarkus.test.services.quarkus.model.QuarkusProperties; @@ -211,13 +219,18 @@ public void applyServicePropertiesUsingTemplate(Service service, String file, Un } /** - * Update the deployment config using the service properties. + * Update the deployment and service using the service properties. * * @param service */ public void applyServicePropertiesToDeployment(Service service) { - Deployment deployment = client.apps().deployments().withName(service.getName()).get(); - Map enrichProperties = enrichProperties(service.getProperties(), deployment); + var serviceName = service.getName(); + Deployment deployment = client.apps().deployments().withName(serviceName).get(); + boolean isQuarkusRuntime = isQuarkusRuntime(deployment.getSpec().getTemplate().getMetadata().getLabels()); + Map enrichProperties = enrichProperties(service.getProperties(), deployment, isQuarkusRuntime); + if (isQuarkusRuntime) { + updateAnnotationsIfNecessary(service, serviceName); + } deployment.getSpec().getTemplate().getSpec().getContainers().forEach(container -> { enrichProperties.forEach((key, value) -> container.getEnv().add(new EnvVar(key, value, null))); @@ -225,6 +238,27 @@ public void applyServicePropertiesToDeployment(Service service) { client.apps().deployments().resource(deployment).createOrReplace(); } + private void updateAnnotationsIfNecessary(Service service, String serviceName) { + var annotations = collectAnnotations(service); + if (!annotations.isEmpty()) { + var k8ServiceResource = client.services().withName(serviceName); + AwaitilityUtils.untilIsNotNull(k8ServiceResource::get); + var k8Service = k8ServiceResource.get(); + boolean edited = false; + for (var keyToVal : annotations) { + var annotationName = keyToVal.key(); + if (!k8Service.getMetadata().getAnnotations().containsKey(annotationName)) { + k8Service = k8Service.edit().editMetadata().addToAnnotations(annotationName, keyToVal.value()).endMetadata() + .build(); + edited = true; + } + } + if (edited) { + k8ServiceResource.patch(k8Service); + } + } + } + /** * Start rollout of the service. * @@ -695,6 +729,30 @@ public KnativeClient getKnClient() { return kn; } + public void removeSecret(String secretName) { + var secret = client.secrets().withName(secretName); + if (secret.get() != null) { + secret.delete(); + } + } + + public boolean isAnyServicePodReady(String serviceName) { + return client.pods().withLabel(LABEL_TO_WATCH_FOR_LOGS, serviceName).resources().anyMatch(Resource::isReady); + } + + private void addAnnotatedConfigMap(String configMapName, String annotationName, String annotationValue) { + var configMapResource = client.configMaps().withName(configMapName); + if (configMapResource.get() == null) { + var configMap = new ConfigMapBuilder() + .editMetadata() + .withName(configMapName) + .addToAnnotations(annotationName, annotationValue) + .endMetadata() + .build(); + client.resource(configMap).create(); + } + } + /** * Delete test resources. */ @@ -711,6 +769,7 @@ private void deleteResourcesByLabel(String labelName, String labelValue) { } private String enrichTemplate(Service service, String template, Map extraTemplateProperties) { + var serviceName = service.getName(); List objs = loadYaml(template); for (HasMetadata obj : objs) { // set namespace @@ -728,14 +787,14 @@ private String enrichTemplate(Service service, String template, Map templateMetadataLabels = deployment.getSpec().getTemplate().getMetadata().getLabels(); - templateMetadataLabels.put(LABEL_TO_WATCH_FOR_LOGS, service.getName()); + templateMetadataLabels.put(LABEL_TO_WATCH_FOR_LOGS, serviceName); templateMetadataLabels.put(LABEL_SCENARIO_ID, getScenarioId()); + final boolean isQuarkusRuntime = isQuarkusRuntime(templateMetadataLabels); // add env var properties - Map enrichProperties = enrichProperties(service.getProperties(), deployment); + Map enrichProperties = enrichProperties(service.getProperties(), deployment, isQuarkusRuntime); final Map environmentVariables; - final boolean isQuarkusRuntime = "quarkus".equals(templateMetadataLabels.get("app.openshift.io/runtime")); if (isQuarkusRuntime) { var propsThatRequireDottedFormat = appPropsThatRequireDottedFormat(enrichProperties); @@ -764,6 +823,15 @@ private String enrichTemplate(Service service, String template, Map { + var annotationKey = keyToVal.key(); + var annotationVal = keyToVal.value(); + k8Service.getMetadata().getAnnotations().put(annotationKey, annotationVal); + }); + } } } @@ -780,6 +848,14 @@ private String enrichTemplate(Service service, String template, Map collectAnnotations(Service service) { + return service.getProperties().values() + .stream() + .filter(OpenShiftPropertiesUtils::isAnnotation) + .map(OpenShiftPropertiesUtils::getServiceAnnotation) + .toList(); + } + private String createAppPropsForPropsThatRequireDottedFormat(Map configProperties) { return configProperties .entrySet() @@ -788,6 +864,10 @@ private String createAppPropsForPropsThatRequireDottedFormat(Map .collect(Collectors.joining()); } + private static boolean isQuarkusRuntime(Map templateMetadataLabels) { + return "quarkus".equals(templateMetadataLabels.get("app.openshift.io/runtime")); + } + private static Map appPropsThatRequireDottedFormat(Map configProperties) { return configProperties .entrySet() @@ -858,11 +938,15 @@ private EnvVar getEnvVarByKey(String key, Container container) { return container.getEnv().stream().filter(env -> StringUtils.equals(key, env.getName())).findFirst().orElse(null); } - private Map enrichProperties(Map properties, Deployment deployment) { + private Map enrichProperties(Map properties, Deployment deployment, + boolean isQuarkusRuntime) { // mount path x volume Map volumes = new HashMap<>(); + // the idea of the 'output' is that if you have quarkus.some.property.key=secret::/path + // then it is turned into quarkus.some.property.key=path Map output = new HashMap<>(); + for (Entry entry : properties.entrySet()) { String propertyValue = entry.getValue(); if (isResource(propertyValue)) { @@ -923,6 +1007,31 @@ private Map enrichProperties(Map properties, Dep volumes.put(mountPath, new CustomVolume(secretName, "", SECRET)); propertyValue = joinMountPathAndFileName(mountPath, filename); + } else if (isQuarkusRuntime && isMountSecret(propertyValue)) { + // mount existing secret + var secretNameToMountPath = getMountSecret(propertyValue); + var secretName = secretNameToMountPath.key(); + var mountPath = secretNameToMountPath.value(); + volumes.put(mountPath, new CustomVolume(secretName, "", SECRET)); + + propertyValue = mountPath; + } else if (isQuarkusRuntime && isMountConfigMap(propertyValue)) { + // mount existing configmap + var configMapNameToMountPath = getMountConfigMap(propertyValue); + var configMapName = configMapNameToMountPath.key(); + var mountPath = configMapNameToMountPath.value(); + volumes.put(mountPath, new CustomVolume(configMapName, "", CONFIG_MAP)); + + propertyValue = mountPath; + } else if (isQuarkusRuntime && isAnnotatedConfigMap(propertyValue)) { + var annotatedConfigMap = getAnnotatedConfigMap(propertyValue); + var configMapName = annotatedConfigMap.key(); + var annotationName = annotatedConfigMap.value().key(); + var annotationVal = annotatedConfigMap.value().value(); + addAnnotatedConfigMap(configMapName, annotationName, annotationVal); + + // no sensible value is expected, just assign something + propertyValue = annotationVal; } output.put(entry.getKey(), propertyValue); diff --git a/quarkus-test-openshift/src/main/java/io/quarkus/test/openshift/utils/OpenShiftPropertiesUtils.java b/quarkus-test-openshift/src/main/java/io/quarkus/test/openshift/utils/OpenShiftPropertiesUtils.java new file mode 100644 index 000000000..1e80f5ab8 --- /dev/null +++ b/quarkus-test-openshift/src/main/java/io/quarkus/test/openshift/utils/OpenShiftPropertiesUtils.java @@ -0,0 +1,184 @@ +package io.quarkus.test.openshift.utils; + +import static io.quarkus.test.bootstrap.OpenShiftExtensionBootstrap.CLIENT; +import static io.quarkus.test.security.certificate.ServingCertificateConfig.isServingCertificateScenario; + +import java.util.random.RandomGenerator; + +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.Assertions; + +import io.quarkus.test.bootstrap.Protocol; +import io.quarkus.test.bootstrap.ServiceContext; +import io.quarkus.test.bootstrap.inject.OpenShiftClient; +import io.quarkus.test.security.certificate.ServingCertificateConfig; +import io.quarkus.test.services.URILike; + +public final class OpenShiftPropertiesUtils { + + public record PropertyToValue(String key, String value) { + } + + public record PropertyToObj(String key, PropertyToValue value) { + } + + public static final int EXTERNAL_SSL_PORT = 443; + /** + * ConfigMap name we use for OpenShift CA certificate injection. + */ + public static final String CA_BUNDLE_CONFIGMAP_NAME = "ca-bundle-configmap"; + /** + * Secret name we use for OpenShift serving certificates. + */ + public static final String SERVING_CERTS_SECRET_NAME = "serving-certificates-secret"; + /** + * Adds service annotation. Adding annotations is currently implemented for Quarkus runtimes only as that's what we + * need. Supporting for all runtimes will require additional test that annotation is added to one service only. + */ + private static final String ANNOTATION_PREFIX = "annotation::"; + /** + * Separates annotation name and value. + */ + private static final String ANNOTATION_SEPARATOR = "|"; + /** + * Mounts existing secret. Mounting secrets is currently implemented for Quarkus runtimes only as that's what we + * need. Supporting for all runtimes will require additional test that secret is mounted to one service only. + */ + private static final String MOUNT_SECRET_PREFIX = "mount-secret::"; + /** + * Separates secret name and mount path. + */ + private static final String MOUNT_SECRET_SEPARATOR = "|"; + /** + * Mounts existing configmap. Mounting configmaps is currently implemented for Quarkus runtimes only as that's + * what we need. Supporting for all runtimes will require additional test that secret is mounted to one service only. + */ + private static final String MOUNT_CONFIGMAP_PREFIX = "mount-configmap::"; + /** + * Separates configmap name and mount path. + */ + private static final String MOUNT_CONFIGMAP_SEPARATOR = "|"; + /** + * Creates empty config map with annotation. + */ + private static final String ANNOTATED_CONFIG_MAP_PREFIX = "annotated-configmap::"; + /** + * Separates configmap name and annotation. + */ + private static final String ANNOTATED_CONFIG_MAP_SEPARATOR = "|"; + /** + * Separates configmap annotation key and value. + */ + private static final String ANNOTATED_CONFIG_ANNOTATION_SEPARATOR = "&"; + private static final int INTERNAL_HTTPS_PORT_DEFAULT = 8443; + private static final String QUARKUS_HTTPS_PORT_PROPERTY = "quarkus.http.ssl-port"; + + private OpenShiftPropertiesUtils() { + // utils + } + + public static String buildAnnotatedConfigMapProp(String configmapName, String annotationKey, String annotationVal) { + return ANNOTATED_CONFIG_MAP_PREFIX + configmapName + ANNOTATED_CONFIG_MAP_SEPARATOR + annotationKey + + ANNOTATED_CONFIG_ANNOTATION_SEPARATOR + annotationVal; + } + + public static String buildMountConfigMapProp(String secretName, String secretValue) { + return MOUNT_CONFIGMAP_PREFIX + secretName + MOUNT_CONFIGMAP_SEPARATOR + secretValue; + } + + public static String buildMountSecretProp(String secretName, String secretValue) { + return MOUNT_SECRET_PREFIX + secretName + MOUNT_SECRET_SEPARATOR + secretValue; + } + + public static String buildAnnotationProp(String annotationName, String annotationValue) { + return ANNOTATION_PREFIX + annotationName + ANNOTATION_SEPARATOR + annotationValue; + } + + public static boolean isMountSecret(String propertyValue) { + return propertyValue.startsWith(MOUNT_SECRET_PREFIX); + } + + public static boolean isMountConfigMap(String propertyValue) { + return propertyValue.startsWith(MOUNT_CONFIGMAP_PREFIX); + } + + public static boolean isAnnotatedConfigMap(String propertyValue) { + return propertyValue.startsWith(ANNOTATED_CONFIG_MAP_PREFIX); + } + + public static boolean isAnnotation(String propertyValue) { + return propertyValue.startsWith(ANNOTATION_PREFIX); + } + + public static String createAnnotationPropertyKey() { + return createRandomPropertyKeyWithPrefix("annotation-"); + } + + public static String createSecretPropertyKey() { + return createRandomPropertyKeyWithPrefix("secret-"); + } + + public static String createConfigMapPropertyKey() { + return createRandomPropertyKeyWithPrefix("configmap-"); + } + + public static String createAnnotatedConfigMapPropertyKey() { + return createRandomPropertyKeyWithPrefix("annotated-configmap-"); + } + + public static PropertyToValue getMountConfigMap(String propertyValue) { + return getPropertyToValue(propertyValue, MOUNT_CONFIGMAP_PREFIX, MOUNT_CONFIGMAP_SEPARATOR, "configmap"); + } + + public static PropertyToValue getMountSecret(String propertyValue) { + return getPropertyToValue(propertyValue, MOUNT_SECRET_PREFIX, MOUNT_SECRET_SEPARATOR, "secret"); + } + + public static PropertyToValue getServiceAnnotation(String propertyValue) { + return getPropertyToValue(propertyValue, ANNOTATION_PREFIX, ANNOTATION_SEPARATOR, "annotation"); + } + + public static PropertyToObj getAnnotatedConfigMap(String propertyValue) { + var configMapNameToAnnotation = getPropertyToValue(propertyValue, ANNOTATED_CONFIG_MAP_PREFIX, + ANNOTATED_CONFIG_MAP_SEPARATOR, "annotated-configmap"); + var annotationKeyToValue = getPropertyToValue(configMapNameToAnnotation.value(), "", + ANNOTATED_CONFIG_ANNOTATION_SEPARATOR, "annotation-configmap-annotation"); + return new PropertyToObj(configMapNameToAnnotation.key(), annotationKeyToValue); + } + + public static boolean checkPodReadinessWithStatusInsteadOfRoute(ServiceContext ctx) { + // using HTTPS & internal service URL & no exposed route => need to trust readiness + return isServingCertificateScenario(ctx) && ServingCertificateConfig.get(ctx).addServiceCertificate(); + } + + public static int getInternalHttpsPort(ServiceContext ctx) { + return ctx.getOwner().getProperty(QUARKUS_HTTPS_PORT_PROPERTY) + .filter(StringUtils::isNotBlank) + .map(Integer::parseInt) + .orElse(INTERNAL_HTTPS_PORT_DEFAULT); + } + + public static URILike getInternalServiceHttpsUrl(ServiceContext ctx) { + var serviceName = ctx.getOwner().getName(); + var projectName = ctx. get(CLIENT).project(); + var host = "%s.%s.svc.cluster.local".formatted(serviceName, projectName); + return new URILike(Protocol.HTTPS.getValue(), host, getInternalHttpsPort(ctx), ""); + } + + private static PropertyToValue getPropertyToValue(String propertyValue, String prefix, String separator, String subject) { + var keyToVal = propertyValue.replace(prefix, StringUtils.EMPTY); + var separatorIdx = keyToVal.indexOf(separator); + if (separatorIdx == -1) { + Assertions.fail("Configuration property defining OpenShift '%s' is missing key to value separator '%s'" + .formatted(subject, separatorIdx)); + } + var key = keyToVal.substring(0, separatorIdx); + var value = keyToVal.substring(separatorIdx + 1); + return new PropertyToValue(key, value); + } + + private static String createRandomPropertyKeyWithPrefix(String x) { + var generator = RandomGenerator.getDefault(); + return x + generator.nextInt() + "-" + generator.nextInt(); + } +} diff --git a/quarkus-test-openshift/src/main/java/io/quarkus/test/services/containers/OpenShiftContainerManagedResource.java b/quarkus-test-openshift/src/main/java/io/quarkus/test/services/containers/OpenShiftContainerManagedResource.java index 6287ea6e7..1c631cb40 100644 --- a/quarkus-test-openshift/src/main/java/io/quarkus/test/services/containers/OpenShiftContainerManagedResource.java +++ b/quarkus-test-openshift/src/main/java/io/quarkus/test/services/containers/OpenShiftContainerManagedResource.java @@ -124,6 +124,7 @@ protected String replaceDeploymentContent(String content) { return content.replaceAll(quote("${IMAGE}"), model.getImage()) .replaceAll(quote("${SERVICE_NAME}"), model.getContext().getName()) .replaceAll(quote("${INTERNAL_PORT}"), "" + model.getPort()) + .replaceAll(quote("${INTERNAL_INGRESS_PORT}"), "" + model.getPort()) .replaceAll(quote("${ARGS}"), args) .replaceAll(quote("${CURRENT_NAMESPACE}"), client.project()); } diff --git a/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/OpenShiftQuarkusApplicationManagedResource.java b/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/OpenShiftQuarkusApplicationManagedResource.java index 5b6f21da2..84256e2d8 100644 --- a/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/OpenShiftQuarkusApplicationManagedResource.java +++ b/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/OpenShiftQuarkusApplicationManagedResource.java @@ -1,5 +1,19 @@ package io.quarkus.test.services.quarkus; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.CA_BUNDLE_CONFIGMAP_NAME; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.EXTERNAL_SSL_PORT; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.SERVING_CERTS_SECRET_NAME; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.buildAnnotatedConfigMapProp; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.buildAnnotationProp; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.buildMountConfigMapProp; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.buildMountSecretProp; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.checkPodReadinessWithStatusInsteadOfRoute; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.createAnnotatedConfigMapPropertyKey; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.createAnnotationPropertyKey; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.createConfigMapPropertyKey; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.createSecretPropertyKey; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.getInternalServiceHttpsUrl; +import static io.quarkus.test.security.certificate.ServingCertificateConfig.isServingCertificateScenario; import static io.quarkus.test.utils.AwaitilityUtils.AwaitilitySettings; import static io.quarkus.test.utils.AwaitilityUtils.untilIsNotNull; import static io.restassured.RestAssured.given; @@ -15,13 +29,13 @@ import io.quarkus.test.bootstrap.inject.OpenShiftClient; import io.quarkus.test.logging.LoggingHandler; import io.quarkus.test.logging.OpenShiftLoggingHandler; +import io.quarkus.test.security.certificate.ServingCertificateConfig; import io.quarkus.test.services.URILike; public abstract class OpenShiftQuarkusApplicationManagedResource extends QuarkusManagedResource { private static final int EXTERNAL_PORT = 80; - private static final int EXTERNAL_SSL_PORT = 443; protected final T model; protected final OpenShiftClient client; @@ -36,6 +50,7 @@ public OpenShiftQuarkusApplicationManagedResource(T model) { super(model.getContext()); this.model = model; this.client = model.getContext().get(OpenShiftExtensionBootstrap.CLIENT); + configureServingCertificates(); } protected abstract void doInit(); @@ -82,7 +97,8 @@ public void stop() { public URILike getURI(Protocol protocol) { final ServiceContext context = model.getContext(); final boolean isServerless = client.isServerlessService(context.getName()); - if (protocol == Protocol.HTTPS && !isServerless) { + final boolean isServingCertSslScenario = isServingCertificateScenario(context) && model.isSslEnabled(); + if (protocol == Protocol.HTTPS && !isServerless && !isServingCertSslScenario) { fail("SSL is not supported for OpenShift tests yet"); } else if (protocol == Protocol.GRPC) { fail("gRPC is not supported for OpenShift tests yet"); @@ -93,6 +109,10 @@ public URILike getURI(Protocol protocol) { return client.url(context.getOwner().getName() + "-management").withPort(EXTERNAL_PORT); } if (this.uri == null) { + if (isServingCertSslScenario) { + this.uri = getInternalServiceHttpsUrl(context); + return this.uri; + } final int port = isServerless ? EXTERNAL_SSL_PORT : EXTERNAL_PORT; this.uri = untilIsNotNull( () -> client.url(context.getOwner()).withPort(port), @@ -110,7 +130,12 @@ public boolean isRunning() { return routeIsReachable(Protocol.HTTPS); } - return super.isRunning() && routeIsReachable(Protocol.HTTP); + if (checkPodReadinessWithStatusInsteadOfRoute(model.getContext())) { + var serviceName = model.getContext().getOwner().getName(); + return super.isRunning() && client.isAnyServicePodReady(serviceName); + } else { + return super.isRunning() && routeIsReachable(Protocol.HTTP); + } } @Override @@ -139,4 +164,53 @@ private boolean routeIsReachable(Protocol protocol) { return given().relaxedHTTPSValidation().baseUri(url.getRestAssuredStyleUri()).basePath("/").port(url.getPort()).get() .getStatusCode() != HttpStatus.SC_SERVICE_UNAVAILABLE; } + + private void configureServingCertificates() { + // this is based on https://quarkus.io/guides/tls-registry-reference#utilizing-openshift-serving-certificates + var ctx = model.getContext(); + if (isServingCertificateScenario(ctx)) { + var config = ServingCertificateConfig.get(ctx); + if (config.addServiceCertificate()) { + // add service annotation + var annotationVal = buildAnnotationProp("service.beta.openshift.io/serving-cert-secret-name", + SERVING_CERTS_SECRET_NAME); + var annotationKey = createAnnotationPropertyKey(); + ctx.withTestScopeConfigProperty(annotationKey, annotationVal); + // mount secret created by OpenShift operator + var mountSecretVal = buildMountSecretProp(SERVING_CERTS_SECRET_NAME, "/etc/tls"); + var mountSecretKey = createSecretPropertyKey(); + ctx.withTestScopeConfigProperty(mountSecretKey, mountSecretVal); + // configure TLS registry with mounted secret + if (config.tlsConfigName() == null) { + ctx.withTestScopeConfigProperty("quarkus.tls.key-store.pem.acme.cert", "/etc/tls/tls.crt"); + ctx.withTestScopeConfigProperty("quarkus.tls.key-store.pem.acme.key", "/etc/tls/tls.key"); + } else { + ctx.withTestScopeConfigProperty("quarkus.tls." + config.tlsConfigName() + ".key-store.pem.acme.cert", + "/etc/tls/tls.crt"); + ctx.withTestScopeConfigProperty("quarkus.tls." + config.tlsConfigName() + ".key-store.pem.acme.key", + "/etc/tls/tls.key"); + ctx.withTestScopeConfigProperty("quarkus.http.tls-configuration-name", config.tlsConfigName()); + } + } + if (config.injectCABundle()) { + var annotationVal = "true"; + var annotationKey = "service.beta.openshift.io/inject-cabundle"; + var annotatedConfigMapVal = buildAnnotatedConfigMapProp(CA_BUNDLE_CONFIGMAP_NAME, annotationKey, annotationVal); + var annotatedConfigMapKey = createAnnotatedConfigMapPropertyKey(); + ctx.withTestScopeConfigProperty(annotatedConfigMapKey, annotatedConfigMapVal); + var caCertDirPath = "/deployments/tls"; + var mountConfigMapVal = buildMountConfigMapProp(CA_BUNDLE_CONFIGMAP_NAME, caCertDirPath); + var mountConfigMapKey = createConfigMapPropertyKey(); + ctx.withTestScopeConfigProperty(mountConfigMapKey, mountConfigMapVal); + // configure TLS registry with mounted configmap + var caCertPath = caCertDirPath + "/service-ca.crt"; + if (config.tlsConfigName() == null) { + ctx.withTestScopeConfigProperty("quarkus.tls.trust-store.pem.certs", caCertPath); + } else { + ctx.withTestScopeConfigProperty("quarkus.tls." + config.tlsConfigName() + ".trust-store.pem.certs", + caCertPath); + } + } + } + } } diff --git a/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/TemplateOpenShiftQuarkusApplicationManagedResource.java b/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/TemplateOpenShiftQuarkusApplicationManagedResource.java index 454653007..e8e1fc030 100644 --- a/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/TemplateOpenShiftQuarkusApplicationManagedResource.java +++ b/quarkus-test-openshift/src/main/java/io/quarkus/test/services/quarkus/TemplateOpenShiftQuarkusApplicationManagedResource.java @@ -1,5 +1,8 @@ package io.quarkus.test.services.quarkus; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.EXTERNAL_SSL_PORT; +import static io.quarkus.test.openshift.utils.OpenShiftPropertiesUtils.getInternalHttpsPort; +import static io.quarkus.test.security.certificate.ServingCertificateConfig.isServingCertificateScenario; import static java.util.regex.Pattern.quote; import java.util.Collections; @@ -41,12 +44,19 @@ protected void doUpdate() { } protected int getInternalPort() { + if (isHttpsScenario()) { + return getInternalHttpsPort(model.getContext()); + } return model.getContext().getOwner().getProperty(QUARKUS_HTTP_PORT_PROPERTY) .filter(StringUtils::isNotBlank) .map(Integer::parseInt) .orElse(INTERNAL_PORT_DEFAULT); } + private boolean isHttpsScenario() { + return model.isSslEnabled() && isServingCertificateScenario(model.getContext()); + } + protected Map addExtraTemplateProperties() { return Collections.emptyMap(); } @@ -68,8 +78,10 @@ private String internalReplaceDeploymentContent(String content) { content = content.replaceAll(quote(customServiceName), model.getContext().getOwner().getName()); } + var ingressInternalPort = (isHttpsScenario() ? EXTERNAL_SSL_PORT : getInternalPort()); content = content.replaceAll(quote("${SERVICE_NAME}"), model.getContext().getName()) .replaceAll(quote("${INTERNAL_PORT}"), "" + getInternalPort()) + .replaceAll(quote("${INTERNAL_INGRESS_PORT}"), "" + ingressInternalPort) .replace("${MANAGEMENT_PORT}", "" + model.getManagementPort()); return replaceDeploymentContent(content); diff --git a/quarkus-test-openshift/src/main/resources/openshift-deployment-template.yml b/quarkus-test-openshift/src/main/resources/openshift-deployment-template.yml index 1021de353..269277174 100644 --- a/quarkus-test-openshift/src/main/resources/openshift-deployment-template.yml +++ b/quarkus-test-openshift/src/main/resources/openshift-deployment-template.yml @@ -9,7 +9,7 @@ items: spec: ports: - name: "http" - port: ${INTERNAL_PORT} + port: ${INTERNAL_INGRESS_PORT} targetPort: ${INTERNAL_PORT} selector: deployment: "${SERVICE_NAME}" diff --git a/quarkus-test-openshift/src/main/resources/quarkus-build-openshift-template.yml b/quarkus-test-openshift/src/main/resources/quarkus-build-openshift-template.yml index 838dcffda..b74716796 100644 --- a/quarkus-test-openshift/src/main/resources/quarkus-build-openshift-template.yml +++ b/quarkus-test-openshift/src/main/resources/quarkus-build-openshift-template.yml @@ -12,7 +12,7 @@ items: spec: ports: - name: "http" - port: 8080 + port: ${INTERNAL_INGRESS_PORT} targetPort: ${INTERNAL_PORT} selector: app.kubernetes.io/name: "${SERVICE_NAME}" diff --git a/quarkus-test-openshift/src/main/resources/quarkus-registry-openshift-template.yml b/quarkus-test-openshift/src/main/resources/quarkus-registry-openshift-template.yml index e23f9b842..c4850914b 100644 --- a/quarkus-test-openshift/src/main/resources/quarkus-registry-openshift-template.yml +++ b/quarkus-test-openshift/src/main/resources/quarkus-registry-openshift-template.yml @@ -9,7 +9,7 @@ items: spec: ports: - name: "http" - port: 8080 + port: ${INTERNAL_INGRESS_PORT} targetPort: ${INTERNAL_PORT} selector: app.kubernetes.io/name: "${SERVICE_NAME}"