[Backport 1.3] Integration Tests for Security Config Initialization

src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java

@@ -31,12 +31,13 @@ public static Path createConfigurationDirectory() {
             String[] configurationFiles = {
                 "config.yml",
                 "action_groups.yml",
-                "config.yml",
                 "internal_users.yml",
+                "nodes_dn.yml",
                 "roles.yml",
                 "roles_mapping.yml",
                 "security_tenants.yml",
-                "tenants.yml" };
+                "tenants.yml",
+                "whitelist.yml"};
             for (String fileName : configurationFiles) {
                 Path configFileDestination = tempDirectory.resolve(fileName);
                 copyResourceToFile(fileName, configFileDestination.toFile());

src/integrationTest/java/org/opensearch/security/SecurityAdminLauncher.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.security;
+
+import java.io.File;
+import java.nio.file.Path;
+
+import org.opensearch.security.tools.SecurityAdmin;
+import org.opensearch.test.framework.certificate.TestCertificates;
+
+import static java.util.Objects.requireNonNull;
+
+class SecurityAdminLauncher {
+
+    private final TestCertificates certificates;
+    private int port;
+
+    public SecurityAdminLauncher(int port, TestCertificates certificates) {
+        this.port = port;
+        this.certificates = requireNonNull(certificates, "Certificates are required to communicate with cluster.");
+    }
+
+    public int updateRoleMappings(File roleMappingsConfigurationFile) throws Exception {
+        String[] commandLineArguments = {
+                "-cacert",
+                certificates.getRootCertificate().getAbsolutePath(),
+                "-cert",
+                certificates.getAdminCertificate().getAbsolutePath(),
+                "-key",
+                certificates.getAdminKey(null).getAbsolutePath(),
+                "-nhnv",
+                "-p",
+                String.valueOf(port),
+                "-f",
+                roleMappingsConfigurationFile.getAbsolutePath(),
+                "-t",
+                "rolesmapping" };
+
+        return SecurityAdmin.execute(commandLineArguments);
+    }
+
+    public int runSecurityAdmin(Path configurationFolder) throws Exception {
+        String[] commandLineArguments = {
+                "-cacert",
+                certificates.getRootCertificate().getAbsolutePath(),
+                "-cert",
+                certificates.getAdminCertificate().getAbsolutePath(),
+                "-key",
+                certificates.getAdminKey(null).getAbsolutePath(),
+                "-nhnv",
+                "-icl",
+                "-p",
+                String.valueOf(port),
+                "-cd",
+                configurationFolder.toString(),
+                "--diagnose"};
+
+        return SecurityAdmin.execute(commandLineArguments);
+    }
+}

src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.security;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.FileUtils;
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.opensearch.client.Client;
+import org.opensearch.security.securityconf.impl.CType;
+import org.opensearch.security.support.ConfigConstants;
+import org.opensearch.security.support.ConfigHelper;
+import org.opensearch.test.framework.TestSecurityConfig.User;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.ContextHeaderDecoratorClient;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+import static org.apache.http.HttpStatus.SC_SERVICE_UNAVAILABLE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.security.configuration.ConfigurationRepository.DEFAULT_CONFIG_VERSION;
+import static org.opensearch.security.support.ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS;
+import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class SecurityConfigurationBootstrapTests {
+
+    private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory();
+    private static final User USER_ADMIN = new User("admin").roles(ALL_ACCESS);
+
+    private static LocalCluster createCluster(final Map<String, Object> nodeSettings) {
+        LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS)
+                .loadConfigurationIntoIndex(false)
+                .defaultConfigurationInitDirectory(configurationFolder.toString())
+                .nodeSettings(
+                        ImmutableMap.<String, Object>builder()
+                                .put(SECURITY_RESTAPI_ROLES_ENABLED, List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()))
+                                .putAll(nodeSettings)
+                                .build()
+                )
+                .build();
+
+        cluster.before(); // normally invoked by JUnit rules when run as a class rule - this starts the cluster
+        return cluster;
+    }
+
+    @AfterClass
+    public static void cleanConfigurationDirectory() throws IOException {
+        FileUtils.deleteDirectory(configurationFolder.toFile());
+    }
+
+    @Test
+    public void testInitializeWithSecurityAdminWhenNoBackgroundInitialization() throws Exception {
+        final Map<String, Object> nodeSettings = ImmutableMap.<String, Object>builder()
+                .put(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false)
+                .put(SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, false)
+                .build();
+        try (final LocalCluster cluster = createCluster(nodeSettings)) {
+            try (final TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+                final TestRestClient.HttpResponse rolesMapsResponse = client.get("_plugins/_security/api/rolesmapping/readall");
+                assertThat(rolesMapsResponse.getStatusCode(), equalTo(SC_SERVICE_UNAVAILABLE));
+                assertThat(rolesMapsResponse.getBody(), containsString("OpenSearch Security not initialized"));
+            }
+
+            final SecurityAdminLauncher securityAdminLauncher = new SecurityAdminLauncher(cluster.getTransportAddress().getPort(), cluster.getTestCertificates());
+            final int exitCode = securityAdminLauncher.runSecurityAdmin(configurationFolder);
+            assertThat(exitCode, equalTo(0));
+
+            try (final TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+                Awaitility.await()
+                        .alias("Waiting for rolemapping 'readall' availability.")
+                        .until(() -> client.get("_plugins/_security/api/rolesmapping/readall").getStatusCode(), equalTo(200));
+            }
+        }
+    }
+
+    @Test
+    public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateRequests() throws Exception {
+        final Map<String, Object> nodeSettings = ImmutableMap.<String, Object>builder()
+                .put(SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true)
+                .put(SECURITY_UNSUPPORTED_DELAY_INITIALIZATION_SECONDS, 5)
+                .build();
+        try (final LocalCluster cluster = createCluster(nodeSettings)) {
+            try (final TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+                cluster.getInternalNodeClient()
+                        .admin()
+                        .cluster()
+                        .health(new ClusterHealthRequest(OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX).waitForGreenStatus())
+                        .actionGet();
+
+                // Make sure the cluster is unavalaible to authenticate with the security plugin even though it is green
+                final TestRestClient.HttpResponse authResponseWhenUnconfigured = client.getAuthInfo();
+                authResponseWhenUnconfigured.assertStatusCode(503);
+
+                final Client internalNodeClient = new ContextHeaderDecoratorClient(
+                        cluster.getInternalNodeClient(),
+                        Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true")
+                );
+                final Map<String, CType> filesToUpload = ImmutableMap.<String, CType>builder()
+                        .put("action_groups.yml", CType.ACTIONGROUPS)
+                        .put("config.yml", CType.CONFIG)
+                        .put("roles.yml", CType.ROLES)
+                        .put("tenants.yml", CType.TENANTS)
+                        .build();
+
+                final String defaultInitDirectory = System.getProperty("security.default_init.dir") + "/";
+                filesToUpload.forEach((fileName, ctype) -> {
+                    try {
+                        ConfigHelper.uploadFile(
+                                internalNodeClient,
+                                defaultInitDirectory + fileName,
+                                OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX,
+                                ctype,
+                                DEFAULT_CONFIG_VERSION
+                        );
+                    } catch (final Exception ex) {
+                        throw new RuntimeException(ex);
+                    }
+                });
+
+                Awaitility.await().alias("Load default configuration").pollInterval(Duration.ofMillis(100)).until(() -> {
+                    // After the configuration has been loaded, the rest clients should be able to connect successfully
+                    cluster.triggerConfigurationReloadForCTypes(
+                            internalNodeClient,
+                            List.of(CType.ACTIONGROUPS, CType.CONFIG, CType.ROLES, CType.TENANTS),
+                            true
+                    );
+                    try (final TestRestClient freshClient = cluster.getRestClient(USER_ADMIN)) {
+                        return client.getAuthInfo().getStatusCode();
+                    }
+                }, equalTo(200));
+            }
+        }
+    }
+}

src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java

@@ -119,11 +119,6 @@ public TestSecurityConfig xff(XffConfig xffConfig) {
         return this;
     }
 
-    public TestSecurityConfig onBehalfOf(OnBehalfOfConfig onBehalfOfConfig) {
-        config.onBehalfOfConfig(onBehalfOfConfig);
-        return this;
-    }
-
     public TestSecurityConfig authc(AuthcDomain authcDomain) {
         config.authc(authcDomain);
         return this;
@@ -180,7 +175,6 @@ public static class Config implements ToXContentObject {
 
         private Boolean doNotFailOnForbidden;
         private XffConfig xffConfig;
-        private OnBehalfOfConfig onBehalfOfConfig;
         private Map<String, AuthcDomain> authcDomainMap = new LinkedHashMap<>();
 
         private AuthFailureListeners authFailureListeners;
@@ -201,11 +195,6 @@ public Config xffConfig(XffConfig xffConfig) {
             return this;
         }
 
-        public Config onBehalfOfConfig(OnBehalfOfConfig onBehalfOfConfig) {
-            this.onBehalfOfConfig = onBehalfOfConfig;
-            return this;
-        }
-
         public Config authc(AuthcDomain authcDomain) {
             authcDomainMap.put(authcDomain.id, authcDomain);
             return this;
@@ -226,10 +215,6 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params
             xContentBuilder.startObject();
             xContentBuilder.startObject("dynamic");
 
-            if (onBehalfOfConfig != null) {
-                xContentBuilder.field("on_behalf_of", onBehalfOfConfig);
-            }
-
             if (anonymousAuth || (xffConfig != null)) {
                 xContentBuilder.startObject("http");
                 xContentBuilder.field("anonymous_auth_enabled", anonymousAuth);

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java

@@ -55,7 +55,6 @@
 import org.opensearch.test.framework.AuditConfiguration;
 import org.opensearch.test.framework.AuthFailureListeners;
 import org.opensearch.test.framework.AuthzDomain;
-import org.opensearch.test.framework.OnBehalfOfConfig;
 import org.opensearch.test.framework.RolesMapping;
 import org.opensearch.test.framework.TestIndex;
 import org.opensearch.test.framework.TestSecurityConfig;
@@ -133,7 +132,7 @@ public String getSnapshotDirPath() {
     }
 
     @Override
-    public void before() throws Throwable {
+    public void before() {
         if (localOpenSearchCluster == null) {
             for (LocalCluster dependency : clusterDependencies) {
                 if (!dependency.isStarted()) {
@@ -297,6 +296,16 @@ private static void triggerConfigurationReload(Client client) {
         }
     }
 
+    public void triggerConfigurationReloadForCTypes(Client client, List<CType> cTypes, boolean ignoreFailures) {
+        ConfigUpdateResponse configUpdateResponse = client.execute(
+                ConfigUpdateAction.INSTANCE,
+                new ConfigUpdateRequest(cTypes.stream().map(CType::toLCString).toArray(String[]::new))
+        ).actionGet();
+        if (!ignoreFailures && configUpdateResponse.hasFailures()) {
+            throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures());
+        }
+    }
+
     public CertificateData getAdminCertificate() {
         return testCertificates.getAdminCertificateData();
     }
@@ -472,11 +481,6 @@ public Builder xff(XffConfig xffConfig) {
             return this;
         }
 
-        public Builder onBehalfOf(OnBehalfOfConfig onBehalfOfConfig) {
-            testSecurityConfig.onBehalfOf(onBehalfOfConfig);
-            return this;
-        }
-
         public Builder loadConfigurationIntoIndex(boolean loadConfigurationIntoIndex) {
             this.loadConfigurationIntoIndex = loadConfigurationIntoIndex;
             return this;

src/integrationTest/resources/config.yml

@@ -15,8 +15,3 @@ config:
         authentication_backend:
           type: "internal"
           config: {}
-    on_behalf_of:
-      # The decoded signing key is: This is the jwt signing key for an on behalf of token authentication backend for testing of extensions
-      # The decoded encryption key is: encryptionKey
-      signing_key: "VGhpcyBpcyB0aGUgand0IHNpZ25pbmcga2V5IGZvciBhbiBvbiBiZWhhbGYgb2YgdG9rZW4gYXV0aGVudGljYXRpb24gYmFja2VuZCBmb3IgdGVzdGluZyBvZiBleHRlbnNpb25z"
-      encryption_key: "ZW5jcnlwdGlvbktleQ=="