Allow TransportConfigUpdateAction when security config initialization has completed

src/integrationTest/java/org/opensearch/security/ConfigurationFiles.java

@@ -33,10 +33,12 @@ public static Path createConfigurationDirectory() {
                 "action_groups.yml",
                 "config.yml",
                 "internal_users.yml",
+                "nodes_dn.yml",
                 "roles.yml",
                 "roles_mapping.yml",
                 "security_tenants.yml",
-                "tenants.yml" };
+                "tenants.yml",
+                "whitelist.yml" };
             for (String fileName : configurationFiles) {
                 Path configFileDestination = tempDirectory.resolve(fileName);
                 copyResourceToFile(fileName, configFileDestination.toFile());

src/integrationTest/java/org/opensearch/security/SecurityAdminLauncher.java

@@ -10,6 +10,7 @@
 package org.opensearch.security;
 
 import java.io.File;
+import java.nio.file.Path;
 
 import org.opensearch.security.tools.SecurityAdmin;
 import org.opensearch.test.framework.certificate.TestCertificates;
@@ -44,4 +45,21 @@ public int updateRoleMappings(File roleMappingsConfigurationFile) throws Excepti
 
         return SecurityAdmin.execute(commandLineArguments);
     }
+
+    public int runSecurityAdmin(Path configurationFolder) throws Exception {
+        String[] commandLineArguments = {
+            "-cacert",
+            certificates.getRootCertificate().getAbsolutePath(),
+            "-cert",
+            certificates.getAdminCertificate().getAbsolutePath(),
+            "-key",
+            certificates.getAdminKey(null).getAbsolutePath(),
+            "-nhnv",
+            "-p",
+            String.valueOf(port),
+            "-cd",
+            configurationFolder.toString() };
+
+        return SecurityAdmin.execute(commandLineArguments);
+    }
 }

src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapTests.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.security;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.commons.io.FileUtils;
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.opensearch.client.Client;
+import org.opensearch.security.securityconf.impl.CType;
+import org.opensearch.security.support.ConfigConstants;
+import org.opensearch.test.framework.TestSecurityConfig.User;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.ContextHeaderDecoratorClient;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED;
+import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class SecurityConfigurationBootstrapTests {
+
+    private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory();
+
+    private static final User USER_ADMIN = new User("admin").roles(ALL_ACCESS);
+
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS)
+        .loadConfigurationIntoIndex(false)
+        .defaultConfigurationInitDirectory(configurationFolder.toString())
+        .nodeSettings(
+            Map.of(
+                SECURITY_RESTAPI_ROLES_ENABLED,
+                List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()),
+                SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX,
+                true
+            )
+        )
+        .build();
+
+    @BeforeClass
+    public static void initData() {
+
+        Client client = new ContextHeaderDecoratorClient(
+            cluster.getInternalNodeClient(),
+            Map.of(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true")
+        );
+        Runnable r = new Runnable() {
+            public void run() {
+                long t = System.currentTimeMillis();
+                long end = t + 10000;
+                while (System.currentTimeMillis() < end) {
+                    cluster.triggerConfigurationReloadForSingleCType(client, CType.CONFIG, true);
+                    try {
+                        Thread.sleep(50);
+                    } catch (InterruptedException e) {
+                        break;
+                    }
+                }
+            }
+        };
+
+        new Thread(r).start();
+    }
+
+    @AfterClass
+    public static void cleanConfigurationDirectory() throws IOException {
+        FileUtils.deleteDirectory(configurationFolder.toFile());
+    }
+
+    @Test
+    public void shouldStillLoadSecurityConfigDuringBootstrapAndActiveConfigUpdateRequests() {
+        try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+            Awaitility.await().alias("Load default configuration").until(() -> client.getAuthInfo().getStatusCode(), equalTo(200));
+
+            TestRestClient.HttpResponse response = client.getAuthInfo();
+
+            response.assertStatusCode(200);
+        }
+    }
+}

src/integrationTest/java/org/opensearch/security/SecurityConfigurationBootstrapWithSecurityAdminTests.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.security;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.commons.io.FileUtils;
+import org.awaitility.Awaitility;
+import org.junit.AfterClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+
+import org.opensearch.test.framework.TestSecurityConfig.User;
+import org.opensearch.test.framework.cluster.ClusterManager;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED;
+import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class SecurityConfigurationBootstrapWithSecurityAdminTests {
+
+    private final static Path configurationFolder = ConfigurationFiles.createConfigurationDirectory();
+
+    @Rule
+    public TemporaryFolder configurationDirectory = new TemporaryFolder();
+
+    private static final User USER_ADMIN = new User("admin").roles(ALL_ACCESS);
+
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS)
+        .loadConfigurationIntoIndex(false)
+        .nodeSettings(
+            Map.of(
+                SECURITY_RESTAPI_ROLES_ENABLED,
+                List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()),
+                SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX,
+                false,
+                SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST,
+                false
+            )
+        )
+        .build();
+
+    @AfterClass
+    public static void cleanConfigurationDirectory() throws IOException {
+        FileUtils.deleteDirectory(configurationFolder.toFile());
+    }
+
+    @Test
+    public void testInitializeWithSecurityAdminWhenNoBackgroundInitialization() throws Exception {
+        SecurityAdminLauncher securityAdminLauncher = new SecurityAdminLauncher(cluster.getHttpPort(), cluster.getTestCertificates());
+
+        int exitCode = securityAdminLauncher.runSecurityAdmin(configurationFolder);
+
+        assertThat(exitCode, equalTo(0));
+        try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) {
+            Awaitility.await()
+                .alias("Waiting for rolemapping 'readall' availability.")
+                .until(() -> client.get("_plugins/_security/api/rolesmapping/readall").getStatusCode(), equalTo(200));
+        }
+    }
+}

src/integrationTest/java/org/opensearch/security/SecurityConfigurationTests.java

@@ -70,7 +70,7 @@ public class SecurityConfigurationTests {
                 SECURITY_RESTAPI_ROLES_ENABLED,
                 List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()),
                 SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST,
-                true
+                false
             )
         )
         .build();

src/integrationTest/java/org/opensearch/security/api/CreateResetPasswordTest.java

@@ -63,7 +63,7 @@ public class CreateResetPasswordTest {
                 SECURITY_RESTAPI_ROLES_ENABLED,
                 List.of("user_" + USER_ADMIN.getName() + "__" + ALL_ACCESS.getName()),
                 SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST,
-                true,
+                false,
                 ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_REGEX,
                 CUSTOM_PASSWORD_REGEX,
                 ConfigConstants.SECURITY_RESTAPI_PASSWORD_VALIDATION_ERROR_MESSAGE,

src/integrationTest/java/org/opensearch/security/http/OnBehalfOfJwtAuthenticationTest.java

@@ -48,7 +48,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.opensearch.security.support.ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX;
+import static org.opensearch.security.support.ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST;
 import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ADMIN_ENABLED;
 import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED;
 import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
@@ -128,8 +128,8 @@ private static OnBehalfOfConfig defaultOnBehalfOfConfig() {
         .users(ADMIN_USER, OBO_USER, OBO_USER_NO_PERM, HOST_MAPPING_OBO_USER)
         .nodeSettings(
             Map.of(
-                SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX,
-                true,
+                SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST,
+                false,
                 SECURITY_RESTAPI_ROLES_ENABLED,
                 ADMIN_USER.getRoleNames(),
                 SECURITY_RESTAPI_ADMIN_ENABLED,

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java

@@ -297,6 +297,16 @@ private static void triggerConfigurationReload(Client client) {
         }
     }
 
+    public void triggerConfigurationReloadForSingleCType(Client client, CType cType, boolean ignoreFailures) {
+        ConfigUpdateResponse configUpdateResponse = client.execute(
+            ConfigUpdateAction.INSTANCE,
+            new ConfigUpdateRequest(new String[] { cType.toLCString() })
+        ).actionGet();
+        if (!ignoreFailures && configUpdateResponse.hasFailures()) {
+            throw new RuntimeException("ConfigUpdateResponse produced failures: " + configUpdateResponse.failures());
+        }
+    }
+
     public CertificateData getAdminCertificate() {
         return testCertificates.getAdminCertificateData();
     }

src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java

@@ -125,8 +125,10 @@ protected ConfigUpdateResponse newResponse(
 
     @Override
     protected ConfigUpdateNodeResponse nodeOperation(final NodeConfigUpdateRequest request) {
-        configurationRepository.reloadConfiguration(CType.fromStringValues((request.request.getConfigTypes())));
-        backendRegistry.get().invalidateCache();
+        if (configurationRepository.isBgThreadComplete()) {
+            configurationRepository.reloadConfiguration(CType.fromStringValues((request.request.getConfigTypes())));
+            backendRegistry.get().invalidateCache();
+        }
         return new ConfigUpdateNodeResponse(clusterService.localNode(), request.request.getConfigTypes(), null);
     }
 

src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java

@@ -28,6 +28,8 @@
 
 import java.io.File;
 import java.nio.file.Path;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -37,6 +39,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Lock;
@@ -91,6 +94,7 @@ public class ConfigurationRepository {
     private final ThreadPool threadPool;
     private DynamicConfigFactory dynamicConfigFactory;
     private static final int DEFAULT_CONFIG_VERSION = 2;
+    private CompletableFuture<Void> bgThreadRunner = new CompletableFuture<>();
     private final Thread bgThread;
     private final AtomicBoolean installDefaultConfig = new AtomicBoolean();
     private final boolean acceptInvalid;
@@ -311,22 +315,23 @@ public void initOnNodeStart() {
             if (settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false)) {
                 LOGGER.info("Will attempt to create index {} and default configs if they are absent", securityIndex);
                 installDefaultConfig.set(true);
-                bgThread.start();
+                bgThreadRunner = CompletableFuture.runAsync(AccessController.doPrivileged((PrivilegedAction<Thread>) () -> bgThread));
             } else if (settings.getAsBoolean(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, true)) {
                 LOGGER.info(
                     "Will not attempt to create index {} and default configs if they are absent. Use securityadmin to initialize cluster",
                     securityIndex
                 );
-                bgThread.start();
+                bgThreadRunner = CompletableFuture.runAsync(AccessController.doPrivileged((PrivilegedAction<Thread>) () -> bgThread));
             } else {
                 LOGGER.info(
                     "Will not attempt to create index {} and default configs if they are absent. Will not perform background initialization",
                     securityIndex
                 );
+                bgThreadRunner.complete(null);
             }
         } catch (Throwable e2) {
             LOGGER.error("Error during node initialization: {}", e2, e2);
-            bgThread.start();
+            bgThreadRunner = CompletableFuture.runAsync(AccessController.doPrivileged((PrivilegedAction<Thread>) () -> bgThread));
         }
     }
 
@@ -372,6 +377,10 @@ public SecurityDynamicConfiguration<?> getConfiguration(CType configurationType)
 
     private final Lock LOCK = new ReentrantLock();
 
+    public boolean isBgThreadComplete() {
+        return bgThreadRunner.isDone();
+    }
+
     public void reloadConfiguration(Collection<CType> configTypes) throws ConfigUpdateAlreadyInProgressException {
         try {
             if (LOCK.tryLock(60, TimeUnit.SECONDS)) {

src/main/java/org/opensearch/security/securityconf/DynamicConfigFactory.java

@@ -315,7 +315,6 @@ public void onChange(Map<CType, SecurityDynamicConfiguration<?>> typeToConfig) {
         }
 
         initialized.set(true);
-
     }
 
     private static ConfigV6 getConfigV6(SecurityDynamicConfiguration<?> sdc) {