diff --git a/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java b/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java index 0aa69f9bee..21ee0248f5 100644 --- a/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java +++ b/src/main/java/org/opensearch/security/action/configupdate/TransportConfigUpdateAction.java @@ -110,8 +110,10 @@ protected ConfigUpdateResponse newResponse(ConfigUpdateRequest request, List configurationChangedListener; private final ConfigurationLoaderSecurity7 cl; private final Settings settings; + private final Path configPath; private final ClusterService clusterService; private final AuditLog auditLog; private final ThreadPool threadPool; private DynamicConfigFactory dynamicConfigFactory; - private static final int DEFAULT_CONFIG_VERSION = 2; - private final Thread bgThread; - private final AtomicBoolean installDefaultConfig = new AtomicBoolean(); + public static final int DEFAULT_CONFIG_VERSION = 2; + private final CompletableFuture initalizeConfigTask = new CompletableFuture<>(); private final boolean acceptInvalid; private ConfigurationRepository(Settings settings, final Path configPath, ThreadPool threadPool, Client client, ClusterService clusterService, AuditLog auditLog) { this.securityIndex = settings.get(ConfigConstants.SECURITY_CONFIG_INDEX_NAME, ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX); this.settings = settings; + this.configPath = configPath; this.client = client; this.threadPool = threadPool; this.clusterService = clusterService; @@ -110,99 +112,132 @@ private ConfigurationRepository(Settings settings, final Path configPath, Thread this.acceptInvalid = settings.getAsBoolean(ConfigConstants.SECURITY_UNSUPPORTED_ACCEPT_INVALID_CONFIG, false); cl = new ConfigurationLoaderSecurity7(client, threadPool, settings, clusterService); - configCache = CacheBuilder - .newBuilder() - .build(); + configCache = CacheBuilder.newBuilder().build(); + } + + private void initalizeClusterConfiguration(final boolean installDefaultConfig) { + try { + LOGGER.info("Background init thread started. Install default config?: " + installDefaultConfig); + // wait for the cluster here until it will finish managed node election + while (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) { + LOGGER.info("Wait for cluster to be available ..."); + TimeUnit.SECONDS.sleep(1); + } + + if (installDefaultConfig) { - bgThread = new Thread(new Runnable() { - @Override - public void run() { try { - LOGGER.info("Background init thread started. Install default config?: "+installDefaultConfig.get()); - while (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) { - LOGGER.info("Wait for cluster to be available ..."); - TimeUnit.SECONDS.sleep(1); - } - if(installDefaultConfig.get()) { - - try { - String lookupDir = System.getProperty("security.default_init.dir"); - final String cd = lookupDir != null? (lookupDir+"/") : new Environment(settings, configPath).pluginsFile().toAbsolutePath().toString()+"/opensearch-security/securityconfig/"; - File confFile = new File(cd+"config.yml"); - if(confFile.exists()) { - final ThreadContext threadContext = threadPool.getThreadContext(); - try(StoredContext ctx = threadContext.stashContext()) { - threadContext.putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true"); - - createSecurityIndexIfAbsent(); - waitForSecurityIndexToBeAtLeastYellow(); - - ConfigHelper.uploadFile(client, cd+"config.yml", securityIndex, CType.CONFIG, DEFAULT_CONFIG_VERSION); - ConfigHelper.uploadFile(client, cd+"roles.yml", securityIndex, CType.ROLES, DEFAULT_CONFIG_VERSION); - ConfigHelper.uploadFile(client, cd+"roles_mapping.yml", securityIndex, CType.ROLESMAPPING, DEFAULT_CONFIG_VERSION); - ConfigHelper.uploadFile(client, cd+"internal_users.yml", securityIndex, CType.INTERNALUSERS, DEFAULT_CONFIG_VERSION); - ConfigHelper.uploadFile(client, cd+"action_groups.yml", securityIndex, CType.ACTIONGROUPS, DEFAULT_CONFIG_VERSION); - if(DEFAULT_CONFIG_VERSION == 2) { - ConfigHelper.uploadFile(client, cd+"tenants.yml", securityIndex, CType.TENANTS, DEFAULT_CONFIG_VERSION); - } - final boolean populateEmptyIfFileMissing = true; - ConfigHelper.uploadFile(client, cd+"nodes_dn.yml", securityIndex, CType.NODESDN, DEFAULT_CONFIG_VERSION, populateEmptyIfFileMissing); - ConfigHelper.uploadFile(client, cd + "whitelist.yml", securityIndex, CType.WHITELIST, DEFAULT_CONFIG_VERSION, populateEmptyIfFileMissing); - - // audit.yml is not packaged by default - final String auditConfigPath = cd + "audit.yml"; - if (new File(auditConfigPath).exists()) { - ConfigHelper.uploadFile(client, auditConfigPath, securityIndex, CType.AUDIT, DEFAULT_CONFIG_VERSION); - } - } - } else { - LOGGER.error("{} does not exist", confFile.getAbsolutePath()); + String lookupDir = System.getProperty("security.default_init.dir"); + final String cd = lookupDir != null? (lookupDir+"/") : new Environment(settings, configPath).pluginsFile().toAbsolutePath().toString()+"/opensearch-security/securityconfig/"; + File confFile = new File(cd + "config.yml"); + if (confFile.exists()) { + final ThreadContext threadContext = threadPool.getThreadContext(); + try (StoredContext ctx = threadContext.stashContext()) { + threadContext.putHeader(ConfigConstants.OPENDISTRO_SECURITY_CONF_REQUEST_HEADER, "true"); + + createSecurityIndexIfAbsent(); + waitForSecurityIndexToBeAtLeastYellow(); + + ConfigHelper.uploadFile(client, cd + "config.yml", securityIndex, CType.CONFIG, DEFAULT_CONFIG_VERSION); + ConfigHelper.uploadFile(client, cd + "roles.yml", securityIndex, CType.ROLES, DEFAULT_CONFIG_VERSION); + ConfigHelper.uploadFile( + client, + cd + "roles_mapping.yml", + securityIndex, + CType.ROLESMAPPING, + DEFAULT_CONFIG_VERSION + ); + ConfigHelper.uploadFile( + client, + cd + "internal_users.yml", + securityIndex, + CType.INTERNALUSERS, + DEFAULT_CONFIG_VERSION + ); + ConfigHelper.uploadFile( + client, + cd + "action_groups.yml", + securityIndex, + CType.ACTIONGROUPS, + DEFAULT_CONFIG_VERSION + ); + if (DEFAULT_CONFIG_VERSION == 2) { + ConfigHelper.uploadFile(client, cd + "tenants.yml", securityIndex, CType.TENANTS, DEFAULT_CONFIG_VERSION); } - } catch (Exception e) { - LOGGER.error("Cannot apply default config (this is maybe not an error!)", e); - } - } - - while(!dynamicConfigFactory.isInitialized()) { - try { - LOGGER.debug("Try to load config ..."); - reloadConfiguration(Arrays.asList(CType.values())); - break; - } catch (Exception e) { - LOGGER.debug("Unable to load configuration due to {}", String.valueOf(ExceptionUtils.getRootCause(e))); - try { - Thread.sleep(3000); - } catch (InterruptedException e1) { - Thread.currentThread().interrupt(); - LOGGER.debug("Thread was interrupted so we cancel initialization"); - break; + final boolean populateEmptyIfFileMissing = true; + ConfigHelper.uploadFile( + client, + cd + "nodes_dn.yml", + securityIndex, + CType.NODESDN, + DEFAULT_CONFIG_VERSION, + populateEmptyIfFileMissing + ); + ConfigHelper.uploadFile( + client, + cd + "whitelist.yml", + securityIndex, + CType.WHITELIST, + DEFAULT_CONFIG_VERSION, + populateEmptyIfFileMissing + ); + + // audit.yml is not packaged by default + final String auditConfigPath = cd + "audit.yml"; + if (new File(auditConfigPath).exists()) { + ConfigHelper.uploadFile(client, auditConfigPath, securityIndex, CType.AUDIT, DEFAULT_CONFIG_VERSION); } } - } - - final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn("Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", deprecatedAuditKeysInSettings); - } - final boolean isAuditConfigDocPresentInIndex = cl.isAuditConfigDocPresentInIndex(); - if (isAuditConfigDocPresentInIndex) { - if (!deprecatedAuditKeysInSettings.isEmpty()) { - LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); - } - LOGGER.info("Hot-reloading of audit configuration is enabled"); } else { - LOGGER.info("Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it."); - auditLog.setConfig(AuditConfig.from(settings)); + LOGGER.error("{} does not exist", confFile.getAbsolutePath()); } + } catch (Exception e) { + LOGGER.error("Cannot apply default config (this is maybe not an error!)", e); + } + } - LOGGER.info("Node '{}' initialized", clusterService.localNode().getName()); - + while (!dynamicConfigFactory.isInitialized()) { + try { + LOGGER.debug("Try to load config ..."); + reloadConfiguration(Arrays.asList(CType.values()), true); + break; } catch (Exception e) { - LOGGER.error("Unexpected exception while initializing node "+e, e); + LOGGER.debug("Unable to load configuration due to {}", String.valueOf(ExceptionUtils.getRootCause(e))); + try { + Thread.sleep(3000); + } catch (InterruptedException e1) { + Thread.currentThread().interrupt(); + LOGGER.debug("Thread was interrupted so we cancel initialization"); + break; + } + } + } + + final Set deprecatedAuditKeysInSettings = AuditConfig.getDeprecatedKeys(settings); + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn( + "Following keys {} are deprecated in opensearch settings. They will be removed in plugin v2.0.0.0", + deprecatedAuditKeysInSettings + ); + } + final boolean isAuditConfigDocPresentInIndex = cl.isAuditConfigDocPresentInIndex(); + if (isAuditConfigDocPresentInIndex) { + if (!deprecatedAuditKeysInSettings.isEmpty()) { + LOGGER.warn("Audit configuration settings found in both index and opensearch settings (deprecated)"); } + LOGGER.info("Hot-reloading of audit configuration is enabled"); + } else { + LOGGER.info( + "Hot-reloading of audit configuration is disabled. Using configuration with defaults from opensearch settings. Populate the configuration in index using audit.yml or securityadmin to enable it." + ); + auditLog.setConfig(AuditConfig.from(settings)); } - }); + LOGGER.info("Node '{}' initialized", clusterService.localNode().getName()); + + } catch (Exception e) { + LOGGER.error("Unexpected exception while initializing node " + e, e); + } } private boolean createSecurityIndexIfAbsent() { @@ -253,23 +288,37 @@ private void waitForSecurityIndexToBeAtLeastYellow() { } } - public void initOnNodeStart() { + public CompletableFuture initOnNodeStart() { + final boolean installDefaultConfig = settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false); + + final Supplier> startInitialization = () -> { + new Thread(() -> { + initalizeClusterConfiguration(installDefaultConfig); + initalizeConfigTask.complete(null); + }).start(); + return initalizeConfigTask.thenApply(result -> installDefaultConfig); + }; try { - if (settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, false)) { + if (installDefaultConfig) { LOGGER.info("Will attempt to create index {} and default configs if they are absent", securityIndex); - installDefaultConfig.set(true); - bgThread.start(); - } else if (settings.getAsBoolean(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, true)){ - LOGGER.info("Will not attempt to create index {} and default configs if they are absent. Use securityadmin to initialize cluster", - securityIndex); - bgThread.start(); + return startInitialization.get(); + } else if (settings.getAsBoolean(ConfigConstants.SECURITY_BACKGROUND_INIT_IF_SECURITYINDEX_NOT_EXIST, true)) { + LOGGER.info( + "Will not attempt to create index {} and default configs if they are absent. Use securityadmin to initialize cluster", + securityIndex + ); + return startInitialization.get(); } else { - LOGGER.info("Will not attempt to create index {} and default configs if they are absent. Will not perform background initialization", - securityIndex); + LOGGER.info( + "Will not attempt to create index {} and default configs if they are absent. Will not perform background initialization", + securityIndex + ); + initalizeConfigTask.complete(null); + return initalizeConfigTask.thenApply(result -> installDefaultConfig); } } catch (Throwable e2) { LOGGER.error("Error during node initialization: {}", e2, e2); - bgThread.start(); + return startInitialization.get(); } } @@ -302,16 +351,26 @@ public SecurityDynamicConfiguration getConfiguration(CType configurationType) private final Lock LOCK = new ReentrantLock(); - public void reloadConfiguration(Collection configTypes) throws ConfigUpdateAlreadyInProgressException { + public boolean reloadConfiguration(final Collection configTypes) throws ConfigUpdateAlreadyInProgressException { + return reloadConfiguration(configTypes, false); + } + + private boolean reloadConfiguration(final Collection configTypes, final boolean fromBackgroundThread) + throws ConfigUpdateAlreadyInProgressException { + if (!fromBackgroundThread && !initalizeConfigTask.isDone()) { + LOGGER.warn("Unable to reload configuration, initalization thread has not yet completed."); + return false; + } try { if (LOCK.tryLock(60, TimeUnit.SECONDS)) { try { reloadConfiguration0(configTypes, this.acceptInvalid); + return true; } finally { LOCK.unlock(); } } else { - throw new ConfigUpdateAlreadyInProgressException("A config update is already imn progress"); + throw new ConfigUpdateAlreadyInProgressException("A config update is already in progress"); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -408,4 +467,5 @@ private static String formatDate(long date) { public static int getDefaultConfigVersion() { return ConfigurationRepository.DEFAULT_CONFIG_VERSION; } + } diff --git a/src/test/java/org/opensearch/security/InitializationIntegrationTests.java b/src/test/java/org/opensearch/security/InitializationIntegrationTests.java index 82b5f73238..72a35e55c9 100644 --- a/src/test/java/org/opensearch/security/InitializationIntegrationTests.java +++ b/src/test/java/org/opensearch/security/InitializationIntegrationTests.java @@ -211,13 +211,10 @@ public void testInvalidDefaultConfig() throws Exception { setup(Settings.EMPTY, null, settings, false); RestHelper rh = nonSslRestHelper(); Thread.sleep(10000); - Assert.assertEquals(HttpStatus.SC_SERVICE_UNAVAILABLE, rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode()); - - System.setProperty("security.default_init.dir", defaultInitDirectory); - restart(Settings.EMPTY, null, settings, false); - rh = nonSslRestHelper(); - Thread.sleep(10000); - Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode()); + Assert.assertEquals( + HttpStatus.SC_SERVICE_UNAVAILABLE, + rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode() + ); } finally { if (defaultInitDirectory != null) { System.setProperty("security.default_init.dir", defaultInitDirectory); diff --git a/src/test/java/org/opensearch/security/SlowIntegrationTests.java b/src/test/java/org/opensearch/security/SlowIntegrationTests.java index df5c6325c1..cbbde23fe1 100644 --- a/src/test/java/org/opensearch/security/SlowIntegrationTests.java +++ b/src/test/java/org/opensearch/security/SlowIntegrationTests.java @@ -31,6 +31,7 @@ package org.opensearch.security; import org.apache.http.HttpStatus; +import org.junit.Ignore; import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; @@ -163,6 +164,7 @@ public void testNodeClientDisallowedWithNonServerCertificate2() throws Exception } @Test + @Ignore public void testDelayInSecurityIndexInitialization() throws Exception { final Settings settings = Settings.builder() .put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true) diff --git a/src/test/java/org/opensearch/security/test/SingleClusterTest.java b/src/test/java/org/opensearch/security/test/SingleClusterTest.java index 81ac6b3705..34b780ae29 100644 --- a/src/test/java/org/opensearch/security/test/SingleClusterTest.java +++ b/src/test/java/org/opensearch/security/test/SingleClusterTest.java @@ -73,13 +73,6 @@ protected void setup(Settings initTransportClientSettings, DynamicSecurityConfig setup(initTransportClientSettings, dynamicSecuritySettings, nodeOverride, initSecurityIndex, ClusterConfiguration.DEFAULT); } - protected void restart(Settings initTransportClientSettings, DynamicSecurityConfig dynamicSecuritySettings, Settings nodeOverride, boolean initOpendistroSecurityIndex) throws Exception { - clusterInfo = clusterHelper.startCluster(minimumSecuritySettings(ccs(nodeOverride)), ClusterConfiguration.DEFAULT); - if(initOpendistroSecurityIndex && dynamicSecuritySettings != null) { - initialize(clusterInfo, initTransportClientSettings, dynamicSecuritySettings); - } - } - private Settings ccs(Settings nodeOverride) throws Exception { if(remoteClusterHelper != null) { Assert.assertNull("No remote clusters", remoteClusterInfo);