DataNode: removing the dependency to the data_dir (#18483)

* removing the dependency to the data_dir * removing the dependency to the data_dir * adding native lib dir
Graylog2 · Mar 6, 2024 · d12a6dd · d12a6dd
1 parent c4afe7d
commit d12a6dd
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 13 deletions.
diff --git a/data-node/src/main/java/org/graylog/datanode/Configuration.java b/data-node/src/main/java/org/graylog/datanode/Configuration.java
@@ -55,7 +55,7 @@
 import java.util.Set;
 
 /**
- * Helper class to hold configuration of Graylog
+ * Helper class to hold configuration of DataNode
  */
 @SuppressWarnings("FieldMayBeFinal")
 public class Configuration extends BaseConfiguration {
@@ -100,6 +100,9 @@ public class Configuration extends BaseConfiguration {
     @Parameter(value = "config_location", validators = DirectoryReadableValidator.class)
     private Path configLocation = null;
 
+    @Parameter(value = "native_lib_dir", required = true)
+    private Path nativeLibDir = Path.of("native_libs");
+
     @Parameter(value = "process_logs_buffer_size")
     private Integer opensearchProcessLogsBufferSize = 500;
 
@@ -245,7 +248,7 @@ public class Configuration extends BaseConfiguration {
 
     @Parameter(value = "node_search_cache_size")
     private String searchCacheSize = "10gb";
-  
+
     /**
      * https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore/#shared-file-system
      */
@@ -437,6 +440,10 @@ public String getMetricsPolicy() {
         return metricsPolicy;
     }
 
+    public Path getNativeLibDir() {
+        return nativeLibDir;
+    }
+
     public static class NodeIdFileValidator implements Validator<String> {
         @Override
         public void validate(String name, String path) throws ValidationException {
@@ -663,7 +670,7 @@ public String getRootPasswordSha2() {
     public String getNodeSearchCacheSize() {
         return searchCacheSize;
     }
-  
+
     public List<String> getPathRepo() {
         return pathRepo;
     }

diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/CmdLineTool.java
@@ -50,6 +50,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.graylog.datanode.Configuration;
 import org.graylog.datanode.bootstrap.commands.MigrateCmd;
 import org.graylog2.bindings.NamedConfigParametersOverrideModule;
 import org.graylog2.bootstrap.CliCommand;
@@ -185,7 +186,7 @@ protected void beforeStart() {
      * Things that have to run before the {@link #startCommand()} method is being called.
      * Please note that this happens *before* the configuration file has been parsed.
      */
-    protected void beforeStart(TLSProtocolsConfiguration configuration, PathConfiguration pathConfiguration) {
+    protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration, Configuration configuration) {
     }
 
     /**
@@ -262,7 +263,7 @@ public void doRun(Level logLevel) {
         installCommandConfig();
 
         beforeStart();
-        beforeStart(parseAndGetTLSConfiguration(), parseAndGetPathConfiguration(configFile));
+        beforeStart(parseAndGetTLSConfiguration(), parseAndGetConfiguration(configFile));
 
         processConfiguration(jadConfig);
 
@@ -306,6 +307,12 @@ public void doRun(Level logLevel) {
         startCommand();
     }
 
+    private Configuration parseAndGetConfiguration(String configFile) {
+        final Configuration configuration = new Configuration();
+        processConfiguration(new JadConfig(getConfigRepositories(configFile), configuration));
+        return configuration;
+    }
+
     // Parse only the TLSConfiguration bean
     // to avoid triggering anything that might initialize the default SSLContext
     private TLSProtocolsConfiguration parseAndGetTLSConfiguration() {

diff --git a/data-node/src/main/java/org/graylog/datanode/bootstrap/ServerBootstrap.java b/data-node/src/main/java/org/graylog/datanode/bootstrap/ServerBootstrap.java
@@ -97,8 +97,8 @@ private void registerFreshInstallation() {
     }
 
     @Override
-    protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration, PathConfiguration pathConfiguration) {
-        super.beforeStart(tlsProtocolsConfiguration, pathConfiguration);
+    protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration, Configuration configuration) {
+        super.beforeStart(tlsProtocolsConfiguration, configuration);
 
         // Do not use a PID file if the user requested not to
         if (!isNoPidFile()) {
@@ -109,8 +109,7 @@ protected void beforeStart(TLSProtocolsConfiguration tlsProtocolsConfiguration,
         applySecuritySettings(tlsProtocolsConfiguration);
 
         // Set these early in the startup because netty's NativeLibraryUtil uses a static initializer
-        setNettyNativeDefaults(pathConfiguration);
-
+        setNettyNativeDefaults(configuration);
     }
 
     @Override
@@ -175,10 +174,10 @@ public void configure(Binder binder) {
                 });
     }
 
-    private void setNettyNativeDefaults(PathConfiguration pathConfiguration) {
+    private void setNettyNativeDefaults(Configuration configuration) {
         // Give netty a better spot than /tmp to unpack its tcnative libraries
         if (System.getProperty("io.netty.native.workdir") == null) {
-            System.setProperty("io.netty.native.workdir", pathConfiguration.getNativeLibDir().toAbsolutePath().toString());
+            System.setProperty("io.netty.native.workdir", configuration.getNativeLibDir().toAbsolutePath().toString());
         }
         // Don't delete the native lib after unpacking, as this confuses needrestart(1) on some distributions
         if (System.getProperty("io.netty.native.deleteLibAfterLoading") == null) {

diff --git a/data-node/src/main/java/org/graylog/datanode/configuration/BaseConfiguration.java b/data-node/src/main/java/org/graylog/datanode/configuration/BaseConfiguration.java
@@ -19,7 +19,7 @@
 import com.github.joschi.jadconfig.Parameter;
 import org.graylog2.configuration.PathConfiguration;
 
-public class BaseConfiguration extends PathConfiguration {
+public class BaseConfiguration {
     @Parameter(value = "async_eventbus_processors")
     private int asyncEventbusProcessors = 2;
 

diff --git a/data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeDevContainerBuilder.java b/data-node/src/test/java/org/graylog/datanode/testinfra/DatanodeDevContainerBuilder.java
@@ -141,7 +141,6 @@ public GenericContainer<?> build() {
         GenericContainer<?> container = new GenericContainer<>(imageSupplier.get())
                 .withExposedPorts(restPort, openSearchHttpPort)
                 .withNetwork(network)
-                .withEnv("GRAYLOG_DATANODE_DATA_DIR", "data")
                 .withEnv("GRAYLOG_DATANODE_OPENSEARCH_LOCATION", IMAGE_WORKING_DIR)
                 .withEnv("GRAYLOG_DATANODE_OPENSEARCH_PLUGINS_LOCATION", IMAGE_WORKING_DIR + "/plugins")
                 .withEnv("GRAYLOG_DATANODE_INSECURE_STARTUP", "true")