chore: add override integration tests (#615)

* add recapturing to immutable, fix broken link * add integration test for overrides with pod overrides * move security.properties evaluation and bump stackable-operator version * fix formatting * remove recapturing for now * rename test step files * test for env overrides * test config overrides * fix yaml lint --------- Co-authored-by: Razvan-Daniel Mihai <[email protected]>
stackabletech · Aug 27, 2024 · 650af78 · 650af78
1 parent 70bc2c6
commit 650af78
Show file tree

Hide file tree

Showing 16 changed files with 545 additions and 197 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.nix b/Cargo.nix
diff --git a/Cargo.toml b/Cargo.toml
@@ -19,13 +19,13 @@ indoc = "2.0"
 openssl = "0.10"
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
 pin-project = "1.1"
-rstest = "0.21"
+rstest = "0.22"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.38", features = ["full"] }
 tracing = "0.1"

diff --git a/crate-hashes.json b/crate-hashes.json
diff --git a/rust/crd/src/affinity.rs b/rust/crd/src/affinity.rs
@@ -5,7 +5,7 @@ use stackable_operator::{
 
 use crate::{DeepStorageSpec, DruidRole, HdfsDeepStorageSpec, APP_NAME};
 
-/// Please have a look at the architecture diagram in <https://druid.apache.org/docs/latest/assets/druid-architecture.png>
+/// Please have a look at the architecture diagram in <https://druid.apache.org/assets/images/druid-architecture-7db1cd79d2d70b2e5ccc73b6bebfcaa4.svg>
 /// to understand which roles do communicate with each other.
 pub fn get_affinity(
     cluster_name: &str,

diff --git a/rust/operator-binary/src/druid_controller.rs b/rust/operator-binary/src/druid_controller.rs
@@ -766,16 +766,37 @@ fn build_rolegroup_config_map(
                     to_java_properties_string(conf.iter()).context(PropertiesWriteSnafu)?;
                 cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties);
             }
+
             PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => {
                 let (heap, direct) = merged_rolegroup_config
                     .resources
                     .get_memory_sizes(&role)
                     .context(DeriveMemorySettingsSnafu)?;
                 let jvm_config = get_jvm_config(&role, heap, direct).context(GetJvmConfigSnafu)?;
-                // the user can set overrides in the config, but currently they have no effect
+                // TODO the user can set overrides in the config, but currently they have no effect
                 // if this is changed in the future, make sure to respect overrides!
                 cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config);
             }
+
+            PropertyNameKind::File(file_name) if file_name == JVM_SECURITY_PROPERTIES_FILE => {
+                let jvm_sec_props: BTreeMap<String, Option<String>> = rolegroup_config
+                    .get(&PropertyNameKind::File(
+                        JVM_SECURITY_PROPERTIES_FILE.to_string(),
+                    ))
+                    .cloned()
+                    .unwrap_or_default()
+                    .into_iter()
+                    .map(|(k, v)| (k, Some(v)))
+                    .collect();
+                cm_conf_data.insert(
+                    JVM_SECURITY_PROPERTIES_FILE.to_string(),
+                    to_java_properties_string(jvm_sec_props.iter()).with_context(|_| {
+                        JvmSecurityPropertiesSnafu {
+                            rolegroup: rolegroup.role_group.clone(),
+                        }
+                    })?,
+                );
+            }
             _ => {}
         }
     }
@@ -802,24 +823,6 @@ fn build_rolegroup_config_map(
         config_map_builder.add_data(filename, file_content);
     }
 
-    let jvm_sec_props: BTreeMap<String, Option<String>> = rolegroup_config
-        .get(&PropertyNameKind::File(
-            JVM_SECURITY_PROPERTIES_FILE.to_string(),
-        ))
-        .cloned()
-        .unwrap_or_default()
-        .into_iter()
-        .map(|(k, v)| (k, Some(v)))
-        .collect();
-    config_map_builder.add_data(
-        JVM_SECURITY_PROPERTIES_FILE,
-        to_java_properties_string(jvm_sec_props.iter()).with_context(|_| {
-            JvmSecurityPropertiesSnafu {
-                rolegroup: rolegroup.role_group.clone(),
-            }
-        })?,
-    );
-
     extend_role_group_config_map(
         rolegroup,
         vector_aggregator_address,

diff --git a/tests/templates/kuttl/overrides/00-assert.yaml.j2 b/tests/templates/kuttl/overrides/00-assert.yaml.j2
@@ -0,0 +1,6 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: druid-zk-server-default
+status:
+  availableReplicas: 1
diff --git a/tests/templates/kuttl/overrides/00-install-zk.yaml.j2 b/tests/templates/kuttl/overrides/00-install-zk.yaml.j2
@@ -0,0 +1,28 @@
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+  name: druid-zk
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}"
+  servers:
+    roleGroups:
+      default:
+        replicas: 1
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+  name: druid-znode
+spec:
+  clusterRef:
+    name: druid-zk
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+  name: hdfs-znode
+spec:
+  clusterRef:
+    name: druid-zk
diff --git a/tests/templates/kuttl/overrides/00-patch-ns.yaml.j2 b/tests/templates/kuttl/overrides/00-patch-ns.yaml.j2
@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+    timeout: 120
+{% endif %}
diff --git a/tests/templates/kuttl/overrides/10-assert.yaml b/tests/templates/kuttl/overrides/10-assert.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hdfs-namenode-default
+status:
+  availableReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hdfs-datanode-default
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hdfs-journalnode-default
+status:
+  availableReplicas: 1
diff --git a/tests/templates/kuttl/overrides/10-install-hdfs.yaml.j2 b/tests/templates/kuttl/overrides/10-install-hdfs.yaml.j2
@@ -0,0 +1,23 @@
+---
+apiVersion: hdfs.stackable.tech/v1alpha1
+kind: HdfsCluster
+metadata:
+  name: hdfs
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['hadoop-latest'] }}"
+  clusterConfig:
+    dfsReplication: 1
+    zookeeperConfigMapName: hdfs-znode
+  nameNodes:
+    roleGroups:
+      default:
+        replicas: 2
+  dataNodes:
+    roleGroups:
+      default:
+        replicas: 1
+  journalNodes:
+    roleGroups:
+      default:
+        replicas: 1
diff --git a/tests/templates/kuttl/overrides/20-assert.yaml b/tests/templates/kuttl/overrides/20-assert.yaml
@@ -0,0 +1,45 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: druid-broker-default
+spec:
+  template:
+    metadata:
+      labels:
+        level: role
+status:
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: druid-coordinator-default
+spec:
+  template:
+    metadata:
+      labels:
+        level: role-group
+status:
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: druid-historical-default
+status:
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: druid-middlemanager-default
+status:
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: druid-router-default
+status:
+  readyReplicas: 1
diff --git a/tests/templates/kuttl/overrides/20-install-druid.yaml.j2 b/tests/templates/kuttl/overrides/20-install-druid.yaml.j2
@@ -0,0 +1,123 @@
+---
+apiVersion: druid.stackable.tech/v1alpha1
+kind: DruidCluster
+metadata:
+  name: druid
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['druid-latest'] }}"
+  clusterConfig:
+    deepStorage:
+      hdfs:
+        configMapName: hdfs
+        directory: /data
+    metadataStorageDatabase:
+      dbType: derby
+      connString: jdbc:derby://localhost:1527/var/druid/metadata.db;create=true
+      host: localhost
+      port: 1527
+    tls: null
+    zookeeperConfigMapName: druid-znode
+  brokers:
+    envOverrides:
+      ROLE_VAR: role-value
+      COMMON_VAR: role-value
+    configOverrides:
+      runtime.properties:
+        prop.common: role-value
+        prop.role: role-value
+    podOverrides:
+      metadata:
+        labels:
+          level: role
+    roleGroups:
+      default:
+        replicas: 1
+        envOverrides:
+          GROUP_VAR: group-value
+          COMMON_VAR: group-value
+        configOverrides:
+          runtime.properties:
+            prop.common: group-value
+            prop.group: group-value
+  coordinators:
+    envOverrides:
+      ROLE_VAR: role-value
+      COMMON_VAR: role-value
+    configOverrides:
+      runtime.properties:
+        prop.common: role-value
+        prop.role: role-value
+    podOverrides:
+      metadata:
+        labels:
+          level: role
+    roleGroups:
+      default:
+        replicas: 1
+        podOverrides:
+          metadata:
+            labels:
+              level: role-group
+        envOverrides:
+          GROUP_VAR: group-value
+          COMMON_VAR: group-value
+        configOverrides:
+          runtime.properties:
+            prop.common: group-value
+            prop.group: group-value
+
+  historicals:
+    envOverrides:
+      ROLE_VAR: role-value
+      COMMON_VAR: role-value
+    configOverrides:
+      runtime.properties:
+        prop.common: role-value
+        prop.role: role-value
+    roleGroups:
+      default:
+        replicas: 1
+        envOverrides:
+          GROUP_VAR: group-value
+          COMMON_VAR: group-value
+        configOverrides:
+          runtime.properties:
+            prop.common: group-value
+            prop.group: group-value
+  middleManagers:
+    envOverrides:
+      ROLE_VAR: role-value
+      COMMON_VAR: role-value
+    configOverrides:
+      runtime.properties:
+        prop.common: role-value
+        prop.role: role-value
+    roleGroups:
+      default:
+        replicas: 1
+        envOverrides:
+          GROUP_VAR: group-value
+          COMMON_VAR: group-value
+        configOverrides:
+          runtime.properties:
+            prop.common: group-value
+            prop.group: group-value
+  routers:
+    envOverrides:
+      ROLE_VAR: role-value
+      COMMON_VAR: role-value
+    configOverrides:
+      runtime.properties:
+        prop.common: role-value
+        prop.role: role-value
+    roleGroups:
+      default:
+        replicas: 1
+        envOverrides:
+          GROUP_VAR: group-value
+          COMMON_VAR: group-value
+        configOverrides:
+          runtime.properties:
+            prop.common: group-value
+            prop.group: group-value
diff --git a/tests/templates/kuttl/overrides/21-assert.yaml b/tests/templates/kuttl/overrides/21-assert.yaml
@@ -0,0 +1,28 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+  #
+  # Test envOverrides
+  #
+  - script: |
+      kubectl -n $NAMESPACE get sts druid-broker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-broker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-broker-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
+  - script: |
+      kubectl -n $NAMESPACE get sts druid-coordinator-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-coordinator-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-coordinator-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
+  - script: |
+      kubectl -n $NAMESPACE get sts druid-historical-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-historical-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-historical-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
+  - script: |
+      kubectl -n $NAMESPACE get sts druid-middlemanager-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-middlemanager-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-middlemanager-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'
+  - script: |
+      kubectl -n $NAMESPACE get sts druid-router-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-router-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")'
+      kubectl -n $NAMESPACE get sts druid-router-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "druid") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")'