cherry picked 8874319

konpyutaika · Aug 26, 2022 · c9dda8f · c9dda8f
1 parent b3a6369
commit c9dda8f
Show file tree

Hide file tree

Showing 55 changed files with 3,550 additions and 22 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,12 @@
 
 ### Removed
 
+## v0.13.1
+
+### Changed
+
+- [PR #146](https://github.com/konpyutaika/nifikop/pull/146) - **[Operator/NifiCluster]** Move from volume prefix to pvc label selection for deletion
+
 ## v0.13.0
 
 ### Added

diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -13,4 +13,4 @@ kind: Kustomization
 images:
   - name: controller
     newName: ghcr.io/konpyutaika/docker-images/nifikop
-    newTag: 0.13.0-master
+    newTag: 0.13.1-master
diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml
@@ -31,7 +31,7 @@ spec:
             - /manager
           args:
             - --leader-elect
-          image: ghcr.io/konpyutaika/docker-images/nifikop:v0.13.0-release
+          image: ghcr.io/konpyutaika/docker-images/nifikop:v0.13.1-release
           name: nifikop
           securityContext:
             allowPrivilegeEscalation: false

diff --git a/config/samples/keycloak-example/step-1/operator.yaml b/config/samples/keycloak-example/step-1/operator.yaml
@@ -1,4 +1,4 @@
-# nifikop 0.13.0
+# nifikop 0.13.1
 rbacEnable: true
 namespaces:
   - nifi
diff --git a/config/samples/nifi_v1alpha1_nificluster.yaml b/config/samples/nifi_v1alpha1_nificluster.yaml
@@ -34,9 +34,9 @@ spec:
   #      imagePullPolicy: IfNotPresent
   #      command: ["sh", "-c", "cp -vr /nifi_lib/* /nifi_lib_volume/"]
   #      volumeMounts:
-  #        - name: nifi-data-extensions-repository
+  #        - name: extensions-repository
   #          mountPath: /nifi_lib_volume
-  #        - name: nifi-data-logs
+  #        - name: logs
   #          mountPath: /logs
   # clusterImage can specify the whole nificluster image in one place
   clusterImage: "apache/nifi:1.12.1"

diff --git a/controllers/nifinodegroupautoscaler_controller.go b/controllers/nifinodegroupautoscaler_controller.go
@@ -237,6 +237,7 @@ func (r *NifiNodeGroupAutoscalerReconciler) updateAutoscalerReplicaState(ctx con
 	return r.Client.Status().Update(ctx, autoscaler)
 }
 
+// TODO : discuss about replacing by looking for NifiCluster.Spec.Nodes instead
 // updateAutoscalerReplicaStatus updates autoscaler replica status to inform the k8s scale subresource
 func (r *NifiNodeGroupAutoscalerReconciler) updateAutoscalerReplicaStatus(ctx context.Context, nifiCluster *v1alpha1.NifiCluster, autoscaler *v1alpha1.NifiNodeGroupAutoscaler) error {
 	podList, err := r.getCurrentReplicaPods(ctx, autoscaler)

diff --git a/helm/nifikop/Chart.yaml b/helm/nifikop/Chart.yaml
@@ -4,8 +4,8 @@ name: nifikop
 home: https://github.com/konpyutaika/nifikop
 sources:
   - https://github.com/konpyutaika/nifikop
-version: 0.13.0
-appVersion: 0.13.0-release
+version: 0.13.1
+appVersion: 0.13.1-release
 icon:
 maintainers:
   - name: erdrix

diff --git a/helm/nifikop/README.md b/helm/nifikop/README.md
@@ -23,7 +23,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm
 | Parameter                        | Description                                                                                                                                                                          | Default                  |
 | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |--------------------------|
 | `image.repository`               | Image                                                                                                                                                                                | `konpyutaika/nifikop`    |
-| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.13.0-release`        |
+| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.13.1-release`        |
 | `image.pullPolicy`               | Image pull policy                                                                                                                                                                    | `Always`                 |
 | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image                                                                                                                                            | `false`                  |
 | `image.imagePullSecrets.name`    | Name of the secret to connect to docker registry                                                                                                                                     | -                        |

diff --git a/pkg/resources/nifi/nifi.go b/pkg/resources/nifi/nifi.go
@@ -345,7 +345,24 @@ OUTERLOOP:
 			}
 
 			for _, volume := range node.Spec.Volumes {
-				if strings.HasPrefix(volume.Name, nifiutil.NifiDataVolumeMount) {
+				if volume.PersistentVolumeClaim == nil {
+					continue
+				}
+				pvcFound := &corev1.PersistentVolumeClaim{}
+				if err := r.Client.Get(context.TODO(),
+					types.NamespacedName{
+						Name:      volume.PersistentVolumeClaim.ClaimName,
+						Namespace: r.NifiCluster.Namespace,
+					},
+					pvcFound,
+				); err != nil {
+					if apierrors.IsNotFound(err) {
+						continue
+					}
+					return errors.WrapIfWithDetails(err, "could not get pvc for node", "id", node.Labels["nodeId"])
+				}
+
+				if pvcFound.Labels[nifiutil.NifiDataVolumeMountKey] == "true" {
 					err = r.Client.Delete(context.TODO(), &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
 						Name:      volume.PersistentVolumeClaim.ClaimName,
 						Namespace: r.NifiCluster.Namespace,

diff --git a/pkg/resources/nifi/pvc.go b/pkg/resources/nifi/pvc.go
@@ -19,11 +19,12 @@ func (r *Reconciler) pvc(id int32, storage v1alpha1.StorageConfig, log zap.Logge
 			util.MergeLabels(
 				nifiutil.LabelsForNifi(r.NifiCluster.Name),
 				map[string]string{
-					"nodeId":      fmt.Sprintf("%d", id),
-					"storageName": storage.Name,
+					"nodeId":                        fmt.Sprintf("%d", id),
+					"storageName":                   storage.Name,
+					nifiutil.NifiDataVolumeMountKey: "true",
 				},
 			),
-			map[string]string{"mountPath": storage.MountPath, "storageName": fmt.Sprintf(templates.StorageNameTemplate, nifiutil.NifiDataVolumeMount, storage.Name)}, r.NifiCluster),
+			map[string]string{"mountPath": storage.MountPath, "storageName": storage.Name}, r.NifiCluster),
 		Spec: *storage.PVCSpec,
 	}
 }
diff --git a/pkg/resources/templates/variables.go b/pkg/resources/templates/variables.go
@@ -2,7 +2,6 @@ package templates
 
 const (
 	NodeConfigTemplate            = "%s-config"
-	StorageNameTemplate           = "%s-%s"
 	NodeStorageTemplate           = "%s-%d-%s-storage-"
 	ExternalClusterSecretTemplate = "%s-basic-secret"
 )
diff --git a/pkg/util/nifi/common.go b/pkg/util/nifi/common.go
@@ -15,8 +15,11 @@ const (
 	NodeNameTemplate       = PrefixNodeNameTemplate + RootNodeNameTemplate + SuffixNodeNameTemplate
 
 	// TimeStampLayout defines the date format used.
-	TimeStampLayout     = "Mon, 2 Jan 2006 15:04:05 GMT"
-	NifiDataVolumeMount = "nifi-data"
+	TimeStampLayout = "Mon, 2 Jan 2006 15:04:05 GMT"
+)
+
+var (
+	NifiDataVolumeMountKey = fmt.Sprintf("%s/nifi-data", v1alpha1.GroupVersion.Group)
 )
 
 // ParseTimeStampToUnixTime parses the given CC timeStamp to time format

diff --git a/site/docs/2_setup/1_getting_started.md b/site/docs/2_setup/1_getting_started.md
@@ -109,8 +109,8 @@ Now deploy the helm chart :
 helm install nifikop \
     oci://ghcr.io/konpyutaika/helm-charts/nifikop \
     --namespace=nifi \
-    --version 0.13.0 \
-    --set image.tag=v0.13.0-release \
+    --version 0.13.1 \
+    --set image.tag=v0.13.1-release \
     --set resources.requests.memory=256Mi \
     --set resources.requests.cpu=250m \
     --set resources.limits.memory=256Mi \

diff --git a/site/docs/2_setup/3_install/1_customizable_install_with_helm.md b/site/docs/2_setup/3_install/1_customizable_install_with_helm.md
@@ -32,7 +32,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm
 | Parameter                        | Description                                                                                                                                                                          | Default                                     |
 |----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------|
 | `image.repository`               | Image                                                                                                                                                                                | `ghcr.io/konpyutaika/docker-images/nifikop` |
-| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.13.0-release`                           |
+| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.13.1-release`                           |
 | `image.pullPolicy`               | Image pull policy                                                                                                                                                                    | `Always`                                    |
 | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image                                                                                                                                            | `false`                                     |
 | `image.imagePullSecrets.name`    | Name of the secret to connect to docker registry                                                                                                                                     | -                                           |

diff --git a/site/website/versioned_docs/version-v0.13.0/2_setup/1_getting_started.md b/site/website/versioned_docs/version-v0.13.0/2_setup/1_getting_started.md
@@ -109,8 +109,8 @@ Now deploy the helm chart :
 helm install nifikop \
     oci://ghcr.io/konpyutaika/helm-charts/nifikop \
     --namespace=nifi \
-    --version 0.13.0 \
-    --set image.tag=v0.13.0-release \
+    --version 0.13.1 \
+    --set image.tag=v0.13.1-release \
     --set resources.requests.memory=256Mi \
     --set resources.requests.cpu=250m \
     --set resources.limits.memory=256Mi \

diff --git a/...oned_docs/version-v0.13.0/2_setup/3_install/1_customizable_install_with_helm.md b/...oned_docs/version-v0.13.0/2_setup/3_install/1_customizable_install_with_helm.md
@@ -32,7 +32,7 @@ The following tables lists the configurable parameters of the NiFi Operator Helm
 | Parameter                        | Description                                                                                                                                                                          | Default                                     |
 |----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------|
 | `image.repository`               | Image                                                                                                                                                                                | `ghcr.io/konpyutaika/docker-images/nifikop` |
-| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.13.0-release`                           |
+| `image.tag`                      | Image tag                                                                                                                                                                            | `v0.13.1-release`                           |
 | `image.pullPolicy`               | Image pull policy                                                                                                                                                                    | `Always`                                    |
 | `image.imagePullSecrets.enabled` | Enable tue use of secret for docker image                                                                                                                                            | `false`                                     |
 | `image.imagePullSecrets.name`    | Name of the secret to connect to docker registry                                                                                                                                     | -                                           |

diff --git a/site/website/versioned_docs/version-v0.13.1/1_concepts/1_introduction.md b/site/website/versioned_docs/version-v0.13.1/1_concepts/1_introduction.md
@@ -0,0 +1,34 @@
+---
+id: 1_introduction
+title: Introduction
+sidebar_label: Introduction
+---
+
+The Konpyūtāika NiFi operator is a Kubernetes operator to automate provisioning, management, autoscaling and operations of [Apache NiFi](https://nifi.apache.org/) clusters deployed to K8s.
+
+## Overview
+
+Apache NiFi is an open-source solution that support powerful and scalable directed graphs of data routing, transformation, and system mediation logic. 
+Some of the high-level capabilities and objectives of Apache NiFi include, and some of the main features of the **NiFiKop** are:
+
+- **Fine grained** node configuration support
+- Graceful rolling upgrade
+- graceful NiFi cluster **scaling**
+- Encrypted communication using SSL
+- the provisioning of secure NiFi clusters
+- Advanced Dataflow and user management via CRD
+
+Some of the roadmap features :
+
+- Monitoring via **Prometheus**
+- Automatic reaction and self healing based on alerts (plugin system, with meaningful default alert plugins)
+- graceful NiFi cluster **scaling and rebalancing**
+
+## Motivation
+
+There are already some approaches to operating NiFi on Kubernetes, however, we did not find them appropriate for use in a highly dynamic environment, nor capable of meeting our needs.
+
+- [Helm chart](https://github.com/cetic/helm-nifi)
+- [Cloudera Nifi Operator](https://blog.cloudera.com/cloudera-flow-management-goes-cloud-native-with-apache-nifi-on-red-hat-openshift-kubernetes-platform/)
+
+Finally, our motivation is to build an open source solution and a community which drives the innovation and features of this operator.
diff --git a/site/website/versioned_docs/version-v0.13.1/1_concepts/2_design_principes.md b/site/website/versioned_docs/version-v0.13.1/1_concepts/2_design_principes.md
@@ -0,0 +1,62 @@
+---
+id: 2_design_principes
+title: Design Principes
+sidebar_label: Design Principes
+---
+
+## Pod level management
+
+NiFi is a stateful application. The first piece of the puzzle is the Node, which is a simple server capable of createing/forming a cluster with other Nodes. Every Node has his own **unique** configuration which differs slightly from all others.
+
+All NiFi on Kubernetes setup use [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) to create a NiFi Cluster. Just to quickly recap from the K8s docs:
+
+>StatefulSet manages the deployment and scaling of a set of Pods, and provide guarantees about their ordering and uniqueness. Like a Deployment, a StatefulSet manages Pods that are based on an identical container spec. Unlike a Deployment, a StatefulSet maintains sticky identities for each of its Pods. These pods are created from the same spec, but are not interchangeable: each has a persistent identifier that is maintained across any rescheduling.
+
+How does this looks from the perspective of Apache NiFi ?
+
+With StatefulSet we get :
+- unique Node IDs generated during Pod startup
+- networking between Nodes with headless services
+- unique Persistent Volumes for Nodes
+
+Using StatefulSet we **lose**  the ability to :
+
+- modify the configuration of unique Nodes
+- remove a specific Node from a cluster (StatefulSet always removes the most recently created Node)
+- use multiple, different Persistent Volumes for each Node
+
+The NiFi Operator uses `simple` Pods, ConfigMaps, and PersistentVolumeClaims, instead of StatefulSet (based on the design used by [Banzai Cloud Kafka Operator](https://github.com/banzaicloud/kafka-operator)). 
+Using these resources allows us to build an Operator which is better suited to NiFi.
+
+With the NiFi operator we can:
+
+- modify the configuration of unique Nodes
+- remove specific Nodes from clusters
+- use multiple Persistent Volumes for each Node
+
+## Dataflow Lifecycle management
+
+The [Dataflow Lifecycle management feature](./3_features.md#dataflow-lifecycle-management-via-crd) introduces 3 new CRDs :
+
+- **NiFiRegistryClient :** Allowing you to declare a [NiFi registry client](https://nifi.apache.org/docs/nifi-registry-docs/html/getting-started.html#connect-nifi-to-the-registry).
+- **NiFiParameterContext :** Allowing you to create parameter context, with two kinds of parameters, a simple `map[string]string` for non-sensitive parameters and a `list of secrets` which contains sensitive parameters.
+- **NiFiDataflow :** Allowing you to declare a Dataflow based on a `NiFiRegistryClient` and optionally a `ParameterContext`, which will be deployed and managed by the operator on the `targeted NiFi cluster`.
+
+The following diagram shows the interactions between all the components : 
+
+![dataflow lifecycle management schema](/img/1_concepts/2_design_principes/dataflow_lifecycle_management_schema.jpg)
+
+With each CRD comes a new controller, with a reconcile loop : 
+
+- **NiFiRegistryClient's controller :** 
+
+![NiFi registry client's reconcile loop](/img/1_concepts/2_design_principes/registry_client_reconcile_loop.jpeg)
+
+- **NiFiParameterContext's controller :** 
+
+![NiFi parameter context's reconcile loop](/img/1_concepts/2_design_principes/parameter_context_reconcile_loop.jpeg)
+
+- **NiFiDataflow's controller :** 
+
+![NiFi dataflow's reconcile loop](/img/1_concepts/2_design_principes/dataflow_reconcile_loop.jpeg)
+