Skip to content

Commit

Permalink
controllers: init container to decide allowing reconciliation of manager
Browse files Browse the repository at this point in the history
manual cherry-pick of red-hat-storage#196

Signed-off-by: Leela Venkaiah G <[email protected]>
  • Loading branch information
leelavg committed Jul 31, 2024
1 parent 50a1c62 commit 3988022
Show file tree
Hide file tree
Showing 11 changed files with 175 additions and 4 deletions.
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /workspace/bin/manager .
COPY --from=builder /workspace/bin/status-reporter .
COPY --from=builder /workspace/bin/deployment-guard .
USER 65532:65532

ENTRYPOINT ["/manager"]
10 changes: 7 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,11 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
$(KUSTOMIZE) build config/crd | kubectl delete -f -

deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} && \
$(KUSTOMIZE) edit set image deployment-guard=$(IMG)
cd config/default && $(KUSTOMIZE) edit set image kube-rbac-proxy=$(RBAC_PROXY_IMG)
cd config/console && $(KUSTOMIZE) edit set image ocs-client-operator-console=$(OCS_CLIENT_CONSOLE_IMG)
cd config/console && $(KUSTOMIZE) edit set image ocs-client-operator-console=$(OCS_CLIENT_CONSOLE_IMG) && \
$(KUSTOMIZE) edit set image deployment-guard=$(IMG)
$(KUSTOMIZE) build config/default | sed "s|STATUS_REPORTER_IMAGE_VALUE|$(IMG)|g" | awk '{print}' | kubectl apply -f -

remove: ## Remove controller from the K8s cluster specified in ~/.kube/config.
Expand All @@ -115,8 +117,10 @@ remove-with-olm: ## Remove controller from the K8s cluster
bundle: manifests kustomize operator-sdk yq ## Generate bundle manifests and metadata, then validate generated files.
rm -rf ./bundle
$(OPERATOR_SDK) generate kustomize manifests -q
cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) && \
$(KUSTOMIZE) edit set image deployment-guard=$(IMG)
cd config/console && $(KUSTOMIZE) edit set image ocs-client-operator-console=$(OCS_CLIENT_CONSOLE_IMG) && \
$(KUSTOMIZE) edit set image deployment-guard=$(IMG) && \
$(KUSTOMIZE) edit set nameprefix $(OPERATOR_NAMEPREFIX)
cd config/default && \
$(KUSTOMIZE) edit set image kube-rbac-proxy=$(RBAC_PROXY_IMG) && \
Expand Down
33 changes: 32 additions & 1 deletion bundle/manifests/ocs-client-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ metadata:
categories: Storage
console.openshift.io/plugins: '["odf-client-console"]'
containerImage: quay.io/ocs-dev/ocs-client-operator:latest
createdAt: "2024-06-11T05:34:18Z"
createdAt: "2024-07-31T09:49:01Z"
description: OpenShift Data Foundation client operator enables consumption of
storage services from a remote centralized OpenShift Data Foundation provider
cluster.
Expand Down Expand Up @@ -237,6 +237,13 @@ spec:
- get
- patch
- update
- apiGroups:
- ocs.openshift.io
resources:
- storageclusters
verbs:
- get
- list
- apiGroups:
- operators.coreos.com
resources:
Expand Down Expand Up @@ -742,6 +749,17 @@ spec:
name: csi-images
- mountPath: /etc/tls/private
name: webhook-cert-secret
initContainers:
- command:
- /deployment-guard
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: quay.io/ocs-dev/ocs-client-operator:latest
name: deployment-guard
resources: {}
securityContext:
runAsNonRoot: true
serviceAccountName: ocs-client-operator-controller-manager
Expand Down Expand Up @@ -802,14 +820,27 @@ spec:
name: ocs-client-operator-console-nginx-log
- mountPath: /var/lib/nginx/tmp
name: ocs-client-operator-console-nginx-tmp
initContainers:
- command:
- /deployment-guard
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: quay.io/ocs-dev/ocs-client-operator:latest
name: deployment-guard
resources: {}
securityContext:
runAsNonRoot: true
serviceAccountName: ocs-client-operator-controller-manager
volumes:
- name: ocs-client-operator-console-serving-cert
secret:
secretName: ocs-client-operator-console-serving-cert
- configMap:
name: ocs-client-operator-console-nginx-conf
optional: true
name: ocs-client-operator-console-nginx-conf
- emptyDir: {}
name: ocs-client-operator-console-nginx-log
Expand Down
12 changes: 12 additions & 0 deletions config/console/console_init.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,16 @@ spec:
labels:
app.kubernetes.io/name: ocs-client-operator-console
spec:
initContainers:
- name: deployment-guard
image: deployment-guard:latest
command:
- /deployment-guard
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
containers:
- name: ocs-client-operator-console
image: ocs-client-operator-console:latest
Expand Down Expand Up @@ -54,9 +64,11 @@ spec:
- name: ocs-client-operator-console-nginx-conf
configMap:
name: ocs-client-operator-console-nginx-conf
optional: true
- name: ocs-client-operator-console-nginx-log
emptyDir: {}
- name: ocs-client-operator-console-nginx-tmp
emptyDir: {}
securityContext:
runAsNonRoot: true
serviceAccountName: ocs-client-operator-controller-manager
3 changes: 3 additions & 0 deletions config/console/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ resources:
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
images:
- name: deployment-guard
newName: quay.io/ocs-dev/ocs-client-operator
newTag: latest
- name: ocs-client-operator-console
newName: quay.io/ocs-dev/ocs-client-console
newTag: latest
Expand Down
3 changes: 3 additions & 0 deletions config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,6 @@ images:
- name: controller
newName: quay.io/ocs-dev/ocs-client-operator
newTag: latest
- name: deployment-guard
newName: quay.io/ocs-dev/ocs-client-operator
newTag: latest
10 changes: 10 additions & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,16 @@ spec:
spec:
securityContext:
runAsNonRoot: true
initContainers:
- name: deployment-guard
image: deployment-guard:latest
command:
- /deployment-guard
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
containers:
- command:
- /manager
Expand Down
7 changes: 7 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,13 @@ rules:
- get
- patch
- update
- apiGroups:
- ocs.openshift.io
resources:
- storageclusters
verbs:
- get
- list
- apiGroups:
- operators.coreos.com
resources:
Expand Down
1 change: 1 addition & 0 deletions controllers/operatorconfigmap_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ func (c *OperatorConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error {
//+kubebuilder:rbac:groups=console.openshift.io,resources=consoleplugins,verbs=*
//+kubebuilder:rbac:groups=operators.coreos.com,resources=subscriptions,verbs=get;list;watch;update
//+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;update;create;watch;delete
//+kubebuilder:rbac:groups=ocs.openshift.io,resources=storageclusters,verbs=get;list

// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
Expand Down
1 change: 1 addition & 0 deletions hack/go-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ set -x

go build -a -o ${GOBIN:-bin}/manager main.go
go build -a -o ${GOBIN:-bin}/status-reporter ./service/status-report/main.go
go build -a -o ${GOBIN:-bin}/deployment-guard ./service/deployment-guard/main.go
98 changes: 98 additions & 0 deletions service/deployment-guard/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package main

import (
"context"
"os"
"time"

"github.com/red-hat-storage/ocs-client-operator/pkg/utils"

extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)

func main() {
// validations
operatorNamespace := os.Getenv(utils.OperatorNamespaceEnvVar)
if operatorNamespace == "" {
klog.Exitf("%s env var is empty", utils.OperatorNamespaceEnvVar)
}

// creation of kube client
scheme := runtime.NewScheme()
cfg, err := config.GetConfig()
if err != nil {
klog.Exitf("Failed to get config: %v", err)
}
cl, err := client.New(cfg, client.Options{Scheme: scheme})
if err != nil {
klog.Exitf("Failed to create controller runtime client: %v", err)
}
ctx := context.Background()

storageClusterCRD := &metav1.PartialObjectMetadata{}
storageClusterCRD.SetGroupVersionKind(
extv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"),
)
storageClusterCRD.Name = "storageclusters.ocs.openshift.io"

// delay exponentially from half a sec and cap at 2 minutes
delayFunc := wait.Backoff{
Duration: 500 * time.Millisecond,
Factor: 2,
Jitter: 0.1,
Steps: 10,
Cap: 2 * time.Minute,
}.DelayFunc()

for !allowOperatorToRun(ctx, cl, operatorNamespace) {
time.Sleep(delayFunc())
}

}

func allowOperatorToRun(ctx context.Context, cl client.Client, namespace string) bool {
// verify presence of StorageCluster CRD
storageClusterCRD := &metav1.PartialObjectMetadata{}
storageClusterCRD.SetGroupVersionKind(
extv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"),
)
storageClusterCRD.Name = "storageclusters.ocs.openshift.io"
if err := cl.Get(ctx, client.ObjectKeyFromObject(storageClusterCRD), storageClusterCRD); client.IgnoreNotFound(err) != nil {
klog.Warning("Failed to find presence of StorageCluster CRD")
return false
}

if storageClusterCRD.UID != "" {
// StorageCluster CRD exists, wait till StorageCluster CR is configured in Provider mode
storageClusters := &metav1.PartialObjectMetadataList{}
storageClusters.SetGroupVersionKind(
schema.GroupVersionKind{
Group: "ocs.openshift.io",
Version: "v1",
Kind: "StorageCluster",
},
)
if err := cl.List(ctx, storageClusters, client.InNamespace(namespace), client.Limit(1)); err != nil {
klog.Warning("Failed to list StorageCluster CR")
return false
}
if len(storageClusters.Items) < 1 {
klog.Info("StorageCluster CR does not exist")
return false
}
klog.Info("Checking if StorageCluster indicates ODF is deployed in provider mode")
if storageClusters.Items[0].GetAnnotations()["ocs.openshift.io/deployment-mode"] != "provider" {
return false
}
}

klog.Info("Condition met to allow operator to run")
return true
}

0 comments on commit 3988022

Please sign in to comment.