From a7482eb1b11bb5b555308e0b1308297cf7664c3b Mon Sep 17 00:00:00 2001 From: Leela Venkaiah G Date: Mon, 21 Oct 2024 05:23:13 +0000 Subject: [PATCH] controllers: remove code which deploys CSI and retain delegation of CSI changes in brief: csi related utils, pkg and templates are not required. usage of csi images manifest is changed and old way of having all images in a single configmap is not required. conditional delegation of csi in controllers is removed and delegation is made default. RBAC related to direct management of deployments and daemonsets is removed. usage of deployCSI in key of operator config is removed since that functionality is now being done by deployment-guard general: concentrated on removal of code rather than refactor to minimize the diff and will be raising a follow up if there is any chance of refactor now. Signed-off-by: Leela Venkaiah G --- .github/workflows/verify-generated.yaml | 3 - Makefile | 9 - ...ient-operator-csi-images_v1_configmap.yaml | 36 -- ...client-operator.clusterserviceversion.yaml | 36 +- config/manager/kustomization.yaml | 3 - config/manager/manager.yaml | 5 - config/rbac/role.yaml | 29 -- go.mod | 4 +- hack/gen-csi-images-manifest.sh | 35 -- hack/make-bundle-vars.mk | 5 - .../operatorconfigmap_controller.go | 350 ++---------------- internal/controller/sliceutils.go | 41 -- internal/controller/sliceutils_test.go | 105 ------ .../controller/storageclaim_controller.go | 111 ++---- .../controller/storageclient_controller.go | 37 +- pkg/csi/cephfsdaemonset.go | 279 -------------- pkg/csi/cephfsdeployment.go | 221 ----------- pkg/csi/csi.go | 101 ----- pkg/csi/csidriver.go | 69 ---- pkg/csi/monconfigmap.go | 194 ---------- pkg/csi/rbddaemonset.go | 348 ----------------- pkg/csi/rbddeployment.go | 264 ------------- pkg/csi/scc.go | 88 ----- pkg/templates/csidriver.go | 42 --- pkg/templates/csisidecars.go | 198 ---------- pkg/templates/defaults.go | 34 -- pkg/utils/csi.go | 33 -- pkg/utils/k8sutils.go | 4 - pkg/utils/placements.go | 32 -- service/status-report/main.go | 19 +- 30 files changed, 69 insertions(+), 2666 deletions(-) delete mode 100644 bundle/manifests/ocs-client-operator-csi-images_v1_configmap.yaml delete mode 100755 hack/gen-csi-images-manifest.sh delete mode 100644 internal/controller/sliceutils.go delete mode 100644 internal/controller/sliceutils_test.go delete mode 100644 pkg/csi/cephfsdaemonset.go delete mode 100644 pkg/csi/cephfsdeployment.go delete mode 100644 pkg/csi/csi.go delete mode 100644 pkg/csi/csidriver.go delete mode 100644 pkg/csi/monconfigmap.go delete mode 100644 pkg/csi/rbddaemonset.go delete mode 100644 pkg/csi/rbddeployment.go delete mode 100644 pkg/csi/scc.go delete mode 100644 pkg/templates/csidriver.go delete mode 100644 pkg/templates/csisidecars.go delete mode 100644 pkg/templates/defaults.go delete mode 100644 pkg/utils/csi.go delete mode 100644 pkg/utils/placements.go diff --git a/.github/workflows/verify-generated.yaml b/.github/workflows/verify-generated.yaml index bdd73a96..610fec02 100644 --- a/.github/workflows/verify-generated.yaml +++ b/.github/workflows/verify-generated.yaml @@ -35,9 +35,6 @@ jobs: with: go-version-file: go.mod - - name: Verify changes to CSI images manifest - run: make verify-csi-images-manifest - - name: Verify go deps run: make godeps-verify diff --git a/Makefile b/Makefile index f5f3628e..4eefecb5 100644 --- a/Makefile +++ b/Makefile @@ -37,15 +37,6 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -csi-images-manifest: ## Generates the YAML manifest of CSI images for each supported environment. - ./hack/gen-csi-images-manifest.sh - -verify-csi-images-manifest: csi-images-manifest ## Verify csi-images-manifest has been run, if required. - @if [[ -n "$$(git status --porcelain $${CSI_IMAGES_MANIFEST})" ]]; then \ - echo -e "\n\033[1;31mError:\033[0m Uncommitted changes to CSI images manifest found. Run \033[1m'make csi-images-manifest'\033[0m and commit the results.\n"; \ - git diff -u $${CSI_IMAGES_MANIFEST}; \ - exit 1; \ - fi fmt: ## Run go fmt against code. go fmt ./... diff --git a/bundle/manifests/ocs-client-operator-csi-images_v1_configmap.yaml b/bundle/manifests/ocs-client-operator-csi-images_v1_configmap.yaml deleted file mode 100644 index 3726fb07..00000000 --- a/bundle/manifests/ocs-client-operator-csi-images_v1_configmap.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: v1 -data: - csi-images.yaml: | - --- - - version: v4.15 - containerImages: - provisionerImageURL: "registry.k8s.io/sig-storage/csi-provisioner:v4.0.0" - attacherImageURL: "registry.k8s.io/sig-storage/csi-attacher:v4.5.0" - resizerImageURL: "registry.k8s.io/sig-storage/csi-resizer:v1.10.0" - snapshotterImageURL: "registry.k8s.io/sig-storage/csi-snapshotter:v7.0.1" - driverRegistrarImageURL: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0" - cephCSIImageURL: "quay.io/cephcsi/cephcsi:v3.10.2" - csiaddonsImageURL: "quay.io/csiaddons/k8s-sidecar:v0.8.0" - - - version: v4.16 - containerImages: - provisionerImageURL: "registry.k8s.io/sig-storage/csi-provisioner:v4.0.0" - attacherImageURL: "registry.k8s.io/sig-storage/csi-attacher:v4.5.0" - resizerImageURL: "registry.k8s.io/sig-storage/csi-resizer:v1.10.0" - snapshotterImageURL: "registry.k8s.io/sig-storage/csi-snapshotter:v7.0.1" - driverRegistrarImageURL: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0" - cephCSIImageURL: "quay.io/cephcsi/cephcsi:v3.10.2" - csiaddonsImageURL: "quay.io/csiaddons/k8s-sidecar:v0.8.0" - - - version: v4.17 - containerImages: - provisionerImageURL: "registry.k8s.io/sig-storage/csi-provisioner:v4.0.0" - attacherImageURL: "registry.k8s.io/sig-storage/csi-attacher:v4.5.0" - resizerImageURL: "registry.k8s.io/sig-storage/csi-resizer:v1.10.0" - snapshotterImageURL: "registry.k8s.io/sig-storage/csi-snapshotter:v7.0.1" - driverRegistrarImageURL: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0" - cephCSIImageURL: "quay.io/cephcsi/cephcsi:v3.10.2" - csiaddonsImageURL: "quay.io/csiaddons/k8s-sidecar:v0.8.0" -kind: ConfigMap -metadata: - name: ocs-client-operator-csi-images diff --git a/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml b/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml index 77cf24ad..f25aaf9e 100644 --- a/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml +++ b/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml @@ -7,7 +7,7 @@ metadata: categories: Storage console.openshift.io/plugins: '["odf-client-console"]' containerImage: quay.io/ocs-dev/ocs-client-operator:latest - createdAt: "2024-10-14T12:21:27Z" + createdAt: "2024-10-21T04:48:51Z" description: OpenShift Data Foundation client operator enables consumption of storage services from a remote centralized OpenShift Data Foundation provider cluster. @@ -106,42 +106,13 @@ spec: - get - list - watch - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - apps - resources: - - daemonsets/finalizers - verbs: - - update - apiGroups: - apps resources: - deployments verbs: - - create - - delete - get - list - - patch - - update - - watch - - apiGroups: - - apps - resources: - - deployments/finalizers - verbs: - - update - apiGroups: - batch resources: @@ -823,8 +794,6 @@ spec: securityContext: allowPrivilegeEscalation: false volumeMounts: - - mountPath: /opt/config - name: csi-images - mountPath: /etc/tls/private name: webhook-cert-secret initContainers: @@ -848,9 +817,6 @@ spec: operator: Equal value: "true" volumes: - - configMap: - name: ocs-client-operator-csi-images - name: csi-images - name: webhook-cert-secret secret: secretName: webhook-cert-secret diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 2c910941..23a2e734 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -9,9 +9,6 @@ configMapGenerator: - files: - controller_manager_config.yaml name: manager-config -- files: - - csi-images.yaml=csi-images.yaml - name: csi-images - name: config apiVersion: kustomize.config.k8s.io/v1beta1 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index f6fcb9c1..2d89e730 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -47,8 +47,6 @@ spec: image: controller:latest name: manager volumeMounts: - - name: csi-images - mountPath: /opt/config - mountPath: /etc/tls/private name: webhook-cert-secret env: @@ -88,9 +86,6 @@ spec: cpu: 10m memory: 256Mi volumes: - - name: csi-images - configMap: - name: csi-images - name: webhook-cert-secret secret: secretName: webhook-cert-secret diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8c2e7ee9..acdb9125 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -63,42 +63,13 @@ rules: - get - list - watch -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - daemonsets/finalizers - verbs: - - update - apiGroups: - apps resources: - deployments verbs: - - create - - delete - get - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments/finalizers - verbs: - - update - apiGroups: - batch resources: diff --git a/go.mod b/go.mod index 45238c78..c0d6f482 100644 --- a/go.mod +++ b/go.mod @@ -25,14 +25,12 @@ require ( github.com/onsi/gomega v1.34.1 github.com/openshift/api v0.0.0-20240828125535-01b3675ba7b3 github.com/operator-framework/api v0.27.0 - github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0 github.com/ramendr/ramen/api v0.0.0-20241001141243-29d6f22ad237 github.com/red-hat-storage/ocs-client-operator/api v0.0.0-00010101000000-000000000000 github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20240917115204-741b9d6f263d github.com/stretchr/testify v1.9.0 google.golang.org/grpc v1.66.0 - gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 k8s.io/apimachinery v0.31.0 @@ -47,8 +45,10 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 // indirect github.com/openshift/custom-resource-status v1.1.3-0.20220503160415-f2fdb4999d87 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/x448/float16 v0.8.4 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.31.0 // indirect sigs.k8s.io/container-object-storage-interface-api v0.1.0 // indirect ) diff --git a/hack/gen-csi-images-manifest.sh b/hack/gen-csi-images-manifest.sh deleted file mode 100755 index 5ea104df..00000000 --- a/hack/gen-csi-images-manifest.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -CSI_IMAGES_MANIFEST="${CSI_IMAGES_MANIFEST:-config/manager/csi-images.yaml}" - -echo "Generating CSI image manifest for OCP versions: ${CSI_OCP_VERSIONS}" - -rm -f "${CSI_IMAGES_MANIFEST}" -echo -n "---" > "${CSI_IMAGES_MANIFEST}" -for version in ${CSI_OCP_VERSIONS}; do - echo "" >> "${CSI_IMAGES_MANIFEST}" - - ver="${version//./_}" - echo -e "- version: $version\n containerImages:" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_PROVISIONER_${ver}" - echo " provisionerImageURL: \"${!csi_var:-${CSI_IMG_PROVISIONER}}\"" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_ATTACHER_${ver}" - echo " attacherImageURL: \"${!csi_var:-${CSI_IMG_ATTACHER}}\"" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_RESIZER_${ver}" - echo " resizerImageURL: \"${!csi_var:-${CSI_IMG_RESIZER}}\"" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_SNAPSHOTTER_${ver}" - echo " snapshotterImageURL: \"${!csi_var:-${CSI_IMG_SNAPSHOTTER}}\"" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_REGISTRAR_${ver}" - echo " driverRegistrarImageURL: \"${!csi_var:-${CSI_IMG_REGISTRAR}}\"" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_CEPH_CSI_${ver}" - echo " cephCSIImageURL: \"${!csi_var:-${CSI_IMG_CEPH_CSI}}\"" >> "${CSI_IMAGES_MANIFEST}" - - csi_var="CSI_IMG_ADDONS_${ver}" - echo " csiaddonsImageURL: \"${!csi_var:-${CSI_IMG_ADDONS}}\"" >> "${CSI_IMAGES_MANIFEST}" -done diff --git a/hack/make-bundle-vars.mk b/hack/make-bundle-vars.mk index d5db9dba..17c71bba 100644 --- a/hack/make-bundle-vars.mk +++ b/hack/make-bundle-vars.mk @@ -108,11 +108,6 @@ endif CSI_ADDONS_PACKAGE_NAME ?= csi-addons CSI_ADDONS_PACKAGE_VERSION ?= 0.9.1 -## CSI driver images -# The following variables define the default CSI container images to deploy -# and the supported versions of OpenShift. -CSI_IMAGES_MANIFEST ?= config/manager/csi-images.yaml - # The following variables are here as a convenience for developers so we don't have # to retype things, because we're lazy. IMAGE_LOCATION_SIG_STORAGE ?= registry.k8s.io/sig-storage diff --git a/internal/controller/operatorconfigmap_controller.go b/internal/controller/operatorconfigmap_controller.go index 004dcf8b..079be790 100644 --- a/internal/controller/operatorconfigmap_controller.go +++ b/internal/controller/operatorconfigmap_controller.go @@ -26,7 +26,6 @@ import ( "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" "github.com/red-hat-storage/ocs-client-operator/pkg/console" - "github.com/red-hat-storage/ocs-client-operator/pkg/csi" "github.com/red-hat-storage/ocs-client-operator/pkg/templates" "github.com/red-hat-storage/ocs-client-operator/pkg/utils" @@ -67,7 +66,6 @@ const ( // ClusterVersionName is the name of the ClusterVersion object in the // openshift cluster. clusterVersionName = "version" - deployCSIKey = "DEPLOY_CSI" manageNoobaaSubKey = "manageNoobaaSubscription" subscriptionLabelKey = "managed-by" subscriptionLabelValue = "webhook.subscription.ocs.openshift.io" @@ -88,11 +86,6 @@ type OperatorConfigMapReconciler struct { ctx context.Context operatorConfigMap *corev1.ConfigMap consoleDeployment *appsv1.Deployment - cephFSDeployment *appsv1.Deployment - cephFSDaemonSet *appsv1.DaemonSet - rbdDeployment *appsv1.Deployment - rbdDaemonSet *appsv1.DaemonSet - scc *secv1.SecurityContextConstraints subscriptionChannel string } @@ -158,31 +151,24 @@ func (c *OperatorConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error { ), ) + generationChangePredicate := predicate.GenerationChangedPredicate{} bldr := ctrl.NewControllerManagedBy(mgr). For(&corev1.ConfigMap{}, configMapPredicates). Owns(&corev1.Service{}, servicePredicate). + Owns(&csiopv1a1.OperatorConfig{}, builder.WithPredicates(generationChangePredicate)). + Owns(&csiopv1a1.Driver{}, builder.WithPredicates(generationChangePredicate)). Watches(&configv1.ClusterVersion{}, enqueueConfigMapRequest, clusterVersionPredicates). Watches(&extv1.CustomResourceDefinition{}, enqueueConfigMapRequest, builder.OnlyMetadata). Watches(&opv1a1.Subscription{}, enqueueConfigMapRequest, subscriptionPredicates). Watches(&admrv1.ValidatingWebhookConfiguration{}, enqueueConfigMapRequest, webhookPredicates). Watches(&v1alpha1.StorageClient{}, enqueueConfigMapRequest, builder.WithPredicates(predicate.AnnotationChangedPredicate{})) - generationChangePredicate := predicate.GenerationChangedPredicate{} - if utils.DelegateCSI { - bldr = bldr. - Owns(&csiopv1a1.OperatorConfig{}, builder.WithPredicates(generationChangePredicate)). - Owns(&csiopv1a1.Driver{}, builder.WithPredicates(generationChangePredicate)) - } - return bldr.Complete(c) } //+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch //+kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list;watch -//+kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="apps",resources=deployments/finalizers,verbs=update -//+kubebuilder:rbac:groups="apps",resources=daemonsets,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups="apps",resources=daemonsets/finalizers,verbs=update +//+kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list //+kubebuilder:rbac:groups="storage.k8s.io",resources=csidrivers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;delete //+kubebuilder:rbac:groups="",resources=configmaps/finalizers,verbs=update @@ -263,41 +249,28 @@ func (c *OperatorConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, err } - if deployCSI, err := c.getDeployCSIConfig(); err != nil { - c.log.Error(err, "failed to perform precheck for deploying CSI") + if err := c.reconcileDelegatedCSI(); err != nil { return ctrl.Result{}, err - } else if deployCSI { - - var err error - if utils.DelegateCSI { - err = c.reconcileDelegatedCSI() - } else { - err = c.reconcileCSI() - } - - if err != nil { - return ctrl.Result{}, err - } - - prometheusRule := &monitoringv1.PrometheusRule{} - if err := k8sYAML.NewYAMLOrJSONDecoder(bytes.NewBufferString(string(pvcPrometheusRules)), 1000).Decode(prometheusRule); err != nil { - c.log.Error(err, "Unable to retrieve prometheus rules.", "prometheusRule", klog.KRef(prometheusRule.Namespace, prometheusRule.Name)) - return ctrl.Result{}, err - } + } - prometheusRule.SetNamespace(c.OperatorNamespace) + prometheusRule := &monitoringv1.PrometheusRule{} + if err := k8sYAML.NewYAMLOrJSONDecoder(bytes.NewBufferString(string(pvcPrometheusRules)), 1000).Decode(prometheusRule); err != nil { + c.log.Error(err, "Unable to retrieve prometheus rules.", "prometheusRule", klog.KRef(prometheusRule.Namespace, prometheusRule.Name)) + return ctrl.Result{}, err + } - err = c.createOrUpdate(prometheusRule, func() error { - applyLabels(c.operatorConfigMap.Data["OCS_METRICS_LABELS"], &prometheusRule.ObjectMeta) - return c.own(prometheusRule) - }) - if err != nil { - c.log.Error(err, "failed to create/update prometheus rules") - return ctrl.Result{}, err - } + prometheusRule.SetNamespace(c.OperatorNamespace) - c.log.Info("prometheus rules deployed", "prometheusRule", klog.KRef(prometheusRule.Namespace, prometheusRule.Name)) + err = c.createOrUpdate(prometheusRule, func() error { + applyLabels(c.operatorConfigMap.Data["OCS_METRICS_LABELS"], &prometheusRule.ObjectMeta) + return c.own(prometheusRule) + }) + if err != nil { + c.log.Error(err, "failed to create/update prometheus rules") + return ctrl.Result{}, err } + + c.log.Info("prometheus rules deployed", "prometheusRule", klog.KRef(prometheusRule.Namespace, prometheusRule.Name)) } else { // deletion phase if err := c.deletionPhase(); err != nil { @@ -316,12 +289,6 @@ func (c *OperatorConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Re } func (c *OperatorConfigMapReconciler) reconcileDelegatedCSI() error { - // remove older CSI deployments and daemonsets as the resources created by csi-operator is different. - // we are guaranteed to use kernel mounts removing daemonsts will not pose any risk - // NOTE: in next minor version this should be removed - if err := c.deleteOlderCSIResources(); err != nil { - return fmt.Errorf("failed to remove older csi resources: %v", err) - } // scc scc := &secv1.SecurityContextConstraints{} @@ -430,214 +397,6 @@ func (c *OperatorConfigMapReconciler) reconcileDelegatedCSI() error { return nil } -func (c *OperatorConfigMapReconciler) getAndDeleteResource(obj client.Object) error { - if err := c.get(obj); err == nil { - if err = c.delete(obj); err != nil { - return fmt.Errorf("failed to delete %s: %v", client.ObjectKeyFromObject(obj), err) - } - } else if client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to get get %s: %v", client.ObjectKeyFromObject(obj), err) - } - return nil -} - -func (c *OperatorConfigMapReconciler) deleteOlderCSIResources() error { - rbdDeployment := &appsv1.Deployment{} - rbdDeployment.Name = csi.RBDDeploymentName - rbdDeployment.Namespace = c.OperatorNamespace - // doing a get hits cache and reduces round trip to api server when trying - // to delete non existing resource in every reconcile - if err := c.getAndDeleteResource(rbdDeployment); err != nil { - return err - } - - rbdDaemonSet := &appsv1.DaemonSet{} - rbdDaemonSet.Name = csi.RBDDaemonSetName - rbdDaemonSet.Namespace = c.OperatorNamespace - if err := c.getAndDeleteResource(rbdDaemonSet); err != nil { - return err - } - - cephFsDeployment := &appsv1.Deployment{} - cephFsDeployment.Name = csi.CephFSDeploymentName - cephFsDeployment.Namespace = c.OperatorNamespace - if err := c.getAndDeleteResource(cephFsDeployment); err != nil { - return err - } - - cephFsDaemonSet := &appsv1.DaemonSet{} - cephFsDaemonSet.Name = csi.CephFSDaemonSetName - cephFsDaemonSet.Namespace = c.OperatorNamespace - if err := c.getAndDeleteResource(cephFsDaemonSet); err != nil { - return err - } - - return nil -} - -func (c *OperatorConfigMapReconciler) reconcileCSI() error { - - clusterVersion := &configv1.ClusterVersion{} - clusterVersion.Name = clusterVersionName - if err := c.get(clusterVersion); err != nil { - c.log.Error(err, "failed to get the clusterVersion version of the OCP cluster") - return err - } - - if err := csi.InitializeSidecars(c.log, clusterVersion.Status.Desired.Version); err != nil { - c.log.Error(err, "unable to initialize sidecars") - return err - } - - c.scc = &secv1.SecurityContextConstraints{ - ObjectMeta: metav1.ObjectMeta{ - Name: csi.SCCName, - }, - } - err := c.createOrUpdate(c.scc, func() error { - // TODO: this is a hack to preserve the resourceVersion of the SCC - resourceVersion := c.scc.ResourceVersion - csi.SetSecurityContextConstraintsDesiredState(c.scc, c.OperatorNamespace) - c.scc.ResourceVersion = resourceVersion - return nil - }) - if err != nil { - c.log.Error(err, "unable to create/update SCC") - return err - } - - // create the monitor configmap for the csi drivers but never updates it. - // This is because the monitor configurations are added to the configmap - // when user creates storageclaims. - monConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: templates.MonConfigMapName, - Namespace: c.OperatorNamespace, - }, - Data: map[string]string{ - "config.json": "[]", - }, - } - if err := c.own(monConfigMap); err != nil { - return err - } - - if err := c.create(monConfigMap); err != nil && !kerrors.IsAlreadyExists(err) { - c.log.Error(err, "failed to create monitor configmap", "name", monConfigMap.Name) - return err - } - - // create the encryption configmap for the csi driver but never updates it. - // This is because the encryption configuration are added to the configmap - // by the users before they create the encryption storageclaims. - encConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: templates.EncryptionConfigMapName, - Namespace: c.OperatorNamespace, - }, - Data: map[string]string{ - "config.json": "[]", - }, - } - if err := c.own(encConfigMap); err != nil { - return err - } - - if err := c.create(encConfigMap); err != nil && !kerrors.IsAlreadyExists(err) { - c.log.Error(err, "failed to create monitor configmap", "name", encConfigMap.Name) - return err - } - - c.cephFSDeployment = &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: csi.CephFSDeploymentName, - Namespace: c.OperatorNamespace, - }, - } - err = c.createOrUpdate(c.cephFSDeployment, func() error { - if err := c.own(c.cephFSDeployment); err != nil { - return err - } - csi.SetCephFSDeploymentDesiredState(c.cephFSDeployment) - return nil - }) - if err != nil { - c.log.Error(err, "failed to create/update cephfs deployment") - return err - } - - c.cephFSDaemonSet = &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: csi.CephFSDaemonSetName, - Namespace: c.OperatorNamespace, - }, - } - err = c.createOrUpdate(c.cephFSDaemonSet, func() error { - if err := c.own(c.cephFSDaemonSet); err != nil { - return err - } - csi.SetCephFSDaemonSetDesiredState(c.cephFSDaemonSet) - return nil - }) - if err != nil { - c.log.Error(err, "failed to create/update cephfs daemonset") - return err - } - - c.rbdDeployment = &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: csi.RBDDeploymentName, - Namespace: c.OperatorNamespace, - }, - } - err = c.createOrUpdate(c.rbdDeployment, func() error { - if err := c.own(c.rbdDeployment); err != nil { - return err - } - csi.SetRBDDeploymentDesiredState(c.rbdDeployment) - return nil - }) - if err != nil { - c.log.Error(err, "failed to create/update rbd deployment") - return err - } - - c.rbdDaemonSet = &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: csi.RBDDaemonSetName, - Namespace: c.OperatorNamespace, - }, - } - err = c.createOrUpdate(c.rbdDaemonSet, func() error { - if err := c.own(c.rbdDaemonSet); err != nil { - return err - } - csi.SetRBDDaemonSetDesiredState(c.rbdDaemonSet) - return nil - }) - if err != nil { - c.log.Error(err, "failed to create/update rbd daemonset") - return err - } - - // Need to handle deletion of the csiDriver object, we cannot set - // ownerReference on it as its cluster scoped resource - cephfsCSIDriver := templates.CephFSCSIDriver.DeepCopy() - cephfsCSIDriver.ObjectMeta.Name = csi.GetCephFSDriverName() - if err := csi.CreateCSIDriver(c.ctx, c.Client, cephfsCSIDriver); err != nil { - c.log.Error(err, "unable to create cephfs CSIDriver") - return err - } - - rbdCSIDriver := templates.RbdCSIDriver.DeepCopy() - rbdCSIDriver.ObjectMeta.Name = csi.GetRBDDriverName() - if err := csi.CreateCSIDriver(c.ctx, c.Client, rbdCSIDriver); err != nil { - c.log.Error(err, "unable to create rbd CSIDriver") - return err - } - return nil -} - func (c *OperatorConfigMapReconciler) deletionPhase() error { claimsList := &v1alpha1.StorageClaimList{} if err := c.list(claimsList, client.Limit(1)); err != nil { @@ -649,13 +408,7 @@ func (c *OperatorConfigMapReconciler) deletionPhase() error { return err } - var err error - if utils.DelegateCSI { - err = c.deleteDelegatedCSI() - } else { - err = c.deleteCSI() - } - if err != nil { + if err := c.deleteDelegatedCSI(); err != nil { return err } @@ -682,10 +435,6 @@ func (c *OperatorConfigMapReconciler) own(obj client.Object) error { return controllerutil.SetControllerReference(c.operatorConfigMap, obj, c.Client.Scheme()) } -func (c *OperatorConfigMapReconciler) create(obj client.Object) error { - return c.Client.Create(c.ctx, obj) -} - // applyLabels adds labels to object meta, overwriting keys that are already defined. func applyLabels(label string, t *metav1.ObjectMeta) { // Create a map to store the configuration @@ -774,42 +523,6 @@ func (c *OperatorConfigMapReconciler) ensureConsolePlugin() error { return nil } -func (c *OperatorConfigMapReconciler) getDeployCSIConfig() (bool, error) { - data := c.operatorConfigMap.Data - if data == nil { - data = map[string]string{} - } - - var deployCSI bool - var err error - if value, ok := data[deployCSIKey]; ok { - deployCSI, err = strconv.ParseBool(value) - if err != nil { - return false, fmt.Errorf("failed to parse value for %q in operator configmap as a boolean: %v", deployCSIKey, err) - } - } else { - // CSI installation is not specified explicitly in the configmap and - // behaviour is different in case we recognize the StorageCluster API on the cluster. - storageClusterCRD := &metav1.PartialObjectMetadata{} - storageClusterCRD.SetGroupVersionKind( - extv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"), - ) - storageClusterCRD.Name = "storageclusters.ocs.openshift.io" - if err = c.get(storageClusterCRD); err != nil { - if !kerrors.IsNotFound(err) { - return false, fmt.Errorf("failed to verify existence of storagecluster crd: %v", err) - } - // storagecluster CRD doesn't exist - deployCSI = true - } else { - // storagecluster CRD exists and don't deploy CSI until explicitly mentioned in the configmap - deployCSI = false - } - } - - return deployCSI, nil -} - func (c *OperatorConfigMapReconciler) getNoobaaSubManagementConfig() bool { valAsString, ok := c.operatorConfigMap.Data[manageNoobaaSubKey] if !ok { @@ -1069,22 +782,3 @@ func (c *OperatorConfigMapReconciler) deleteDelegatedCSI() error { } return nil } - -func (c *OperatorConfigMapReconciler) deleteCSI() error { - if err := csi.DeleteCSIDriver(c.ctx, c.Client, csi.GetCephFSDriverName()); err != nil && !kerrors.IsNotFound(err) { - c.log.Error(err, "unable to delete cephfs CSIDriver") - return err - } - if err := csi.DeleteCSIDriver(c.ctx, c.Client, csi.GetRBDDriverName()); err != nil && !kerrors.IsNotFound(err) { - c.log.Error(err, "unable to delete rbd CSIDriver") - return err - } - - c.scc = &secv1.SecurityContextConstraints{} - c.scc.Name = csi.SCCName - if err := c.delete(c.scc); err != nil { - c.log.Error(err, "unable to delete SCC") - return err - } - return nil -} diff --git a/internal/controller/sliceutils.go b/internal/controller/sliceutils.go deleted file mode 100644 index e2787fa6..00000000 --- a/internal/controller/sliceutils.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2020 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -//nolint:deadcode,unused -func contains(slice []string, s string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - -// Removes a given string from a slice and returns the new slice -// -//nolint:deadcode,unused -func remove(slice []string, s string) (result []string) { - result = []string{} - for _, item := range slice { - if item == s { - continue - } - result = append(result, item) - } - return -} diff --git a/internal/controller/sliceutils_test.go b/internal/controller/sliceutils_test.go deleted file mode 100644 index fc847d4e..00000000 --- a/internal/controller/sliceutils_test.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSliceContains(t *testing.T) { - - testCases := []struct { - label string - slice []string - findString string - isContained bool - }{ - { - label: "string exists in slice", - slice: []string{"foo", "bar"}, - findString: "bar", - isContained: true, - }, - { - label: "string not in slice", - slice: []string{"foo", "bar"}, - findString: "baz", - isContained: false, - }, - { - label: "empty string not in slice", - slice: []string{"foo", "bar"}, - findString: "", - isContained: false, - }, - { - label: "string not in empty slice", - slice: []string{}, - findString: "foo", - isContained: false, - }, - } - - for i, tc := range testCases { - t.Logf("Case %d: %s\n", i+1, tc.label) - checkContain := contains(tc.slice, tc.findString) - assert.Equal(t, tc.isContained, checkContain) - } -} - -func TestSliceRemove(t *testing.T) { - - testCases := []struct { - label string - slice []string - findString string - expectedSlice []string - }{ - { - label: "string exists in slice", - slice: []string{"foo", "bar"}, - findString: "foo", - expectedSlice: []string{"bar"}, - }, - { - label: "string not in slice", - slice: []string{"foo", "bar"}, - findString: "baz", - expectedSlice: []string{"foo", "bar"}, - }, - { - label: "empty string not in slice", - slice: []string{"foo", "bar"}, - findString: "", - expectedSlice: []string{"foo", "bar"}, - }, - { - label: "string not in empty slice", - slice: []string{}, - findString: "foo", - expectedSlice: []string{}, - }, - } - - for i, tc := range testCases { - t.Logf("Case %d: %s\n", i+1, tc.label) - changedSlice := remove(tc.slice, tc.findString) - assert.Equal(t, tc.expectedSlice, changedSlice) - } -} diff --git a/internal/controller/storageclaim_controller.go b/internal/controller/storageclaim_controller.go index 33d0bdda..6c3d8e47 100644 --- a/internal/controller/storageclaim_controller.go +++ b/internal/controller/storageclaim_controller.go @@ -21,13 +21,12 @@ import ( "encoding/json" "fmt" "reflect" - "sigs.k8s.io/controller-runtime/pkg/handler" "slices" "strings" "time" v1alpha1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" - "github.com/red-hat-storage/ocs-client-operator/pkg/csi" + "github.com/red-hat-storage/ocs-client-operator/pkg/templates" "github.com/red-hat-storage/ocs-client-operator/pkg/utils" csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" @@ -47,6 +46,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -81,7 +81,7 @@ type StorageClaimReconciler struct { // SetupWithManager sets up the controller with the Manager. func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { ctx := context.Background() - csiDrivers := []string{csi.GetRBDDriverName(), csi.GetCephFSDriverName()} + csiDrivers := []string{templates.RBDDriverName, templates.CephFsDriverName} if err := mgr.GetCache().IndexField(ctx, &corev1.PersistentVolume{}, pvClusterIDIndexName, func(o client.Object) []string { pv := o.(*corev1.PersistentVolume) if pv != nil && @@ -125,11 +125,8 @@ func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { utils.CrdCreateAndDeletePredicate(&r.log, drClusterConfigCRDName, r.AvailableCrds[drClusterConfigCRDName]), ), builder.OnlyMetadata, - ) - - if utils.DelegateCSI { - bldr = bldr.Owns(&csiopv1a1.ClientProfile{}, builder.WithPredicates(generationChangePredicate)) - } + ). + Owns(&csiopv1a1.ClientProfile{}, builder.WithPredicates(generationChangePredicate)) if r.AvailableCrds[drClusterConfigCRDName] { bldr = bldr.Owns(&ramenv1alpha1.DRClusterConfig{}, builder.WithPredicates(generationChangePredicate)) @@ -257,12 +254,6 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { // Close client-side connections. defer providerClient.Close() - cc := csi.ClusterConfig{ - Client: r.Client, - Namespace: r.OperatorNamespace, - Ctx: r.ctx, - } - if r.storageClaim.GetDeletionTimestamp().IsZero() { // TODO: Phases do not have checks at the moment, in order to make them more predictable and less error-prone, at the expense of increased computation cost. @@ -346,20 +337,6 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { return reconcile.Result{}, fmt.Errorf("no configuration data received") } - var csiClusterConfigEntry = new(csi.ClusterConfigEntry) - scResponse, err := providerClient.GetStorageConfig(r.ctx, r.storageClient.Status.ConsumerID) - if err != nil { - return reconcile.Result{}, fmt.Errorf("failed to get StorageConfig: %v", err) - } - for _, eResource := range scResponse.ExternalResource { - if eResource.Kind == "ConfigMap" && eResource.Name == "rook-ceph-mon-endpoints" { - monitorIps, err := csi.ExtractMonitor(eResource.Data) - if err != nil { - return reconcile.Result{}, fmt.Errorf("failed to extract monitor data: %v", err) - } - csiClusterConfigEntry.Monitors = append(csiClusterConfigEntry.Monitors, monitorIps...) - } - } // Go over the received objects and operate on them accordingly. for _, resource := range resources { @@ -397,21 +374,12 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { if err != nil { return reconcile.Result{}, fmt.Errorf("failed to unmarshal StorageClaim configuration response: %v", err) } - if rns, ok := data["radosnamespace"]; ok { - csiClusterConfigEntry.CephRBD = new(csi.CephRBDSpec) - csiClusterConfigEntry.CephRBD.RadosNamespace = rns - delete(data, "radosnamespace") - } + // we are now using clientprofile from csi-operator for getting this info. + // until provider stops sending this info we'll just need to drop the field + // we'll make changes to provider at some version when all clients are dropping this field + delete(data, "radosnamespace") + delete(data, "subvolumegroupname") - // The clusterID is an opaque value used by the CSI driver - // to identify the cluster config (e.g. mon IPs) to use - // for a volume from a given StorageClass. We set it to - // the claim name for ease of identification. - // - // NOTE: This is distinct from the notion of a "clusterID" - // used within Ceph and Rook-Ceph, despite sharing the - // same name. - csiClusterConfigEntry.ClusterID = r.storageClaimHash var storageClass *storagev1.StorageClass data["csi.storage.k8s.io/provisioner-secret-namespace"] = r.OperatorNamespace data["csi.storage.k8s.io/node-stage-secret-namespace"] = r.OperatorNamespace @@ -419,10 +387,6 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { data["clusterID"] = r.storageClaimHash if resource.Name == "cephfs" { - csiClusterConfigEntry.CephFS = new(csi.CephFSSpec) - csiClusterConfigEntry.CephFS.SubvolumeGroup = data["subvolumegroupname"] - // delete groupname from data as its not required in storageclass - delete(data, "subvolumegroupname") storageClass = r.getCephFSStorageClass(data) } else if resource.Name == "ceph-rbd" { storageClass = r.getCephRBDStorageClass(data) @@ -454,35 +418,26 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { return reconcile.Result{}, fmt.Errorf("failed to create or update VolumeSnapshotClass: %s", err) } case "ClientProfile": - if utils.DelegateCSI { - clientProfile := &csiopv1a1.ClientProfile{} - clientProfile.Name = r.storageClaimHash - clientProfile.Namespace = r.OperatorNamespace - if _, err := controllerutil.CreateOrUpdate(r.ctx, r.Client, clientProfile, func() error { - if err := r.own(clientProfile); err != nil { - return fmt.Errorf("failed to own clientProfile resource: %v", err) - } - if err := json.Unmarshal(resource.Data, &clientProfile.Spec); err != nil { - return fmt.Errorf("failed to unmarshall clientProfile spec: %v", err) - } - clientProfile.Spec.CephConnectionRef = corev1.LocalObjectReference{ - Name: r.storageClient.Name, - } - return nil - }); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to reconcile clientProfile: %v", err) + clientProfile := &csiopv1a1.ClientProfile{} + clientProfile.Name = r.storageClaimHash + clientProfile.Namespace = r.OperatorNamespace + if _, err := controllerutil.CreateOrUpdate(r.ctx, r.Client, clientProfile, func() error { + if err := r.own(clientProfile); err != nil { + return fmt.Errorf("failed to own clientProfile resource: %v", err) } + if err := json.Unmarshal(resource.Data, &clientProfile.Spec); err != nil { + return fmt.Errorf("failed to unmarshall clientProfile spec: %v", err) + } + clientProfile.Spec.CephConnectionRef = corev1.LocalObjectReference{ + Name: r.storageClient.Name, + } + return nil + }); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to reconcile clientProfile: %v", err) } } } - // update monitor configuration for cephcsi - if !utils.DelegateCSI { - err = cc.UpdateMonConfigMap(csiClusterConfigEntry.ClusterID, r.storageClient.Status.ConsumerID, csiClusterConfigEntry) - if err != nil { - return reconcile.Result{}, fmt.Errorf("failed to update mon configmap: %v", err) - } - } // Readiness phase. // Update the StorageClaim status. r.storageClaim.Status.Phase = v1alpha1.StorageClaimReady @@ -505,14 +460,6 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { return reconcile.Result{}, fmt.Errorf("one or more volumesnapshotcontents exist that are dependent on storageclaim %s", r.storageClaim.Name) } - // Delete configmap entry for cephcsi - if !utils.DelegateCSI { - err = cc.UpdateMonConfigMap(r.storageClaimHash, r.storageClient.Status.ConsumerID, nil) - if err != nil { - return reconcile.Result{}, fmt.Errorf("failed to update mon configmap: %v", err) - } - } - // Call `RevokeStorageClaim` service on the provider server with StorageClaim as a request message. // Check if StorageClaim is still exists (it might have been manually removed during the StorageClass // removal above). @@ -547,7 +494,7 @@ func (r *StorageClaimReconciler) getCephFSStorageClass(data map[string]string) * }, ReclaimPolicy: &pvReclaimPolicy, AllowVolumeExpansion: &allowVolumeExpansion, - Provisioner: csi.GetCephFSDriverName(), + Provisioner: templates.CephFsDriverName, Parameters: data, } return storageClass @@ -566,7 +513,7 @@ func (r *StorageClaimReconciler) getCephRBDStorageClass(data map[string]string) }, ReclaimPolicy: &pvReclaimPolicy, AllowVolumeExpansion: &allowVolumeExpansion, - Provisioner: csi.GetRBDDriverName(), + Provisioner: templates.RBDDriverName, Parameters: data, } @@ -581,7 +528,7 @@ func (r *StorageClaimReconciler) getCephFSVolumeSnapshotClass(data map[string]st ObjectMeta: metav1.ObjectMeta{ Name: r.storageClaim.Name, }, - Driver: csi.GetCephFSDriverName(), + Driver: templates.CephFsDriverName, DeletionPolicy: snapapi.VolumeSnapshotContentDelete, Parameters: data, } @@ -593,7 +540,7 @@ func (r *StorageClaimReconciler) getCephRBDVolumeSnapshotClass(data map[string]s ObjectMeta: metav1.ObjectMeta{ Name: r.storageClaim.Name, }, - Driver: csi.GetRBDDriverName(), + Driver: templates.RBDDriverName, DeletionPolicy: snapapi.VolumeSnapshotContentDelete, Parameters: data, } diff --git a/internal/controller/storageclient_controller.go b/internal/controller/storageclient_controller.go index 6497f6d1..74d1c92a 100644 --- a/internal/controller/storageclient_controller.go +++ b/internal/controller/storageclient_controller.go @@ -102,11 +102,8 @@ func (r *StorageClientReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&batchv1.CronJob{}). Owns("av1.ClusterResourceQuota{}, builder.WithPredicates(generationChangePredicate)). Owns(&nbv1.NooBaa{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). - Owns(&corev1.Secret{}) - - if utils.DelegateCSI { - bldr = bldr.Owns(&csiopv1a1.CephConnection{}, builder.WithPredicates(generationChangePredicate)) - } + Owns(&corev1.Secret{}). + Owns(&csiopv1a1.CephConnection{}, builder.WithPredicates(generationChangePredicate)) return bldr.Complete(r) } @@ -224,21 +221,19 @@ func (r *StorageClientReconciler) reconcilePhases() (ctrl.Result, error) { return reconcile.Result{}, err } case "CephConnection": - if utils.DelegateCSI { - cephConnection := &csiopv1a1.CephConnection{} - cephConnection.Name = r.storageClient.Name - cephConnection.Namespace = r.OperatorNamespace - if err := r.createOrUpdate(cephConnection, func() error { - if err := r.own(cephConnection); err != nil { - return fmt.Errorf("failed to own cephConnection resource: %v", err) - } - if err := json.Unmarshal(eResource.Data, &cephConnection.Spec); err != nil { - return fmt.Errorf("failed to unmarshall cephConnectionSpec: %v", err) - } - return nil - }); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to reconcile cephConnection: %v", err) + cephConnection := &csiopv1a1.CephConnection{} + cephConnection.Name = r.storageClient.Name + cephConnection.Namespace = r.OperatorNamespace + if err := r.createOrUpdate(cephConnection, func() error { + if err := r.own(cephConnection); err != nil { + return fmt.Errorf("failed to own cephConnection resource: %v", err) + } + if err := json.Unmarshal(eResource.Data, &cephConnection.Spec); err != nil { + return fmt.Errorf("failed to unmarshall cephConnectionSpec: %v", err) } + return nil + }); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to reconcile cephConnection: %v", err) } case "Secret": data := map[string]string{} @@ -591,10 +586,6 @@ func (r *StorageClientReconciler) reconcileClientStatusReporterJob() (reconcile. Name: utils.OperatorNamespaceEnvVar, Value: r.OperatorNamespace, }, - { - Name: utils.CSIReconcileEnvVar, - Value: os.Getenv(utils.CSIReconcileEnvVar), - }, }, }, }, diff --git a/pkg/csi/cephfsdaemonset.go b/pkg/csi/cephfsdaemonset.go deleted file mode 100644 index 89d90c92..00000000 --- a/pkg/csi/cephfsdaemonset.go +++ /dev/null @@ -1,279 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "fmt" - - "github.com/red-hat-storage/ocs-client-operator/pkg/templates" - "github.com/red-hat-storage/ocs-client-operator/pkg/utils" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" -) - -const ( - CephFSDaemonSetName = "csi-cephfsplugin" - - cephFSDaemonSetContainerName = "csi-cephfsplugin" -) - -var ( - cephfsDaemonsetLabels = map[string]string{ - "app": "csi-cephfsplugin", - } - - biDirectionalMount = corev1.MountPropagationBidirectional - hostPathDirectoryorCreate = corev1.HostPathDirectoryOrCreate - hostPathDirectory = corev1.HostPathDirectory -) - -var cephFSDaemonSetSpec = appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: cephfsDaemonsetLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: cephfsDaemonsetLabels, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: cephFSPluginServiceAccountName, - HostNetwork: true, - PriorityClassName: "system-node-critical", - Containers: []corev1.Container{ - {Name: templates.DriverRegistrar.Name}, - { - Name: cephFSDaemonSetContainerName, - ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - AllowPrivilegeEscalation: ptr.To(true), - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_ADMIN", - }, - }, - }, - Args: []string{ - "--nodeid=$(NODE_ID)", - "--endpoint=$(CSI_ENDPOINT)", - "--v=5", - "--pidlimit=-1", - "--type=cephfs", - "--nodeserver=true", - fmt.Sprintf("--drivername=%s", GetCephFSDriverName()), - }, - Env: []corev1.EnvVar{ - { - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "CSI_ENDPOINT", - Value: templates.DefaultPluginSocketPath, - }, - { - Name: "NODE_ID", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: templates.DefaultSocketDir, - }, - { - Name: "host-dev", - MountPath: "/dev", - }, - { - Name: "host-sys", - MountPath: "/sys", - }, - { - Name: "lib-modules", - MountPath: "/lib/modules", - ReadOnly: true, - }, - { - Name: "ceph-csi-configs", - MountPath: "/etc/ceph-csi-config", - }, - { - Name: "keys-tmp-dir", - MountPath: "/tmp/csi/keys", - }, - { - Name: "host-run-mount", - MountPath: "/run/mount", - }, - { - Name: "csi-plugins-dir", - MountPath: fmt.Sprintf("%s/plugins/", templates.DefaultKubeletDirPath), - MountPropagation: &biDirectionalMount, - }, - { - Name: "pods-mount-dir", - MountPath: fmt.Sprintf("%s/pods", templates.DefaultKubeletDirPath), - MountPropagation: &biDirectionalMount, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "host-dev", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - { - Name: "host-sys", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, - { - Name: "lib-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }, - { - Name: "host-run-mount", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/mount", - }, - }, - }, - { - Name: "keys-tmp-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "ceph-csi-configs", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: templates.MonConfigMapName, - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/plugins/%s", templates.DefaultKubeletDirPath, GetCephFSDriverName()), - Type: &hostPathDirectoryorCreate, - }, - }, - }, - { - Name: "csi-plugins-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/plugins/", templates.DefaultKubeletDirPath), - Type: &hostPathDirectory, - }, - }, - }, - { - Name: "registration-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/plugins_registry/", templates.DefaultKubeletDirPath), - Type: &hostPathDirectory, - }, - }, - }, - { - Name: "pods-mount-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/pods", templates.DefaultKubeletDirPath), - Type: &hostPathDirectory, - }, - }, - }, - }, - Tolerations: []corev1.Toleration{ - utils.GetTolerationForCSIPods(), - }, - }, - }, -} - -func SetCephFSDaemonSetDesiredState(ds *appsv1.DaemonSet) { - // Copy required labels - utils.AddLabels(ds, cephfsDaemonsetLabels) - - // Update the demaon set with desired spec - cephFSDaemonSetSpec.DeepCopyInto(&ds.Spec) - - // Update containers spec with desired state - for i := range ds.Spec.Template.Spec.Containers { - c := &ds.Spec.Template.Spec.Containers[i] - switch c.Name { - case templates.DriverRegistrar.Name: - templates.DriverRegistrar.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.DriverRegistrarImageURL - c.Args = append(c.Args, fmt.Sprintf( - "--kubelet-registration-path=%s/plugins/%s/csi.sock", - templates.DefaultKubeletDirPath, - GetCephFSDriverName(), - )) - - case cephFSDaemonSetContainerName: - c.Image = sidecarImages.ContainerImages.CephCSIImageURL - } - } -} diff --git a/pkg/csi/cephfsdeployment.go b/pkg/csi/cephfsdeployment.go deleted file mode 100644 index ae7d0a01..00000000 --- a/pkg/csi/cephfsdeployment.go +++ /dev/null @@ -1,221 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "fmt" - - "github.com/red-hat-storage/ocs-client-operator/pkg/templates" - "github.com/red-hat-storage/ocs-client-operator/pkg/utils" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" -) - -const ( - CephFSDeploymentName = "csi-cephfsplugin-provisioner" - - cephFSDeploymentContainerName = "csi-cephfsplugin" -) - -var cephfsDeploymentLabels = map[string]string{ - "app": "csi-cephfsplugin-provisioner", -} - -var cephFSDeploymentSpec = appsv1.DeploymentSpec{ - Replicas: ptr.To(int32(2)), - Selector: &metav1.LabelSelector{ - MatchLabels: cephfsDeploymentLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: cephfsDeploymentLabels, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: cephFSProvisionerServiceAccountName, - Containers: []corev1.Container{ - {Name: templates.ProvisionerContainer.Name}, - {Name: templates.AttacherContainer.Name}, - {Name: templates.ResizerContainer.Name}, - {Name: templates.SnapshotterContainer.Name}, - { - Name: cephFSDeploymentContainerName, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--nodeid=$(NODE_ID)", - "--endpoint=$(CSI_ENDPOINT)", - "--v=5", - "--pidlimit=-1", - "--type=cephfs", - "--controllerserver=true", - fmt.Sprintf("--drivername=%s", GetCephFSDriverName()), - }, - Env: []corev1.EnvVar{ - { - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "CSI_ENDPOINT", - Value: templates.DefaultProvisionerSocketPath, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: templates.DefaultSocketDir, - }, - { - Name: "host-dev", - MountPath: "/dev", - }, - { - Name: "host-sys", - MountPath: "/sys", - }, - { - Name: "lib-modules", - MountPath: "/lib/modules", - ReadOnly: true, - }, - { - Name: "ceph-csi-configs", - MountPath: "/etc/ceph-csi-config", - }, - { - Name: "keys-tmp-dir", - MountPath: "/tmp/csi/keys", - }, - }, - }, - }, - PriorityClassName: "system-cluster-critical", - Volumes: []corev1.Volume{ - { - Name: "host-dev", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - { - Name: "host-sys", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, - { - Name: "lib-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }, - { - Name: "socket-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - - { - Name: "keys-tmp-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "ceph-csi-configs", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: templates.MonConfigMapName, - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - }, - }, - }, -} - -func SetCephFSDeploymentDesiredState(deploy *appsv1.Deployment) { - // Copy required labels - utils.AddLabels(deploy, cephfsDeploymentLabels) - - // Update the deployment set with desired spec - cephFSDeploymentSpec.DeepCopyInto(&deploy.Spec) - - // Find and Update placeholder containers with desired state - leaderElectionArg := fmt.Sprintf("--leader-election-namespace=%s", deploy.Namespace) - for i := range deploy.Spec.Template.Spec.Containers { - c := &deploy.Spec.Template.Spec.Containers[i] - - switch c.Name { - case templates.ProvisionerContainer.Name: - templates.ProvisionerContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.ProvisionerImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.AttacherContainer.Name: - templates.AttacherContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.AttacherImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.ResizerContainer.Name: - templates.ResizerContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.ResizerImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.SnapshotterContainer.Name: - templates.SnapshotterContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.SnapshotterImageURL - c.Args = append(c.Args, leaderElectionArg) - - case cephFSDeploymentContainerName: - c.Image = sidecarImages.ContainerImages.CephCSIImageURL - } - } - -} diff --git a/pkg/csi/csi.go b/pkg/csi/csi.go deleted file mode 100644 index b31240fd..00000000 --- a/pkg/csi/csi.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "fmt" - "os" - - "github.com/go-logr/logr" - "gopkg.in/yaml.v2" - "k8s.io/apimachinery/pkg/util/version" -) - -const ( - sidecarsConfigPath = "/opt/config/csi-images.yaml" -) - -type containerImages struct { - ProvisionerImageURL string `yaml:"provisionerImageURL"` - AttacherImageURL string `yaml:"attacherImageURL"` - ResizerImageURL string `yaml:"resizerImageURL"` - SnapshotterImageURL string `yaml:"snapshotterImageURL"` - DriverRegistrarImageURL string `yaml:"driverRegistrarImageURL"` - CephCSIImageURL string `yaml:"cephCSIImageURL"` - CSIADDONSImageURL string `yaml:"csiaddonsImageURL"` -} - -type SidecarImages struct { - Version string `yaml:"version"` - ContainerImages containerImages `yaml:"containerImages"` -} - -var sidecarImages *SidecarImages - -func InitializeSidecars(log logr.Logger, ver string) error { - // ready yaml files and yaml unmarshal to SidecarImages - // and set to csiSidecarImages - si := []SidecarImages{} - yamlFile, err := os.ReadFile(sidecarsConfigPath) - if err != nil { - return err - } - err = yaml.Unmarshal(yamlFile, &si) - if err != nil { - return err - } - - pltVersion := version.MustParseGeneric(ver) - - closestMinor := int64(-1) - for idx := range si { - siVersion := version.MustParseGeneric(si[idx].Version) - log.Info("searching for the most compatible CSI image version", "CSI", siVersion, "Platform", pltVersion) - - // only check sidecar image versions that are not higher than platform - if siVersion.Major() == pltVersion.Major() && siVersion.Minor() <= pltVersion.Minor() { - // filter sidecar closest to platform version - if int64(siVersion.Minor()) > closestMinor { - sidecarImages = &si[idx] - closestMinor = int64(siVersion.Minor()) - } - if closestMinor == int64(pltVersion.Minor()) { // exact match and early exit - break - } - } else { - log.Info("skipping sidecar images: version greater than platform version") - } - } - if sidecarImages == nil { - // happens only if all sidecars image versions are greater than platform - return fmt.Errorf("failed to find container details suitable for %v platform version", pltVersion) - } - - log.Info("selected sidecar images", "version", sidecarImages.Version) - - return nil -} - -// GetCephFSDriverName returns the cephfs driver name -func GetCephFSDriverName() string { - return "openshift-storage.cephfs.csi.ceph.com" -} - -// GetRBDDriverName returns the rbd driver name -func GetRBDDriverName() string { - return "openshift-storage.rbd.csi.ceph.com" -} diff --git a/pkg/csi/csidriver.go b/pkg/csi/csidriver.go deleted file mode 100644 index dad6f59e..00000000 --- a/pkg/csi/csidriver.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "context" - "fmt" - "reflect" - - v1k8scsi "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func CreateCSIDriver(ctx context.Context, client client.Client, csiDriver *v1k8scsi.CSIDriver) error { - actualDriver := &v1k8scsi.CSIDriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: csiDriver.Name, - }, - } - needCreation := false - err := client.Get(ctx, types.NamespacedName{Name: csiDriver.Name}, actualDriver) - if err == nil { - // check if the spec is the same for the existing object and the new one - if !reflect.DeepEqual(csiDriver.Spec, actualDriver.Spec) { - needCreation = true - err = client.Delete(ctx, actualDriver) - if err != nil { - return fmt.Errorf("error deleting CSIDriver %s: %v", csiDriver.Name, err) - } - } - } - - if errors.IsNotFound(err) || needCreation { - err = client.Create(ctx, csiDriver) - } - - return err -} - -func DeleteCSIDriver(ctx context.Context, client client.Client, name string) error { - csiDriver := &v1k8scsi.CSIDriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } - err := client.Delete(ctx, csiDriver) - if err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("error deleting CSIDriver %s: %v", csiDriver.Name, err) - } - - return nil -} diff --git a/pkg/csi/monconfigmap.go b/pkg/csi/monconfigmap.go deleted file mode 100644 index 771d58e8..00000000 --- a/pkg/csi/monconfigmap.go +++ /dev/null @@ -1,194 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package csi - -import ( - "context" - "encoding/json" - "sync" - - "github.com/pkg/errors" - "github.com/red-hat-storage/ocs-client-operator/pkg/templates" - "github.com/red-hat-storage/ocs-client-operator/pkg/utils" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - // configMutex is used to prevent the config map from being updated - // for multiple clusters simultaneously. - configMutex = &sync.Mutex{} -) - -type ClusterConfigEntry struct { - ClusterID string `json:"clusterID"` - StorageClientID string `json:"storageClientID"` - Monitors []string `json:"monitors"` - CephFS *CephFSSpec `json:"cephFS,omitempty"` - CephRBD *CephRBDSpec `json:"rbd,omitempty"` -} - -type CephRBDSpec struct { - RadosNamespace string `json:"radosNamespace,omitempty"` -} - -type CephFSSpec struct { - SubvolumeGroup string `json:"subvolumeGroup,omitempty"` -} - -type ClusterConfig struct { - client.Client - Ctx context.Context - Namespace string -} - -type csiClusterConfig []ClusterConfigEntry - -func parseCsiClusterConfig(c string) (csiClusterConfig, error) { - var cc csiClusterConfig - err := json.Unmarshal([]byte(c), &cc) - if err != nil { - return cc, errors.Wrap(err, "failed to parse csi cluster config") - } - return cc, nil -} - -func formatCsiClusterConfig(cc csiClusterConfig) (string, error) { - ccJSON, err := json.Marshal(cc) - if err != nil { - return "", errors.Wrap(err, "failed to marshal csi cluster config") - } - return string(ccJSON), nil -} - -// updateCsiClusterConfig returns a json-formatted string containing -// the cluster-to-mon mapping required to configure ceph csi. -func updateCsiClusterConfig(curr, clusterKey, storageClientID string, newClusterConfigEntry *ClusterConfigEntry) (string, error) { - var ( - cc csiClusterConfig - centry ClusterConfigEntry - found bool - ) - - cc, err := parseCsiClusterConfig(curr) - if err != nil { - return "", errors.Wrap(err, "failed to parse current csi cluster config") - } - - // Regardless of which controllers call updateCsiClusterConfig(), the values will be preserved since - // a lock is acquired for the update operation. So concurrent updates (rare event) will block and - // wait for the other update to complete. Monitors and Subvolumegroup will be updated - // independently and won't collide. - if newClusterConfigEntry != nil { - for i, centry := range cc { - // If the clusterID belongs to the same cluster, update the entry. - if storageClientID == cc[i].StorageClientID || clusterKey == newClusterConfigEntry.ClusterID { - centry.Monitors = newClusterConfigEntry.Monitors - centry.StorageClientID = storageClientID - cc[i] = centry - } - } - } - for i, centry := range cc { - if centry.ClusterID == clusterKey { - // If the new entry is nil, this means the entry is being deleted so remove it from the list - if newClusterConfigEntry == nil { - cc = append(cc[:i], cc[i+1:]...) - found = true - break - } - centry.StorageClientID = storageClientID - centry.Monitors = newClusterConfigEntry.Monitors - if newClusterConfigEntry.CephFS != nil && (newClusterConfigEntry.CephFS.SubvolumeGroup != "") { - centry.CephFS = newClusterConfigEntry.CephFS - } - if newClusterConfigEntry.CephRBD != nil && (newClusterConfigEntry.CephRBD.RadosNamespace != "") { - centry.CephRBD = newClusterConfigEntry.CephRBD - } - found = true - cc[i] = centry - break - } - } - if !found { - // If it's the first time we create the cluster, the entry does not exist, so the removal - // will fail with a dangling pointer - if newClusterConfigEntry != nil && clusterKey != "" { - centry.ClusterID = clusterKey - centry.StorageClientID = storageClientID - centry.Monitors = newClusterConfigEntry.Monitors - // Add a condition not to fill with empty values - if newClusterConfigEntry.CephFS != nil && (newClusterConfigEntry.CephFS.SubvolumeGroup != "") { - centry.CephFS = newClusterConfigEntry.CephFS - } - if newClusterConfigEntry.CephRBD != nil && (newClusterConfigEntry.CephRBD.RadosNamespace != "") { - centry.CephRBD = newClusterConfigEntry.CephRBD - } - cc = append(cc, centry) - } - } - - return formatCsiClusterConfig(cc) -} - -// UpdateMonConfigMap updates the config map used to provide ceph-csi with -// basic cluster configuration. The clusterID and storageClientID are -// used to determine what "cluster" in the config map will be updated and -// the clusterID value is expected to match the clusterID -// value that is provided to ceph-csi uses in the storage class. -// The locker configMutex is typically a mutex and is used to prevent the config -// map from being updated for multiple clusters simultaneously. -func (c *ClusterConfig) UpdateMonConfigMap(clusterID, storageClientID string, newClusterConfigEntry *ClusterConfigEntry) error { - ConfigKey := "config.json" - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: templates.MonConfigMapName, - Namespace: c.Namespace, - }, - Data: map[string]string{ - ConfigKey: "[]", - }, - } - - configMutex.Lock() - defer configMutex.Unlock() - - // fetch current ConfigMap contents - err := c.Get(c.Ctx, types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace}, configMap) - if err != nil { - return errors.Wrap(err, "failed to fetch current csi config map") - } - - // update ConfigMap contents for current cluster - currData := configMap.Data[ConfigKey] - newData, err := updateCsiClusterConfig(currData, clusterID, storageClientID, newClusterConfigEntry) - if err != nil { - return errors.Wrap(err, "failed to update csi config map data") - } - configMap.Data[ConfigKey] = newData - - err = c.Update(c.Ctx, configMap) - if err != nil { - return errors.Wrapf(err, "failed to update monitor configmap %q", configMap.Name) - } - - return nil -} - -var ExtractMonitor = utils.ExtractMonitor diff --git a/pkg/csi/rbddaemonset.go b/pkg/csi/rbddaemonset.go deleted file mode 100644 index 258bb043..00000000 --- a/pkg/csi/rbddaemonset.go +++ /dev/null @@ -1,348 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "fmt" - - "github.com/red-hat-storage/ocs-client-operator/pkg/templates" - "github.com/red-hat-storage/ocs-client-operator/pkg/utils" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" -) - -var ( - rbdDaemonsetLabels = map[string]string{ - "app": "csi-rbdplugin", - } -) - -const ( - RBDDaemonSetName = "csi-rbdplugin" - - rbdDaemonSetContainerName = "csi-rbdplugin" -) - -var rbdDaemonSetSpec = appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: rbdDaemonsetLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: rbdDaemonsetLabels, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: rbdPluginServiceAccountName, - HostNetwork: true, - HostPID: true, - PriorityClassName: "system-node-critical", - Containers: []corev1.Container{ - {Name: templates.DriverRegistrar.Name}, - {Name: templates.CSIAddonsContainer.Name}, - { - Name: rbdDaemonSetContainerName, - ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - AllowPrivilegeEscalation: ptr.To(true), - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_ADMIN", - }, - }, - }, - Args: []string{ - "--nodeid=$(NODE_ID)", - "--endpoint=$(CSI_ENDPOINT)", - "--v=5", - "--pidlimit=-1", - "--type=rbd", - "--nodeserver=true", - fmt.Sprintf("--drivername=%s", GetRBDDriverName()), - fmt.Sprintf("--stagingpath=%s/plugins/kubernetes.io/csi/", templates.DefaultKubeletDirPath), - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)", - }, - Env: []corev1.EnvVar{ - { - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "CSI_ENDPOINT", - Value: templates.DefaultPluginSocketPath, - }, - { - Name: "NODE_ID", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "CSIADDONS_ENDPOINT", - Value: "unix:///csi/csi-addons.sock", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: templates.DefaultSocketDir, - }, - { - Name: "host-dev", - MountPath: "/dev", - }, - { - Name: "host-sys", - MountPath: "/sys", - }, - { - Name: "lib-modules", - MountPath: "/lib/modules", - ReadOnly: true, - }, - { - Name: "ceph-csi-configs", - MountPath: "/etc/ceph-csi-config", - }, - { - Name: "keys-tmp-dir", - MountPath: "/tmp/csi/keys", - }, - { - Name: "host-run-mount", - MountPath: "/run/mount", - }, - { - Name: "csi-plugins-dir", - MountPath: fmt.Sprintf("%s/plugins/", templates.DefaultKubeletDirPath), - MountPropagation: &biDirectionalMount, - }, - { - Name: "pods-mount-dir", - MountPath: fmt.Sprintf("%s/pods", templates.DefaultKubeletDirPath), - MountPropagation: &biDirectionalMount, - }, - { - Name: "ceph-csi-kms-config", - MountPath: "/etc/ceph-csi-encryption-kms-config/", - ReadOnly: true, - }, - { - Name: "oidc-token", - MountPath: "/run/secrets/tokens", - ReadOnly: true, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "host-dev", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - { - Name: "host-sys", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, - { - Name: "lib-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }, - { - Name: "host-run-mount", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/mount", - }, - }, - }, - { - Name: "keys-tmp-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "ceph-csi-configs", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: templates.MonConfigMapName, - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/plugins/%s", templates.DefaultKubeletDirPath, GetRBDDriverName()), - Type: &hostPathDirectoryorCreate, - }, - }, - }, - { - Name: "csi-plugins-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/plugins/", templates.DefaultKubeletDirPath), - Type: &hostPathDirectory, - }, - }, - }, - { - Name: "registration-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/plugins_registry/", templates.DefaultKubeletDirPath), - Type: &hostPathDirectory, - }, - }, - }, - { - Name: "pods-mount-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: fmt.Sprintf("%s/pods", templates.DefaultKubeletDirPath), - Type: &hostPathDirectory, - }, - }, - }, - { - Name: "ceph-csi-kms-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: templates.EncryptionConfigMapName, - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "oidc-token", VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Path: "oidc-token", - ExpirationSeconds: ptr.To(int64(3600)), - Audience: "ceph-csi-kms", - }, - }, - }, - }, - }, - }, - }, - Tolerations: []corev1.Toleration{ - utils.GetTolerationForCSIPods(), - }, - }, - }, -} - -func SetRBDDaemonSetDesiredState(ds *appsv1.DaemonSet) { - // Copy required labels - utils.AddLabels(ds, rbdDaemonsetLabels) - - // Update the demaon set with desired state - rbdDaemonSetSpec.DeepCopyInto(&ds.Spec) - - // Update containers spec with desired state - for i := range ds.Spec.Template.Spec.Containers { - c := &ds.Spec.Template.Spec.Containers[i] - switch c.Name { - case templates.DriverRegistrar.Name: - templates.DriverRegistrar.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.DriverRegistrarImageURL - c.Args = append(c.Args, fmt.Sprintf( - "--kubelet-registration-path=%s/plugins/%s/csi.sock", - templates.DefaultKubeletDirPath, - GetRBDDriverName(), - )) - - case templates.CSIAddonsContainer.Name: - templates.CSIAddonsContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.CSIADDONSImageURL - vol := utils.Find(c.VolumeMounts, func(vol *corev1.VolumeMount) bool { - return vol.Name == "socket-dir" - }) - vol.Name = "plugin-dir" - - case rbdDaemonSetContainerName: - c.Image = sidecarImages.ContainerImages.CephCSIImageURL - } - } -} diff --git a/pkg/csi/rbddeployment.go b/pkg/csi/rbddeployment.go deleted file mode 100644 index 68bdcdf3..00000000 --- a/pkg/csi/rbddeployment.go +++ /dev/null @@ -1,264 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "fmt" - - "github.com/red-hat-storage/ocs-client-operator/pkg/templates" - "github.com/red-hat-storage/ocs-client-operator/pkg/utils" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" -) - -var rbdDeploymentLabels = map[string]string{ - "app": "csi-rbdplugin-provisioner", -} - -const ( - RBDDeploymentName = "csi-rbdplugin-provisioner" - - rbdDeploymentContainerName = "csi-rbdplugin" -) - -var rbdDeploymentSpec = appsv1.DeploymentSpec{ - Replicas: ptr.To(int32(2)), - Selector: &metav1.LabelSelector{ - MatchLabels: rbdDeploymentLabels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: rbdDeploymentLabels, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: rbdProvisionerServiceAccountName, - Containers: []corev1.Container{ - {Name: templates.ProvisionerContainer.Name}, - {Name: templates.AttacherContainer.Name}, - {Name: templates.ResizerContainer.Name}, - {Name: templates.SnapshotterContainer.Name}, - {Name: templates.CSIAddonsContainer.Name}, - { - Name: rbdDeploymentContainerName, - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--nodeid=$(NODE_ID)", - "--endpoint=$(CSI_ENDPOINT)", - "--v=5", - "--pidlimit=-1", - "--type=rbd", - "--controllerserver=true", - fmt.Sprintf("--csi-addons-endpoint=%s", templates.DefaultCSIAddonsSocketPath), - fmt.Sprintf("--drivername=%s", GetRBDDriverName()), - }, - Env: []corev1.EnvVar{ - { - Name: "POD_IP", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "CSI_ENDPOINT", - Value: templates.DefaultProvisionerSocketPath, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: templates.DefaultSocketDir, - }, - { - Name: "host-dev", - MountPath: "/dev", - }, - { - Name: "host-sys", - MountPath: "/sys", - }, - { - Name: "lib-modules", - MountPath: "/lib/modules", - ReadOnly: true, - }, - { - Name: "ceph-csi-configs", - MountPath: "/etc/ceph-csi-config", - }, - { - Name: "keys-tmp-dir", - MountPath: "/tmp/csi/keys", - }, - { - Name: "ceph-csi-kms-config", - MountPath: "/etc/ceph-csi-encryption-kms-config/", - ReadOnly: true, - }, - { - Name: "oidc-token", - MountPath: "/run/secrets/tokens", - ReadOnly: true, - }, - }, - }, - }, - PriorityClassName: "system-cluster-critical", - Volumes: []corev1.Volume{ - { - Name: "host-dev", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/dev", - }, - }, - }, - { - Name: "host-sys", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/sys", - }, - }, - }, - { - Name: "lib-modules", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/lib/modules", - }, - }, - }, - { - Name: "socket-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - - { - Name: "keys-tmp-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - }, - }, - }, - { - Name: "ceph-csi-configs", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: templates.MonConfigMapName, - }, - }, - }, - }, - { - Name: "ceph-csi-kms-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: templates.EncryptionConfigMapName, - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "oidc-token", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Path: "oidc-token", - ExpirationSeconds: ptr.To(int64(3600)), - Audience: "ceph-csi-kms", - }, - }, - }, - }, - }, - }, - }, - }, - }, -} - -func SetRBDDeploymentDesiredState(deploy *appsv1.Deployment) { - // Copy required labels - utils.AddLabels(deploy, rbdDeploymentLabels) - - // Update the deployment set with desired spec - rbdDeploymentSpec.DeepCopyInto(&deploy.Spec) - - // Find and Update placeholder containers with desired state - leaderElectionArg := fmt.Sprintf("--leader-election-namespace=%s", deploy.Namespace) - - for i := range deploy.Spec.Template.Spec.Containers { - c := &deploy.Spec.Template.Spec.Containers[i] - - switch c.Name { - case templates.ProvisionerContainer.Name: - templates.ProvisionerContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.ProvisionerImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.AttacherContainer.Name: - templates.AttacherContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.AttacherImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.ResizerContainer.Name: - templates.ResizerContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.ResizerImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.SnapshotterContainer.Name: - templates.SnapshotterContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.SnapshotterImageURL - c.Args = append(c.Args, leaderElectionArg) - - case templates.CSIAddonsContainer.Name: - templates.CSIAddonsContainer.DeepCopyInto(c) - c.Image = sidecarImages.ContainerImages.CSIADDONSImageURL - c.Args = append(c.Args, leaderElectionArg) - - case rbdDeploymentContainerName: - c.Image = sidecarImages.ContainerImages.CephCSIImageURL - } - } -} diff --git a/pkg/csi/scc.go b/pkg/csi/scc.go deleted file mode 100644 index 527535b8..00000000 --- a/pkg/csi/scc.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package csi - -import ( - "fmt" - - secv1 "github.com/openshift/api/security/v1" - corev1 "k8s.io/api/core/v1" -) - -const ( - SCCName = "ocs-csi-scc" -) - -var ( - // serviceaccount names - cephFSProvisionerServiceAccountName = "ocs-client-operator-csi-cephfs-provisioner-sa" - cephFSPluginServiceAccountName = "ocs-client-operator-csi-cephfs-plugin-sa" - rbdProvisionerServiceAccountName = "ocs-client-operator-csi-rbd-provisioner-sa" - rbdPluginServiceAccountName = "ocs-client-operator-csi-rbd-plugin-sa" -) - -var securityContext = secv1.SecurityContextConstraints{ - // CSI daemonset pod needs to run as privileged - AllowPrivilegedContainer: true, - // CSI daemonset pod needs hostnetworking - AllowHostNetwork: true, - // This need to be set to true as we use HostPath - AllowHostDirVolumePlugin: true, - // Required for csi addons - AllowHostPorts: true, - // Needed as we are setting this in RBD plugin pod - AllowHostPID: true, - // Required for multus and encryption - AllowHostIPC: true, - // SYS_ADMIN is needed for rbd to execute rbd map command - AllowedCapabilities: []corev1.Capability{"SYS_ADMIN"}, - // # Set to false as we write to RootFilesystem inside csi containers - ReadOnlyRootFilesystem: false, - RunAsUser: secv1.RunAsUserStrategyOptions{ - Type: secv1.RunAsUserStrategyRunAsAny, - }, - SELinuxContext: secv1.SELinuxContextStrategyOptions{ - Type: secv1.SELinuxStrategyRunAsAny, - }, - FSGroup: secv1.FSGroupStrategyOptions{ - Type: secv1.FSGroupStrategyRunAsAny, - }, - SupplementalGroups: secv1.SupplementalGroupsStrategyOptions{ - Type: secv1.SupplementalGroupsStrategyRunAsAny, - }, - Volumes: []secv1.FSType{ - secv1.FSTypeHostPath, - secv1.FSTypeConfigMap, - secv1.FSTypeEmptyDir, - secv1.FSProjected, - }, -} - -func SetSecurityContextConstraintsDesiredState(scc *secv1.SecurityContextConstraints, ns string) { - // Make sure metadata is preserved - metadata := scc.ObjectMeta - securityContext.DeepCopyInto(scc) - scc.ObjectMeta = metadata - - // Adding users based on namespace - scc.Users = []string{ - fmt.Sprintf("system:serviceaccount:%s:%s", ns, cephFSProvisionerServiceAccountName), - fmt.Sprintf("system:serviceaccount:%s:%s", ns, cephFSPluginServiceAccountName), - fmt.Sprintf("system:serviceaccount:%s:%s", ns, rbdProvisionerServiceAccountName), - fmt.Sprintf("system:serviceaccount:%s:%s", ns, rbdPluginServiceAccountName), - } -} diff --git a/pkg/templates/csidriver.go b/pkg/templates/csidriver.go deleted file mode 100644 index 6c224a62..00000000 --- a/pkg/templates/csidriver.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - v1k8scsi "k8s.io/api/storage/v1" - "k8s.io/utils/ptr" -) - -var ( - fileFSGroupPolicy = v1k8scsi.FileFSGroupPolicy -) - -var CephFSCSIDriver = v1k8scsi.CSIDriver{ - Spec: v1k8scsi.CSIDriverSpec{ - AttachRequired: ptr.To(true), - PodInfoOnMount: ptr.To(false), - FSGroupPolicy: &fileFSGroupPolicy, - }, -} - -var RbdCSIDriver = v1k8scsi.CSIDriver{ - Spec: v1k8scsi.CSIDriverSpec{ - AttachRequired: ptr.To(true), - PodInfoOnMount: ptr.To(false), - FSGroupPolicy: &fileFSGroupPolicy, - }, -} diff --git a/pkg/templates/csisidecars.go b/pkg/templates/csisidecars.go deleted file mode 100644 index 2250a0e3..00000000 --- a/pkg/templates/csisidecars.go +++ /dev/null @@ -1,198 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" -) - -var ProvisionerContainer = corev1.Container{ - Name: "csi-provisioner", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - fmt.Sprintf("--csi-address=%s", DefaultProvisionerSocketPath), - "--v=5", - "--timeout=150s", - "--retry-interval-start=500ms", - "--leader-election=true", - "--default-fstype=ext4", - "--extra-create-metadata=true", - }, - Env: []corev1.EnvVar{}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: DefaultSocketDir, - }, - }, -} - -var ResizerContainer = corev1.Container{ - Name: "csi-resizer", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - fmt.Sprintf("--csi-address=%s", DefaultProvisionerSocketPath), - "--v=5", - "--timeout=150s", - "--leader-election=true", - "--handle-volume-inuse-error=false", - }, - Env: []corev1.EnvVar{}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: DefaultSocketDir, - }, - }, -} - -var AttacherContainer = corev1.Container{ - Name: "csi-attacher", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - fmt.Sprintf("--csi-address=%s", DefaultProvisionerSocketPath), - "--v=5", - "--timeout=150s", - "--leader-election=true", - }, - Env: []corev1.EnvVar{}, - - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: DefaultSocketDir, - }, - }, -} - -var SnapshotterContainer = corev1.Container{ - Name: "csi-snapshotter", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - fmt.Sprintf("--csi-address=%s", DefaultProvisionerSocketPath), - "--v=5", - "--timeout=150s", - "--leader-election=true", - "--extra-create-metadata=true", - }, - Env: []corev1.EnvVar{}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: DefaultSocketDir, - }, - }, -} - -var CSIAddonsContainer = corev1.Container{ - Name: "csi-addons", - Args: []string{ - "--node-id=$(NODE_ID)", - "--v=5", - fmt.Sprintf("--csi-addons-address=%s", DefaultCSIAddonsSocketPath), - fmt.Sprintf("--controller-port=%v", DefaultCSIAddonsContainerPort), - "--pod=$(POD_NAME)", - "--namespace=$(POD_NAMESPACE)", - "--pod-uid=$(POD_UID)", - fmt.Sprintf("--stagingpath=%s", DefaultStagingPath), - }, - Ports: []corev1.ContainerPort{ - { - ContainerPort: DefaultCSIAddonsContainerPort, - }, - }, - EnvFrom: nil, - Env: []corev1.EnvVar{ - { - Name: "NODE_ID", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - { - Name: "POD_UID", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.uid", - }, - }, - }, - { - Name: "POD_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }, - }, - { - Name: "POD_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "socket-dir", - MountPath: DefaultSocketDir, - }, - }, - ImagePullPolicy: corev1.PullIfNotPresent, -} - -var DriverRegistrar = corev1.Container{ - Name: "csi-driver-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: &corev1.SecurityContext{ - Privileged: ptr.To(true), - AllowPrivilegeEscalation: ptr.To(true), - }, - Args: []string{ - fmt.Sprintf("--csi-address=%s", DefaultPluginSocketPath), - "--v=5", - }, - - Env: []corev1.EnvVar{ - { - Name: "KUBE_NODE_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: DefaultSocketDir, - }, - { - Name: "registration-dir", - MountPath: "/registration", - }, - }, -} diff --git a/pkg/templates/defaults.go b/pkg/templates/defaults.go deleted file mode 100644 index ccfbce7c..00000000 --- a/pkg/templates/defaults.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2022 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package templates - -const ( - // kubelet directory path - DefaultKubeletDirPath = "/var/lib/kubelet" - DefaultProvisionerSocketPath = "unix:///csi/csi-provisioner.sock" - DefaultPluginSocketPath = "unix:///csi/csi.sock" - DefaultCSIAddonsSocketPath = "unix:///csi/csi-addons.sock" - DefaultSocketDir = "/csi" - DefaultStagingPath = "/var/lib/kubelet/plugins/kubernetes.io/csi/" - - // configmap names - MonConfigMapName = "ceph-csi-configs" - EncryptionConfigMapName = "ceph-csi-kms-config" - - // default port numbers - DefaultCSIAddonsContainerPort = int32(9070) -) diff --git a/pkg/utils/csi.go b/pkg/utils/csi.go deleted file mode 100644 index 6570864a..00000000 --- a/pkg/utils/csi.go +++ /dev/null @@ -1,33 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - "os" - "slices" - "strings" -) - -var DelegateCSI = func() bool { - return strings.ToLower(os.Getenv(CSIReconcileEnvVar)) != "self" -}() - -func ExtractMonitor(monitorData []byte) ([]string, error) { - data := map[string]string{} - monitorIPs := []string{} - err := json.Unmarshal(monitorData, &data) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal data: %v", err) - } - // Ip will be in the format of "b=172.30.60.238:6789","c=172.30.162.124:6789","a=172.30.1.100:6789" - monIPs := strings.Split(data["data"], ",") - for _, monIP := range monIPs { - ip := strings.Split(monIP, "=") - if len(ip) != 2 { - return nil, fmt.Errorf("invalid mon ips: %s", monIPs) - } - monitorIPs = append(monitorIPs, ip[1]) - } - slices.Sort(monitorIPs) - return monitorIPs, nil -} diff --git a/pkg/utils/k8sutils.go b/pkg/utils/k8sutils.go index 068f127f..8f1a98a4 100644 --- a/pkg/utils/k8sutils.go +++ b/pkg/utils/k8sutils.go @@ -44,12 +44,8 @@ const DesiredSubscriptionChannelAnnotationKey = "ocs.openshift.io/subscription.c // Value corresponding to annotation key has desired client hash const DesiredConfigHashAnnotationKey = "ocs.openshift.io/provider-side-state" -const runCSIDaemonsetOnMaster = "RUN_CSI_DAEMONSET_ON_MASTER" - const CronScheduleWeekly = "@weekly" -const CSIReconcileEnvVar = "CSI_RECONCILE" - const ExitCodeThatShouldRestartTheProcess = 42 // GetOperatorNamespace returns the namespace where the operator is deployed. diff --git a/pkg/utils/placements.go b/pkg/utils/placements.go deleted file mode 100644 index 16808368..00000000 --- a/pkg/utils/placements.go +++ /dev/null @@ -1,32 +0,0 @@ -package utils - -import ( - "log" - "os" - "strconv" - - corev1 "k8s.io/api/core/v1" -) - -func GetTolerationForCSIPods() corev1.Toleration { - - runOnMaster := true - var err error - rom := os.Getenv(runCSIDaemonsetOnMaster) - if rom != "" { - runOnMaster, err = strconv.ParseBool(rom) - if err != nil { - log.Fatal(err) - } - } - - if runOnMaster { - toleration := corev1.Toleration{ - Key: "node-role.kubernetes.io/master", - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - } - return toleration - } - return corev1.Toleration{} -} diff --git a/service/status-report/main.go b/service/status-report/main.go index bdb5beef..27381e10 100644 --- a/service/status-report/main.go +++ b/service/status-report/main.go @@ -28,7 +28,6 @@ import ( "github.com/red-hat-storage/ocs-operator/services/provider/api/v4/interfaces" "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" - "github.com/red-hat-storage/ocs-client-operator/pkg/csi" "github.com/red-hat-storage/ocs-client-operator/pkg/utils" csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" @@ -147,23 +146,7 @@ func updateCSIConfig(ctx context.Context, return fmt.Errorf("failed to get StorageConfig of storageClient %v: %v", storageClient.Status.ConsumerID, err) } for _, eResource := range scResponse.ExternalResource { - if !utils.DelegateCSI && eResource.Kind == "ConfigMap" && eResource.Name == "rook-ceph-mon-endpoints" { - monitorIps, err := csi.ExtractMonitor(eResource.Data) - if err != nil { - return fmt.Errorf("failed to extract monitor data for storageClient %v: %v", storageClient.Status.ConsumerID, err) - } - var csiClusterConfigEntry = new(csi.ClusterConfigEntry) - csiClusterConfigEntry.Monitors = append(csiClusterConfigEntry.Monitors, monitorIps...) - cc := csi.ClusterConfig{ - Client: cl, - Namespace: operatorNamespace, - Ctx: ctx, - } - err = cc.UpdateMonConfigMap("", storageClient.Status.ConsumerID, csiClusterConfigEntry) - if err != nil { - return fmt.Errorf("failed to update mon configmap for storageClient %v: %v", storageClient.Status.ConsumerID, err) - } - } else if utils.DelegateCSI && eResource.Kind == "CephConnection" { + if eResource.Kind == "CephConnection" { desiredCephConnectionSpec := &csiopv1a1.CephConnectionSpec{} if err := json.Unmarshal(eResource.Data, &desiredCephConnectionSpec); err != nil { return fmt.Errorf("failed to unmarshall cephConnectionSpec: %v", err)