From d513dc4bc597ca8fc0e9b332e6c66e38bdd6a587 Mon Sep 17 00:00:00 2001 From: Yug Date: Fri, 2 Jul 2021 10:19:53 +0530 Subject: [PATCH] ceph: update CSIDriver object from betav1 to v1 Update the CSIDriver object to v1 from betav1 as it will be deprecated in v1.19+ and become unavailable in v1.22+ Signed-off-by: Yug --- pkg/operator/ceph/csi/betav1csidriver.go | 116 +++++++++++++++++++++++ pkg/operator/ceph/csi/csidriver.go | 116 +++++++++++++++++++++++ pkg/operator/ceph/csi/spec.go | 112 +++++++--------------- 3 files changed, 265 insertions(+), 79 deletions(-) create mode 100644 pkg/operator/ceph/csi/betav1csidriver.go create mode 100644 pkg/operator/ceph/csi/csidriver.go diff --git a/pkg/operator/ceph/csi/betav1csidriver.go b/pkg/operator/ceph/csi/betav1csidriver.go new file mode 100644 index 000000000..3d33ecd14 --- /dev/null +++ b/pkg/operator/ceph/csi/betav1csidriver.go @@ -0,0 +1,116 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +TODO: storage.k8s.io/v1beta1 CSIDriver is deprecated in Kubernetes v1.19+, unavailable in v1.22+; +Once the support of older Kubernetes releases are removed in Rook, delete the file to +remove the support for the betav1 CSIDriver object. +*/ + +package csi + +import ( + "context" + + "github.com/pkg/errors" + betav1k8scsi "k8s.io/api/storage/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/typed/storage/v1beta1" +) + +type beta1CsiDriver struct { + csiDriver *betav1k8scsi.CSIDriver + csiClient v1beta1.CSIDriverInterface +} + +// createCSIDriverInfo Registers CSI driver by creating a CSIDriver object +func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error { + attach := true + mountInfo := false + // Create CSIDriver object + csiDriver := &betav1k8scsi.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: betav1k8scsi.CSIDriverSpec{ + AttachRequired: &attach, + PodInfoOnMount: &mountInfo, + }, + } + if fsGroupPolicy != "" { + policy := betav1k8scsi.FSGroupPolicy(fsGroupPolicy) + csiDriver.Spec.FSGroupPolicy = &policy + } + csidrivers := clientset.StorageV1beta1().CSIDrivers() + driver, err := csidrivers.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + _, err = csidrivers.Create(ctx, csiDriver, metav1.CreateOptions{}) + if err != nil { + return err + } + logger.Infof("CSIDriver object created for driver %q", name) + } + return err + } + + // As FSGroupPolicy field is immutable, should be set only during create time. + // if the request is to change the FSGroupPolicy, we are deleting the CSIDriver object and creating it. + if driver.Spec.FSGroupPolicy != nil && csiDriver.Spec.FSGroupPolicy != nil && *driver.Spec.FSGroupPolicy != *csiDriver.Spec.FSGroupPolicy { + return d.reCreateCSIDriverInfo(ctx) + } + + // For csidriver we need to provide the resourceVersion when updating the object. + // From the docs (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) + // > "This value MUST be treated as opaque by clients and passed unmodified back to the server" + csiDriver.ObjectMeta.ResourceVersion = driver.ObjectMeta.ResourceVersion + _, err = csidrivers.Update(ctx, csiDriver, metav1.UpdateOptions{}) + if err != nil { + return err + } + logger.Infof("CSIDriver object updated for driver %q", name) + return nil +} + +func (d beta1CsiDriver) reCreateCSIDriverInfo(ctx context.Context) error { + csiDriver := d.csiDriver + csiClient := d.csiClient + err := csiClient.Delete(ctx, csiDriver.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", csiDriver.Name) + } + logger.Infof("CSIDriver object deleted for driver %q", csiDriver.Name) + _, err = csiClient.Create(ctx, csiDriver, metav1.CreateOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", csiDriver.Name) + } + logger.Infof("CSIDriver object recreated for driver %q", csiDriver.Name) + return nil +} + +// deleteCSIDriverInfo deletes CSIDriverInfo and returns the error if any +func (d beta1CsiDriver) deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name string) error { + err := clientset.StorageV1beta1().CSIDrivers().Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + logger.Debug("%q CSIDriver not found; skipping deletion.", name) + return nil + } + } + return err +} diff --git a/pkg/operator/ceph/csi/csidriver.go b/pkg/operator/ceph/csi/csidriver.go new file mode 100644 index 000000000..f30267462 --- /dev/null +++ b/pkg/operator/ceph/csi/csidriver.go @@ -0,0 +1,116 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + "github.com/pkg/errors" + v1k8scsi "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + v1 "k8s.io/client-go/kubernetes/typed/storage/v1" +) + +type csiDriver interface { + createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error + reCreateCSIDriverInfo(ctx context.Context) error + deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name string) error +} + +type v1CsiDriver struct { + csiDriver *v1k8scsi.CSIDriver + csiClient v1.CSIDriverInterface +} + +// createCSIDriverInfo Registers CSI driver by creating a CSIDriver object +func (d v1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error { + attach := true + mountInfo := false + // Create CSIDriver object + csiDriver := &v1k8scsi.CSIDriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1k8scsi.CSIDriverSpec{ + AttachRequired: &attach, + PodInfoOnMount: &mountInfo, + }, + } + if fsGroupPolicy != "" { + policy := v1k8scsi.FSGroupPolicy(fsGroupPolicy) + csiDriver.Spec.FSGroupPolicy = &policy + } + csidrivers := clientset.StorageV1().CSIDrivers() + driver, err := csidrivers.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + _, err = csidrivers.Create(ctx, csiDriver, metav1.CreateOptions{}) + if err != nil { + return err + } + logger.Infof("CSIDriver object created for driver %q", name) + } + return err + } + + // As FSGroupPolicy field is immutable, should be set only during create time. + // if the request is to change the FSGroupPolicy, we are deleting the CSIDriver object and creating it. + if driver.Spec.FSGroupPolicy != nil && csiDriver.Spec.FSGroupPolicy != nil && *driver.Spec.FSGroupPolicy != *csiDriver.Spec.FSGroupPolicy { + return d.reCreateCSIDriverInfo(ctx) + } + + // For csidriver we need to provide the resourceVersion when updating the object. + // From the docs (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) + // > "This value MUST be treated as opaque by clients and passed unmodified back to the server" + csiDriver.ObjectMeta.ResourceVersion = driver.ObjectMeta.ResourceVersion + _, err = csidrivers.Update(ctx, csiDriver, metav1.UpdateOptions{}) + if err != nil { + return err + } + logger.Infof("CSIDriver object updated for driver %q", name) + return nil +} + +func (d v1CsiDriver) reCreateCSIDriverInfo(ctx context.Context) error { + csiDriver := d.csiDriver + csiClient := d.csiClient + err := csiClient.Delete(ctx, csiDriver.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", csiDriver.Name) + } + logger.Infof("CSIDriver object deleted for driver %q", csiDriver.Name) + _, err = csiClient.Create(ctx, d.csiDriver, metav1.CreateOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", csiDriver.Name) + } + logger.Infof("CSIDriver object recreated for driver %q", csiDriver.Name) + return nil +} + +// deleteCSIDriverInfo deletes CSIDriverInfo and returns the error if any +func (d v1CsiDriver) deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name string) error { + err := clientset.StorageV1().CSIDrivers().Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + logger.Debug("%s CSIDriver not found; skipping deletion.", name) + return nil + } + } + return err +} diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 28cc9fb05..573aa9d38 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -33,11 +33,9 @@ import ( apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8scsi "k8s.io/api/storage/v1beta1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/typed/storage/v1beta1" ) type Param struct { @@ -100,7 +98,8 @@ var ( ConfigName = "rook-ceph-csi-config" ConfigKey = "csi-cluster-config-json" - csiLock sync.Mutex + csiLock sync.Mutex + csiDriverobj csiDriver ) // Specify default images as var instead of const so that they can be overridden with the Go @@ -124,6 +123,8 @@ const ( kubeMinVerForFilesystemRestore = "15" kubeMinVerForBlockRestore = "16" kubeMinVerForSnapshot = "17" + kubeMinVerForV1csiDriver = "18" + kubeMaxVerForBeta1csiDriver = "21" // common tolerations and node affinity provisionerTolerationsEnv = "CSI_PROVISIONER_TOLERATIONS" @@ -246,6 +247,28 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter CephFSDriverName = tp.DriverNamePrefix + "cephfs.csi.ceph.com" RBDDriverName = tp.DriverNamePrefix + "rbd.csi.ceph.com" + csiDriverobj = beta1CsiDriver{} + if ver.Major > KubeMinMajor || ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForV1csiDriver { + csiDriverobj = v1CsiDriver{} + // In case of an k8s version upgrade, delete the beta CSIDriver object; + // before the creation of updated v1 object to avoid conflicts. + // Also, attempt betav1 driver object deletion only if version is less + // than maximum supported version for betav1 object.(unavailable in v1.22+) + // Ignore if not found. + if EnableRBD && ver.Minor <= kubeMaxVerForBeta1csiDriver { + err = beta1CsiDriver{}.deleteCSIDriverInfo(ctx, clientset, RBDDriverName) + if err != nil { + logger.Errorf("failed to delete %q Driver Info. %v", RBDDriverName, err) + } + } + if EnableCephFS && ver.Minor <= kubeMaxVerForBeta1csiDriver { + err = beta1CsiDriver{}.deleteCSIDriverInfo(ctx, clientset, CephFSDriverName) + if err != nil { + logger.Errorf("failed to delete %q Driver Info. %v", CephFSDriverName, err) + } + } + } + tp.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) // If not set or set to anything but "false", the kernel client will be enabled @@ -579,7 +602,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter // logging a warning and intentionally continuing with the default log level logger.Warningf("failed to parse CSI_RBD_FSGROUPPOLICY. Defaulting to %q. %v", k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy, err) } - err = createCSIDriverInfo(ctx, clientset, RBDDriverName, fsGroupPolicyForRBD) + err = csiDriverobj.createCSIDriverInfo(ctx, clientset, RBDDriverName, fsGroupPolicyForRBD) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName) } @@ -591,7 +614,7 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter // log level logger.Warningf("failed to parse CSI_CEPHFS_FSGROUPPOLICY. Defaulting to %q. %v", k8scsi.ReadWriteOnceWithFSTypeFSGroupPolicy, err) } - err = createCSIDriverInfo(ctx, clientset, CephFSDriverName, fsGroupPolicyForCephFS) + err = csiDriverobj.createCSIDriverInfo(ctx, clientset, CephFSDriverName, fsGroupPolicyForCephFS) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName) } @@ -626,6 +649,10 @@ func deleteCSIDriverResources( clientset kubernetes.Interface, ver *version.Info, namespace, daemonset, deployment, service, driverName string) bool { ctx := context.TODO() succeeded := true + csiDriverobj = beta1CsiDriver{} + if ver.Major > KubeMinMajor || ver.Major == KubeMinMajor && ver.Minor >= kubeMinVerForV1csiDriver { + csiDriverobj = v1CsiDriver{} + } err := k8sutil.DeleteDaemonset(clientset, namespace, daemonset) if err != nil { logger.Errorf("failed to delete the %q. %v", daemonset, err) @@ -644,7 +671,7 @@ func deleteCSIDriverResources( succeeded = false } - err = deleteCSIDriverInfo(ctx, clientset, driverName) + err = csiDriverobj.deleteCSIDriverInfo(ctx, clientset, driverName) if err != nil { logger.Errorf("failed to delete %q Driver Info. %v", driverName, err) succeeded = false @@ -671,79 +698,6 @@ func applyCephClusterNetworkConfig(ctx context.Context, objectMeta *metav1.Objec return isMultusApplied, nil } -// createCSIDriverInfo Registers CSI driver by creating a CSIDriver object -func createCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name, fsGroupPolicy string) error { - attach := true - mountInfo := false - // Create CSIDriver object - csiDriver := &k8scsi.CSIDriver{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: k8scsi.CSIDriverSpec{ - AttachRequired: &attach, - PodInfoOnMount: &mountInfo, - }, - } - if fsGroupPolicy != "" { - policy := k8scsi.FSGroupPolicy(fsGroupPolicy) - csiDriver.Spec.FSGroupPolicy = &policy - } - csidrivers := clientset.StorageV1beta1().CSIDrivers() - driver, err := csidrivers.Get(ctx, name, metav1.GetOptions{}) - if err == nil { - // As FSGroupPolicy field is immutable, should be set only during create time. - // if the request is to change the FSGroupPolicy, we are deleting the CSIDriver object and creating it. - if driver.Spec.FSGroupPolicy != nil && csiDriver.Spec.FSGroupPolicy != nil && *driver.Spec.FSGroupPolicy != *csiDriver.Spec.FSGroupPolicy { - return reCreateCSIDriverInfo(ctx, csidrivers, csiDriver) - } - - // For csidriver we need to provide the resourceVersion when updating the object. - // From the docs (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) - // > "This value MUST be treated as opaque by clients and passed unmodified back to the server" - csiDriver.ObjectMeta.ResourceVersion = driver.ObjectMeta.ResourceVersion - _, err = csidrivers.Update(ctx, csiDriver, metav1.UpdateOptions{}) - if err == nil { - logger.Infof("CSIDriver object updated for driver %q", name) - } - return err - } - - if apierrors.IsNotFound(err) { - _, err = csidrivers.Create(ctx, csiDriver, metav1.CreateOptions{}) - if err == nil { - logger.Infof("CSIDriver object created for driver %q", name) - } - } - - return err -} - -func reCreateCSIDriverInfo(ctx context.Context, csiClient v1beta1.CSIDriverInterface, csiDriver *k8scsi.CSIDriver) error { - err := csiClient.Delete(ctx, csiDriver.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", csiDriver.Name) - } - logger.Infof("CSIDriver object deleted for driver %q", csiDriver.Name) - _, err = csiClient.Create(ctx, csiDriver, metav1.CreateOptions{}) - if err != nil { - return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", csiDriver.Name) - } - logger.Infof("CSIDriver object recreated for driver %q", csiDriver.Name) - return nil -} - -// deleteCSIDriverInfo deletes CSIDriverInfo and returns the error if any -func deleteCSIDriverInfo(ctx context.Context, clientset kubernetes.Interface, name string) error { - err := clientset.StorageV1beta1().CSIDrivers().Delete(ctx, name, metav1.DeleteOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - } - return err -} - // ValidateCSIVersion checks if the configured ceph-csi image is supported func validateCSIVersion(clientset kubernetes.Interface, namespace, rookImage, serviceAccountName string, ownerInfo *k8sutil.OwnerInfo) (*CephCSIVersion, error) { timeout := 15 * time.Minute