From f39a32fcdbd8a83693127686b0c6451aa2cb8671 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Thu, 23 Feb 2023 17:10:22 +0530 Subject: [PATCH] osd: add capability to reconcile key rotation cron jobs This commits adds code to reconcile key rotation cron jobs. Signed-off-by: Rakshith R --- .../rook-ceph/templates/clusterrole.yaml | 9 +++ deploy/examples/common.yaml | 9 +++ pkg/operator/ceph/cluster/osd/key_rotation.go | 71 +++++++++++++++++++ pkg/operator/ceph/cluster/osd/osd.go | 5 ++ pkg/operator/k8sutil/deployment.go | 25 +++++++ 5 files changed, 119 insertions(+) diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml index a5d421f75..f2af4fb7c 100644 --- a/deploy/charts/rook-ceph/templates/clusterrole.yaml +++ b/deploy/charts/rook-ceph/templates/clusterrole.yaml @@ -121,6 +121,7 @@ rules: - create - update - delete + - deletecollection # The Rook operator must be able to watch all ceph.rook.io resources to reconcile them. - apiGroups: ["ceph.rook.io"] resources: @@ -207,6 +208,14 @@ rules: - update - delete - deletecollection +- apiGroups: + - apps + resources: + # This is to add osd deployment owner ref on key rotation + # cron jobs. + - deployments/finalizers + verbs: + - update - apiGroups: - healthchecking.openshift.io resources: diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml index c27c858e3..811a49066 100644 --- a/deploy/examples/common.yaml +++ b/deploy/examples/common.yaml @@ -260,6 +260,7 @@ rules: - create - update - delete + - deletecollection # The Rook operator must be able to watch all ceph.rook.io resources to reconcile them. - apiGroups: ["ceph.rook.io"] resources: @@ -346,6 +347,14 @@ rules: - update - delete - deletecollection + - apiGroups: + - apps + resources: + # This is to add osd deployment owner ref on key rotation + # cron jobs. + - deployments/finalizers + verbs: + - update - apiGroups: - healthchecking.openshift.io resources: diff --git a/pkg/operator/ceph/cluster/osd/key_rotation.go b/pkg/operator/ceph/cluster/osd/key_rotation.go index e1255ffe1..b32edd82d 100644 --- a/pkg/operator/ceph/cluster/osd/key_rotation.go +++ b/pkg/operator/ceph/cluster/osd/key_rotation.go @@ -28,6 +28,8 @@ import ( batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrl "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) const ( @@ -211,3 +213,72 @@ func (c *Cluster) makeKeyRotationCronJob(pvcName string, osd OSDInfo, osdProps o return cronJob, nil } + +// reconcileKeyRotationCronJobs reconciles the key rotation cron jobs for the OSDs. +func (c *Cluster) reconcileKeyRotationCronJob() error { + if !c.spec.Security.KeyRotation.Enabled { + listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, keyRotationCronJobAppName)} + err := c.context.Clientset.BatchV1(). + CronJobs(c.clusterInfo.Namespace). + DeleteCollection(c.clusterInfo.Context, + metav1.DeleteOptions{}, + listOpts) + if err != nil { + if client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "failed to delete key rotation cron jobs") + } + return nil + } + logger.Debugf("successfully deleted key rotation cron jobs") + + return nil + } + + // Get the list of OSDs backed by pvc. + osdListOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s", k8sutil.AppAttr, AppName, OSDOverPVCLabelKey)} + deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, osdListOpts) + if err != nil { + return errors.Wrap(err, "failed to query existing OSD deployments") + } + + logger.Debugf("found %d osd deployments", len(deployments.Items)) + for i := range deployments.Items { + osdDep := deployments.Items[i] + osd, err := c.getOSDInfo(&osdDep) + if err != nil { + return errors.Wrapf(err, "failed to get osd info for osd %q", osdDep.Name) + } + pvcName := osdDep.Labels[OSDOverPVCLabelKey] + if pvcName == "" { + return errors.Errorf("pvc name label %q for osd %q is empty", + OSDOverPVCLabelKey, osdDep.Name) + } + osdProps, err := c.getOSDPropsForPVC(pvcName, osd.DeviceClass) + if err != nil { + return errors.Wrapf(err, "failed to generate config for osd %q", osdDep.Name) + } + if !osdProps.encrypted { + continue + } + + logger.Infof("starting OSD key rotation cron job for osd %q", osd.ID) + cj, err := c.makeKeyRotationCronJob(pvcName, osd, osdProps) + if err != nil { + return errors.Wrap(err, "failed to make key rotation cron job") + } + + err = ctrl.SetOwnerReference(&osdDep, cj, c.context.Client.Scheme()) + if err != nil { + return errors.Wrapf(err, "failed to set controllerReference on cron job %q", cj.Name) + } + + _, err = k8sutil.CreateOrUpdateCronJob(c.clusterInfo.Context, c.context.Clientset, cj) + if err != nil { + return errors.Wrapf(err, "failed to create or update key rotation cron job %q", cj.Name) + } + logger.Infof("started OSD key rotation cron job %q", cj.Name) + } + logger.Infof("successfully started OSD key rotation cron jobs") + + return nil +} diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index a028931c4..efaf14492 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -241,6 +241,11 @@ func (c *Cluster) Start() error { // The following block is used to apply any command(s) required by an upgrade c.applyUpgradeOSDFunctionality() + err = c.reconcileKeyRotationCronJob() + if err != nil { + return errors.Wrapf(err, "failed to reconcile key rotation cron jobs") + } + logger.Infof("finished running OSDs in namespace %q", namespace) return nil } diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index 4fef1b99f..b03c4b670 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -522,6 +522,17 @@ func CreateDeployment(ctx context.Context, clientset kubernetes.Interface, dep * return clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) } +// createCronJob creates a cron job with a last applied hash annotation added +func createCronJob(ctx context.Context, clientset kubernetes.Interface, cj *batchv1.CronJob) (*batchv1.CronJob, error) { + // Set hash annotation to the newly generated deployment + err := patch.DefaultAnnotator.SetLastAppliedAnnotation(cj) + if err != nil { + return nil, errors.Wrapf(err, "failed to set hash annotation on cron job %q", cj.Name) + } + + return clientset.BatchV1().CronJobs(cj.Namespace).Create(ctx, cj, metav1.CreateOptions{}) +} + func CreateOrUpdateDeployment(ctx context.Context, clientset kubernetes.Interface, dep *appsv1.Deployment) (*appsv1.Deployment, error) { newDep, err := CreateDeployment(ctx, clientset, dep) if err != nil { @@ -536,6 +547,20 @@ func CreateOrUpdateDeployment(ctx context.Context, clientset kubernetes.Interfac return newDep, nil } +func CreateOrUpdateCronJob(ctx context.Context, clientset kubernetes.Interface, cj *batchv1.CronJob) (*batchv1.CronJob, error) { + newCJ, err := createCronJob(ctx, clientset, cj) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + // annotation was added in CreateCronJob to cj passed by reference + newCJ, err = clientset.BatchV1().CronJobs(cj.Namespace).Update(ctx, cj, metav1.UpdateOptions{}) + } + if err != nil { + return nil, errors.Wrapf(err, "failed to create or update cronjob %q: %+v", cj.Name, cj) + } + } + return newCJ, nil +} + func maxInt32Ptr(a, b *int32) *int32 { var ret int32 if a == nil && b == nil {