From ef38aea2a4fcd8037eb7c59d8bd8b3280c58e2f8 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Thu, 23 Feb 2023 16:55:03 +0530 Subject: [PATCH] osd: add util function to create key rotation cron job This commit adds util function to create key rotation cron job such as getKeyRotationContainer, getKeyRotationCronPodTemplateSpec and makeKeyRotationCronJob. Signed-off-by: Rakshith R --- pkg/apis/ceph.rook.io/v1/annotations.go | 5 + pkg/apis/ceph.rook.io/v1/keys.go | 1 + pkg/apis/ceph.rook.io/v1/labels.go | 5 + pkg/operator/ceph/cluster/osd/key_rotation.go | 213 ++++++++++++++++++ .../ceph/cluster/osd/key_rotation_test.go | 116 ++++++++++ 5 files changed, 340 insertions(+) create mode 100644 pkg/operator/ceph/cluster/osd/key_rotation.go create mode 100644 pkg/operator/ceph/cluster/osd/key_rotation_test.go diff --git a/pkg/apis/ceph.rook.io/v1/annotations.go b/pkg/apis/ceph.rook.io/v1/annotations.go index 6942907fd..10b8f407c 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations.go +++ b/pkg/apis/ceph.rook.io/v1/annotations.go @@ -42,6 +42,11 @@ func GetMonAnnotations(a AnnotationsSpec) Annotations { return mergeAllAnnotationsWithKey(a, KeyMon) } +// GetKeyRotationAnnotations returns the annotations for the key rotation job +func GetKeyRotationAnnotations(a AnnotationsSpec) Annotations { + return mergeAllAnnotationsWithKey(a, KeyRotation) +} + // GetOSDPrepareAnnotations returns the annotations for the OSD service func GetOSDPrepareAnnotations(a AnnotationsSpec) Annotations { return mergeAllAnnotationsWithKey(a, KeyOSDPrepare) diff --git a/pkg/apis/ceph.rook.io/v1/keys.go b/pkg/apis/ceph.rook.io/v1/keys.go index cf3b4873d..72e18b30f 100644 --- a/pkg/apis/ceph.rook.io/v1/keys.go +++ b/pkg/apis/ceph.rook.io/v1/keys.go @@ -24,6 +24,7 @@ const ( KeyMonArbiter KeyType = "arbiter" KeyMgr KeyType = "mgr" KeyOSDPrepare KeyType = "prepareosd" + KeyRotation KeyType = "keyrotation" KeyOSD KeyType = "osd" KeyCleanup KeyType = "cleanup" KeyMonitoring KeyType = "monitoring" diff --git a/pkg/apis/ceph.rook.io/v1/labels.go b/pkg/apis/ceph.rook.io/v1/labels.go index 78ca5c524..43a3018b2 100644 --- a/pkg/apis/ceph.rook.io/v1/labels.go +++ b/pkg/apis/ceph.rook.io/v1/labels.go @@ -48,6 +48,11 @@ func GetMonLabels(a LabelsSpec) Labels { return mergeAllLabelsWithKey(a, KeyMon) } +// GetKeyRotationLabels returns labels for the key Rotation job +func GetKeyRotationLabels(a LabelsSpec) Labels { + return mergeAllLabelsWithKey(a, KeyRotation) +} + // GetOSDPrepareLabels returns the Labels for the OSD prepare job func GetOSDPrepareLabels(a LabelsSpec) Labels { return mergeAllLabelsWithKey(a, KeyOSDPrepare) diff --git a/pkg/operator/ceph/cluster/osd/key_rotation.go b/pkg/operator/ceph/cluster/osd/key_rotation.go new file mode 100644 index 000000000..e1255ffe1 --- /dev/null +++ b/pkg/operator/ceph/cluster/osd/key_rotation.go @@ -0,0 +1,213 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package osd + +import ( + "fmt" + "path/filepath" + + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms" + "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/k8sutil" + batch "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + keyRotationCronJobAppName = "rook-ceph-osd-key-rotation" + keyRotationCronJobAppNameFmt = "rook-ceph-osd-key-rotation-%d" +) + +// keyRotationCronJobName returns the name of the key rotation cron job for the given OSD ID. +func keyRotationCronJobName(osdID int) string { + return fmt.Sprintf(keyRotationCronJobAppNameFmt, osdID) +} + +// applyKeyRotationPlacement applies the placement settings for the key rotation job +// so that it is scheduled on the same node as the OSD for which the key rotation is scheduled. +func applyKeyRotationPlacement(spec *v1.PodSpec, labels map[string]string) { + spec.TopologySpreadConstraints = nil + if spec.Affinity == nil { + spec.Affinity = &v1.Affinity{} + } else { + spec.Affinity.PodAntiAffinity = nil + } + spec.Affinity.PodAffinity = &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: v1.LabelHostname, + }, + }, + } +} + +// getKeyRotationContainer returns the container spec for the key rotation job. +func (c *Cluster) getKeyRotationContainer(osdProps osdProperties, volumeMounts []v1.VolumeMount, devices []string) (v1.Container, error) { + envVars := c.getConfigEnvVars(osdProps, k8sutil.DataDir, true) + + // enable debug logging + envVars = append(envVars, setDebugLogLevelEnvVar(true)) + envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()}) + envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...) + + // run privileged always since we always mount /dev + privileged := true + runAsUser := int64(0) + runAsNonRoot := false + readOnlyRootFilesystem := false + + args := []string{"key-management", "rotate-key", osdProps.pvc.ClaimName} + args = append(args, devices...) + + osdProvisionContainer := v1.Container{ + Args: args, + Name: keyRotationCronJobAppName, + Image: c.rookVersion, + ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy), + VolumeMounts: volumeMounts, + Env: envVars, + EnvFrom: getEnvFromSources(), + SecurityContext: &v1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + RunAsNonRoot: &runAsNonRoot, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + }, + Resources: osdProps.resources, + } + + return osdProvisionContainer, nil +} + +// getKeyRotationPodTemplateSpec returns the pod template spec for the key rotation job. +func (c *Cluster) getKeyRotationPodTemplateSpec(osdProps osdProperties, osd OSDInfo, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) { + // create a volume on /dev so the pod can access devices on the host + devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}} + udevVolume := v1.Volume{Name: "udev", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/run/udev"}}} + hostPathType := v1.HostPathDirectory + hostPath := filepath.Join(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdProps.pvc.ClaimName, fmt.Sprintf("ceph-%d", osd.ID)) + hostPathVolume := v1.Volume{ + Name: "bridge", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: hostPath, + Type: &hostPathType, + }, + }, + } + devicesBasePath := "/var/lib/ceph/osd/" + volumes := []v1.Volume{ + udevVolume, + devVolume, + hostPathVolume, + } + volumeMounts := []v1.VolumeMount{ + {Name: "devices", MountPath: "/dev"}, + {Name: "udev", MountPath: "/run/udev"}, + {Name: "bridge", MountPath: devicesBasePath}, + } + + devices := []string{encryptionBlockDestinationCopy(devicesBasePath, bluestoreBlockName)} + if osdProps.metadataPVC.ClaimName != "" { + devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreMetadataName)) + } + if osdProps.walPVC.ClaimName != "" { + devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreWalName)) + } + + keyRotationContainer, err := c.getKeyRotationContainer(osdProps, volumeMounts, devices) + if err != nil { + return nil, errors.Wrap(err, "failed to generate key rotation container") + } + + podTemplateSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: keyRotationCronJobAppName, + Labels: map[string]string{ + k8sutil.AppAttr: keyRotationCronJobAppName, + k8sutil.ClusterAttr: c.clusterInfo.Namespace, + }, + Annotations: map[string]string{}, + }, + Spec: v1.PodSpec{ + ServiceAccountName: serviceAccountName, + Containers: []v1.Container{ + keyRotationContainer, + }, + RestartPolicy: restart, + Volumes: volumes, + HostNetwork: c.spec.Network.IsHost(), + PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames), + SchedulerName: osdProps.schedulerName, + }, + } + if c.spec.Network.IsHost() { + podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } else if c.spec.Network.IsMultus() { + if err := k8sutil.ApplyMultus(c.spec.Network, &podTemplateSpec.ObjectMeta); err != nil { + return nil, err + } + } + + cephv1.GetKeyRotationAnnotations(c.spec.Annotations).ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) + cephv1.GetKeyRotationLabels(c.spec.Labels).ApplyToObjectMeta(&podTemplateSpec.ObjectMeta) + + c.applyAllPlacementIfNeeded(&podTemplateSpec.Spec) + // apply storageClassDeviceSets.Placement + osdProps.placement.ApplyToPodSpec(&podTemplateSpec.Spec) + applyKeyRotationPlacement(&podTemplateSpec.Spec, c.getOSDLabels(osd, osdProps.crushHostname, osdProps.portable)) + + // cryptsetup synchronizes with udev on host through semaphore + podTemplateSpec.Spec.HostIPC = true + + k8sutil.RemoveDuplicateEnvVars(&podTemplateSpec.Spec) + return &podTemplateSpec, nil +} + +// makeKeyRotationCronJob creates a key rotation cron job for the given OSD. +func (c *Cluster) makeKeyRotationCronJob(pvcName string, osd OSDInfo, osdProps osdProperties) (*batch.CronJob, error) { + podSpec, err := c.getKeyRotationPodTemplateSpec(osdProps, osd, v1.RestartPolicyOnFailure) + if err != nil { + return nil, err + } + c.applyResourcesToAllContainers(&podSpec.Spec, cephv1.GetOSDResources(c.spec.Resources, osd.DeviceClass)) + cronJob := &batch.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: keyRotationCronJobName(osd.ID), + Namespace: c.clusterInfo.Namespace, + Labels: podSpec.Labels, + Annotations: podSpec.Annotations, + }, + Spec: batch.CronJobSpec{ + ConcurrencyPolicy: batch.ForbidConcurrent, + Schedule: c.spec.Security.KeyRotation.Schedule, + JobTemplate: batch.JobTemplateSpec{ + Spec: batch.JobSpec{ + Template: *podSpec, + }, + }, + }, + } + + return cronJob, nil +} diff --git a/pkg/operator/ceph/cluster/osd/key_rotation_test.go b/pkg/operator/ceph/cluster/osd/key_rotation_test.go new file mode 100644 index 000000000..8b52433fb --- /dev/null +++ b/pkg/operator/ceph/cluster/osd/key_rotation_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package osd + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_keyRotationCronJobName(t *testing.T) { + type args struct { + osdID int + } + tests := []struct { + name string + args args + want string + }{ + { + name: "OSD ID 0", + args: args{ + osdID: 0, + }, + want: fmt.Sprintf(keyRotationCronJobAppNameFmt, 0), + }, + { + name: "OSD ID 1", + args: args{ + osdID: 1, + }, + want: fmt.Sprintf(keyRotationCronJobAppNameFmt, 1), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := keyRotationCronJobName(tt.args.osdID); got != tt.want { + t.Errorf("keyRotationCronJobName() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_applyKeyRotationPlacement(t *testing.T) { + type args struct { + spec *v1.PodSpec + labels map[string]string + } + tests := []struct { + name string + args args + }{ + { + name: "test 1", + args: args{ + spec: &v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAffinity: &v1.PodAffinity{}, + PodAntiAffinity: &v1.PodAntiAffinity{}, + }, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + }, + labels: map[string]string{ + "app": "rook-ceph-osd", + "osd-id": "0", + }, + }, + }, + { + name: "Affinity is nil", + args: args{ + spec: &v1.PodSpec{ + Affinity: nil, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{}, + }, + labels: map[string]string{ + "app": "rook-ceph-osd", + "osd-id": "0", + }, + }, + }, + } + for _, tt := range tests { + currentTT := tt + t.Run(currentTT.name, func(t *testing.T) { + applyKeyRotationPlacement(currentTT.args.spec, currentTT.args.labels) + assert.Nil(t, currentTT.args.spec.Affinity.PodAntiAffinity) + assert.Nil(t, currentTT.args.spec.TopologySpreadConstraints) + assert.Equal(t, currentTT.args.spec.Affinity.PodAffinity, &v1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchLabels: currentTT.args.labels}, + TopologyKey: v1.LabelHostname, + }, + }, + }) + }) + } +}