my-rook-config/pkg/operator/ceph/cluster/osd/key_rotation.go

/*
Copyright 2023 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package osd

import (
	"fmt"
	"path/filepath"

	"github.com/pkg/errors"
	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
	"github.com/rook/rook/pkg/operator/ceph/controller"
	"github.com/rook/rook/pkg/operator/k8sutil"
	batch "k8s.io/api/batch/v1"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	ctrl "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

const (
	keyRotationCronJobAppName    = "rook-ceph-osd-key-rotation"
	keyRotationCronJobAppNameFmt = "rook-ceph-osd-key-rotation-%d"
)

// keyRotationCronJobName returns the name of the key rotation cron job for the given OSD ID.
func keyRotationCronJobName(osdID int) string {
	return fmt.Sprintf(keyRotationCronJobAppNameFmt, osdID)
}

// applyKeyRotationPlacement applies the placement settings for the key rotation job
// so that it is scheduled on the same node as the OSD for which the key rotation is scheduled.
func applyKeyRotationPlacement(spec *v1.PodSpec, labels map[string]string) {
	spec.TopologySpreadConstraints = nil
	if spec.Affinity == nil {
		spec.Affinity = &v1.Affinity{}
	} else {
		spec.Affinity.PodAntiAffinity = nil
	}
	spec.Affinity.PodAffinity = &v1.PodAffinity{
		RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
			{
				LabelSelector: &metav1.LabelSelector{
					MatchLabels: labels,
				},
				TopologyKey: v1.LabelHostname,
			},
		},
	}
}

// getKeyRotationContainer returns the container spec for the key rotation job.
func (c *Cluster) getKeyRotationContainer(osdProps osdProperties, volumeMounts []v1.VolumeMount, devices []string) (v1.Container, error) {
	envVars := c.getConfigEnvVars(osdProps, k8sutil.DataDir, true)

	// enable debug logging
	envVars = append(envVars, setDebugLogLevelEnvVar(true))
	envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()})
	envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...)

	// run privileged always since we always mount /dev
	privileged := true
	runAsUser := int64(0)
	runAsNonRoot := false
	readOnlyRootFilesystem := false

	args := []string{"key-management", "rotate-key", osdProps.pvc.ClaimName}
	args = append(args, devices...)

	osdProvisionContainer := v1.Container{
		Args:            args,
		Name:            keyRotationCronJobAppName,
		Image:           c.rookVersion,
		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
		VolumeMounts:    volumeMounts,
		Env:             envVars,
		EnvFrom:         getEnvFromSources(),
		SecurityContext: &v1.SecurityContext{
			Privileged:             &privileged,
			RunAsUser:              &runAsUser,
			RunAsNonRoot:           &runAsNonRoot,
			ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
		},
		Resources: osdProps.resources,
	}

	return osdProvisionContainer, nil
}

// getKeyRotationPodTemplateSpec returns the pod template spec for the key rotation job.
func (c *Cluster) getKeyRotationPodTemplateSpec(osdProps osdProperties, osd OSDInfo, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
	// create a volume on /dev so the pod can access devices on the host
	devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}
	udevVolume := v1.Volume{Name: "udev", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/run/udev"}}}
	hostPathType := v1.HostPathDirectory
	hostPath := filepath.Join(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdProps.pvc.ClaimName, fmt.Sprintf("ceph-%d", osd.ID))
	hostPathVolume := v1.Volume{
		Name: "bridge",
		VolumeSource: v1.VolumeSource{
			HostPath: &v1.HostPathVolumeSource{
				Path: hostPath,
				Type: &hostPathType,
			},
		},
	}
	devicesBasePath := "/var/lib/ceph/osd/"
	volumes := []v1.Volume{
		udevVolume,
		devVolume,
		hostPathVolume,
	}
	volumeMounts := []v1.VolumeMount{
		{Name: "devices", MountPath: "/dev"},
		{Name: "udev", MountPath: "/run/udev"},
		{Name: "bridge", MountPath: devicesBasePath},
	}

	devices := []string{encryptionBlockDestinationCopy(devicesBasePath, bluestoreBlockName)}
	if osdProps.metadataPVC.ClaimName != "" {
		devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreMetadataName))
	}
	if osdProps.walPVC.ClaimName != "" {
		devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreWalName))
	}

	keyRotationContainer, err := c.getKeyRotationContainer(osdProps, volumeMounts, devices)
	if err != nil {
		return nil, errors.Wrap(err, "failed to generate key rotation container")
	}

	podTemplateSpec := v1.PodTemplateSpec{
		ObjectMeta: metav1.ObjectMeta{
			Name: keyRotationCronJobAppName,
			Labels: map[string]string{
				k8sutil.AppAttr:     keyRotationCronJobAppName,
				k8sutil.ClusterAttr: c.clusterInfo.Namespace,
			},
			Annotations: map[string]string{},
		},
		Spec: v1.PodSpec{
			ServiceAccountName: serviceAccountName,
			Containers: []v1.Container{
				keyRotationContainer,
			},
			RestartPolicy:     restart,
			Volumes:           volumes,
			HostNetwork:       c.spec.Network.IsHost(),
			PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames),
			SchedulerName:     osdProps.schedulerName,
		},
	}
	if c.spec.Network.IsHost() {
		podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
	} else if c.spec.Network.IsMultus() {
		if err := k8sutil.ApplyMultus(c.clusterInfo.Namespace, &c.spec.Network, &podTemplateSpec.ObjectMeta); err != nil {
			return nil, err
		}
	}

	cephv1.GetKeyRotationAnnotations(c.spec.Annotations).ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
	cephv1.GetKeyRotationLabels(c.spec.Labels).ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)

	c.applyAllPlacementIfNeeded(&podTemplateSpec.Spec)
	// apply storageClassDeviceSets.Placement
	osdProps.placement.ApplyToPodSpec(&podTemplateSpec.Spec)
	applyKeyRotationPlacement(&podTemplateSpec.Spec, c.getOSDLabels(osd, osdProps.crushHostname, osdProps.portable))

	// cryptsetup synchronizes with udev on host through semaphore
	podTemplateSpec.Spec.HostIPC = true

	k8sutil.RemoveDuplicateEnvVars(&podTemplateSpec.Spec)
	return &podTemplateSpec, nil
}

// makeKeyRotationCronJob creates a key rotation cron job for the given OSD.
func (c *Cluster) makeKeyRotationCronJob(pvcName string, osd OSDInfo, osdProps osdProperties) (*batch.CronJob, error) {
	podSpec, err := c.getKeyRotationPodTemplateSpec(osdProps, osd, v1.RestartPolicyOnFailure)
	if err != nil {
		return nil, err
	}
	c.applyResourcesToAllContainers(&podSpec.Spec, cephv1.GetOSDResources(c.spec.Resources, osd.DeviceClass))
	schedule := c.spec.Security.KeyRotation.Schedule
	if schedule == "" {
		// default to rotate keyrings weekly (default is in code since default in crds causes issues)
		schedule = "@weekly"
	}
	cronJob := &batch.CronJob{
		ObjectMeta: metav1.ObjectMeta{
			Name:        keyRotationCronJobName(osd.ID),
			Namespace:   c.clusterInfo.Namespace,
			Labels:      podSpec.Labels,
			Annotations: podSpec.Annotations,
		},
		Spec: batch.CronJobSpec{
			ConcurrencyPolicy: batch.ForbidConcurrent,
			Schedule:          schedule,
			JobTemplate: batch.JobTemplateSpec{
				Spec: batch.JobSpec{
					Template: *podSpec,
				},
			},
		},
	}

	return cronJob, nil
}

// reconcileKeyRotationCronJobs reconciles the key rotation cron jobs for the OSDs.
func (c *Cluster) reconcileKeyRotationCronJob() error {
	if !c.spec.Security.KeyRotation.Enabled {
		listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, keyRotationCronJobAppName)}
		err := c.context.Clientset.BatchV1().
			CronJobs(c.clusterInfo.Namespace).
			DeleteCollection(c.clusterInfo.Context,
				metav1.DeleteOptions{},
				listOpts)
		if err != nil {
			if client.IgnoreNotFound(err) != nil {
				return errors.Wrap(err, "failed to delete key rotation cron jobs")
			}
			return nil
		}
		logger.Debugf("successfully deleted key rotation cron jobs")

		return nil
	}

	// Get the list of OSDs backed by pvc.
	osdListOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s,%s", k8sutil.AppAttr, AppName, OSDOverPVCLabelKey)}
	deployments, err := c.context.Clientset.AppsV1().Deployments(c.clusterInfo.Namespace).List(c.clusterInfo.Context, osdListOpts)
	if err != nil {
		return errors.Wrap(err, "failed to query existing OSD deployments")
	}

	logger.Debugf("found %d osd deployments", len(deployments.Items))
	for i := range deployments.Items {
		osdDep := deployments.Items[i]
		osd, err := c.getOSDInfo(&osdDep)
		if err != nil {
			return errors.Wrapf(err, "failed to get osd info for osd %q", osdDep.Name)
		}
		pvcName := osdDep.Labels[OSDOverPVCLabelKey]
		if pvcName == "" {
			return errors.Errorf("pvc name label %q for osd %q is empty",
				OSDOverPVCLabelKey, osdDep.Name)
		}
		osdProps, err := c.getOSDPropsForPVC(pvcName, osd.DeviceClass)
		if err != nil {
			return errors.Wrapf(err, "failed to generate config for osd %q", osdDep.Name)
		}
		if !osdProps.encrypted {
			continue
		}

		logger.Infof("starting OSD key rotation cron job for osd %q", osd.ID)
		cj, err := c.makeKeyRotationCronJob(pvcName, osd, osdProps)
		if err != nil {
			return errors.Wrap(err, "failed to make key rotation cron job")
		}

		err = ctrl.SetOwnerReference(&osdDep, cj, c.context.Client.Scheme())
		if err != nil {
			return errors.Wrapf(err, "failed to set controllerReference on cron job %q", cj.Name)
		}

		_, err = k8sutil.CreateOrUpdateCronJob(c.clusterInfo.Context, c.context.Clientset, cj)
		if err != nil {
			return errors.Wrapf(err, "failed to create or update key rotation cron job %q", cj.Name)
		}
		logger.Infof("started OSD key rotation cron job %q", cj.Name)
	}
	logger.Infof("successfully started OSD key rotation cron jobs")

	return nil
}