osd: add util function to create key rotation cron job
This commit adds util function to create key rotation cron job such as getKeyRotationContainer, getKeyRotationCronPodTemplateSpec and makeKeyRotationCronJob. Signed-off-by: Rakshith R <rar@redhat.com>
This commit is contained in:
1 parent
71e011f731
commit
ef38aea2a4
5 files changed
+340
No files matched your search
@@ -42,6 +42,11 @@ func GetMonAnnotations(a AnnotationsSpec) Annotations {
|
||||
return mergeAllAnnotationsWithKey(a, KeyMon)
|
||||
}
|
||||
|
||||
// GetKeyRotationAnnotations returns the annotations for the key rotation job
|
||||
func GetKeyRotationAnnotations(a AnnotationsSpec) Annotations {
|
||||
return mergeAllAnnotationsWithKey(a, KeyRotation)
|
||||
}
|
||||
|
||||
// GetOSDPrepareAnnotations returns the annotations for the OSD service
|
||||
func GetOSDPrepareAnnotations(a AnnotationsSpec) Annotations {
|
||||
return mergeAllAnnotationsWithKey(a, KeyOSDPrepare)
|
||||
|
||||
@@ -24,6 +24,7 @@ const (
|
||||
KeyMonArbiter KeyType = "arbiter"
|
||||
KeyMgr KeyType = "mgr"
|
||||
KeyOSDPrepare KeyType = "prepareosd"
|
||||
KeyRotation KeyType = "keyrotation"
|
||||
KeyOSD KeyType = "osd"
|
||||
KeyCleanup KeyType = "cleanup"
|
||||
KeyMonitoring KeyType = "monitoring"
|
||||
|
||||
@@ -48,6 +48,11 @@ func GetMonLabels(a LabelsSpec) Labels {
|
||||
return mergeAllLabelsWithKey(a, KeyMon)
|
||||
}
|
||||
|
||||
// GetKeyRotationLabels returns labels for the key Rotation job
|
||||
func GetKeyRotationLabels(a LabelsSpec) Labels {
|
||||
return mergeAllLabelsWithKey(a, KeyRotation)
|
||||
}
|
||||
|
||||
// GetOSDPrepareLabels returns the Labels for the OSD prepare job
|
||||
func GetOSDPrepareLabels(a LabelsSpec) Labels {
|
||||
return mergeAllLabelsWithKey(a, KeyOSDPrepare)
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
Copyright 2023 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package osd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
kms "github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
||||
"github.com/rook/rook/pkg/operator/ceph/controller"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
keyRotationCronJobAppName = "rook-ceph-osd-key-rotation"
|
||||
keyRotationCronJobAppNameFmt = "rook-ceph-osd-key-rotation-%d"
|
||||
)
|
||||
|
||||
// keyRotationCronJobName returns the name of the key rotation cron job for the given OSD ID.
|
||||
func keyRotationCronJobName(osdID int) string {
|
||||
return fmt.Sprintf(keyRotationCronJobAppNameFmt, osdID)
|
||||
}
|
||||
|
||||
// applyKeyRotationPlacement applies the placement settings for the key rotation job
|
||||
// so that it is scheduled on the same node as the OSD for which the key rotation is scheduled.
|
||||
func applyKeyRotationPlacement(spec *v1.PodSpec, labels map[string]string) {
|
||||
spec.TopologySpreadConstraints = nil
|
||||
if spec.Affinity == nil {
|
||||
spec.Affinity = &v1.Affinity{}
|
||||
} else {
|
||||
spec.Affinity.PodAntiAffinity = nil
|
||||
}
|
||||
spec.Affinity.PodAffinity = &v1.PodAffinity{
|
||||
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
|
||||
{
|
||||
LabelSelector: &metav1.LabelSelector{
|
||||
MatchLabels: labels,
|
||||
},
|
||||
TopologyKey: v1.LabelHostname,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// getKeyRotationContainer returns the container spec for the key rotation job.
|
||||
func (c *Cluster) getKeyRotationContainer(osdProps osdProperties, volumeMounts []v1.VolumeMount, devices []string) (v1.Container, error) {
|
||||
envVars := c.getConfigEnvVars(osdProps, k8sutil.DataDir, true)
|
||||
|
||||
// enable debug logging
|
||||
envVars = append(envVars, setDebugLogLevelEnvVar(true))
|
||||
envVars = append(envVars, v1.EnvVar{Name: "ROOK_CEPH_VERSION", Value: c.clusterInfo.CephVersion.CephVersionFormatted()})
|
||||
envVars = append(envVars, kms.ConfigToEnvVar(c.spec)...)
|
||||
|
||||
// run privileged always since we always mount /dev
|
||||
privileged := true
|
||||
runAsUser := int64(0)
|
||||
runAsNonRoot := false
|
||||
readOnlyRootFilesystem := false
|
||||
|
||||
args := []string{"key-management", "rotate-key", osdProps.pvc.ClaimName}
|
||||
args = append(args, devices...)
|
||||
|
||||
osdProvisionContainer := v1.Container{
|
||||
Args: args,
|
||||
Name: keyRotationCronJobAppName,
|
||||
Image: c.rookVersion,
|
||||
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: envVars,
|
||||
EnvFrom: getEnvFromSources(),
|
||||
SecurityContext: &v1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
RunAsUser: &runAsUser,
|
||||
RunAsNonRoot: &runAsNonRoot,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
},
|
||||
Resources: osdProps.resources,
|
||||
}
|
||||
|
||||
return osdProvisionContainer, nil
|
||||
}
|
||||
|
||||
// getKeyRotationPodTemplateSpec returns the pod template spec for the key rotation job.
|
||||
func (c *Cluster) getKeyRotationPodTemplateSpec(osdProps osdProperties, osd OSDInfo, restart v1.RestartPolicy) (*v1.PodTemplateSpec, error) {
|
||||
// create a volume on /dev so the pod can access devices on the host
|
||||
devVolume := v1.Volume{Name: "devices", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/dev"}}}
|
||||
udevVolume := v1.Volume{Name: "udev", VolumeSource: v1.VolumeSource{HostPath: &v1.HostPathVolumeSource{Path: "/run/udev"}}}
|
||||
hostPathType := v1.HostPathDirectory
|
||||
hostPath := filepath.Join(c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdProps.pvc.ClaimName, fmt.Sprintf("ceph-%d", osd.ID))
|
||||
hostPathVolume := v1.Volume{
|
||||
Name: "bridge",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: hostPath,
|
||||
Type: &hostPathType,
|
||||
},
|
||||
},
|
||||
}
|
||||
devicesBasePath := "/var/lib/ceph/osd/"
|
||||
volumes := []v1.Volume{
|
||||
udevVolume,
|
||||
devVolume,
|
||||
hostPathVolume,
|
||||
}
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{Name: "devices", MountPath: "/dev"},
|
||||
{Name: "udev", MountPath: "/run/udev"},
|
||||
{Name: "bridge", MountPath: devicesBasePath},
|
||||
}
|
||||
|
||||
devices := []string{encryptionBlockDestinationCopy(devicesBasePath, bluestoreBlockName)}
|
||||
if osdProps.metadataPVC.ClaimName != "" {
|
||||
devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreMetadataName))
|
||||
}
|
||||
if osdProps.walPVC.ClaimName != "" {
|
||||
devices = append(devices, encryptionBlockDestinationCopy(devicesBasePath, bluestoreWalName))
|
||||
}
|
||||
|
||||
keyRotationContainer, err := c.getKeyRotationContainer(osdProps, volumeMounts, devices)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to generate key rotation container")
|
||||
}
|
||||
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: keyRotationCronJobAppName,
|
||||
Labels: map[string]string{
|
||||
k8sutil.AppAttr: keyRotationCronJobAppName,
|
||||
k8sutil.ClusterAttr: c.clusterInfo.Namespace,
|
||||
},
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
ServiceAccountName: serviceAccountName,
|
||||
Containers: []v1.Container{
|
||||
keyRotationContainer,
|
||||
},
|
||||
RestartPolicy: restart,
|
||||
Volumes: volumes,
|
||||
HostNetwork: c.spec.Network.IsHost(),
|
||||
PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames),
|
||||
SchedulerName: osdProps.schedulerName,
|
||||
},
|
||||
}
|
||||
if c.spec.Network.IsHost() {
|
||||
podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
} else if c.spec.Network.IsMultus() {
|
||||
if err := k8sutil.ApplyMultus(c.spec.Network, &podTemplateSpec.ObjectMeta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
cephv1.GetKeyRotationAnnotations(c.spec.Annotations).ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
||||
cephv1.GetKeyRotationLabels(c.spec.Labels).ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
||||
|
||||
c.applyAllPlacementIfNeeded(&podTemplateSpec.Spec)
|
||||
// apply storageClassDeviceSets.Placement
|
||||
osdProps.placement.ApplyToPodSpec(&podTemplateSpec.Spec)
|
||||
applyKeyRotationPlacement(&podTemplateSpec.Spec, c.getOSDLabels(osd, osdProps.crushHostname, osdProps.portable))
|
||||
|
||||
// cryptsetup synchronizes with udev on host through semaphore
|
||||
podTemplateSpec.Spec.HostIPC = true
|
||||
|
||||
k8sutil.RemoveDuplicateEnvVars(&podTemplateSpec.Spec)
|
||||
return &podTemplateSpec, nil
|
||||
}
|
||||
|
||||
// makeKeyRotationCronJob creates a key rotation cron job for the given OSD.
|
||||
func (c *Cluster) makeKeyRotationCronJob(pvcName string, osd OSDInfo, osdProps osdProperties) (*batch.CronJob, error) {
|
||||
podSpec, err := c.getKeyRotationPodTemplateSpec(osdProps, osd, v1.RestartPolicyOnFailure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.applyResourcesToAllContainers(&podSpec.Spec, cephv1.GetOSDResources(c.spec.Resources, osd.DeviceClass))
|
||||
cronJob := &batch.CronJob{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: keyRotationCronJobName(osd.ID),
|
||||
Namespace: c.clusterInfo.Namespace,
|
||||
Labels: podSpec.Labels,
|
||||
Annotations: podSpec.Annotations,
|
||||
},
|
||||
Spec: batch.CronJobSpec{
|
||||
ConcurrencyPolicy: batch.ForbidConcurrent,
|
||||
Schedule: c.spec.Security.KeyRotation.Schedule,
|
||||
JobTemplate: batch.JobTemplateSpec{
|
||||
Spec: batch.JobSpec{
|
||||
Template: *podSpec,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return cronJob, nil
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
Copyright 2023 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package osd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func Test_keyRotationCronJobName(t *testing.T) {
|
||||
type args struct {
|
||||
osdID int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "OSD ID 0",
|
||||
args: args{
|
||||
osdID: 0,
|
||||
},
|
||||
want: fmt.Sprintf(keyRotationCronJobAppNameFmt, 0),
|
||||
},
|
||||
{
|
||||
name: "OSD ID 1",
|
||||
args: args{
|
||||
osdID: 1,
|
||||
},
|
||||
want: fmt.Sprintf(keyRotationCronJobAppNameFmt, 1),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := keyRotationCronJobName(tt.args.osdID); got != tt.want {
|
||||
t.Errorf("keyRotationCronJobName() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_applyKeyRotationPlacement(t *testing.T) {
|
||||
type args struct {
|
||||
spec *v1.PodSpec
|
||||
labels map[string]string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
}{
|
||||
{
|
||||
name: "test 1",
|
||||
args: args{
|
||||
spec: &v1.PodSpec{
|
||||
Affinity: &v1.Affinity{
|
||||
PodAffinity: &v1.PodAffinity{},
|
||||
PodAntiAffinity: &v1.PodAntiAffinity{},
|
||||
},
|
||||
TopologySpreadConstraints: []v1.TopologySpreadConstraint{},
|
||||
},
|
||||
labels: map[string]string{
|
||||
"app": "rook-ceph-osd",
|
||||
"osd-id": "0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Affinity is nil",
|
||||
args: args{
|
||||
spec: &v1.PodSpec{
|
||||
Affinity: nil,
|
||||
TopologySpreadConstraints: []v1.TopologySpreadConstraint{},
|
||||
},
|
||||
labels: map[string]string{
|
||||
"app": "rook-ceph-osd",
|
||||
"osd-id": "0",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
currentTT := tt
|
||||
t.Run(currentTT.name, func(t *testing.T) {
|
||||
applyKeyRotationPlacement(currentTT.args.spec, currentTT.args.labels)
|
||||
assert.Nil(t, currentTT.args.spec.Affinity.PodAntiAffinity)
|
||||
assert.Nil(t, currentTT.args.spec.TopologySpreadConstraints)
|
||||
assert.Equal(t, currentTT.args.spec.Affinity.PodAffinity, &v1.PodAffinity{
|
||||
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
|
||||
{
|
||||
LabelSelector: &metav1.LabelSelector{MatchLabels: currentTT.args.labels},
|
||||
TopologyKey: v1.LabelHostname,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user