From d0d4333f856dd7093d15bfa6de185a8053cd7fc5 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 15 Aug 2024 14:06:33 -0600 Subject: [PATCH] core: add annotations and labels to detect version jobs The jobs to detect the ceph and csi versions now can have custom annotationso and labels added to them. They are very short-lived jobs, but they may need custom annotations in some environments to run. Signed-off-by: Travis Nielsen (cherry picked from commit 5014d5bcf271aa952e1ce8d4da7cead5853c5b4b) --- Documentation/CRDs/specification.md | 2 ++ deploy/examples/cluster.yaml | 5 +++++ pkg/apis/ceph.rook.io/v1/annotations.go | 5 +++++ pkg/apis/ceph.rook.io/v1/annotations_test.go | 9 +++++++-- pkg/apis/ceph.rook.io/v1/keys.go | 1 + pkg/apis/ceph.rook.io/v1/labels.go | 4 ++++ pkg/apis/ceph.rook.io/v1/labels_test.go | 10 ++++++++-- pkg/operator/ceph/controller/network.go | 2 ++ pkg/operator/ceph/controller/version.go | 3 +++ pkg/operator/ceph/csi/controller.go | 6 ++++++ pkg/operator/ceph/csi/spec.go | 5 +++++ 11 files changed, 48 insertions(+), 4 deletions(-) diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index f03bc8648..d99706bb0 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7394,6 +7394,8 @@ string

"clusterMetadata"

+

"cmdreporter"

+

"crashcollector"

"dashboard"

diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index de5cd17ce..07771cbd3 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -194,9 +194,14 @@ spec: annotations: # all: # mon: + # mgr: # osd: + # exporter: + # crashcollector: # cleanup: # prepareosd: + # cmdreporter is for jobs to detect ceph and csi versions, and check network status + # cmdreporter: # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets. # And clusterMetadata annotations will not be merged with `all` annotations. # clusterMetadata: diff --git a/pkg/apis/ceph.rook.io/v1/annotations.go b/pkg/apis/ceph.rook.io/v1/annotations.go index b2d77dc2d..9610420be 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations.go +++ b/pkg/apis/ceph.rook.io/v1/annotations.go @@ -72,6 +72,11 @@ func GetCephExporterAnnotations(a AnnotationsSpec) Annotations { return mergeAllAnnotationsWithKey(a, KeyCephExporter) } +// GetCmdReporterAnnotations returns the Annotations for jobs detecting versions +func GetCmdReporterAnnotations(a AnnotationsSpec) Annotations { + return mergeAllAnnotationsWithKey(a, KeyCmdReporter) +} + func GetClusterMetadataAnnotations(a AnnotationsSpec) Annotations { return a[KeyClusterMetadata] } diff --git a/pkg/apis/ceph.rook.io/v1/annotations_test.go b/pkg/apis/ceph.rook.io/v1/annotations_test.go index e3012a482..b832690c3 100644 --- a/pkg/apis/ceph.rook.io/v1/annotations_test.go +++ b/pkg/apis/ceph.rook.io/v1/annotations_test.go @@ -58,8 +58,9 @@ func TestCephAnnotationsMerge(t *testing.T) { // Merge with "all" testAnnotations = AnnotationsSpec{ - "all": {"allkey1": "allval1", "allkey2": "allval2"}, - "mgr": {"mgrkey": "mgrval"}, + "all": {"allkey1": "allval1", "allkey2": "allval2"}, + "mgr": {"mgrkey": "mgrval"}, + "cmdreporter": {"myversions": "detect"}, } a = GetMonAnnotations(testAnnotations) assert.Equal(t, "allval1", a["allkey1"]) @@ -70,6 +71,10 @@ func TestCephAnnotationsMerge(t *testing.T) { assert.Equal(t, "allval1", a["allkey1"]) assert.Equal(t, "allval2", a["allkey2"]) assert.Equal(t, 3, len(a)) + b := GetCmdReporterAnnotations(testAnnotations) + assert.Equal(t, "detect", b["myversions"]) + assert.Equal(t, "allval1", b["allkey1"]) + assert.Equal(t, "allval2", b["allkey2"]) } func TestAnnotationsSpec(t *testing.T) { diff --git a/pkg/apis/ceph.rook.io/v1/keys.go b/pkg/apis/ceph.rook.io/v1/keys.go index 05395f1e3..88467d31f 100644 --- a/pkg/apis/ceph.rook.io/v1/keys.go +++ b/pkg/apis/ceph.rook.io/v1/keys.go @@ -32,4 +32,5 @@ const ( KeyCrashCollector KeyType = "crashcollector" KeyClusterMetadata KeyType = "clusterMetadata" KeyCephExporter KeyType = "exporter" + KeyCmdReporter KeyType = "cmdreporter" ) diff --git a/pkg/apis/ceph.rook.io/v1/labels.go b/pkg/apis/ceph.rook.io/v1/labels.go index ad4bb1466..6825838f8 100644 --- a/pkg/apis/ceph.rook.io/v1/labels.go +++ b/pkg/apis/ceph.rook.io/v1/labels.go @@ -87,6 +87,10 @@ func GetCephExporterLabels(a LabelsSpec) Labels { return mergeAllLabelsWithKey(a, KeyCephExporter) } +func GetCmdReporterLabels(a LabelsSpec) Labels { + return mergeAllLabelsWithKey(a, KeyCmdReporter) +} + func mergeAllLabelsWithKey(a LabelsSpec, name KeyType) Labels { all := a.All() if all != nil { diff --git a/pkg/apis/ceph.rook.io/v1/labels_test.go b/pkg/apis/ceph.rook.io/v1/labels_test.go index c48100453..86668fb25 100644 --- a/pkg/apis/ceph.rook.io/v1/labels_test.go +++ b/pkg/apis/ceph.rook.io/v1/labels_test.go @@ -58,8 +58,9 @@ func TestCephLabelsMerge(t *testing.T) { // Merge with "all" testLabels = LabelsSpec{ - "all": {"allkey1": "allval1", "allkey2": "allval2"}, - "mgr": {"mgrkey": "mgrval"}, + "all": {"allkey1": "allval1", "allkey2": "allval2"}, + "mgr": {"mgrkey": "mgrval"}, + "cmdreporter": {"detect": "myversion"}, } a = GetMonLabels(testLabels) assert.Equal(t, "allval1", a["allkey1"]) @@ -70,6 +71,11 @@ func TestCephLabelsMerge(t *testing.T) { assert.Equal(t, "allval1", a["allkey1"]) assert.Equal(t, "allval2", a["allkey2"]) assert.Equal(t, 3, len(a)) + a = GetCmdReporterLabels(testLabels) + assert.Equal(t, "myversion", a["detect"]) + assert.Equal(t, "allval1", a["allkey1"]) + assert.Equal(t, "allval2", a["allkey2"]) + assert.Equal(t, 3, len(a)) } func TestLabelsSpec(t *testing.T) { diff --git a/pkg/operator/ceph/controller/network.go b/pkg/operator/ceph/controller/network.go index 4c64d37ab..b97cb454a 100644 --- a/pkg/operator/ceph/controller/network.go +++ b/pkg/operator/ceph/controller/network.go @@ -264,6 +264,8 @@ func discoverAddressRanges( job.Spec.Template.Annotations = map[string]string{ nadv1.NetworkAttachmentAnnot: netSelectionValue, } + cephv1.GetCmdReporterAnnotations(clusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(clusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) // use osd placement for net canaries b/c osd pods are present on both public and cluster nets cephv1.GetOSDPlacement(clusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) diff --git a/pkg/operator/ceph/controller/version.go b/pkg/operator/ceph/controller/version.go index 1585cef77..915672fda 100644 --- a/pkg/operator/ceph/controller/version.go +++ b/pkg/operator/ceph/controller/version.go @@ -86,6 +86,9 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string cephv1.GetMonPlacement(cephClusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) job.Spec.Template.Spec.Affinity.PodAntiAffinity = nil + cephv1.GetCmdReporterAnnotations(cephClusterSpec.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(cephClusterSpec.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + stdout, stderr, retcode, err := versionReporter.Run(ctx, detectCephVersionTimeout) if err != nil { return nil, errors.Wrap(err, "failed to complete ceph version job") diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index 87070a2a4..63a2fd638 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -56,6 +56,8 @@ type ReconcileCSI struct { opManagerContext context.Context opConfig opcontroller.OperatorConfig clustersWithHolder []ClusterDetail + // the first cluster CR which will determine some settings for the csi driver + firstCephCluster *cephv1.ClusterSpec } // ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like @@ -255,6 +257,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, nil } + if r.firstCephCluster == nil { + r.firstCephCluster = &cephClusters.Items[i].Spec + } + // Load cluster info for later use in updating the ceph-csi configmap clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace, &cephClusters.Items[i].Spec) if err != nil { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 636601522..5d7f7fa4b 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -24,6 +24,7 @@ import ( "strings" "time" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/cluster/telemetry" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" @@ -784,6 +785,10 @@ func (r *ReconcileCSI) validateCSIVersion(ownerInfo *k8sutil.OwnerInfo) (*CephCS job.Spec.Template.Spec.Affinity = &corev1.Affinity{ NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), } + if r.firstCephCluster != nil { + cephv1.GetCmdReporterAnnotations(r.firstCephCluster.Annotations).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + cephv1.GetCmdReporterLabels(r.firstCephCluster.Labels).ApplyToObjectMeta(&job.Spec.Template.ObjectMeta) + } stdout, _, retcode, err := versionReporter.Run(r.opManagerContext, timeout) if err != nil {