From 06c21954e123aa3bb0001bf89e958ea22c012a8d Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Tue, 4 Aug 2020 15:44:12 -0600 Subject: [PATCH] ceph: labels are immutable for match selectors The labels cannot be updated on a deployment match selector. In v1.4 a new label was added for ceph_daemon_type that is intended to be on the pod labels, but cannot be applied to the match selectors. Therefore, we suppress any new labels that are added to the daemons. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/mgr/spec.go | 10 +++++----- pkg/operator/ceph/cluster/mon/service.go | 2 +- pkg/operator/ceph/cluster/mon/spec.go | 12 ++++++------ pkg/operator/ceph/cluster/osd/labels.go | 3 +-- pkg/operator/ceph/cluster/rbd/spec.go | 4 ++-- pkg/operator/ceph/controller/spec.go | 8 ++++++-- pkg/operator/ceph/file/mds/spec.go | 10 +++++----- pkg/operator/ceph/nfs/nfs.go | 2 +- pkg/operator/ceph/nfs/spec.go | 12 ++++++------ pkg/operator/ceph/object/spec.go | 18 +++++++++--------- pkg/operator/ceph/object/spec_test.go | 2 +- 11 files changed, 43 insertions(+), 40 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index cf360d7b1..6aaaab4fb 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -50,7 +50,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mgrConfig.ResourceName, - Labels: c.getPodLabels(mgrConfig.DaemonID), + Labels: c.getPodLabels(mgrConfig.DaemonID, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -96,11 +96,11 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) ObjectMeta: metav1.ObjectMeta{ Name: mgrConfig.ResourceName, Namespace: c.clusterInfo.Namespace, - Labels: c.getPodLabels(mgrConfig.DaemonID), + Labels: c.getPodLabels(mgrConfig.DaemonID, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: c.getPodLabels(mgrConfig.DaemonID), + MatchLabels: c.getPodLabels(mgrConfig.DaemonID, false), }, Template: podSpec, Replicas: &replicas, @@ -322,8 +322,8 @@ func (c *Cluster) makeDashboardService(name string) *v1.Service { return svc } -func (c *Cluster) getPodLabels(daemonName string) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName) +func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName, includeNewLabels) // leave "instance" key for legacy usage labels["instance"] = daemonName return labels diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 7f13fe12e..54c8fe404 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -27,7 +27,7 @@ import ( ) func (c *Cluster) createService(mon *monConfig) (string, error) { - labels := c.getLabels(mon.DaemonName, false, "") + labels := c.getLabels(mon.DaemonName, false, "", true) svcDef := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: mon.ResourceName, diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 3fc514adb..b8b29176f 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -43,10 +43,10 @@ const ( monmapFile = "monmap" ) -func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string) map[string]string { +func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string, includeNewLabels bool) map[string]string { // Mons have a service for each mon, so the additional pod data is relevant for its services // Use pod labels to keep "mon: id" for legacy - labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName) + labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName, includeNewLabels) // Add "mon_cluster: " for legacy labels[monClusterAttr] = c.Namespace if canary { @@ -64,7 +64,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, Namespace: c.Namespace, - Labels: c.getLabels(monConfig.DaemonName, canary, ""), + Labels: c.getLabels(monConfig.DaemonName, canary, "", true), }, } k8sutil.AddRookVersionLabelToDeployment(d) @@ -79,7 +79,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo replicaCount := int32(1) d.Spec = apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: c.getLabels(monConfig.DaemonName, canary, ""), + MatchLabels: c.getLabels(monConfig.DaemonName, canary, "", false), }, Template: v1.PodTemplateSpec{ ObjectMeta: pod.ObjectMeta, @@ -101,7 +101,7 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*v1.PersistentVo ObjectMeta: metav1.ObjectMeta{ Name: m.ResourceName, Namespace: c.Namespace, - Labels: c.getLabels(m.DaemonName, canary, m.ResourceName), + Labels: c.getLabels(m.DaemonName, canary, m.ResourceName, true), }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ @@ -167,7 +167,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string) ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, Namespace: c.Namespace, - Labels: c.getLabels(monConfig.DaemonName, canary, PVCName), + Labels: c.getLabels(monConfig.DaemonName, canary, PVCName, true), }, Spec: podSpec, } diff --git a/pkg/operator/ceph/cluster/osd/labels.go b/pkg/operator/ceph/cluster/osd/labels.go index c0764d16b..e3800fd81 100644 --- a/pkg/operator/ceph/cluster/osd/labels.go +++ b/pkg/operator/ceph/cluster/osd/labels.go @@ -44,8 +44,7 @@ func makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClas func (c *Cluster) getOSDLabels(osdID int, failureDomainValue string, portable bool) map[string]string { stringID := fmt.Sprintf("%d", osdID) - labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID) - // Add "ceph-osd-id: " for legacy + labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID, true) labels[OsdIdLabelKey] = stringID labels[FailureDomainKey] = failureDomainValue labels[portableKey] = strconv.FormatBool(portable) diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index f94f4219e..bbce315b6 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -31,7 +31,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: daemonConfig.ResourceName, - Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID), + Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -64,7 +64,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM Name: daemonConfig.ResourceName, Namespace: rbdMirror.Namespace, Annotations: rbdMirror.Spec.Annotations, - Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID), + Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index c4791deed..e48f0d3a1 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -310,9 +310,13 @@ func AppLabels(appName, namespace string) map[string]string { // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc. // Daemon type is the Ceph daemon type: "mon", "mgr", "osd", "mds", "rgw" // Daemon ID is the ID portion of the Ceph daemon name: "a" for "mon.a"; "c" for "mds.c" -func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string) map[string]string { +func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string, includeNewLabels bool) map[string]string { labels := AppLabels(appName, namespace) - labels["ceph_daemon_type"] = daemonType + + // New labels cannot be applied to match selectors during upgrade + if includeNewLabels { + labels["ceph_daemon_type"] = daemonType + } labels["ceph_daemon_id"] = daemonID // Also report the daemon id keyed by its daemon type: "mon: a", "mds: c", etc. labels[daemonType] = daemonID diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 311a6420e..035b419af 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -42,7 +42,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error) podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, - Labels: c.podLabels(mdsConfig), + Labels: c.podLabels(mdsConfig, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -68,11 +68,11 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error) ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, Namespace: c.fs.Namespace, - Labels: c.podLabels(mdsConfig), + Labels: c.podLabels(mdsConfig, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: podSpec.Labels, + MatchLabels: c.podLabels(mdsConfig, false), }, Template: podSpec, Replicas: &replicas, @@ -132,8 +132,8 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { return container } -func (c *Cluster) podLabels(mdsConfig *mdsConfig) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID) +func (c *Cluster) podLabels(mdsConfig *mdsConfig, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID, includeNewLabels) labels["rook_file_system"] = c.fs.Name return labels } diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index be052b735..72af6cc30 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -175,7 +175,7 @@ func (r *ReconcileCephNFS) generateConfigMap(n *cephv1.CephNFS, name string) *v1 ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s-%s", AppName, n.Name, name), Namespace: n.Namespace, - Labels: getLabels(n, name), + Labels: getLabels(n, name, true), }, Data: data, } diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index feef4b5be..3c3272a9d 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -41,7 +41,7 @@ const ( ) func (r *ReconcileCephNFS) generateCephNFSService(nfs *cephv1.CephNFS, cfg daemonConfig) *v1.Service { - labels := getLabels(nfs, cfg.ID) + labels := getLabels(nfs, cfg.ID, true) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -96,7 +96,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) ObjectMeta: metav1.ObjectMeta{ Name: instanceName(nfs, cfg.ID), Namespace: nfs.Namespace, - Labels: getLabels(nfs, cfg.ID), + Labels: getLabels(nfs, cfg.ID, true), }, } k8sutil.AddRookVersionLabelToDeployment(deployment) @@ -139,7 +139,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: instanceName(nfs, cfg.ID), - Labels: getLabels(nfs, cfg.ID), + Labels: getLabels(nfs, cfg.ID, true), }, Spec: podSpec, } @@ -158,7 +158,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) replicas := int32(1) deployment.Spec = apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: getLabels(nfs, cfg.ID), + MatchLabels: getLabels(nfs, cfg.ID, false), }, Template: podTemplateSpec, Replicas: &replicas, @@ -239,8 +239,8 @@ func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { } } -func getLabels(n *cephv1.CephNFS, name string) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name) +func getLabels(n *cephv1.CephNFS, name string, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name, includeNewLabels) labels["ceph_nfs"] = n.Name labels["instance"] = name return labels diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index fd6cde97d..0f597d86b 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -47,11 +47,11 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment ObjectMeta: metav1.ObjectMeta{ Name: rgwConfig.ResourceName, Namespace: c.store.Namespace, - Labels: getLabels(c.store.Name, c.store.Namespace), + Labels: getLabels(c.store.Name, c.store.Namespace, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: getLabels(c.store.Name, c.store.Namespace), + MatchLabels: getLabels(c.store.Name, c.store.Namespace, false), }, Template: pod, Replicas: &replicas, @@ -106,13 +106,13 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec // If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons preferredDuringScheduling := true - k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, getLabels(c.store.Name, c.store.Namespace), - nil) + labels := getLabels(c.store.Name, c.store.Namespace, false) + k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, labels, nil) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: rgwConfig.ResourceName, - Labels: getLabels(c.store.Name, c.store.Namespace), + Labels: getLabels(c.store.Name, c.store.Namespace, true), }, Spec: podSpec, } @@ -239,7 +239,7 @@ func (c *clusterConfig) generateLiveProbePort() intstr.IntOrString { } func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) *v1.Service { - labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace) + labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: instanceName(cephObjectStore.Name), @@ -268,7 +268,7 @@ func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) } func (c *clusterConfig) generateEndpoint(cephObjectStore *cephv1.CephObjectStore) *v1.Endpoints { - labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace) + labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true) endpoints := &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ @@ -348,8 +348,8 @@ func addPortToEndpoint(endpoints *v1.Endpoints, name string, port int32) { ) } -func getLabels(name, namespace string) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name) +func getLabels(name, namespace string, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name, includeNewLabels) labels["rook_object_store"] = name return labels } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 87203cc5e..d9c1a6734 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -75,7 +75,7 @@ func TestPodSpecs(t *testing.T) { 1, len(s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)) assert.Equal(t, - getLabels(c.store.Name, c.store.Namespace), + getLabels(c.store.Name, c.store.Namespace, false), s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels) podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)