diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index cf360d7b1..6aaaab4fb 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -50,7 +50,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mgrConfig.ResourceName, - Labels: c.getPodLabels(mgrConfig.DaemonID), + Labels: c.getPodLabels(mgrConfig.DaemonID, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -96,11 +96,11 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) ObjectMeta: metav1.ObjectMeta{ Name: mgrConfig.ResourceName, Namespace: c.clusterInfo.Namespace, - Labels: c.getPodLabels(mgrConfig.DaemonID), + Labels: c.getPodLabels(mgrConfig.DaemonID, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: c.getPodLabels(mgrConfig.DaemonID), + MatchLabels: c.getPodLabels(mgrConfig.DaemonID, false), }, Template: podSpec, Replicas: &replicas, @@ -322,8 +322,8 @@ func (c *Cluster) makeDashboardService(name string) *v1.Service { return svc } -func (c *Cluster) getPodLabels(daemonName string) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName) +func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName, includeNewLabels) // leave "instance" key for legacy usage labels["instance"] = daemonName return labels diff --git a/pkg/operator/ceph/cluster/mon/service.go b/pkg/operator/ceph/cluster/mon/service.go index 7f13fe12e..54c8fe404 100644 --- a/pkg/operator/ceph/cluster/mon/service.go +++ b/pkg/operator/ceph/cluster/mon/service.go @@ -27,7 +27,7 @@ import ( ) func (c *Cluster) createService(mon *monConfig) (string, error) { - labels := c.getLabels(mon.DaemonName, false, "") + labels := c.getLabels(mon.DaemonName, false, "", true) svcDef := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: mon.ResourceName, diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 3fc514adb..b8b29176f 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -43,10 +43,10 @@ const ( monmapFile = "monmap" ) -func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string) map[string]string { +func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string, includeNewLabels bool) map[string]string { // Mons have a service for each mon, so the additional pod data is relevant for its services // Use pod labels to keep "mon: id" for legacy - labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName) + labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName, includeNewLabels) // Add "mon_cluster: " for legacy labels[monClusterAttr] = c.Namespace if canary { @@ -64,7 +64,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, Namespace: c.Namespace, - Labels: c.getLabels(monConfig.DaemonName, canary, ""), + Labels: c.getLabels(monConfig.DaemonName, canary, "", true), }, } k8sutil.AddRookVersionLabelToDeployment(d) @@ -79,7 +79,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo replicaCount := int32(1) d.Spec = apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: c.getLabels(monConfig.DaemonName, canary, ""), + MatchLabels: c.getLabels(monConfig.DaemonName, canary, "", false), }, Template: v1.PodTemplateSpec{ ObjectMeta: pod.ObjectMeta, @@ -101,7 +101,7 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*v1.PersistentVo ObjectMeta: metav1.ObjectMeta{ Name: m.ResourceName, Namespace: c.Namespace, - Labels: c.getLabels(m.DaemonName, canary, m.ResourceName), + Labels: c.getLabels(m.DaemonName, canary, m.ResourceName, true), }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ @@ -167,7 +167,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string) ObjectMeta: metav1.ObjectMeta{ Name: monConfig.ResourceName, Namespace: c.Namespace, - Labels: c.getLabels(monConfig.DaemonName, canary, PVCName), + Labels: c.getLabels(monConfig.DaemonName, canary, PVCName, true), }, Spec: podSpec, } diff --git a/pkg/operator/ceph/cluster/osd/labels.go b/pkg/operator/ceph/cluster/osd/labels.go index c0764d16b..e3800fd81 100644 --- a/pkg/operator/ceph/cluster/osd/labels.go +++ b/pkg/operator/ceph/cluster/osd/labels.go @@ -44,8 +44,7 @@ func makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClas func (c *Cluster) getOSDLabels(osdID int, failureDomainValue string, portable bool) map[string]string { stringID := fmt.Sprintf("%d", osdID) - labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID) - // Add "ceph-osd-id: " for legacy + labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID, true) labels[OsdIdLabelKey] = stringID labels[FailureDomainKey] = failureDomainValue labels[portableKey] = strconv.FormatBool(portable) diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index f94f4219e..bbce315b6 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -31,7 +31,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: daemonConfig.ResourceName, - Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID), + Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -64,7 +64,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM Name: daemonConfig.ResourceName, Namespace: rbdMirror.Namespace, Annotations: rbdMirror.Spec.Annotations, - Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID), + Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index c4791deed..e48f0d3a1 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -310,9 +310,13 @@ func AppLabels(appName, namespace string) map[string]string { // App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc. // Daemon type is the Ceph daemon type: "mon", "mgr", "osd", "mds", "rgw" // Daemon ID is the ID portion of the Ceph daemon name: "a" for "mon.a"; "c" for "mds.c" -func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string) map[string]string { +func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string, includeNewLabels bool) map[string]string { labels := AppLabels(appName, namespace) - labels["ceph_daemon_type"] = daemonType + + // New labels cannot be applied to match selectors during upgrade + if includeNewLabels { + labels["ceph_daemon_type"] = daemonType + } labels["ceph_daemon_id"] = daemonID // Also report the daemon id keyed by its daemon type: "mon: a", "mds: c", etc. labels[daemonType] = daemonID diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 311a6420e..035b419af 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -42,7 +42,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error) podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, - Labels: c.podLabels(mdsConfig), + Labels: c.podLabels(mdsConfig, true), }, Spec: v1.PodSpec{ InitContainers: []v1.Container{ @@ -68,11 +68,11 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error) ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, Namespace: c.fs.Namespace, - Labels: c.podLabels(mdsConfig), + Labels: c.podLabels(mdsConfig, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: podSpec.Labels, + MatchLabels: c.podLabels(mdsConfig, false), }, Template: podSpec, Replicas: &replicas, @@ -132,8 +132,8 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { return container } -func (c *Cluster) podLabels(mdsConfig *mdsConfig) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID) +func (c *Cluster) podLabels(mdsConfig *mdsConfig, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID, includeNewLabels) labels["rook_file_system"] = c.fs.Name return labels } diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index be052b735..72af6cc30 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -175,7 +175,7 @@ func (r *ReconcileCephNFS) generateConfigMap(n *cephv1.CephNFS, name string) *v1 ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s-%s", AppName, n.Name, name), Namespace: n.Namespace, - Labels: getLabels(n, name), + Labels: getLabels(n, name, true), }, Data: data, } diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index feef4b5be..3c3272a9d 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -41,7 +41,7 @@ const ( ) func (r *ReconcileCephNFS) generateCephNFSService(nfs *cephv1.CephNFS, cfg daemonConfig) *v1.Service { - labels := getLabels(nfs, cfg.ID) + labels := getLabels(nfs, cfg.ID, true) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -96,7 +96,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) ObjectMeta: metav1.ObjectMeta{ Name: instanceName(nfs, cfg.ID), Namespace: nfs.Namespace, - Labels: getLabels(nfs, cfg.ID), + Labels: getLabels(nfs, cfg.ID, true), }, } k8sutil.AddRookVersionLabelToDeployment(deployment) @@ -139,7 +139,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: instanceName(nfs, cfg.ID), - Labels: getLabels(nfs, cfg.ID), + Labels: getLabels(nfs, cfg.ID, true), }, Spec: podSpec, } @@ -158,7 +158,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig) replicas := int32(1) deployment.Spec = apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: getLabels(nfs, cfg.ID), + MatchLabels: getLabels(nfs, cfg.ID, false), }, Template: podTemplateSpec, Replicas: &replicas, @@ -239,8 +239,8 @@ func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { } } -func getLabels(n *cephv1.CephNFS, name string) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name) +func getLabels(n *cephv1.CephNFS, name string, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name, includeNewLabels) labels["ceph_nfs"] = n.Name labels["instance"] = name return labels diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index fd6cde97d..0f597d86b 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -47,11 +47,11 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment ObjectMeta: metav1.ObjectMeta{ Name: rgwConfig.ResourceName, Namespace: c.store.Namespace, - Labels: getLabels(c.store.Name, c.store.Namespace), + Labels: getLabels(c.store.Name, c.store.Namespace, true), }, Spec: apps.DeploymentSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: getLabels(c.store.Name, c.store.Namespace), + MatchLabels: getLabels(c.store.Name, c.store.Namespace, false), }, Template: pod, Replicas: &replicas, @@ -106,13 +106,13 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec // If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons preferredDuringScheduling := true - k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, getLabels(c.store.Name, c.store.Namespace), - nil) + labels := getLabels(c.store.Name, c.store.Namespace, false) + k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, labels, nil) podTemplateSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: rgwConfig.ResourceName, - Labels: getLabels(c.store.Name, c.store.Namespace), + Labels: getLabels(c.store.Name, c.store.Namespace, true), }, Spec: podSpec, } @@ -239,7 +239,7 @@ func (c *clusterConfig) generateLiveProbePort() intstr.IntOrString { } func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) *v1.Service { - labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace) + labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: instanceName(cephObjectStore.Name), @@ -268,7 +268,7 @@ func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) } func (c *clusterConfig) generateEndpoint(cephObjectStore *cephv1.CephObjectStore) *v1.Endpoints { - labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace) + labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true) endpoints := &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ @@ -348,8 +348,8 @@ func addPortToEndpoint(endpoints *v1.Endpoints, name string, port int32) { ) } -func getLabels(name, namespace string) map[string]string { - labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name) +func getLabels(name, namespace string, includeNewLabels bool) map[string]string { + labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name, includeNewLabels) labels["rook_object_store"] = name return labels } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 87203cc5e..d9c1a6734 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -75,7 +75,7 @@ func TestPodSpecs(t *testing.T) { 1, len(s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)) assert.Equal(t, - getLabels(c.store.Name, c.store.Namespace), + getLabels(c.store.Name, c.store.Namespace, false), s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels) podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)