ceph: labels are immutable for match selectors

The labels cannot be updated on a deployment match selector.
In v1.4 a new label was added for ceph_daemon_type that is intended
to be on the pod labels, but cannot be applied to the match selectors.
Therefore, we suppress any new labels that are added to the daemons.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2020-08-04 16:47:00 -06:00
1 parent 772b7990db
commit 06c21954e1
11 files changed
+43 -40

No files matched your search

+5 -5
View File
@@ -50,7 +50,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: mgrConfig.ResourceName,
Labels: c.getPodLabels(mgrConfig.DaemonID),
Labels: c.getPodLabels(mgrConfig.DaemonID, true),
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
@@ -96,11 +96,11 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
ObjectMeta: metav1.ObjectMeta{
Name: mgrConfig.ResourceName,
Namespace: c.clusterInfo.Namespace,
Labels: c.getPodLabels(mgrConfig.DaemonID),
Labels: c.getPodLabels(mgrConfig.DaemonID, true),
},
Spec: apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: c.getPodLabels(mgrConfig.DaemonID),
MatchLabels: c.getPodLabels(mgrConfig.DaemonID, false),
},
Template: podSpec,
Replicas: &replicas,
@@ -322,8 +322,8 @@ func (c *Cluster) makeDashboardService(name string) *v1.Service {
return svc
}
func (c *Cluster) getPodLabels(daemonName string) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName)
func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName, includeNewLabels)
// leave "instance" key for legacy usage
labels["instance"] = daemonName
return labels
+1 -1
View File
@@ -27,7 +27,7 @@ import (
)
func (c *Cluster) createService(mon *monConfig) (string, error) {
labels := c.getLabels(mon.DaemonName, false, "")
labels := c.getLabels(mon.DaemonName, false, "", true)
svcDef := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: mon.ResourceName,
+6 -6
View File
@@ -43,10 +43,10 @@ const (
monmapFile = "monmap"
)
func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string) map[string]string {
func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string, includeNewLabels bool) map[string]string {
// Mons have a service for each mon, so the additional pod data is relevant for its services
// Use pod labels to keep "mon: id" for legacy
labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName)
labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName, includeNewLabels)
// Add "mon_cluster: <namespace>" for legacy
labels[monClusterAttr] = c.Namespace
if canary {
@@ -64,7 +64,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo
ObjectMeta: metav1.ObjectMeta{
Name: monConfig.ResourceName,
Namespace: c.Namespace,
Labels: c.getLabels(monConfig.DaemonName, canary, ""),
Labels: c.getLabels(monConfig.DaemonName, canary, "", true),
},
}
k8sutil.AddRookVersionLabelToDeployment(d)
@@ -79,7 +79,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo
replicaCount := int32(1)
d.Spec = apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: c.getLabels(monConfig.DaemonName, canary, ""),
MatchLabels: c.getLabels(monConfig.DaemonName, canary, "", false),
},
Template: v1.PodTemplateSpec{
ObjectMeta: pod.ObjectMeta,
@@ -101,7 +101,7 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*v1.PersistentVo
ObjectMeta: metav1.ObjectMeta{
Name: m.ResourceName,
Namespace: c.Namespace,
Labels: c.getLabels(m.DaemonName, canary, m.ResourceName),
Labels: c.getLabels(m.DaemonName, canary, m.ResourceName, true),
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{
@@ -167,7 +167,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string)
ObjectMeta: metav1.ObjectMeta{
Name: monConfig.ResourceName,
Namespace: c.Namespace,
Labels: c.getLabels(monConfig.DaemonName, canary, PVCName),
Labels: c.getLabels(monConfig.DaemonName, canary, PVCName, true),
},
Spec: podSpec,
}
+1 -2
View File
@@ -44,8 +44,7 @@ func makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClas
func (c *Cluster) getOSDLabels(osdID int, failureDomainValue string, portable bool) map[string]string {
stringID := fmt.Sprintf("%d", osdID)
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID)
// Add "ceph-osd-id: <id>" for legacy
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID, true)
labels[OsdIdLabelKey] = stringID
labels[FailureDomainKey] = failureDomainValue
labels[portableKey] = strconv.FormatBool(portable)
+2 -2
View File
@@ -31,7 +31,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: daemonConfig.ResourceName,
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID),
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true),
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
@@ -64,7 +64,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM
Name: daemonConfig.ResourceName,
Namespace: rbdMirror.Namespace,
Annotations: rbdMirror.Spec.Annotations,
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID),
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true),
},
Spec: apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
+6 -2
View File
@@ -310,9 +310,13 @@ func AppLabels(appName, namespace string) map[string]string {
// App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc.
// Daemon type is the Ceph daemon type: "mon", "mgr", "osd", "mds", "rgw"
// Daemon ID is the ID portion of the Ceph daemon name: "a" for "mon.a"; "c" for "mds.c"
func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string) map[string]string {
func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string, includeNewLabels bool) map[string]string {
labels := AppLabels(appName, namespace)
labels["ceph_daemon_type"] = daemonType
// New labels cannot be applied to match selectors during upgrade
if includeNewLabels {
labels["ceph_daemon_type"] = daemonType
}
labels["ceph_daemon_id"] = daemonID
// Also report the daemon id keyed by its daemon type: "mon: a", "mds: c", etc.
labels[daemonType] = daemonID
+5 -5
View File
@@ -42,7 +42,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error)
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: mdsConfig.ResourceName,
Labels: c.podLabels(mdsConfig),
Labels: c.podLabels(mdsConfig, true),
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
@@ -68,11 +68,11 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error)
ObjectMeta: metav1.ObjectMeta{
Name: mdsConfig.ResourceName,
Namespace: c.fs.Namespace,
Labels: c.podLabels(mdsConfig),
Labels: c.podLabels(mdsConfig, true),
},
Spec: apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: podSpec.Labels,
MatchLabels: c.podLabels(mdsConfig, false),
},
Template: podSpec,
Replicas: &replicas,
@@ -132,8 +132,8 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
return container
}
func (c *Cluster) podLabels(mdsConfig *mdsConfig) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID)
func (c *Cluster) podLabels(mdsConfig *mdsConfig, includeNewLabels bool) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID, includeNewLabels)
labels["rook_file_system"] = c.fs.Name
return labels
}
+1 -1
View File
@@ -175,7 +175,7 @@ func (r *ReconcileCephNFS) generateConfigMap(n *cephv1.CephNFS, name string) *v1
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s-%s", AppName, n.Name, name),
Namespace: n.Namespace,
Labels: getLabels(n, name),
Labels: getLabels(n, name, true),
},
Data: data,
}
+6 -6
View File
@@ -41,7 +41,7 @@ const (
)
func (r *ReconcileCephNFS) generateCephNFSService(nfs *cephv1.CephNFS, cfg daemonConfig) *v1.Service {
labels := getLabels(nfs, cfg.ID)
labels := getLabels(nfs, cfg.ID, true)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
@@ -96,7 +96,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
ObjectMeta: metav1.ObjectMeta{
Name: instanceName(nfs, cfg.ID),
Namespace: nfs.Namespace,
Labels: getLabels(nfs, cfg.ID),
Labels: getLabels(nfs, cfg.ID, true),
},
}
k8sutil.AddRookVersionLabelToDeployment(deployment)
@@ -139,7 +139,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: instanceName(nfs, cfg.ID),
Labels: getLabels(nfs, cfg.ID),
Labels: getLabels(nfs, cfg.ID, true),
},
Spec: podSpec,
}
@@ -158,7 +158,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
replicas := int32(1)
deployment.Spec = apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: getLabels(nfs, cfg.ID),
MatchLabels: getLabels(nfs, cfg.ID, false),
},
Template: podTemplateSpec,
Replicas: &replicas,
@@ -239,8 +239,8 @@ func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container {
}
}
func getLabels(n *cephv1.CephNFS, name string) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name)
func getLabels(n *cephv1.CephNFS, name string, includeNewLabels bool) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name, includeNewLabels)
labels["ceph_nfs"] = n.Name
labels["instance"] = name
return labels
+9 -9
View File
@@ -47,11 +47,11 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment
ObjectMeta: metav1.ObjectMeta{
Name: rgwConfig.ResourceName,
Namespace: c.store.Namespace,
Labels: getLabels(c.store.Name, c.store.Namespace),
Labels: getLabels(c.store.Name, c.store.Namespace, true),
},
Spec: apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: getLabels(c.store.Name, c.store.Namespace),
MatchLabels: getLabels(c.store.Name, c.store.Namespace, false),
},
Template: pod,
Replicas: &replicas,
@@ -106,13 +106,13 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
// If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons
preferredDuringScheduling := true
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, getLabels(c.store.Name, c.store.Namespace),
nil)
labels := getLabels(c.store.Name, c.store.Namespace, false)
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, labels, nil)
podTemplateSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: rgwConfig.ResourceName,
Labels: getLabels(c.store.Name, c.store.Namespace),
Labels: getLabels(c.store.Name, c.store.Namespace, true),
},
Spec: podSpec,
}
@@ -239,7 +239,7 @@ func (c *clusterConfig) generateLiveProbePort() intstr.IntOrString {
}
func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) *v1.Service {
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace)
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: instanceName(cephObjectStore.Name),
@@ -268,7 +268,7 @@ func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore)
}
func (c *clusterConfig) generateEndpoint(cephObjectStore *cephv1.CephObjectStore) *v1.Endpoints {
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace)
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true)
endpoints := &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
@@ -348,8 +348,8 @@ func addPortToEndpoint(endpoints *v1.Endpoints, name string, port int32) {
)
}
func getLabels(name, namespace string) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name)
func getLabels(name, namespace string, includeNewLabels bool) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name, includeNewLabels)
labels["rook_object_store"] = name
return labels
}
+1 -1
View File
@@ -75,7 +75,7 @@ func TestPodSpecs(t *testing.T) {
1,
len(s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution))
assert.Equal(t,
getLabels(c.store.Name, c.store.Namespace),
getLabels(c.store.Name, c.store.Namespace, false),
s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels)
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)