ceph: labels are immutable for match selectors
The labels cannot be updated on a deployment match selector. In v1.4 a new label was added for ceph_daemon_type that is intended to be on the pod labels, but cannot be applied to the match selectors. Therefore, we suppress any new labels that are added to the daemons. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
11 files changed
+43
-40
No files matched your search
@@ -50,7 +50,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mgrConfig.ResourceName,
|
||||
Labels: c.getPodLabels(mgrConfig.DaemonID),
|
||||
Labels: c.getPodLabels(mgrConfig.DaemonID, true),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
@@ -96,11 +96,11 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mgrConfig.ResourceName,
|
||||
Namespace: c.clusterInfo.Namespace,
|
||||
Labels: c.getPodLabels(mgrConfig.DaemonID),
|
||||
Labels: c.getPodLabels(mgrConfig.DaemonID, true),
|
||||
},
|
||||
Spec: apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: c.getPodLabels(mgrConfig.DaemonID),
|
||||
MatchLabels: c.getPodLabels(mgrConfig.DaemonID, false),
|
||||
},
|
||||
Template: podSpec,
|
||||
Replicas: &replicas,
|
||||
@@ -322,8 +322,8 @@ func (c *Cluster) makeDashboardService(name string) *v1.Service {
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *Cluster) getPodLabels(daemonName string) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName)
|
||||
func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName, includeNewLabels)
|
||||
// leave "instance" key for legacy usage
|
||||
labels["instance"] = daemonName
|
||||
return labels
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
)
|
||||
|
||||
func (c *Cluster) createService(mon *monConfig) (string, error) {
|
||||
labels := c.getLabels(mon.DaemonName, false, "")
|
||||
labels := c.getLabels(mon.DaemonName, false, "", true)
|
||||
svcDef := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mon.ResourceName,
|
||||
|
||||
@@ -43,10 +43,10 @@ const (
|
||||
monmapFile = "monmap"
|
||||
)
|
||||
|
||||
func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string) map[string]string {
|
||||
func (c *Cluster) getLabels(daemonName string, canary bool, pvcName string, includeNewLabels bool) map[string]string {
|
||||
// Mons have a service for each mon, so the additional pod data is relevant for its services
|
||||
// Use pod labels to keep "mon: id" for legacy
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName)
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", daemonName, includeNewLabels)
|
||||
// Add "mon_cluster: <namespace>" for legacy
|
||||
labels[monClusterAttr] = c.Namespace
|
||||
if canary {
|
||||
@@ -64,7 +64,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: monConfig.ResourceName,
|
||||
Namespace: c.Namespace,
|
||||
Labels: c.getLabels(monConfig.DaemonName, canary, ""),
|
||||
Labels: c.getLabels(monConfig.DaemonName, canary, "", true),
|
||||
},
|
||||
}
|
||||
k8sutil.AddRookVersionLabelToDeployment(d)
|
||||
@@ -79,7 +79,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo
|
||||
replicaCount := int32(1)
|
||||
d.Spec = apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: c.getLabels(monConfig.DaemonName, canary, ""),
|
||||
MatchLabels: c.getLabels(monConfig.DaemonName, canary, "", false),
|
||||
},
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: pod.ObjectMeta,
|
||||
@@ -101,7 +101,7 @@ func (c *Cluster) makeDeploymentPVC(m *monConfig, canary bool) (*v1.PersistentVo
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: m.ResourceName,
|
||||
Namespace: c.Namespace,
|
||||
Labels: c.getLabels(m.DaemonName, canary, m.ResourceName),
|
||||
Labels: c.getLabels(m.DaemonName, canary, m.ResourceName, true),
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{
|
||||
@@ -167,7 +167,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool, PVCName string)
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: monConfig.ResourceName,
|
||||
Namespace: c.Namespace,
|
||||
Labels: c.getLabels(monConfig.DaemonName, canary, PVCName),
|
||||
Labels: c.getLabels(monConfig.DaemonName, canary, PVCName, true),
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
|
||||
@@ -44,8 +44,7 @@ func makeStorageClassDeviceSetPVCLabel(storageClassDeviceSetName, pvcStorageClas
|
||||
|
||||
func (c *Cluster) getOSDLabels(osdID int, failureDomainValue string, portable bool) map[string]string {
|
||||
stringID := fmt.Sprintf("%d", osdID)
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID)
|
||||
// Add "ceph-osd-id: <id>" for legacy
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "osd", stringID, true)
|
||||
labels[OsdIdLabelKey] = stringID
|
||||
labels[FailureDomainKey] = failureDomainValue
|
||||
labels[portableKey] = strconv.FormatBool(portable)
|
||||
|
||||
@@ -31,7 +31,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: daemonConfig.ResourceName,
|
||||
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID),
|
||||
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
@@ -64,7 +64,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM
|
||||
Name: daemonConfig.ResourceName,
|
||||
Namespace: rbdMirror.Namespace,
|
||||
Annotations: rbdMirror.Spec.Annotations,
|
||||
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID),
|
||||
Labels: controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, true),
|
||||
},
|
||||
Spec: apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
|
||||
@@ -310,9 +310,13 @@ func AppLabels(appName, namespace string) map[string]string {
|
||||
// App name is the name of the application: e.g., 'rook-ceph-mon', 'rook-ceph-mgr', etc.
|
||||
// Daemon type is the Ceph daemon type: "mon", "mgr", "osd", "mds", "rgw"
|
||||
// Daemon ID is the ID portion of the Ceph daemon name: "a" for "mon.a"; "c" for "mds.c"
|
||||
func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string) map[string]string {
|
||||
func CephDaemonAppLabels(appName, namespace, daemonType, daemonID string, includeNewLabels bool) map[string]string {
|
||||
labels := AppLabels(appName, namespace)
|
||||
labels["ceph_daemon_type"] = daemonType
|
||||
|
||||
// New labels cannot be applied to match selectors during upgrade
|
||||
if includeNewLabels {
|
||||
labels["ceph_daemon_type"] = daemonType
|
||||
}
|
||||
labels["ceph_daemon_id"] = daemonID
|
||||
// Also report the daemon id keyed by its daemon type: "mon: a", "mds: c", etc.
|
||||
labels[daemonType] = daemonID
|
||||
|
||||
@@ -42,7 +42,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error)
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mdsConfig.ResourceName,
|
||||
Labels: c.podLabels(mdsConfig),
|
||||
Labels: c.podLabels(mdsConfig, true),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
@@ -68,11 +68,11 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) (*apps.Deployment, error)
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mdsConfig.ResourceName,
|
||||
Namespace: c.fs.Namespace,
|
||||
Labels: c.podLabels(mdsConfig),
|
||||
Labels: c.podLabels(mdsConfig, true),
|
||||
},
|
||||
Spec: apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: podSpec.Labels,
|
||||
MatchLabels: c.podLabels(mdsConfig, false),
|
||||
},
|
||||
Template: podSpec,
|
||||
Replicas: &replicas,
|
||||
@@ -132,8 +132,8 @@ func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
|
||||
return container
|
||||
}
|
||||
|
||||
func (c *Cluster) podLabels(mdsConfig *mdsConfig) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID)
|
||||
func (c *Cluster) podLabels(mdsConfig *mdsConfig, includeNewLabels bool) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID, includeNewLabels)
|
||||
labels["rook_file_system"] = c.fs.Name
|
||||
return labels
|
||||
}
|
||||
|
||||
@@ -175,7 +175,7 @@ func (r *ReconcileCephNFS) generateConfigMap(n *cephv1.CephNFS, name string) *v1
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-%s-%s", AppName, n.Name, name),
|
||||
Namespace: n.Namespace,
|
||||
Labels: getLabels(n, name),
|
||||
Labels: getLabels(n, name, true),
|
||||
},
|
||||
Data: data,
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ const (
|
||||
)
|
||||
|
||||
func (r *ReconcileCephNFS) generateCephNFSService(nfs *cephv1.CephNFS, cfg daemonConfig) *v1.Service {
|
||||
labels := getLabels(nfs, cfg.ID)
|
||||
labels := getLabels(nfs, cfg.ID, true)
|
||||
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -96,7 +96,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(nfs, cfg.ID),
|
||||
Namespace: nfs.Namespace,
|
||||
Labels: getLabels(nfs, cfg.ID),
|
||||
Labels: getLabels(nfs, cfg.ID, true),
|
||||
},
|
||||
}
|
||||
k8sutil.AddRookVersionLabelToDeployment(deployment)
|
||||
@@ -139,7 +139,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(nfs, cfg.ID),
|
||||
Labels: getLabels(nfs, cfg.ID),
|
||||
Labels: getLabels(nfs, cfg.ID, true),
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
@@ -158,7 +158,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
|
||||
replicas := int32(1)
|
||||
deployment.Spec = apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: getLabels(nfs, cfg.ID),
|
||||
MatchLabels: getLabels(nfs, cfg.ID, false),
|
||||
},
|
||||
Template: podTemplateSpec,
|
||||
Replicas: &replicas,
|
||||
@@ -239,8 +239,8 @@ func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container {
|
||||
}
|
||||
}
|
||||
|
||||
func getLabels(n *cephv1.CephNFS, name string) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name)
|
||||
func getLabels(n *cephv1.CephNFS, name string, includeNewLabels bool) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, n.Namespace, "nfs", name, includeNewLabels)
|
||||
labels["ceph_nfs"] = n.Name
|
||||
labels["instance"] = name
|
||||
return labels
|
||||
|
||||
@@ -47,11 +47,11 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: rgwConfig.ResourceName,
|
||||
Namespace: c.store.Namespace,
|
||||
Labels: getLabels(c.store.Name, c.store.Namespace),
|
||||
Labels: getLabels(c.store.Name, c.store.Namespace, true),
|
||||
},
|
||||
Spec: apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: getLabels(c.store.Name, c.store.Namespace),
|
||||
MatchLabels: getLabels(c.store.Name, c.store.Namespace, false),
|
||||
},
|
||||
Template: pod,
|
||||
Replicas: &replicas,
|
||||
@@ -106,13 +106,13 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
|
||||
|
||||
// If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons
|
||||
preferredDuringScheduling := true
|
||||
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, getLabels(c.store.Name, c.store.Namespace),
|
||||
nil)
|
||||
labels := getLabels(c.store.Name, c.store.Namespace, false)
|
||||
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.Gateway.Placement, c.clusterSpec.Network.IsHost(), preferredDuringScheduling, labels, nil)
|
||||
|
||||
podTemplateSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: rgwConfig.ResourceName,
|
||||
Labels: getLabels(c.store.Name, c.store.Namespace),
|
||||
Labels: getLabels(c.store.Name, c.store.Namespace, true),
|
||||
},
|
||||
Spec: podSpec,
|
||||
}
|
||||
@@ -239,7 +239,7 @@ func (c *clusterConfig) generateLiveProbePort() intstr.IntOrString {
|
||||
}
|
||||
|
||||
func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) *v1.Service {
|
||||
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace)
|
||||
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: instanceName(cephObjectStore.Name),
|
||||
@@ -268,7 +268,7 @@ func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore)
|
||||
}
|
||||
|
||||
func (c *clusterConfig) generateEndpoint(cephObjectStore *cephv1.CephObjectStore) *v1.Endpoints {
|
||||
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace)
|
||||
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true)
|
||||
|
||||
endpoints := &v1.Endpoints{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -348,8 +348,8 @@ func addPortToEndpoint(endpoints *v1.Endpoints, name string, port int32) {
|
||||
)
|
||||
}
|
||||
|
||||
func getLabels(name, namespace string) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name)
|
||||
func getLabels(name, namespace string, includeNewLabels bool) map[string]string {
|
||||
labels := controller.CephDaemonAppLabels(AppName, namespace, "rgw", name, includeNewLabels)
|
||||
labels["rook_object_store"] = name
|
||||
return labels
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
1,
|
||||
len(s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution))
|
||||
assert.Equal(t,
|
||||
getLabels(c.store.Name, c.store.Namespace),
|
||||
getLabels(c.store.Name, c.store.Namespace, false),
|
||||
s.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels)
|
||||
|
||||
podTemplate := cephtest.NewPodTemplateSpecTester(t, &s)
|
||||
|
||||
Reference in new issue
Block a user