Merge pull request #7622 from leseb/rgw-single-deployment

ceph: use replicaset instead of deployments for rgw instances
This commit is contained in:
Sébastien Han authored and GitHub committed 2021-04-15 09:43:55 +02:00
commit b6eea0cb33
8 files changed
+43 -13

No files matched your search

+1
View File
@@ -34,3 +34,4 @@ v1.6...
up updates and upgrades for larger Ceph clusters
* Disable CSI GRPC metrics by default
* Monitors failover can be [disabled](Documentation/ceph-mon-health.md#failing-over-a-monitor), this is useful for monitor going under planned maintenance where automatic failover is not desired
* RGW: Rook has started using deployment's replicatset functionality instead of deploying multiple deployments for each rgw
@@ -6399,7 +6399,7 @@ spec:
nullable: true
type: array
instances:
description: The number of pods in the rgw replicaset. If "allNodes" is specified, a daemonset is created.
description: The number of pods in the rgw replicaset.
format: int32
minimum: 1
type: integer
+1 -1
View File
@@ -6402,7 +6402,7 @@ spec:
nullable: true
type: array
instances:
description: The number of pods in the rgw replicaset. If "allNodes" is specified, a daemonset is created.
description: The number of pods in the rgw replicaset.
format: int32
minimum: 1
type: integer
@@ -54,16 +54,22 @@ spec:
# securePort: 443
# The number of pods in the rgw deployment
instances: 1
# The affinity rules to apply to the rgw deployment or daemonset.
# The affinity rules to apply to the rgw deployment
placement:
# nodeAffinity:
# requiredDuringSchedulingIgnoredDuringExecution:
# nodeSelectorTerms:
# - matchExpressions:
# - key: role
# operator: In
# values:
# - rgw-node
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- rook-ceph-rgw
# topologyKey: */zone can be used to spread RGW across different AZ
# Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
# Use <topologyKey: topology.kubernetes.io/zone> in k8s cluster is v1.17 or upper
topologyKey: kubernetes.io/hostname
# topologySpreadConstraints:
# tolerations:
# - key: rgw-node
+16 -1
View File
@@ -54,8 +54,23 @@ spec:
# securePort: 443
# The number of pods in the rgw deployment
instances: 1
# The affinity rules to apply to the rgw deployment or daemonset.
# The affinity rules to apply to the rgw deployment.
placement:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: app
operator: In
values:
- rook-ceph-rgw
# topologyKey: */zone can be used to spread RGW across different AZ
# Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
# Use <topologyKey: topology.kubernetes.io/zone> in k8s cluster is v1.17 or upper
topologyKey: kubernetes.io/hostname
# A key/value list of annotations
# nodeAffinity:
# requiredDuringSchedulingIgnoredDuringExecution:
# nodeSelectorTerms:
+1 -1
View File
@@ -1062,7 +1062,7 @@ type GatewaySpec struct {
// +optional
SecurePort int32 `json:"securePort,omitempty"`
// The number of pods in the rgw replicaset. If "allNodes" is specified, a daemonset is created.
// The number of pods in the rgw replicaset.
// +kubebuilder:validation:Minimum=1
Instances int32 `json:"instances"`
+4
View File
@@ -85,6 +85,10 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string)
// start a new deployment and scale up
desiredRgwInstances := int(c.store.Spec.Gateway.Instances)
// If running on Pacific we force a single deployment and later set the deployment replica to the "instances" value
if c.clusterInfo.CephVersion.IsAtLeastPacific() {
desiredRgwInstances = 1
}
for i := 0; i < desiredRgwInstances; i++ {
var err error
+4
View File
@@ -58,6 +58,10 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment
return nil, err
}
replicas := int32(1)
// On Pacific, we can use the same keyring and have dedicated rgw instances reflected in the service map
if c.clusterInfo.CephVersion.IsAtLeastPacific() {
replicas = c.store.Spec.Gateway.Instances
}
d := &apps.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: rgwConfig.ResourceName,