diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index eb7490ef2..d3e22dddd 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -34,3 +34,4 @@ v1.6... up updates and upgrades for larger Ceph clusters * Disable CSI GRPC metrics by default * Monitors failover can be [disabled](Documentation/ceph-mon-health.md#failing-over-a-monitor), this is useful for monitor going under planned maintenance where automatic failover is not desired +* RGW: Rook has started using deployment's replicatset functionality instead of deploying multiple deployments for each rgw diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index 2e37dca65..0eeb85786 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -6399,7 +6399,7 @@ spec: nullable: true type: array instances: - description: The number of pods in the rgw replicaset. If "allNodes" is specified, a daemonset is created. + description: The number of pods in the rgw replicaset. format: int32 minimum: 1 type: integer diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 9b9d0513d..102a413ce 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -6402,7 +6402,7 @@ spec: nullable: true type: array instances: - description: The number of pods in the rgw replicaset. If "allNodes" is specified, a daemonset is created. + description: The number of pods in the rgw replicaset. format: int32 minimum: 1 type: integer diff --git a/cluster/examples/kubernetes/ceph/object-openshift.yaml b/cluster/examples/kubernetes/ceph/object-openshift.yaml index 07fba4426..7ba94661f 100644 --- a/cluster/examples/kubernetes/ceph/object-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/object-openshift.yaml @@ -54,16 +54,22 @@ spec: # securePort: 443 # The number of pods in the rgw deployment instances: 1 - # The affinity rules to apply to the rgw deployment or daemonset. + # The affinity rules to apply to the rgw deployment placement: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: role - # operator: In - # values: - # - rgw-node + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-rgw + # topologyKey: */zone can be used to spread RGW across different AZ + # Use in k8s cluster if your cluster is v1.16 or lower + # Use in k8s cluster is v1.17 or upper + topologyKey: kubernetes.io/hostname # topologySpreadConstraints: # tolerations: # - key: rgw-node diff --git a/cluster/examples/kubernetes/ceph/object.yaml b/cluster/examples/kubernetes/ceph/object.yaml index 3d1a24bf5..2b59c94d4 100644 --- a/cluster/examples/kubernetes/ceph/object.yaml +++ b/cluster/examples/kubernetes/ceph/object.yaml @@ -54,8 +54,23 @@ spec: # securePort: 443 # The number of pods in the rgw deployment instances: 1 - # The affinity rules to apply to the rgw deployment or daemonset. + # The affinity rules to apply to the rgw deployment. placement: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-rgw + # topologyKey: */zone can be used to spread RGW across different AZ + # Use in k8s cluster if your cluster is v1.16 or lower + # Use in k8s cluster is v1.17 or upper + topologyKey: kubernetes.io/hostname + # A key/value list of annotations # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: # nodeSelectorTerms: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index fad83f2c3..aa33bf673 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1062,7 +1062,7 @@ type GatewaySpec struct { // +optional SecurePort int32 `json:"securePort,omitempty"` - // The number of pods in the rgw replicaset. If "allNodes" is specified, a daemonset is created. + // The number of pods in the rgw replicaset. // +kubebuilder:validation:Minimum=1 Instances int32 `json:"instances"` diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 1249a8c8d..a2a1625a3 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -85,6 +85,10 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) // start a new deployment and scale up desiredRgwInstances := int(c.store.Spec.Gateway.Instances) + // If running on Pacific we force a single deployment and later set the deployment replica to the "instances" value + if c.clusterInfo.CephVersion.IsAtLeastPacific() { + desiredRgwInstances = 1 + } for i := 0; i < desiredRgwInstances; i++ { var err error diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 2451845ad..42e02d11a 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -58,6 +58,10 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment return nil, err } replicas := int32(1) + // On Pacific, we can use the same keyring and have dedicated rgw instances reflected in the service map + if c.clusterInfo.CephVersion.IsAtLeastPacific() { + replicas = c.store.Spec.Gateway.Instances + } d := &apps.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: rgwConfig.ResourceName,