diff --git a/cluster/charts/rook-ceph/templates/resources.yaml b/cluster/charts/rook-ceph/templates/resources.yaml index bb478ba73..10af4cd5f 100644 --- a/cluster/charts/rook-ceph/templates/resources.yaml +++ b/cluster/charts/rook-ceph/templates/resources.yaml @@ -6557,6 +6557,10 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + caBundleRef: + description: The name of the secret that stores custom ca-bundle with root and intermediate certificates. + nullable: true + type: string externalRgwEndpoints: description: ExternalRgwEndpoints points to external rgw endpoint(s) items: diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 00065717b..814e5e7dd 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -6552,6 +6552,10 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + caBundleRef: + description: The name of the secret that stores custom ca-bundle with root and intermediate certificates. + nullable: true + type: string externalRgwEndpoints: description: ExternalRgwEndpoints points to external rgw endpoint(s) items: diff --git a/cluster/examples/kubernetes/ceph/object.yaml b/cluster/examples/kubernetes/ceph/object.yaml index 7f64026dc..4fd04a387 100644 --- a/cluster/examples/kubernetes/ceph/object.yaml +++ b/cluster/examples/kubernetes/ceph/object.yaml @@ -46,6 +46,8 @@ spec: gateway: # A reference to the secret in the rook namespace where the ssl certificate is stored # sslCertificateRef: + # A reference to the secret in the rook namespace where the ca bundle is stored + # caBundleRef: # The port that RGW pods will listen on (http) port: 80 # The port that RGW pods will listen on (https). An ssl certificate is required. diff --git a/design/ceph/object/store.md b/design/ceph/object/store.md index e34316440..bfac0bf73 100644 --- a/design/ceph/object/store.md +++ b/design/ceph/object/store.md @@ -80,6 +80,7 @@ If there is a `zone` section in object-store configuration, then the pool sectio The gateway settings correspond to the RGW service. - `type`: Can be `s3`. In the future support for `swift` can be added. - `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." If the certificate is not specified, SSL will not be configured. +- `caBundleRef`: If specified, this is the name of the Kubernetes secret (type `opaque`) that contains ca-bundle to use. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cabundle` key name. - `port`: The service port where the RGW service will be listening (http) - `securePort`: The service port where the RGW service will be listening (https) - `instances`: The number of RGW pods that will be started for this object store diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 29141e436..6bb4b7950 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1319,6 +1319,11 @@ type GatewaySpec struct { // +optional SSLCertificateRef string `json:"sslCertificateRef,omitempty"` + // The name of the secret that stores custom ca-bundle with root and intermediate certificates. + // +nullable + // +optional + CaBundleRef string `json:"caBundleRef,omitempty"` + // The affinity to place the rgw pods (default is to place on any available node) // +kubebuilder:pruning:PreserveUnknownFields // +nullable diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index be60afe25..6f2456e18 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -37,14 +37,21 @@ caps mon = "allow rw" caps osd = "allow rwx" ` - certVolumeName = "rook-ceph-rgw-cert" - certDir = "/etc/ceph/private" - certKeyName = "cert" - certFilename = "rgw-cert.pem" - certKeyFileName = "rgw-key.pem" - rgwPortInternalPort int32 = 8080 - ServiceServingCertCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" - HttpTimeOut = time.Second * 15 + caBundleVolumeName = "rook-ceph-custom-ca-bundle" + caBundleUpdatedVolumeName = "rook-ceph-ca-bundle-updated" + caBundleTrustedDir = "/etc/pki/ca-trust/" + caBundleSourceCustomDir = caBundleTrustedDir + "source/anchors/" + caBundleExtractedDir = caBundleTrustedDir + "extracted/" + caBundleKeyName = "cabundle" + caBundleFileName = "custom-ca-bundle.crt" + certVolumeName = "rook-ceph-rgw-cert" + certDir = "/etc/ceph/private" + certKeyName = "cert" + certFilename = "rgw-cert.pem" + certKeyFileName = "rgw-key.pem" + rgwPortInternalPort int32 = 8080 + ServiceServingCertCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + HttpTimeOut = time.Second * 15 ) var ( @@ -72,7 +79,7 @@ func (c *clusterConfig) portString() string { portString = fmt.Sprintf("ssl_port=%d ssl_certificate=%s", c.store.Spec.Gateway.SecurePort, certPath) } - secretType, _ := c.rgwTLSSecretType() + secretType, _ := c.rgwTLSSecretType(c.store.Spec.Gateway.SSLCertificateRef) if c.store.Spec.GetServiceServingCert() != "" || secretType == v1.SecretTypeTLS { privateKey := path.Join(certDir, certKeyFileName) portString = fmt.Sprintf("%s ssl_private_key=%s", portString, privateKey) diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 9e37b9150..1aef1d5d2 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -131,6 +131,27 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec }} podSpec.Volumes = append(podSpec.Volumes, certVol) } + // Check custom caBundle provided + if c.store.Spec.Gateway.CaBundleRef != "" { + customCaBundleVolSrc, err := c.generateVolumeSourceWithCaBundleSecret() + if err != nil { + return v1.PodTemplateSpec{}, err + } + customCaBundleVol := v1.Volume{ + Name: caBundleVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: customCaBundleVolSrc, + }} + podSpec.Volumes = append(podSpec.Volumes, customCaBundleVol) + updatedCaBundleVol := v1.Volume{ + Name: caBundleUpdatedVolumeName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }} + podSpec.Volumes = append(podSpec.Volumes, updatedCaBundleVol) + podSpec.InitContainers = append(podSpec.InitContainers, + c.createCaBundleUpdateInitContainer(rgwConfig)) + } kmsEnabled, err := c.CheckRGWKMS() if err != nil { return v1.PodTemplateSpec{}, err @@ -170,6 +191,26 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec return podTemplateSpec, nil } +func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) v1.Container { + caBundleMount := v1.VolumeMount{Name: caBundleVolumeName, MountPath: caBundleSourceCustomDir, ReadOnly: true} + volumeMounts := append(controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), caBundleMount) + updatedCaBundleDir := "/tmp/new-ca-bundle/" + updatedBundleMount := v1.VolumeMount{Name: caBundleUpdatedVolumeName, MountPath: updatedCaBundleDir, ReadOnly: false} + volumeMounts = append(volumeMounts, updatedBundleMount) + return v1.Container{ + Name: "update-ca-bundle-initcontainer", + Command: []string{"/bin/bash", "-c"}, + // copy all content of caBundleExtractedDir to avoid directory mount itself + Args: []string{ + fmt.Sprintf("/usr/bin/update-ca-trust extract; cp -rf %s/* %s", caBundleExtractedDir, updatedCaBundleDir), + }, + Image: c.clusterSpec.CephVersion.Image, + VolumeMounts: volumeMounts, + Resources: c.store.Spec.Gateway.Resources, + SecurityContext: controller.PodSecurityContext(), + } +} + // The vault token is passed as Secret for rgw container. So it is mounted as read only. // RGW has restrictions over vault token file, it should owned by same user(ceph) which // rgw daemon runs and all other permission should be nil or zero. Here ownership can be @@ -242,6 +283,10 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container { mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true} container.VolumeMounts = append(container.VolumeMounts, mount) } + if c.store.Spec.Gateway.CaBundleRef != "" { + updatedBundleMount := v1.VolumeMount{Name: caBundleUpdatedVolumeName, MountPath: caBundleExtractedDir, ReadOnly: true} + container.VolumeMounts = append(container.VolumeMounts, updatedBundleMount) + } kmsEnabled, err := c.CheckRGWKMS() if err != nil { logger.Errorf("failed to enable KMS. %v", err) @@ -511,7 +556,7 @@ func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSou secretVolSrc = &v1.SecretVolumeSource{ SecretName: c.store.Spec.Gateway.SSLCertificateRef, } - secretType, err := c.rgwTLSSecretType() + secretType, err := c.rgwTLSSecretType(c.store.Spec.Gateway.SSLCertificateRef) if err != nil { return nil, err } @@ -540,8 +585,28 @@ func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSou return secretVolSrc, nil } -func (c *clusterConfig) rgwTLSSecretType() (v1.SecretType, error) { - rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), c.store.Spec.Gateway.SSLCertificateRef, metav1.GetOptions{}) +func (c *clusterConfig) generateVolumeSourceWithCaBundleSecret() (*v1.SecretVolumeSource, error) { + // Keep the ca-bundle as secure as possible in the container. Give only user read perms. + // Same as above for generateVolumeSourceWithTLSSecret function. + userReadOnly := int32(0400) + caBundleVolSrc := &v1.SecretVolumeSource{ + SecretName: c.store.Spec.Gateway.CaBundleRef, + } + secretType, err := c.rgwTLSSecretType(c.store.Spec.Gateway.CaBundleRef) + if err != nil { + return nil, err + } + if secretType != v1.SecretTypeOpaque { + return nil, errors.New("CaBundle secret should be 'Opaque' type") + } + caBundleVolSrc.Items = []v1.KeyToPath{ + {Key: caBundleKeyName, Path: caBundleFileName, Mode: &userReadOnly}, + } + return caBundleVolSrc, nil +} + +func (c *clusterConfig) rgwTLSSecretType(secretName string) (v1.SecretType, error) { + rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) if rgwTlsSecret != nil { return rgwTlsSecret.Type, nil } diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index ac4bb76d7..f47b7a2bc 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -189,6 +189,24 @@ func TestSSLPodSpec(t *testing.T) { secretVolSrc, err = c.generateVolumeSourceWithTLSSecret() assert.NoError(t, err) assert.Equal(t, secretVolSrc.SecretName, "rgw-cert") + // Using caBundleRef + // Opaque Secret + c.store.Spec.Gateway.CaBundleRef = "mycabundle" + cabundlesecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.store.Spec.Gateway.CaBundleRef, + Namespace: c.store.Namespace, + }, + Data: map[string][]byte{ + "cabundle": []byte("cabundletesting"), + }, + Type: v1.SecretTypeOpaque, + } + _, err = c.context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, cabundlesecret, metav1.CreateOptions{}) + assert.NoError(t, err) + caBundleVolSrc, err := c.generateVolumeSourceWithCaBundleSecret() + assert.NoError(t, err) + assert.Equal(t, caBundleVolSrc.SecretName, "mycabundle") s, err = c.makeRGWPodSpec(rgwConfig) assert.NoError(t, err) podTemplate = cephtest.NewPodTemplateSpecTester(t, &s)