Merge pull request #8492 from degorenko/rgw-ca-bundle

ceph: add ability to specify ca bundle for rgw

cluster/charts/rook-ceph/templates/resources.yaml

@@ -6557,6 +6557,10 @@ spec:
                       nullable: true
                       type: object
                       x-kubernetes-preserve-unknown-fields: true
+                    caBundleRef:
+                      description: The name of the secret that stores custom ca-bundle with root and intermediate certificates.
+                      nullable: true
+                      type: string
                     externalRgwEndpoints:
                       description: ExternalRgwEndpoints points to external rgw endpoint(s)
                       items:

cluster/examples/kubernetes/ceph/crds.yaml

@@ -6552,6 +6552,10 @@ spec:
                       nullable: true
                       type: object
                       x-kubernetes-preserve-unknown-fields: true
+                    caBundleRef:
+                      description: The name of the secret that stores custom ca-bundle with root and intermediate certificates.
+                      nullable: true
+                      type: string
                     externalRgwEndpoints:
                       description: ExternalRgwEndpoints points to external rgw endpoint(s)
                       items:

cluster/examples/kubernetes/ceph/object.yaml

@@ -46,6 +46,8 @@ spec:
   gateway:
     # A reference to the secret in the rook namespace where the ssl certificate is stored
     # sslCertificateRef:
+    # A reference to the secret in the rook namespace where the ca bundle is stored
+    # caBundleRef:
     # The port that RGW pods will listen on (http)
     port: 80
     # The port that RGW pods will listen on (https). An ssl certificate is required.

design/ceph/object/store.md

@@ -80,6 +80,7 @@ If there is a `zone` section in object-store configuration, then the pool sectio
 The gateway settings correspond to the RGW service.
 - `type`: Can be `s3`. In the future support for `swift` can be added.
 - `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the SSL certificate to be used for secure connections to the object store. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in pem form." If the certificate is not specified, SSL will not be configured.
+- `caBundleRef`: If specified, this is the name of the Kubernetes secret (type `opaque`) that contains ca-bundle to use. The secret must be in the same namespace as the Rook cluster. Rook will look in the secret provided at the `cabundle` key name.
 - `port`: The service port where the RGW service will be listening (http)
 - `securePort`: The service port where the RGW service will be listening (https)
 - `instances`: The number of RGW pods that will be started for this object store

pkg/apis/ceph.rook.io/v1/types.go

@@ -1319,6 +1319,11 @@ type GatewaySpec struct {
 	// +optional
 	SSLCertificateRef string `json:"sslCertificateRef,omitempty"`
 
+	// The name of the secret that stores custom ca-bundle with root and intermediate certificates.
+	// +nullable
+	// +optional
+	CaBundleRef string `json:"caBundleRef,omitempty"`
+
 	// The affinity to place the rgw pods (default is to place on any available node)
 	// +kubebuilder:pruning:PreserveUnknownFields
 	// +nullable

pkg/operator/ceph/object/config.go

@@ -37,14 +37,21 @@ caps mon = "allow rw"
 caps osd = "allow rwx"
 `
 
-	certVolumeName                 = "rook-ceph-rgw-cert"
-	certDir                        = "/etc/ceph/private"
-	certKeyName                    = "cert"
-	certFilename                   = "rgw-cert.pem"
-	certKeyFileName                = "rgw-key.pem"
-	rgwPortInternalPort      int32 = 8080
-	ServiceServingCertCAFile       = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
-	HttpTimeOut                    = time.Second * 15
+	caBundleVolumeName              = "rook-ceph-custom-ca-bundle"
+	caBundleUpdatedVolumeName       = "rook-ceph-ca-bundle-updated"
+	caBundleTrustedDir              = "/etc/pki/ca-trust/"
+	caBundleSourceCustomDir         = caBundleTrustedDir + "source/anchors/"
+	caBundleExtractedDir            = caBundleTrustedDir + "extracted/"
+	caBundleKeyName                 = "cabundle"
+	caBundleFileName                = "custom-ca-bundle.crt"
+	certVolumeName                  = "rook-ceph-rgw-cert"
+	certDir                         = "/etc/ceph/private"
+	certKeyName                     = "cert"
+	certFilename                    = "rgw-cert.pem"
+	certKeyFileName                 = "rgw-key.pem"
+	rgwPortInternalPort       int32 = 8080
+	ServiceServingCertCAFile        = "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt"
+	HttpTimeOut                     = time.Second * 15
 )
 
 var (
@@ -72,7 +79,7 @@ func (c *clusterConfig) portString() string {
 			portString = fmt.Sprintf("ssl_port=%d ssl_certificate=%s",
 				c.store.Spec.Gateway.SecurePort, certPath)
 		}
-		secretType, _ := c.rgwTLSSecretType()
+		secretType, _ := c.rgwTLSSecretType(c.store.Spec.Gateway.SSLCertificateRef)
 		if c.store.Spec.GetServiceServingCert() != "" || secretType == v1.SecretTypeTLS {
 			privateKey := path.Join(certDir, certKeyFileName)
 			portString = fmt.Sprintf("%s ssl_private_key=%s", portString, privateKey)

pkg/operator/ceph/object/spec.go

@@ -131,6 +131,27 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
 			}}
 		podSpec.Volumes = append(podSpec.Volumes, certVol)
 	}
+	// Check custom caBundle provided
+	if c.store.Spec.Gateway.CaBundleRef != "" {
+		customCaBundleVolSrc, err := c.generateVolumeSourceWithCaBundleSecret()
+		if err != nil {
+			return v1.PodTemplateSpec{}, err
+		}
+		customCaBundleVol := v1.Volume{
+			Name: caBundleVolumeName,
+			VolumeSource: v1.VolumeSource{
+				Secret: customCaBundleVolSrc,
+			}}
+		podSpec.Volumes = append(podSpec.Volumes, customCaBundleVol)
+		updatedCaBundleVol := v1.Volume{
+			Name: caBundleUpdatedVolumeName,
+			VolumeSource: v1.VolumeSource{
+				EmptyDir: &v1.EmptyDirVolumeSource{},
+			}}
+		podSpec.Volumes = append(podSpec.Volumes, updatedCaBundleVol)
+		podSpec.InitContainers = append(podSpec.InitContainers,
+			c.createCaBundleUpdateInitContainer(rgwConfig))
+	}
 	kmsEnabled, err := c.CheckRGWKMS()
 	if err != nil {
 		return v1.PodTemplateSpec{}, err
@@ -170,6 +191,26 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
 	return podTemplateSpec, nil
 }
 
+func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) v1.Container {
+	caBundleMount := v1.VolumeMount{Name: caBundleVolumeName, MountPath: caBundleSourceCustomDir, ReadOnly: true}
+	volumeMounts := append(controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName), caBundleMount)
+	updatedCaBundleDir := "/tmp/new-ca-bundle/"
+	updatedBundleMount := v1.VolumeMount{Name: caBundleUpdatedVolumeName, MountPath: updatedCaBundleDir, ReadOnly: false}
+	volumeMounts = append(volumeMounts, updatedBundleMount)
+	return v1.Container{
+		Name:    "update-ca-bundle-initcontainer",
+		Command: []string{"/bin/bash", "-c"},
+		// copy all content of caBundleExtractedDir to avoid directory mount itself
+		Args: []string{
+			fmt.Sprintf("/usr/bin/update-ca-trust extract; cp -rf %s/* %s", caBundleExtractedDir, updatedCaBundleDir),
+		},
+		Image:           c.clusterSpec.CephVersion.Image,
+		VolumeMounts:    volumeMounts,
+		Resources:       c.store.Spec.Gateway.Resources,
+		SecurityContext: controller.PodSecurityContext(),
+	}
+}
+
 // The vault token is passed as Secret for rgw container. So it is mounted as read only.
 // RGW has restrictions over vault token file, it should owned by same user(ceph) which
 // rgw daemon runs and all other permission should be nil or zero. Here ownership can be
@@ -242,6 +283,10 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) v1.Container {
 		mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true}
 		container.VolumeMounts = append(container.VolumeMounts, mount)
 	}
+	if c.store.Spec.Gateway.CaBundleRef != "" {
+		updatedBundleMount := v1.VolumeMount{Name: caBundleUpdatedVolumeName, MountPath: caBundleExtractedDir, ReadOnly: true}
+		container.VolumeMounts = append(container.VolumeMounts, updatedBundleMount)
+	}
 	kmsEnabled, err := c.CheckRGWKMS()
 	if err != nil {
 		logger.Errorf("failed to enable KMS. %v", err)
@@ -511,7 +556,7 @@ func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSou
 		secretVolSrc = &v1.SecretVolumeSource{
 			SecretName: c.store.Spec.Gateway.SSLCertificateRef,
 		}
-		secretType, err := c.rgwTLSSecretType()
+		secretType, err := c.rgwTLSSecretType(c.store.Spec.Gateway.SSLCertificateRef)
 		if err != nil {
 			return nil, err
 		}
@@ -540,8 +585,28 @@ func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSou
 	return secretVolSrc, nil
 }
 
-func (c *clusterConfig) rgwTLSSecretType() (v1.SecretType, error) {
-	rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), c.store.Spec.Gateway.SSLCertificateRef, metav1.GetOptions{})
+func (c *clusterConfig) generateVolumeSourceWithCaBundleSecret() (*v1.SecretVolumeSource, error) {
+	// Keep the ca-bundle as secure as possible in the container. Give only user read perms.
+	// Same as above for generateVolumeSourceWithTLSSecret function.
+	userReadOnly := int32(0400)
+	caBundleVolSrc := &v1.SecretVolumeSource{
+		SecretName: c.store.Spec.Gateway.CaBundleRef,
+	}
+	secretType, err := c.rgwTLSSecretType(c.store.Spec.Gateway.CaBundleRef)
+	if err != nil {
+		return nil, err
+	}
+	if secretType != v1.SecretTypeOpaque {
+		return nil, errors.New("CaBundle secret should be 'Opaque' type")
+	}
+	caBundleVolSrc.Items = []v1.KeyToPath{
+		{Key: caBundleKeyName, Path: caBundleFileName, Mode: &userReadOnly},
+	}
+	return caBundleVolSrc, nil
+}
+
+func (c *clusterConfig) rgwTLSSecretType(secretName string) (v1.SecretType, error) {
+	rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
 	if rgwTlsSecret != nil {
 		return rgwTlsSecret.Type, nil
 	}

pkg/operator/ceph/object/spec_test.go

@@ -189,6 +189,24 @@ func TestSSLPodSpec(t *testing.T) {
 	secretVolSrc, err = c.generateVolumeSourceWithTLSSecret()
 	assert.NoError(t, err)
 	assert.Equal(t, secretVolSrc.SecretName, "rgw-cert")
+	// Using caBundleRef
+	// Opaque Secret
+	c.store.Spec.Gateway.CaBundleRef = "mycabundle"
+	cabundlesecret := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      c.store.Spec.Gateway.CaBundleRef,
+			Namespace: c.store.Namespace,
+		},
+		Data: map[string][]byte{
+			"cabundle": []byte("cabundletesting"),
+		},
+		Type: v1.SecretTypeOpaque,
+	}
+	_, err = c.context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, cabundlesecret, metav1.CreateOptions{})
+	assert.NoError(t, err)
+	caBundleVolSrc, err := c.generateVolumeSourceWithCaBundleSecret()
+	assert.NoError(t, err)
+	assert.Equal(t, caBundleVolSrc.SecretName, "mycabundle")
 	s, err = c.makeRGWPodSpec(rgwConfig)
 	assert.NoError(t, err)
 	podTemplate = cephtest.NewPodTemplateSpecTester(t, &s)