From 1665ad6ea7d7e58ec7a4d8e5a35720452bee3d2a Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Fri, 2 Jul 2021 18:34:22 +0530 Subject: [PATCH] ceph: add support for tls certs via k8s tls secrets for rgw With this PR the RGW can accept TLS certs as K8s TLS secrets Fixes: 2079 Signed-off-by: Jiffin Tony Thottan --- Documentation/ceph-object-store-crd.md | 2 +- PendingReleaseNotes.md | 1 + pkg/operator/ceph/object/config.go | 4 +- pkg/operator/ceph/object/config_test.go | 17 +++++---- pkg/operator/ceph/object/rgw.go | 7 +++- pkg/operator/ceph/object/spec.go | 42 ++++++++++++++++----- pkg/operator/ceph/object/spec_test.go | 50 +++++++++++++++++++++++-- 7 files changed, 100 insertions(+), 23 deletions(-) diff --git a/Documentation/ceph-object-store-crd.md b/Documentation/ceph-object-store-crd.md index a43c17c5f..ee7c40fca 100644 --- a/Documentation/ceph-object-store-crd.md +++ b/Documentation/ceph-object-store-crd.md @@ -91,7 +91,7 @@ When the `zone` section is set pools with the object stores name will not be cre The gateway settings correspond to the RGW daemon settings. * `type`: `S3` is supported -* `sslCertificateRef`: If specified, this is the name of the Kubernetes secret that contains the TLS certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in PEM form." +* `sslCertificateRef`: If specified, this is the name of the Kubernetes secret(`opaque` or `tls` type) that contains the TLS certificate to be used for secure connections to the object store. Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be in the format expected by the [RGW service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): "The server key, server certificate, and any other CA or intermediate certificates be supplied in one file. Each of these items must be in PEM form." * `port`: The port on which the Object service will be reachable. If host networking is enabled, the RGW daemons will also listen on that port. If running on SDN, the RGW daemon listening port will be 8080 internally. * `securePort`: The secure port on which RGW pods will be listening. A TLS certificate must be specified either via `sslCerticateRef` or `service.annotations` * `instances`: The number of pods that will be started to load balance this object store. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 491c1accc..b878635e6 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -31,3 +31,4 @@ v1.7... - Checkout the [ceph docs](https://docs.ceph.com/en/latest/rados/operations/crush-map/#custom-crush-rules) for detailed information. - Add support cephfs mirroring peer configuration, refer to the [configuration](Documentation/ceph-filesystem-crd.md#mirroring) for more details +- Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server. diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index cc2ff4b36..be60afe25 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" + v1 "k8s.io/api/core/v1" ) const ( @@ -71,7 +72,8 @@ func (c *clusterConfig) portString() string { portString = fmt.Sprintf("ssl_port=%d ssl_certificate=%s", c.store.Spec.Gateway.SecurePort, certPath) } - if c.store.Spec.GetServiceServingCert() != "" { + secretType, _ := c.rgwTLSSecretType() + if c.store.Spec.GetServiceServingCert() != "" || secretType == v1.SecretTypeTLS { privateKey := path.Join(certDir, certKeyFileName) portString = fmt.Sprintf("%s ssl_private_key=%s", portString, privateKey) } diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index 029892fd7..478df1d1d 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -20,12 +20,14 @@ import ( "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" ) -func newConfig() *clusterConfig { +func newConfig(t *testing.T) *clusterConfig { clusterInfo := &cephclient.ClusterInfo{ CephVersion: cephver.Nautilus, } @@ -41,17 +43,18 @@ func newConfig() *clusterConfig { }}, clusterInfo: clusterInfo, clusterSpec: clusterSpec, + context: &clusterd.Context{Clientset: test.New(t, 3)}, } } func TestPortString(t *testing.T) { // No port or secure port on beast - cfg := newConfig() + cfg := newConfig(t) result := cfg.portString() assert.Equal(t, "", result) // Insecure port on beast - cfg = newConfig() + cfg = newConfig(t) // Set host networking cfg.clusterSpec.Network.HostNetwork = true cfg.store.Spec.Gateway.Port = 80 @@ -59,14 +62,14 @@ func TestPortString(t *testing.T) { assert.Equal(t, "port=80", result) // Secure port on beast - cfg = newConfig() + cfg = newConfig(t) cfg.store.Spec.Gateway.SecurePort = 443 cfg.store.Spec.Gateway.SSLCertificateRef = "some-k8s-key-secret" result = cfg.portString() assert.Equal(t, "ssl_port=443 ssl_certificate=/etc/ceph/private/rgw-cert.pem", result) // Both ports on beast - cfg = newConfig() + cfg = newConfig(t) // Set host networking cfg.clusterSpec.Network.HostNetwork = true cfg.store.Spec.Gateway.Port = 80 @@ -76,13 +79,13 @@ func TestPortString(t *testing.T) { assert.Equal(t, "port=80 ssl_port=443 ssl_certificate=/etc/ceph/private/rgw-cert.pem", result) // Secure port requires the cert on beast - cfg = newConfig() + cfg = newConfig(t) cfg.store.Spec.Gateway.SecurePort = 443 result = cfg.portString() assert.Equal(t, "", result) // Using SDN, no host networking so the rgw port internal is not the same - cfg = newConfig() + cfg = newConfig(t) cfg.store.Spec.Gateway.Port = 80 result = cfg.portString() assert.Equal(t, "port=8080", result) diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index bac7fffbe..10adfa279 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -35,6 +35,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/exec" + v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -333,7 +334,11 @@ func GetTlsCaCert(objContext *Context, objectStoreSpec *cephv1.ObjectStoreSpec) if err != nil { return nil, errors.Wrapf(err, "failed to get secret %s containing TLS certificate defined in %s", objectStoreSpec.Gateway.SSLCertificateRef, objContext.Name) } - tlsCert = tlsSecretCert.Data[certKeyName] + if tlsSecretCert.Type == v1.SecretTypeOpaque { + tlsCert = tlsSecretCert.Data[certKeyName] + } else if tlsSecretCert.Type == v1.SecretTypeTLS { + tlsCert = tlsSecretCert.Data[v1.TLSCertKey] + } } else if objectStoreSpec.GetServiceServingCert() != "" { tlsCert, err = ioutil.ReadFile(ServiceServingCertCAFile) if err != nil { diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index 5ddc2a4b7..f1f1ffa04 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -17,6 +17,7 @@ limitations under the License. package object import ( + "context" "fmt" "path" "reflect" @@ -119,7 +120,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec // Set the ssl cert if specified if c.store.Spec.Gateway.SecurePort != 0 { - secretVolSrc, err := generateVolumeSourceWithTLSSecret(c.store.Spec) + secretVolSrc, err := c.generateVolumeSourceWithTLSSecret() if err != nil { return v1.PodTemplateSpec{}, err } @@ -492,25 +493,38 @@ func getLabels(name, namespace string, includeNewLabels bool) map[string]string return labels } -func generateVolumeSourceWithTLSSecret(objSpec cephv1.ObjectStoreSpec) (*v1.SecretVolumeSource, error) { +func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSource, error) { // Keep the TLS secret as secure as possible in the container. Give only user read perms. // Because the Secret mount is owned by "root" and fsGroup breaks on OCP since we cannot predict it // Also, we don't want to change the SCC for fsGroup to RunAsAny since it has a major broader impact // Let's open the permissions a bit more so that everyone can read the cert. userReadOnly := int32(0444) var secretVolSrc *v1.SecretVolumeSource - if objSpec.Gateway.SSLCertificateRef != "" { + if c.store.Spec.Gateway.SSLCertificateRef != "" { secretVolSrc = &v1.SecretVolumeSource{ - SecretName: objSpec.Gateway.SSLCertificateRef, - Items: []v1.KeyToPath{ + SecretName: c.store.Spec.Gateway.SSLCertificateRef, + } + secretType, err := c.rgwTLSSecretType() + if err != nil { + return nil, err + } + switch secretType { + case v1.SecretTypeOpaque: + secretVolSrc.Items = []v1.KeyToPath{ {Key: certKeyName, Path: certFilename, Mode: &userReadOnly}, - }} - } else if objSpec.GetServiceServingCert() != "" { + } + case v1.SecretTypeTLS: + secretVolSrc.Items = []v1.KeyToPath{ + {Key: v1.TLSCertKey, Path: certFilename, Mode: &userReadOnly}, + {Key: v1.TLSPrivateKeyKey, Path: certKeyFileName, Mode: &userReadOnly}, + } + } + } else if c.store.Spec.GetServiceServingCert() != "" { secretVolSrc = &v1.SecretVolumeSource{ - SecretName: objSpec.GetServiceServingCert(), + SecretName: c.store.Spec.GetServiceServingCert(), Items: []v1.KeyToPath{ - {Key: "tls.crt", Path: certFilename, Mode: &userReadOnly}, - {Key: "tls.key", Path: certKeyFileName, Mode: &userReadOnly}, + {Key: v1.TLSCertKey, Path: certFilename, Mode: &userReadOnly}, + {Key: v1.TLSPrivateKeyKey, Path: certKeyFileName, Mode: &userReadOnly}, }} } else { return nil, errors.New("no TLS certificates found") @@ -518,3 +532,11 @@ func generateVolumeSourceWithTLSSecret(objSpec cephv1.ObjectStoreSpec) (*v1.Secr return secretVolSrc, nil } + +func (c *clusterConfig) rgwTLSSecretType() (v1.SecretType, error) { + rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(context.TODO(), c.store.Spec.Gateway.SSLCertificateRef, metav1.GetOptions{}) + if rgwTlsSecret != nil { + return rgwTlsSecret.Type, nil + } + return "", errors.Wrapf(err, "no Kubernetes secrets referring TLS certificates found") +} diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go index 821c72925..5874e4707 100644 --- a/pkg/operator/ceph/object/spec_test.go +++ b/pkg/operator/ceph/object/spec_test.go @@ -91,6 +91,10 @@ func TestPodSpecs(t *testing.T) { } func TestSSLPodSpec(t *testing.T) { + ctx := context.TODO() + // Placeholder + context := &clusterd.Context{Clientset: test.New(t, 3)} + store := simpleStore() store.Spec.Gateway.Resources = v1.ResourceRequirements{ Limits: v1.ResourceList{ @@ -105,12 +109,14 @@ func TestSSLPodSpec(t *testing.T) { store.Spec.Gateway.PriorityClassName = "my-priority-class" info := clienttest.CreateTestClusterInfo(1) info.CephVersion = cephver.Nautilus + info.Namespace = store.Namespace data := cephconfig.NewStatelessDaemonDataPathMap(cephconfig.RgwType, "default", "rook-ceph", "/var/lib/rook/") store.Spec.Gateway.SecurePort = 443 c := &clusterConfig{ clusterInfo: info, store: store, + context: context, rookVersion: "rook/rook:myversion", clusterSpec: &cephv1.ClusterSpec{ CephVersion: cephv1.CephVersionSpec{Image: "ceph/ceph:v15"}, @@ -130,8 +136,22 @@ func TestSSLPodSpec(t *testing.T) { assert.Error(t, err) // Using SSLCertificateRef + // Opaque Secret c.store.Spec.Gateway.SSLCertificateRef = "mycert" - secretVolSrc, _ := generateVolumeSourceWithTLSSecret(c.store.Spec) + rgwtlssecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.store.Spec.Gateway.SSLCertificateRef, + Namespace: c.store.Namespace, + }, + Data: map[string][]byte{ + "cert": []byte("tlssecrettesting"), + }, + Type: v1.SecretTypeOpaque, + } + _, err = c.context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, rgwtlssecret, metav1.CreateOptions{}) + assert.NoError(t, err) + secretVolSrc, err := c.generateVolumeSourceWithTLSSecret() + assert.NoError(t, err) assert.Equal(t, secretVolSrc.SecretName, "mycert") s, err := c.makeRGWPodSpec(rgwConfig) assert.NoError(t, err) @@ -139,11 +159,35 @@ func TestSSLPodSpec(t *testing.T) { podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", "200", "100", "1337", "500", /* resources */ "my-priority-class") - + // TLS Secret + c.store.Spec.Gateway.SSLCertificateRef = "tlscert" + rgwtlssecret = &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.store.Spec.Gateway.SSLCertificateRef, + Namespace: c.store.Namespace, + }, + Data: map[string][]byte{ + "tls.crt": []byte("tlssecrettestingcert"), + "tls.key": []byte("tlssecrettestingkey"), + }, + Type: v1.SecretTypeTLS, + } + _, err = c.context.Clientset.CoreV1().Secrets(store.Namespace).Create(ctx, rgwtlssecret, metav1.CreateOptions{}) + assert.NoError(t, err) + secretVolSrc, err = c.generateVolumeSourceWithTLSSecret() + assert.NoError(t, err) + assert.Equal(t, secretVolSrc.SecretName, "tlscert") + s, err = c.makeRGWPodSpec(rgwConfig) + assert.NoError(t, err) + podTemplate = cephtest.NewPodTemplateSpecTester(t, &s) + podTemplate.RunFullSuite(cephconfig.RgwType, "default", "rook-ceph-rgw", "mycluster", "ceph/ceph:myversion", + "200", "100", "1337", "500", /* resources */ + "my-priority-class") // Using service serving cert c.store.Spec.Gateway.SSLCertificateRef = "" c.store.Spec.Gateway.Service = &(cephv1.RGWServiceSpec{Annotations: rook.Annotations{cephv1.ServiceServingCertKey: "rgw-cert"}}) - secretVolSrc, _ = generateVolumeSourceWithTLSSecret(c.store.Spec) + secretVolSrc, err = c.generateVolumeSourceWithTLSSecret() + assert.NoError(t, err) assert.Equal(t, secretVolSrc.SecretName, "rgw-cert") s, err = c.makeRGWPodSpec(rgwConfig) assert.NoError(t, err)