Merge pull request #4423 from leseb/ceph-crash-key

ceph: add crash key

pkg/operator/ceph/cluster/cluster.go

@@ -32,6 +32,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
+	"github.com/rook/rook/pkg/operator/ceph/cluster/crash"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
@@ -39,6 +40,7 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/csi"
 	cephspec "github.com/rook/rook/pkg/operator/ceph/spec"
+	"github.com/rook/rook/pkg/operator/ceph/version"
 	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -431,6 +433,15 @@ func (c *cluster) postMonStartupActions() error {
 		return errors.Wrapf(err, "failed to create csi kubernetes secrets")
 	}
 
+	// Create Crash Collector Secret
+	// In 14.2.5 the crash daemon will read the client.crash key instead of the admin key
+	if c.Info.CephVersion.IsAtLeast(version.CephVersion{Major: 14, Minor: 2, Extra: 5}) {
+		err = crash.CreateCrashCollectorSecret(c.context, c.Namespace, &c.ownerRef)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create crash collector kubernetes secret")
+		}
+	}
+
 	// Enable Ceph messenger 2 protocol on Nautilus
 	if c.Info.CephVersion.IsAtLeastNautilus() {
 		v, err := client.GetCephMonVersion(c.context, c.Info.Name)

pkg/operator/ceph/cluster/crash/crash.go

@@ -24,6 +24,8 @@ import (
 
 	"github.com/pkg/errors"
 	opkit "github.com/rook/operator-kit"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
+
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
@@ -42,6 +44,11 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 )
 
+const (
+	crashCollectorKeyringUsername = "client.crash"
+	crashCollectorSecretName      = "rook-ceph-crash-collector-keyring"
+)
+
 // ClusterResource operator-kit Custom Resource Definition
 var clusterResource = opkit.CustomResource{
 	Group:   cephv1.CustomResourceGroup,
@@ -65,6 +72,9 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations []
 		},
 	}
 
+	volumes := opspec.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "")
+	volumes = append(volumes, getVolumes(*cephVersion))
+
 	mutateFunc := func() error {
 
 		// labels for the pod, the deployment, and the deploymentSelector
@@ -108,12 +118,12 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations []
 					getCrashChownInitContainer(cephCluster),
 				},
 				Containers: []corev1.Container{
-					getCrashDaemonContainer(cephCluster),
+					getCrashDaemonContainer(cephCluster, *cephVersion),
 				},
 				Tolerations:   tolerations,
 				RestartPolicy: corev1.RestartPolicyAlways,
 				HostNetwork:   cephCluster.Spec.Network.IsHost(),
-				Volumes:       append(opspec.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), ""), keyring.Volume().Admin()),
+				Volumes:       volumes,
 			},
 		}
 
@@ -156,11 +166,13 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container
 	)
 }
 
-func getCrashDaemonContainer(cephCluster cephv1.CephCluster) corev1.Container {
+func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion version.CephVersion) corev1.Container {
 	cephImage := cephCluster.Spec.CephVersion.Image
 	dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
-	crashEnvVar := corev1.EnvVar{Name: "CEPH_ARGS", Value: "-m $(ROOK_CEPH_MON_HOST) -k /etc/ceph/admin-keyring-store/keyring -n client.admin"}
+	crashEnvVar := generateCrashEnvVar(cephVersion)
 	envVars := append(opspec.DaemonEnvVars(cephImage), crashEnvVar)
+	volumeMounts := opspec.DaemonVolumeMounts(dataPathMap, "")
+	volumeMounts = append(volumeMounts, getVolumeMounts(cephVersion))
 
 	container := corev1.Container{
 		Name: "ceph-crash",
@@ -169,7 +181,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster) corev1.Container {
 		},
 		Image:        cephImage,
 		Env:          envVars,
-		VolumeMounts: append(opspec.DaemonVolumeMounts(dataPathMap, ""), keyring.VolumeMount().Admin()),
+		VolumeMounts: volumeMounts,
 		Resources:    cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources),
 	}
 
@@ -186,3 +198,39 @@ func clusterOwnerRef(clusterName, clusterID string) metav1.OwnerReference {
 		BlockOwnerDeletion: &blockOwner,
 	}
 }
+
+func generateCrashEnvVar(v cephver.CephVersion) corev1.EnvVar {
+	val := fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())
+	if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}) {
+		val = fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().CrashCollectorKeyringFilePath())
+	}
+
+	env := corev1.EnvVar{Name: "CEPH_ARGS", Value: val}
+	return env
+}
+
+func getVolumeMounts(v cephver.CephVersion) corev1.VolumeMount {
+	volumeMounts := keyring.VolumeMount().Admin()
+
+	// As of Ceph Nautilus 14.2.5, the crash collector has its own key
+	// If not running on on at least this version let's use the Ceph admin key
+	// Thanks to https://github.com/ceph/ceph/pull/30844
+	if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}) {
+		volumeMounts = keyring.VolumeMount().CrashCollector()
+	}
+
+	return volumeMounts
+}
+
+func getVolumes(v cephver.CephVersion) corev1.Volume {
+	volumes := keyring.Volume().Admin()
+
+	// As of Ceph Nautilus 14.2.5, the crash collector has its own key
+	// If not running on on at least this version let's use the Ceph admin key
+	// Thanks to https://github.com/ceph/ceph/pull/30844
+	if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}) {
+		volumes = keyring.Volume().CrashCollector()
+	}
+
+	return volumes
+}

pkg/operator/ceph/cluster/crash/crash_test.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2019 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crash
+
+import (
+	"testing"
+
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerateCrashEnvVar(t *testing.T) {
+	v := cephver.Nautilus
+	env := generateCrashEnvVar(v)
+	assert.Equal(t, "CEPH_ARGS", env.Name)
+	assert.Equal(t, "-m $(ROOK_CEPH_MON_HOST) -k /etc/ceph/admin-keyring-store/keyring", env.Value)
+
+	v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}
+	env = generateCrashEnvVar(v)
+	assert.Equal(t, "CEPH_ARGS", env.Name)
+	assert.Equal(t, "-m $(ROOK_CEPH_MON_HOST) -k /etc/ceph/crash-collector-keyring-store/keyring", env.Value)
+}
+
+func TestGetVolumes(t *testing.T) {
+	v := cephver.Nautilus
+	vol := getVolumes(v)
+	assert.Equal(t, "rook-ceph-admin-keyring", vol.Name)
+
+	v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}
+	vol = getVolumes(v)
+	assert.Equal(t, "rook-ceph-crash-collector-keyring", vol.Name)
+}
+
+func TestGetVolumeMount(t *testing.T) {
+	v := cephver.Nautilus
+	volMounts := getVolumeMounts(v)
+	assert.Equal(t, "rook-ceph-admin-keyring", volMounts.Name)
+
+	v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}
+	volMounts = getVolumeMounts(v)
+	assert.Equal(t, "rook-ceph-crash-collector-keyring", volMounts.Name)
+}

pkg/operator/ceph/cluster/crash/keyring.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2019 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crash
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/rook/rook/pkg/clusterd"
+	"github.com/rook/rook/pkg/operator/ceph/config/keyring"
+	"github.com/rook/rook/pkg/operator/k8sutil"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	crashKeyringTemplate = `
+[client.crash]
+	key = %s
+	caps mon = "allow profile crash"
+	caps mgr = "allow profile crash"
+`
+)
+
+// CreateCrashCollectorSecret creates the Kubernetes Crash Collector Secret
+func CreateCrashCollectorSecret(context *clusterd.Context, clusterName string, ownerRef *metav1.OwnerReference) error {
+	k := keyring.GetSecretStore(context, clusterName, ownerRef)
+
+	// Create CrashCollector Ceph key
+	crashCollectorSecretKey, err := createCrashCollectorKeyring(k)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create %q ceph keyring", crashCollectorKeyringUsername)
+	}
+
+	// Create or update Kubernetes CSI secret
+	if err := createOrUpdateCrashCollectorSecret(clusterName, crashCollectorSecretKey, k, ownerRef); err != nil {
+		return errors.Wrapf(err, "failed to create kubernetes csi secret")
+	}
+
+	return nil
+}
+
+func cephCrashCollectorKeyringCaps() []string {
+	return []string{
+		"mon", "allow profile crash",
+		"mgr", "allow profile crash",
+	}
+}
+
+func createCrashCollectorKeyring(s *keyring.SecretStore) (string, error) {
+	key, err := s.GenerateKey(crashCollectorKeyringUsername, cephCrashCollectorKeyringCaps())
+	if err != nil {
+		return "", err
+	}
+
+	return key, nil
+}
+
+func createOrUpdateCrashCollectorSecret(namespace, crashCollectorSecretKey string, k *keyring.SecretStore, ownerRef *metav1.OwnerReference) error {
+
+	keyring := fmt.Sprintf(crashKeyringTemplate, crashCollectorSecretKey)
+
+	crashCollectorSecret := map[string][]byte{
+		"keyring": []byte(keyring),
+	}
+
+	s := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      crashCollectorSecretName,
+			Namespace: namespace,
+		},
+		Data: crashCollectorSecret,
+		Type: k8sutil.RookType,
+	}
+	k8sutil.SetOwnerRef(&s.ObjectMeta, ownerRef)
+
+	// Create Kubernetes Secret
+	err := k.CreateSecret(s)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", crashCollectorSecret, namespace)
+	}
+
+	logger.Infof("created kubernetes crash collector secret for cluster %q", namespace)
+	return nil
+}

pkg/operator/ceph/cluster/crash/keyring_test.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2019 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crash
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCephCrashCollectorKeyringCaps(t *testing.T) {
+	caps := cephCrashCollectorKeyringCaps()
+	assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow profile crash"})
+}

pkg/operator/ceph/config/keyring/admin.go

@@ -23,8 +23,8 @@ import (
 )
 
 const (
-	adminKeyringResourceName = "rook-ceph-admin"
-	adminKeyringFileName     = "ceph.client.admin.keyring"
+	adminKeyringResourceName          = "rook-ceph-admin"
+	crashCollectorKeyringResourceName = "rook-ceph-crash-collector"
 
 	adminKeyringTemplate = `
 [client.admin]

pkg/operator/ceph/config/keyring/volume.go

@@ -3,7 +3,7 @@ package keyring
 import (
 	"path"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 const (
@@ -11,7 +11,8 @@ const (
 
 	// admin keyring path must be different from keyring path so that the two keyrings can be
 	// mounted independently
-	adminKeyringDir = "/etc/ceph/admin-keyring-store/"
+	adminKeyringDir          = "/etc/ceph/admin-keyring-store/"
+	crashCollectorKeyringDir = "/etc/ceph/crash-collector-keyring-store/"
 )
 
 // VolumeBuilder is a helper for creating Kubernetes pod volumes with content sourced by keyrings
@@ -41,6 +42,11 @@ func (v *VolumeBuilder) Admin() v1.Volume {
 	return v.Resource(adminKeyringResourceName)
 }
 
+// CrashCollector returns a kubernetes pod volume whose content is sourced by the SecretStore crash collector keyring.
+func (v *VolumeBuilder) CrashCollector() v1.Volume {
+	return v.Resource(crashCollectorKeyringResourceName)
+}
+
 // VolumeMount returns a VolumeMountBuilder.
 func VolumeMount() *VolumeMountBuilder { return &VolumeMountBuilder{} }
 
@@ -64,6 +70,16 @@ func (*VolumeMountBuilder) Admin() v1.VolumeMount {
 	}
 }
 
+// CrashCollector returns a Kubernetes container volume mount that mounts the content from the matching
+// VolumeBuilder Crash Collector volume.
+func (*VolumeMountBuilder) CrashCollector() v1.VolumeMount {
+	return v1.VolumeMount{
+		Name:      keyringSecretName(crashCollectorKeyringResourceName),
+		ReadOnly:  true, // should be no reason to write to the keyring in pods, so enforce this
+		MountPath: crashCollectorKeyringDir,
+	}
+}
+
 // KeyringFilePath returns the full path to the regular keyring file within a container.
 func (*VolumeMountBuilder) KeyringFilePath() string {
 	return path.Join(keyringDir, keyringFileName)
@@ -73,3 +89,8 @@ func (*VolumeMountBuilder) KeyringFilePath() string {
 func (*VolumeMountBuilder) AdminKeyringFilePath() string {
 	return path.Join(adminKeyringDir, keyringFileName)
 }
+
+// CrashCollectorKeyringFilePath returns the full path to the admin keyring file within a container.
+func (*VolumeMountBuilder) CrashCollectorKeyringFilePath() string {
+	return path.Join(crashCollectorKeyringDir, keyringFileName)
+}

pkg/operator/ceph/csi/secrets.go

@@ -152,7 +152,7 @@ func createOrUpdateCSISecret(namespace, csiRBDProvisionerSecretKey, csiRBDNodeSe
 		// Create Kubernetes Secret
 		err := k.CreateSecret(s)
 		if err != nil {
-			return errors.Errorf("failed to create kubernetes secret %q for cluster %q", secret, namespace)
+			return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", secret, namespace)
 		}
 
 	}