diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index eb073e361..c36900e75 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -32,6 +32,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/cluster/crash" "github.com/rook/rook/pkg/operator/ceph/cluster/mgr" "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd" @@ -39,6 +40,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/csi" cephspec "github.com/rook/rook/pkg/operator/ceph/spec" + "github.com/rook/rook/pkg/operator/ceph/version" cephver "github.com/rook/rook/pkg/operator/ceph/version" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -431,6 +433,15 @@ func (c *cluster) postMonStartupActions() error { return errors.Wrapf(err, "failed to create csi kubernetes secrets") } + // Create Crash Collector Secret + // In 14.2.5 the crash daemon will read the client.crash key instead of the admin key + if c.Info.CephVersion.IsAtLeast(version.CephVersion{Major: 14, Minor: 2, Extra: 5}) { + err = crash.CreateCrashCollectorSecret(c.context, c.Namespace, &c.ownerRef) + if err != nil { + return errors.Wrapf(err, "failed to create crash collector kubernetes secret") + } + } + // Enable Ceph messenger 2 protocol on Nautilus if c.Info.CephVersion.IsAtLeastNautilus() { v, err := client.GetCephMonVersion(c.context, c.Info.Name) diff --git a/pkg/operator/ceph/cluster/crash/crash.go b/pkg/operator/ceph/cluster/crash/crash.go index 56db52389..1171888fc 100644 --- a/pkg/operator/ceph/cluster/crash/crash.go +++ b/pkg/operator/ceph/cluster/crash/crash.go @@ -24,6 +24,8 @@ import ( "github.com/pkg/errors" opkit "github.com/rook/operator-kit" + cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" @@ -42,6 +44,11 @@ import ( "k8s.io/apimachinery/pkg/types" ) +const ( + crashCollectorKeyringUsername = "client.crash" + crashCollectorSecretName = "rook-ceph-crash-collector-keyring" +) + // ClusterResource operator-kit Custom Resource Definition var clusterResource = opkit.CustomResource{ Group: cephv1.CustomResourceGroup, @@ -65,6 +72,9 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations [] }, } + volumes := opspec.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "") + volumes = append(volumes, getVolumes(*cephVersion)) + mutateFunc := func() error { // labels for the pod, the deployment, and the deploymentSelector @@ -108,12 +118,12 @@ func (r *ReconcileNode) createOrUpdateCephCrash(node corev1.Node, tolerations [] getCrashChownInitContainer(cephCluster), }, Containers: []corev1.Container{ - getCrashDaemonContainer(cephCluster), + getCrashDaemonContainer(cephCluster, *cephVersion), }, Tolerations: tolerations, RestartPolicy: corev1.RestartPolicyAlways, HostNetwork: cephCluster.Spec.Network.IsHost(), - Volumes: append(opspec.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), ""), keyring.Volume().Admin()), + Volumes: volumes, }, } @@ -156,11 +166,13 @@ func getCrashChownInitContainer(cephCluster cephv1.CephCluster) corev1.Container ) } -func getCrashDaemonContainer(cephCluster cephv1.CephCluster) corev1.Container { +func getCrashDaemonContainer(cephCluster cephv1.CephCluster, cephVersion version.CephVersion) corev1.Container { cephImage := cephCluster.Spec.CephVersion.Image dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) - crashEnvVar := corev1.EnvVar{Name: "CEPH_ARGS", Value: "-m $(ROOK_CEPH_MON_HOST) -k /etc/ceph/admin-keyring-store/keyring -n client.admin"} + crashEnvVar := generateCrashEnvVar(cephVersion) envVars := append(opspec.DaemonEnvVars(cephImage), crashEnvVar) + volumeMounts := opspec.DaemonVolumeMounts(dataPathMap, "") + volumeMounts = append(volumeMounts, getVolumeMounts(cephVersion)) container := corev1.Container{ Name: "ceph-crash", @@ -169,7 +181,7 @@ func getCrashDaemonContainer(cephCluster cephv1.CephCluster) corev1.Container { }, Image: cephImage, Env: envVars, - VolumeMounts: append(opspec.DaemonVolumeMounts(dataPathMap, ""), keyring.VolumeMount().Admin()), + VolumeMounts: volumeMounts, Resources: cephv1.GetCrashCollectorResources(cephCluster.Spec.Resources), } @@ -186,3 +198,39 @@ func clusterOwnerRef(clusterName, clusterID string) metav1.OwnerReference { BlockOwnerDeletion: &blockOwner, } } + +func generateCrashEnvVar(v cephver.CephVersion) corev1.EnvVar { + val := fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath()) + if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}) { + val = fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().CrashCollectorKeyringFilePath()) + } + + env := corev1.EnvVar{Name: "CEPH_ARGS", Value: val} + return env +} + +func getVolumeMounts(v cephver.CephVersion) corev1.VolumeMount { + volumeMounts := keyring.VolumeMount().Admin() + + // As of Ceph Nautilus 14.2.5, the crash collector has its own key + // If not running on on at least this version let's use the Ceph admin key + // Thanks to https://github.com/ceph/ceph/pull/30844 + if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}) { + volumeMounts = keyring.VolumeMount().CrashCollector() + } + + return volumeMounts +} + +func getVolumes(v cephver.CephVersion) corev1.Volume { + volumes := keyring.Volume().Admin() + + // As of Ceph Nautilus 14.2.5, the crash collector has its own key + // If not running on on at least this version let's use the Ceph admin key + // Thanks to https://github.com/ceph/ceph/pull/30844 + if v.IsAtLeast(cephver.CephVersion{Major: 14, Minor: 2, Extra: 5}) { + volumes = keyring.Volume().CrashCollector() + } + + return volumes +} diff --git a/pkg/operator/ceph/cluster/crash/crash_test.go b/pkg/operator/ceph/cluster/crash/crash_test.go new file mode 100644 index 000000000..af1884156 --- /dev/null +++ b/pkg/operator/ceph/cluster/crash/crash_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crash + +import ( + "testing" + + cephver "github.com/rook/rook/pkg/operator/ceph/version" + "github.com/stretchr/testify/assert" +) + +func TestGenerateCrashEnvVar(t *testing.T) { + v := cephver.Nautilus + env := generateCrashEnvVar(v) + assert.Equal(t, "CEPH_ARGS", env.Name) + assert.Equal(t, "-m $(ROOK_CEPH_MON_HOST) -k /etc/ceph/admin-keyring-store/keyring", env.Value) + + v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 5} + env = generateCrashEnvVar(v) + assert.Equal(t, "CEPH_ARGS", env.Name) + assert.Equal(t, "-m $(ROOK_CEPH_MON_HOST) -k /etc/ceph/crash-collector-keyring-store/keyring", env.Value) +} + +func TestGetVolumes(t *testing.T) { + v := cephver.Nautilus + vol := getVolumes(v) + assert.Equal(t, "rook-ceph-admin-keyring", vol.Name) + + v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 5} + vol = getVolumes(v) + assert.Equal(t, "rook-ceph-crash-collector-keyring", vol.Name) +} + +func TestGetVolumeMount(t *testing.T) { + v := cephver.Nautilus + volMounts := getVolumeMounts(v) + assert.Equal(t, "rook-ceph-admin-keyring", volMounts.Name) + + v = cephver.CephVersion{Major: 14, Minor: 2, Extra: 5} + volMounts = getVolumeMounts(v) + assert.Equal(t, "rook-ceph-crash-collector-keyring", volMounts.Name) +} diff --git a/pkg/operator/ceph/cluster/crash/keyring.go b/pkg/operator/ceph/cluster/crash/keyring.go new file mode 100644 index 000000000..38424b0eb --- /dev/null +++ b/pkg/operator/ceph/cluster/crash/keyring.go @@ -0,0 +1,99 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crash + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/config/keyring" + "github.com/rook/rook/pkg/operator/k8sutil" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + crashKeyringTemplate = ` +[client.crash] + key = %s + caps mon = "allow profile crash" + caps mgr = "allow profile crash" +` +) + +// CreateCrashCollectorSecret creates the Kubernetes Crash Collector Secret +func CreateCrashCollectorSecret(context *clusterd.Context, clusterName string, ownerRef *metav1.OwnerReference) error { + k := keyring.GetSecretStore(context, clusterName, ownerRef) + + // Create CrashCollector Ceph key + crashCollectorSecretKey, err := createCrashCollectorKeyring(k) + if err != nil { + return errors.Wrapf(err, "failed to create %q ceph keyring", crashCollectorKeyringUsername) + } + + // Create or update Kubernetes CSI secret + if err := createOrUpdateCrashCollectorSecret(clusterName, crashCollectorSecretKey, k, ownerRef); err != nil { + return errors.Wrapf(err, "failed to create kubernetes csi secret") + } + + return nil +} + +func cephCrashCollectorKeyringCaps() []string { + return []string{ + "mon", "allow profile crash", + "mgr", "allow profile crash", + } +} + +func createCrashCollectorKeyring(s *keyring.SecretStore) (string, error) { + key, err := s.GenerateKey(crashCollectorKeyringUsername, cephCrashCollectorKeyringCaps()) + if err != nil { + return "", err + } + + return key, nil +} + +func createOrUpdateCrashCollectorSecret(namespace, crashCollectorSecretKey string, k *keyring.SecretStore, ownerRef *metav1.OwnerReference) error { + + keyring := fmt.Sprintf(crashKeyringTemplate, crashCollectorSecretKey) + + crashCollectorSecret := map[string][]byte{ + "keyring": []byte(keyring), + } + + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: crashCollectorSecretName, + Namespace: namespace, + }, + Data: crashCollectorSecret, + Type: k8sutil.RookType, + } + k8sutil.SetOwnerRef(&s.ObjectMeta, ownerRef) + + // Create Kubernetes Secret + err := k.CreateSecret(s) + if err != nil { + return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", crashCollectorSecret, namespace) + } + + logger.Infof("created kubernetes crash collector secret for cluster %q", namespace) + return nil +} diff --git a/pkg/operator/ceph/cluster/crash/keyring_test.go b/pkg/operator/ceph/cluster/crash/keyring_test.go new file mode 100644 index 000000000..b558e69e3 --- /dev/null +++ b/pkg/operator/ceph/cluster/crash/keyring_test.go @@ -0,0 +1,28 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crash + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCephCrashCollectorKeyringCaps(t *testing.T) { + caps := cephCrashCollectorKeyringCaps() + assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow profile crash"}) +} diff --git a/pkg/operator/ceph/config/keyring/admin.go b/pkg/operator/ceph/config/keyring/admin.go index 70f200ffd..38cd0cbb1 100644 --- a/pkg/operator/ceph/config/keyring/admin.go +++ b/pkg/operator/ceph/config/keyring/admin.go @@ -23,8 +23,8 @@ import ( ) const ( - adminKeyringResourceName = "rook-ceph-admin" - adminKeyringFileName = "ceph.client.admin.keyring" + adminKeyringResourceName = "rook-ceph-admin" + crashCollectorKeyringResourceName = "rook-ceph-crash-collector" adminKeyringTemplate = ` [client.admin] diff --git a/pkg/operator/ceph/config/keyring/volume.go b/pkg/operator/ceph/config/keyring/volume.go index 2aa1baa44..67e1b89ab 100644 --- a/pkg/operator/ceph/config/keyring/volume.go +++ b/pkg/operator/ceph/config/keyring/volume.go @@ -3,7 +3,7 @@ package keyring import ( "path" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) const ( @@ -11,7 +11,8 @@ const ( // admin keyring path must be different from keyring path so that the two keyrings can be // mounted independently - adminKeyringDir = "/etc/ceph/admin-keyring-store/" + adminKeyringDir = "/etc/ceph/admin-keyring-store/" + crashCollectorKeyringDir = "/etc/ceph/crash-collector-keyring-store/" ) // VolumeBuilder is a helper for creating Kubernetes pod volumes with content sourced by keyrings @@ -41,6 +42,11 @@ func (v *VolumeBuilder) Admin() v1.Volume { return v.Resource(adminKeyringResourceName) } +// CrashCollector returns a kubernetes pod volume whose content is sourced by the SecretStore crash collector keyring. +func (v *VolumeBuilder) CrashCollector() v1.Volume { + return v.Resource(crashCollectorKeyringResourceName) +} + // VolumeMount returns a VolumeMountBuilder. func VolumeMount() *VolumeMountBuilder { return &VolumeMountBuilder{} } @@ -64,6 +70,16 @@ func (*VolumeMountBuilder) Admin() v1.VolumeMount { } } +// CrashCollector returns a Kubernetes container volume mount that mounts the content from the matching +// VolumeBuilder Crash Collector volume. +func (*VolumeMountBuilder) CrashCollector() v1.VolumeMount { + return v1.VolumeMount{ + Name: keyringSecretName(crashCollectorKeyringResourceName), + ReadOnly: true, // should be no reason to write to the keyring in pods, so enforce this + MountPath: crashCollectorKeyringDir, + } +} + // KeyringFilePath returns the full path to the regular keyring file within a container. func (*VolumeMountBuilder) KeyringFilePath() string { return path.Join(keyringDir, keyringFileName) @@ -73,3 +89,8 @@ func (*VolumeMountBuilder) KeyringFilePath() string { func (*VolumeMountBuilder) AdminKeyringFilePath() string { return path.Join(adminKeyringDir, keyringFileName) } + +// CrashCollectorKeyringFilePath returns the full path to the admin keyring file within a container. +func (*VolumeMountBuilder) CrashCollectorKeyringFilePath() string { + return path.Join(crashCollectorKeyringDir, keyringFileName) +} diff --git a/pkg/operator/ceph/csi/secrets.go b/pkg/operator/ceph/csi/secrets.go index a1bc30eeb..36df4acf3 100644 --- a/pkg/operator/ceph/csi/secrets.go +++ b/pkg/operator/ceph/csi/secrets.go @@ -152,7 +152,7 @@ func createOrUpdateCSISecret(namespace, csiRBDProvisionerSecretKey, csiRBDNodeSe // Create Kubernetes Secret err := k.CreateSecret(s) if err != nil { - return errors.Errorf("failed to create kubernetes secret %q for cluster %q", secret, namespace) + return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", secret, namespace) } }