diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index bc395ecb3..f0a453714 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -464,6 +464,12 @@ func (c *cluster) postMonStartupActions() error { return errors.Wrap(err, "failed to create crash collector kubernetes secret") } + // Create exporter Kubernetes Secret + err = nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo) + if err != nil { + return errors.Wrap(err, "failed to create exporter kubernetes secret") + } + if err := c.configureMsgr2(); err != nil { return errors.Wrap(err, "failed to configure msgr2") } diff --git a/pkg/operator/ceph/cluster/nodedaemon/exporter.go b/pkg/operator/ceph/cluster/nodedaemon/exporter.go index df922503f..1c01a151c 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/exporter.go +++ b/pkg/operator/ceph/cluster/nodedaemon/exporter.go @@ -46,6 +46,8 @@ const ( statsPeriod = "5" DefaultMetricsPort uint16 = 9926 exporterServiceMetricName = "ceph-exporter-http-metrics" + exporterKeyringUsername = "client.ceph-exporter" + exporterKeyName = "rook-ceph-exporter-keyring" ) var ( @@ -82,7 +84,7 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations volumes := append( controller.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "", cephCluster.Spec.DataDirHostPath), - keyring.Volume().Admin()) + keyring.Volume().Exporter()) mutateFunc := func() error { @@ -166,12 +168,10 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion cephImage := cephCluster.Spec.CephVersion.Image dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath) volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath) - // FIX: Use an exporter keyring instead of the admin keyring - volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin()) + volumeMounts = append(volumeMounts, keyring.VolumeMount().Exporter()) - envVars := append( - controller.DaemonEnvVars(&cephCluster.Spec), - corev1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())}) + exporterEnvVar := generateExporterEnvVar() + envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), exporterEnvVar) args := []string{ "--sock-dir", sockDir, @@ -278,3 +278,10 @@ func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *meta t.ApplyToObjectMeta(objectMeta) } } + +func generateExporterEnvVar() corev1.EnvVar { + val := fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -n %s -k %s", exporterKeyringUsername, keyring.VolumeMount().ExporterKeyringFilePath()) + env := corev1.EnvVar{Name: "CEPH_ARGS", Value: val} + + return env +} diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring.go b/pkg/operator/ceph/cluster/nodedaemon/keyring.go index af59694bf..5ef767be4 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring.go @@ -30,11 +30,20 @@ import ( const ( crashClient = `client.crash` + exporterClient = `client.ceph-exporter` crashKeyringTemplate = ` [client.crash] key = %s caps mon = "allow profile crash" caps mgr = "allow rw" +` + exporterKeyringTemplate = ` +[client.ceph-exporter] + key = %s + caps mon = "allow profile ceph-exporter" + caps mgr = "allow r" + caps osd = "allow r" + caps mds = "allow r" ` ) @@ -102,3 +111,69 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace) return nil } + +func CreateExporterSecret(context *clusterd.Context, clusterInfo *client.ClusterInfo) error { + k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo) + + // Create exporter Ceph key + exporterSecretKey, err := createExporterKeyring(k) + if err != nil { + return errors.Wrapf(err, "failed to create %q ceph keyring", exporterKeyringUsername) + } + + // Create or update Kubernetes CSI secret + if err := createOrUpdateExporterSecret(clusterInfo, exporterSecretKey, k); err != nil { + return errors.Wrap(err, "failed to create kubernetes csi secret") + } + + return nil +} + +func createExporterKeyringCaps() []string { + return []string{ + "mon", "allow profile ceph-exporter", + "mgr", "allow r", + "osd", "allow r", + "mds", "allow r", + } +} + +func createExporterKeyring(s *keyring.SecretStore) (string, error) { + key, err := s.GenerateKey(exporterKeyringUsername, createExporterKeyringCaps()) + if err != nil { + return "", err + } + + return key, nil +} + +func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecretKey string, k *keyring.SecretStore) error { + + keyring := fmt.Sprintf(exporterKeyringTemplate, exporterSecretKey) + + exporterSecret := map[string][]byte{ + "keyring": []byte(keyring), + } + + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: exporterKeyName, + Namespace: clusterInfo.Namespace, + }, + Data: exporterSecret, + Type: k8sutil.RookType, + } + err := clusterInfo.OwnerInfo.SetControllerReference(s) + if err != nil { + return errors.Wrapf(err, "failed to set owner reference to exporter controller secret %q", s.Name) + } + + // Create Kubernetes Secret + err = k.CreateSecret(s) + if err != nil { + return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace) + } + + logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace) + return nil +} diff --git a/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go b/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go index 4dc9f244c..7d98cd4a1 100644 --- a/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go +++ b/pkg/operator/ceph/cluster/nodedaemon/keyring_test.go @@ -26,3 +26,8 @@ func TestCephCrashCollectorKeyringCaps(t *testing.T) { caps := cephCrashCollectorKeyringCaps() assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"}) } + +func TestExporterKeyringCaps(t *testing.T) { + caps := createExporterKeyringCaps() + assert.Equal(t, caps, []string{"mon", "allow profile ceph-exporter", "mgr", "allow r", "osd", "allow r", "mds", "allow r"}) +} diff --git a/pkg/operator/ceph/config/keyring/admin.go b/pkg/operator/ceph/config/keyring/admin.go index cd3ac2d13..d202d5dfd 100644 --- a/pkg/operator/ceph/config/keyring/admin.go +++ b/pkg/operator/ceph/config/keyring/admin.go @@ -29,6 +29,7 @@ import ( const ( adminKeyringResourceName = "rook-ceph-admin" crashCollectorKeyringResourceName = "rook-ceph-crash-collector" + exporterKeyringResourceName = "rook-ceph-exporter" adminKeyringTemplate = ` [client.admin] diff --git a/pkg/operator/ceph/config/keyring/volume.go b/pkg/operator/ceph/config/keyring/volume.go index 67e1b89ab..5700d07cb 100644 --- a/pkg/operator/ceph/config/keyring/volume.go +++ b/pkg/operator/ceph/config/keyring/volume.go @@ -13,6 +13,7 @@ const ( // mounted independently adminKeyringDir = "/etc/ceph/admin-keyring-store/" crashCollectorKeyringDir = "/etc/ceph/crash-collector-keyring-store/" + exporterKeyringDir = "/etc/ceph/exporter-keyring-store/" ) // VolumeBuilder is a helper for creating Kubernetes pod volumes with content sourced by keyrings @@ -47,6 +48,11 @@ func (v *VolumeBuilder) CrashCollector() v1.Volume { return v.Resource(crashCollectorKeyringResourceName) } +// Exporter returns a kubernetes pod volume whose content is sourced by the SecretStore exporter keyring. +func (v *VolumeBuilder) Exporter() v1.Volume { + return v.Resource(exporterKeyringResourceName) +} + // VolumeMount returns a VolumeMountBuilder. func VolumeMount() *VolumeMountBuilder { return &VolumeMountBuilder{} } @@ -80,6 +86,16 @@ func (*VolumeMountBuilder) CrashCollector() v1.VolumeMount { } } +// Exporter returns a Kubernetes container volume mount that mounts the content from the matching +// VolumeBuilder Exporter volume. +func (*VolumeMountBuilder) Exporter() v1.VolumeMount { + return v1.VolumeMount{ + Name: keyringSecretName(exporterKeyringResourceName), + ReadOnly: true, // should be no reason to write to the keyring in pods, so enforce this + MountPath: exporterKeyringDir, + } +} + // KeyringFilePath returns the full path to the regular keyring file within a container. func (*VolumeMountBuilder) KeyringFilePath() string { return path.Join(keyringDir, keyringFileName) @@ -94,3 +110,8 @@ func (*VolumeMountBuilder) AdminKeyringFilePath() string { func (*VolumeMountBuilder) CrashCollectorKeyringFilePath() string { return path.Join(crashCollectorKeyringDir, keyringFileName) } + +// ExporterKeyringFilePath returns the full path to the admin keyring file within a container. +func (*VolumeMountBuilder) ExporterKeyringFilePath() string { + return path.Join(exporterKeyringDir, keyringFileName) +}