Merge pull request #13239 from avanthakkar/run-exporter-with-specific-keyring

exporter: run exporter with specific keyring
This commit is contained in:
Travis Nielsen authored and GitHub committed 2023-11-21 11:59:05 -07:00
commit e3c9ed6768
6 files changed
+121 -6

No files matched your search

+6
View File
@@ -464,6 +464,12 @@ func (c *cluster) postMonStartupActions() error {
return errors.Wrap(err, "failed to create crash collector kubernetes secret")
}
// Create exporter Kubernetes Secret
err = nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo)
if err != nil {
return errors.Wrap(err, "failed to create exporter kubernetes secret")
}
if err := c.configureMsgr2(); err != nil {
return errors.Wrap(err, "failed to configure msgr2")
}
@@ -46,6 +46,8 @@ const (
statsPeriod = "5"
DefaultMetricsPort uint16 = 9926
exporterServiceMetricName = "ceph-exporter-http-metrics"
exporterKeyringUsername = "client.ceph-exporter"
exporterKeyName = "rook-ceph-exporter-keyring"
)
var (
@@ -82,7 +84,7 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations
volumes := append(
controller.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "", cephCluster.Spec.DataDirHostPath),
keyring.Volume().Admin())
keyring.Volume().Exporter())
mutateFunc := func() error {
@@ -166,12 +168,10 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
cephImage := cephCluster.Spec.CephVersion.Image
dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
// FIX: Use an exporter keyring instead of the admin keyring
volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin())
volumeMounts = append(volumeMounts, keyring.VolumeMount().Exporter())
envVars := append(
controller.DaemonEnvVars(&cephCluster.Spec),
corev1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())})
exporterEnvVar := generateExporterEnvVar()
envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), exporterEnvVar)
args := []string{
"--sock-dir", sockDir,
@@ -278,3 +278,10 @@ func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *meta
t.ApplyToObjectMeta(objectMeta)
}
}
func generateExporterEnvVar() corev1.EnvVar {
val := fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -n %s -k %s", exporterKeyringUsername, keyring.VolumeMount().ExporterKeyringFilePath())
env := corev1.EnvVar{Name: "CEPH_ARGS", Value: val}
return env
}
@@ -30,11 +30,20 @@ import (
const (
crashClient = `client.crash`
exporterClient = `client.ceph-exporter`
crashKeyringTemplate = `
[client.crash]
key = %s
caps mon = "allow profile crash"
caps mgr = "allow rw"
`
exporterKeyringTemplate = `
[client.ceph-exporter]
key = %s
caps mon = "allow profile ceph-exporter"
caps mgr = "allow r"
caps osd = "allow r"
caps mds = "allow r"
`
)
@@ -102,3 +111,69 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo
logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace)
return nil
}
func CreateExporterSecret(context *clusterd.Context, clusterInfo *client.ClusterInfo) error {
k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo)
// Create exporter Ceph key
exporterSecretKey, err := createExporterKeyring(k)
if err != nil {
return errors.Wrapf(err, "failed to create %q ceph keyring", exporterKeyringUsername)
}
// Create or update Kubernetes CSI secret
if err := createOrUpdateExporterSecret(clusterInfo, exporterSecretKey, k); err != nil {
return errors.Wrap(err, "failed to create kubernetes csi secret")
}
return nil
}
func createExporterKeyringCaps() []string {
return []string{
"mon", "allow profile ceph-exporter",
"mgr", "allow r",
"osd", "allow r",
"mds", "allow r",
}
}
func createExporterKeyring(s *keyring.SecretStore) (string, error) {
key, err := s.GenerateKey(exporterKeyringUsername, createExporterKeyringCaps())
if err != nil {
return "", err
}
return key, nil
}
func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecretKey string, k *keyring.SecretStore) error {
keyring := fmt.Sprintf(exporterKeyringTemplate, exporterSecretKey)
exporterSecret := map[string][]byte{
"keyring": []byte(keyring),
}
s := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: exporterKeyName,
Namespace: clusterInfo.Namespace,
},
Data: exporterSecret,
Type: k8sutil.RookType,
}
err := clusterInfo.OwnerInfo.SetControllerReference(s)
if err != nil {
return errors.Wrapf(err, "failed to set owner reference to exporter controller secret %q", s.Name)
}
// Create Kubernetes Secret
err = k.CreateSecret(s)
if err != nil {
return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
}
logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
return nil
}
@@ -26,3 +26,8 @@ func TestCephCrashCollectorKeyringCaps(t *testing.T) {
caps := cephCrashCollectorKeyringCaps()
assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"})
}
func TestExporterKeyringCaps(t *testing.T) {
caps := createExporterKeyringCaps()
assert.Equal(t, caps, []string{"mon", "allow profile ceph-exporter", "mgr", "allow r", "osd", "allow r", "mds", "allow r"})
}
@@ -29,6 +29,7 @@ import (
const (
adminKeyringResourceName = "rook-ceph-admin"
crashCollectorKeyringResourceName = "rook-ceph-crash-collector"
exporterKeyringResourceName = "rook-ceph-exporter"
adminKeyringTemplate = `
[client.admin]
@@ -13,6 +13,7 @@ const (
// mounted independently
adminKeyringDir = "/etc/ceph/admin-keyring-store/"
crashCollectorKeyringDir = "/etc/ceph/crash-collector-keyring-store/"
exporterKeyringDir = "/etc/ceph/exporter-keyring-store/"
)
// VolumeBuilder is a helper for creating Kubernetes pod volumes with content sourced by keyrings
@@ -47,6 +48,11 @@ func (v *VolumeBuilder) CrashCollector() v1.Volume {
return v.Resource(crashCollectorKeyringResourceName)
}
// Exporter returns a kubernetes pod volume whose content is sourced by the SecretStore exporter keyring.
func (v *VolumeBuilder) Exporter() v1.Volume {
return v.Resource(exporterKeyringResourceName)
}
// VolumeMount returns a VolumeMountBuilder.
func VolumeMount() *VolumeMountBuilder { return &VolumeMountBuilder{} }
@@ -80,6 +86,16 @@ func (*VolumeMountBuilder) CrashCollector() v1.VolumeMount {
}
}
// Exporter returns a Kubernetes container volume mount that mounts the content from the matching
// VolumeBuilder Exporter volume.
func (*VolumeMountBuilder) Exporter() v1.VolumeMount {
return v1.VolumeMount{
Name: keyringSecretName(exporterKeyringResourceName),
ReadOnly: true, // should be no reason to write to the keyring in pods, so enforce this
MountPath: exporterKeyringDir,
}
}
// KeyringFilePath returns the full path to the regular keyring file within a container.
func (*VolumeMountBuilder) KeyringFilePath() string {
return path.Join(keyringDir, keyringFileName)
@@ -94,3 +110,8 @@ func (*VolumeMountBuilder) AdminKeyringFilePath() string {
func (*VolumeMountBuilder) CrashCollectorKeyringFilePath() string {
return path.Join(crashCollectorKeyringDir, keyringFileName)
}
// ExporterKeyringFilePath returns the full path to the admin keyring file within a container.
func (*VolumeMountBuilder) ExporterKeyringFilePath() string {
return path.Join(exporterKeyringDir, keyringFileName)
}