Merge pull request #13239 from avanthakkar/run-exporter-with-specific-keyring
exporter: run exporter with specific keyring
This commit is contained in:
6 files changed
+121
-6
No files matched your search
@@ -464,6 +464,12 @@ func (c *cluster) postMonStartupActions() error {
|
||||
return errors.Wrap(err, "failed to create crash collector kubernetes secret")
|
||||
}
|
||||
|
||||
// Create exporter Kubernetes Secret
|
||||
err = nodedaemon.CreateExporterSecret(c.context, c.ClusterInfo)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create exporter kubernetes secret")
|
||||
}
|
||||
|
||||
if err := c.configureMsgr2(); err != nil {
|
||||
return errors.Wrap(err, "failed to configure msgr2")
|
||||
}
|
||||
|
||||
@@ -46,6 +46,8 @@ const (
|
||||
statsPeriod = "5"
|
||||
DefaultMetricsPort uint16 = 9926
|
||||
exporterServiceMetricName = "ceph-exporter-http-metrics"
|
||||
exporterKeyringUsername = "client.ceph-exporter"
|
||||
exporterKeyName = "rook-ceph-exporter-keyring"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -82,7 +84,7 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations
|
||||
|
||||
volumes := append(
|
||||
controller.DaemonVolumesBase(config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath), "", cephCluster.Spec.DataDirHostPath),
|
||||
keyring.Volume().Admin())
|
||||
keyring.Volume().Exporter())
|
||||
|
||||
mutateFunc := func() error {
|
||||
|
||||
@@ -166,12 +168,10 @@ func getCephExporterDaemonContainer(cephCluster cephv1.CephCluster, cephVersion
|
||||
cephImage := cephCluster.Spec.CephVersion.Image
|
||||
dataPathMap := config.NewDatalessDaemonDataPathMap(cephCluster.GetNamespace(), cephCluster.Spec.DataDirHostPath)
|
||||
volumeMounts := controller.DaemonVolumeMounts(dataPathMap, "", cephCluster.Spec.DataDirHostPath)
|
||||
// FIX: Use an exporter keyring instead of the admin keyring
|
||||
volumeMounts = append(volumeMounts, keyring.VolumeMount().Admin())
|
||||
volumeMounts = append(volumeMounts, keyring.VolumeMount().Exporter())
|
||||
|
||||
envVars := append(
|
||||
controller.DaemonEnvVars(&cephCluster.Spec),
|
||||
corev1.EnvVar{Name: "CEPH_ARGS", Value: fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -k %s", keyring.VolumeMount().AdminKeyringFilePath())})
|
||||
exporterEnvVar := generateExporterEnvVar()
|
||||
envVars := append(controller.DaemonEnvVars(&cephCluster.Spec), exporterEnvVar)
|
||||
|
||||
args := []string{
|
||||
"--sock-dir", sockDir,
|
||||
@@ -278,3 +278,10 @@ func applyPrometheusAnnotations(cephCluster cephv1.CephCluster, objectMeta *meta
|
||||
t.ApplyToObjectMeta(objectMeta)
|
||||
}
|
||||
}
|
||||
|
||||
func generateExporterEnvVar() corev1.EnvVar {
|
||||
val := fmt.Sprintf("-m $(ROOK_CEPH_MON_HOST) -n %s -k %s", exporterKeyringUsername, keyring.VolumeMount().ExporterKeyringFilePath())
|
||||
env := corev1.EnvVar{Name: "CEPH_ARGS", Value: val}
|
||||
|
||||
return env
|
||||
}
|
||||
@@ -30,11 +30,20 @@ import (
|
||||
|
||||
const (
|
||||
crashClient = `client.crash`
|
||||
exporterClient = `client.ceph-exporter`
|
||||
crashKeyringTemplate = `
|
||||
[client.crash]
|
||||
key = %s
|
||||
caps mon = "allow profile crash"
|
||||
caps mgr = "allow rw"
|
||||
`
|
||||
exporterKeyringTemplate = `
|
||||
[client.ceph-exporter]
|
||||
key = %s
|
||||
caps mon = "allow profile ceph-exporter"
|
||||
caps mgr = "allow r"
|
||||
caps osd = "allow r"
|
||||
caps mds = "allow r"
|
||||
`
|
||||
)
|
||||
|
||||
@@ -102,3 +111,69 @@ func createOrUpdateCrashCollectorSecret(clusterInfo *client.ClusterInfo, crashCo
|
||||
logger.Infof("created kubernetes crash collector secret for cluster %q", clusterInfo.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
func CreateExporterSecret(context *clusterd.Context, clusterInfo *client.ClusterInfo) error {
|
||||
k := keyring.GetSecretStore(context, clusterInfo, clusterInfo.OwnerInfo)
|
||||
|
||||
// Create exporter Ceph key
|
||||
exporterSecretKey, err := createExporterKeyring(k)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create %q ceph keyring", exporterKeyringUsername)
|
||||
}
|
||||
|
||||
// Create or update Kubernetes CSI secret
|
||||
if err := createOrUpdateExporterSecret(clusterInfo, exporterSecretKey, k); err != nil {
|
||||
return errors.Wrap(err, "failed to create kubernetes csi secret")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createExporterKeyringCaps() []string {
|
||||
return []string{
|
||||
"mon", "allow profile ceph-exporter",
|
||||
"mgr", "allow r",
|
||||
"osd", "allow r",
|
||||
"mds", "allow r",
|
||||
}
|
||||
}
|
||||
|
||||
func createExporterKeyring(s *keyring.SecretStore) (string, error) {
|
||||
key, err := s.GenerateKey(exporterKeyringUsername, createExporterKeyringCaps())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func createOrUpdateExporterSecret(clusterInfo *client.ClusterInfo, exporterSecretKey string, k *keyring.SecretStore) error {
|
||||
|
||||
keyring := fmt.Sprintf(exporterKeyringTemplate, exporterSecretKey)
|
||||
|
||||
exporterSecret := map[string][]byte{
|
||||
"keyring": []byte(keyring),
|
||||
}
|
||||
|
||||
s := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: exporterKeyName,
|
||||
Namespace: clusterInfo.Namespace,
|
||||
},
|
||||
Data: exporterSecret,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
err := clusterInfo.OwnerInfo.SetControllerReference(s)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to set owner reference to exporter controller secret %q", s.Name)
|
||||
}
|
||||
|
||||
// Create Kubernetes Secret
|
||||
err = k.CreateSecret(s)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create kubernetes secret %q for cluster %q", s.Name, clusterInfo.Namespace)
|
||||
}
|
||||
|
||||
logger.Infof("created kubernetes exporter secret for cluster %q", clusterInfo.Namespace)
|
||||
return nil
|
||||
}
|
||||
@@ -26,3 +26,8 @@ func TestCephCrashCollectorKeyringCaps(t *testing.T) {
|
||||
caps := cephCrashCollectorKeyringCaps()
|
||||
assert.Equal(t, caps, []string{"mon", "allow profile crash", "mgr", "allow rw"})
|
||||
}
|
||||
|
||||
func TestExporterKeyringCaps(t *testing.T) {
|
||||
caps := createExporterKeyringCaps()
|
||||
assert.Equal(t, caps, []string{"mon", "allow profile ceph-exporter", "mgr", "allow r", "osd", "allow r", "mds", "allow r"})
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
const (
|
||||
adminKeyringResourceName = "rook-ceph-admin"
|
||||
crashCollectorKeyringResourceName = "rook-ceph-crash-collector"
|
||||
exporterKeyringResourceName = "rook-ceph-exporter"
|
||||
|
||||
adminKeyringTemplate = `
|
||||
[client.admin]
|
||||
|
||||
@@ -13,6 +13,7 @@ const (
|
||||
// mounted independently
|
||||
adminKeyringDir = "/etc/ceph/admin-keyring-store/"
|
||||
crashCollectorKeyringDir = "/etc/ceph/crash-collector-keyring-store/"
|
||||
exporterKeyringDir = "/etc/ceph/exporter-keyring-store/"
|
||||
)
|
||||
|
||||
// VolumeBuilder is a helper for creating Kubernetes pod volumes with content sourced by keyrings
|
||||
@@ -47,6 +48,11 @@ func (v *VolumeBuilder) CrashCollector() v1.Volume {
|
||||
return v.Resource(crashCollectorKeyringResourceName)
|
||||
}
|
||||
|
||||
// Exporter returns a kubernetes pod volume whose content is sourced by the SecretStore exporter keyring.
|
||||
func (v *VolumeBuilder) Exporter() v1.Volume {
|
||||
return v.Resource(exporterKeyringResourceName)
|
||||
}
|
||||
|
||||
// VolumeMount returns a VolumeMountBuilder.
|
||||
func VolumeMount() *VolumeMountBuilder { return &VolumeMountBuilder{} }
|
||||
|
||||
@@ -80,6 +86,16 @@ func (*VolumeMountBuilder) CrashCollector() v1.VolumeMount {
|
||||
}
|
||||
}
|
||||
|
||||
// Exporter returns a Kubernetes container volume mount that mounts the content from the matching
|
||||
// VolumeBuilder Exporter volume.
|
||||
func (*VolumeMountBuilder) Exporter() v1.VolumeMount {
|
||||
return v1.VolumeMount{
|
||||
Name: keyringSecretName(exporterKeyringResourceName),
|
||||
ReadOnly: true, // should be no reason to write to the keyring in pods, so enforce this
|
||||
MountPath: exporterKeyringDir,
|
||||
}
|
||||
}
|
||||
|
||||
// KeyringFilePath returns the full path to the regular keyring file within a container.
|
||||
func (*VolumeMountBuilder) KeyringFilePath() string {
|
||||
return path.Join(keyringDir, keyringFileName)
|
||||
@@ -94,3 +110,8 @@ func (*VolumeMountBuilder) AdminKeyringFilePath() string {
|
||||
func (*VolumeMountBuilder) CrashCollectorKeyringFilePath() string {
|
||||
return path.Join(crashCollectorKeyringDir, keyringFileName)
|
||||
}
|
||||
|
||||
// ExporterKeyringFilePath returns the full path to the admin keyring file within a container.
|
||||
func (*VolumeMountBuilder) ExporterKeyringFilePath() string {
|
||||
return path.Join(exporterKeyringDir, keyringFileName)
|
||||
}
|
||||
Reference in new issue
Block a user