From c1fd2f2ee86be4d32e0d1a6c3a06910df06f13a3 Mon Sep 17 00:00:00 2001 From: Artem Torubarov Date: Fri, 28 Mar 2025 16:07:38 +0100 Subject: [PATCH] rgw: use pod name in ops log filename Signed-off-by: Artem Torubarov --- pkg/operator/ceph/cluster/mgr/spec.go | 2 +- pkg/operator/ceph/cluster/mon/spec.go | 2 +- pkg/operator/ceph/cluster/osd/spec.go | 2 +- pkg/operator/ceph/cluster/rbd/spec.go | 2 +- pkg/operator/ceph/controller/spec.go | 6 ++-- pkg/operator/ceph/controller/spec_test.go | 18 +++++----- pkg/operator/ceph/file/mds/spec.go | 2 +- pkg/operator/ceph/file/mirror/spec.go | 2 +- pkg/operator/ceph/object/config.go | 6 ---- pkg/operator/ceph/object/config_test.go | 1 - pkg/operator/ceph/object/spec.go | 43 ++++++++++++++++++----- 11 files changed, 54 insertions(+), 32 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index b004bdaa2..01e747320 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -103,7 +103,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) if c.spec.LogCollector.Enabled { shareProcessNamespace := true podSpec.Spec.ShareProcessNamespace = &shareProcessNamespace - podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-mgr.%s", mgrConfig.DaemonID), c.clusterInfo.Namespace, c.spec)) + podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-mgr.%s", mgrConfig.DaemonID), c.clusterInfo.Namespace, c.spec, nil)) } // Replace default unreachable node toleration diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index 92c547989..09ad8bc85 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -200,7 +200,7 @@ func (c *Cluster) makeMonPod(monConfig *monConfig, canary bool) (*corev1.Pod, er if c.spec.LogCollector.Enabled { shareProcessNamespace := true podSpec.ShareProcessNamespace = &shareProcessNamespace - podSpec.Containers = append(podSpec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("%s.%s", cephMonCommand, monConfig.DaemonName), c.ClusterInfo.Namespace, c.spec)) + podSpec.Containers = append(podSpec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("%s.%s", cephMonCommand, monConfig.DaemonName), c.ClusterInfo.Namespace, c.spec, nil)) } // Replace default unreachable node toleration diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 850e13a97..0fdd7f040 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -715,7 +715,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision shareProcessNamespace := true podTemplateSpec.Spec.ShareProcessNamespace = &shareProcessNamespace } - podTemplateSpec.Spec.Containers = append(podTemplateSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-osd.%s", osdID), c.clusterInfo.Namespace, c.spec)) + podTemplateSpec.Spec.Containers = append(podTemplateSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-osd.%s", osdID), c.clusterInfo.Namespace, c.spec, nil)) } podTemplateSpec.Spec.Containers[0] = opconfig.ConfigureStartupProbe(podTemplateSpec.Spec.Containers[0], c.spec.HealthCheck.StartupProbe[cephv1.KeyOSD]) diff --git a/pkg/operator/ceph/cluster/rbd/spec.go b/pkg/operator/ceph/cluster/rbd/spec.go index b7b7a2ccf..cb78011da 100644 --- a/pkg/operator/ceph/cluster/rbd/spec.go +++ b/pkg/operator/ceph/cluster/rbd/spec.go @@ -57,7 +57,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM // ceph-client.rbd-mirror.a.log // -client.rbd-mirror-peer.log logRotationFilter := "*-client.rbd-mirror*" - podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(logRotationFilter, r.clusterInfo.Namespace, *r.cephClusterSpec)) + podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(logRotationFilter, r.clusterInfo.Namespace, *r.cephClusterSpec, nil)) } // Replace default unreachable node toleration diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 3b299f05a..bf2d623bf 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -799,7 +799,7 @@ func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) { } // LogCollectorContainer rotate logs -func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, additionalLogFiles ...string) *v1.Container { +func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, env []v1.EnvVar, additionalLogFiles ...string) *v1.Container { maxLogSize, periodicity := GetLogRotateConfig(c) rotation := "7" @@ -829,11 +829,12 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, additional Resources: cephv1.GetLogCollectorResources(c.Resources), // We need a TTY for the bash job control (enabled by -m) TTY: true, + Env: env, } } // rgw operations will be logged in sidecar ops-log -func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, Resources v1.ResourceRequirements) *v1.Container { +func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, env []v1.EnvVar, Resources v1.ResourceRequirements) *v1.Container { return &v1.Container{ Name: "ops-log", Command: []string{ @@ -849,6 +850,7 @@ func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, Reso Resources: Resources, // We need a TTY for the bash job control (enabled by -m) TTY: true, + Env: env, } } diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index ea3e13c44..0c0cc2025 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -392,7 +392,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("Periodicity 1d and no MaxlogSize", func(t *testing.T) { c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d"}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "0", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -400,7 +400,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("Periodicity 1h and MaxlogSize 1M", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("1M") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1h", MaxLogSize: &maxsize}} - got := LogCollectorContainer("ceph-client.rbd-mirror.a", ns, c) + got := LogCollectorContainer("ceph-client.rbd-mirror.a", ns, c, nil) want := fmt.Sprintf(cronLogRotate, "ceph-client.rbd-mirror.a", "hourly", "1M", "28", "") assert.Equal(t, want, got.Command[5]) }) @@ -408,7 +408,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("Periodicity weekly and 1G MaxlogSize", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("1Gi") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "weekly", MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "1073M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -416,7 +416,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("MaxlogSize 1M and no Periodicity", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("1Mi") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -424,7 +424,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("10G MaxlogSize and Periodicity 1d", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("10G") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "10G", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -432,7 +432,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("10M MaxlogSize and Periodicity weekly", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("10Mi") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "weekly", MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "10M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -440,7 +440,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("1M MaxlogSize and Periodicity 1d", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("1M") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -448,7 +448,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("500k MaxlogSize and Periodicity 1d", func(t *testing.T) { maxsize, _ := resource.ParseQuantity("500K") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c) + got := LogCollectorContainer(daemonId, ns, c, nil) want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -457,7 +457,7 @@ func TestLogCollectorContainer(t *testing.T) { additionalLogFile := "/tmp/ceph_test_log_500KB.log" maxsize, _ := resource.ParseQuantity("500K") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} - got := LogCollectorContainer(daemonId, ns, c, additionalLogFile) + got := LogCollectorContainer(daemonId, ns, c, nil, additionalLogFile) want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", additionalLogFile) assert.Equal(t, want, got.Command[5]) diff --git a/pkg/operator/ceph/file/mds/spec.go b/pkg/operator/ceph/file/mds/spec.go index 9f93ef4b4..ac54eba58 100644 --- a/pkg/operator/ceph/file/mds/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -76,7 +76,7 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, fsNamespacedname types.Na if c.clusterSpec.LogCollector.Enabled { shareProcessNamespace := true podSpec.Spec.ShareProcessNamespace = &shareProcessNamespace - podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-mds.%s", mdsConfig.DaemonID), c.clusterInfo.Namespace, *c.clusterSpec)) + podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-mds.%s", mdsConfig.DaemonID), c.clusterInfo.Namespace, *c.clusterSpec, nil)) } c.fs.Spec.MetadataServer.Annotations.ApplyToObjectMeta(&podSpec.ObjectMeta) diff --git a/pkg/operator/ceph/file/mirror/spec.go b/pkg/operator/ceph/file/mirror/spec.go index 2facdceb3..45fdc9a38 100644 --- a/pkg/operator/ceph/file/mirror/spec.go +++ b/pkg/operator/ceph/file/mirror/spec.go @@ -55,7 +55,7 @@ func (r *ReconcileFilesystemMirror) makeDeployment(daemonConfig *daemonConfig, f if r.cephClusterSpec.LogCollector.Enabled { shareProcessNamespace := true podSpec.Spec.ShareProcessNamespace = &shareProcessNamespace - podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-%s", user), r.clusterInfo.Namespace, *r.cephClusterSpec)) + podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-%s", user), r.clusterInfo.Namespace, *r.cephClusterSpec, nil)) } // Replace default unreachable node toleration diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 80c8a3473..2e83f5ed0 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -182,12 +182,6 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri configOptions["rgw_zone"] = rgwConfig.Zone configOptions["rgw_zonegroup"] = rgwConfig.ZoneGroup - if c.store.Spec.Gateway.OpsLogSidecar != nil { - configOptions["rgw_enable_ops_log"] = "true" - } else { - configOptions["rgw_enable_ops_log"] = "false" - } - configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions) if err != nil { return configOptions, err diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index d435f917c..7036b66ca 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -107,7 +107,6 @@ func TestGenerateCephXUser(t *testing.T) { // RGW configs determined here are applied to the running RGW func Test_clusterConfig_generateMonConfigOptions(t *testing.T) { defaultConfigs := map[string]string{ - "rgw_enable_ops_log": "false", "rgw_enable_usage_log": "true", "rgw_log_nonexistent_bucket": "true", "rgw_log_object_name_utc": "true", diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index d8222978d..bae2a1e87 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -82,6 +82,31 @@ var ( rgwProbeScriptTemplate string rgwAPIwithoutS3 = []string{"s3website", "swift", "swift_auth", "admin", "sts", "iam", "notifications"} + + // used to pass Pod name and namespaces to Pod container arguments. + podNameEnvVars = []v1.EnvVar{ + { + Name: "POD_NS", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + } + // Filename to store RGW ops log per pod. + // Relies on the on environment variables to get the pod name and namespace. + // podNameEnvVars have to be passed to to log collector container. + opsLogFilename = "ops-log.$(POD_NS).$(POD_NAME).log" + opsLogAbsFilename = path.Join(cephconfig.VarLogCephDir, opsLogFilename) ) type ( @@ -168,16 +193,11 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec ServiceAccountName: serviceAccountName, } - opsLogFile := "" if opsLogSidecar := c.store.Spec.Gateway.OpsLogSidecar; opsLogSidecar != nil { - // Generate ops log file name based on rgw daemon ID - opsLogName := fmt.Sprintf("ops-log-%s.log", getDaemonName(rgwConfig)) - opsLogFile = path.Join(cephconfig.VarLogCephDir, opsLogName) - // Add the side-car container named ops-log podSpec.Containers = append(podSpec.Containers, - *controller.RgwOpsLogSidecarContainer(opsLogName, - c.clusterInfo.Namespace, *c.clusterSpec, + *controller.RgwOpsLogSidecarContainer(opsLogFilename, + c.clusterInfo.Namespace, *c.clusterSpec, podNameEnvVars, opsLogSidecar.Resources)) } @@ -187,7 +207,7 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec podSpec.ShareProcessNamespace = &shareProcessNamespace podSpec.Containers = append(podSpec.Containers, *controller.LogCollectorContainer(getDaemonName(rgwConfig), - c.clusterInfo.Namespace, *c.clusterSpec, opsLogFile)) + c.clusterInfo.Namespace, *c.clusterSpec, podNameEnvVars, opsLogAbsFilename)) } // Replace default unreachable node toleration @@ -431,6 +451,13 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, container.Args = append(container.Args, c.sseKMSVaultTLSOptions(kmsEnabled)...) } } + if c.store.Spec.Gateway.OpsLogSidecar != nil { + container.Env = append(container.Env, podNameEnvVars...) + container.Args = append(container.Args, + cephconfig.NewFlag("rgw_enable_ops_log", "true"), + cephconfig.NewFlag("rgw_ops_log_file_path", opsLogAbsFilename), + ) + } if flags := buildRGWConfigFlags(c.store); len(flags) != 0 { container.Args = append(container.Args, flags...)