diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index ec1f60a4f..79289c2e8 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -235,6 +235,24 @@ The gateway settings correspond to the RGW daemon settings. service.beta.openshift.io/serving-cert-secret-name: ``` +* `opsLogSidecar`: By default, all RGW logs are included in the main RGW container logs. + For enhanced observability of the operations, the [RGW operations log](https://docs.ceph.com/en/latest/radosgw/config-ref/#confval-rgw_enable_ops_log) can be separated from the main RGW container logs by enabling a sidecar. + OpsLogSidecar can be enabled by the following example: + +```yaml + gateway: + opsLogSidecar: + resources: + requests: {} + limits: {} +``` + +Once enabled, logs can be accessed in RGW pod `ops-log` sidecar containers. For example: + +```sh +kubectl --namespace rook-ceph logs rook-ceph-rgw-my-store-a-59d48474d8-jv7ps --container ops-log +``` + ## Zone Settings The [zone](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) settings allow the object store to join custom created [ceph-object-zone](ceph-object-zone-crd.md). diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index c086097bf..99e2000fd 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -7134,6 +7134,20 @@ RGWServiceSpec +opsLogSidecar
+ + +OpsLogSidecar + + + + +(Optional) +

Enable enhanced operation Logs for S3 in a sidecar named ops-log

+ + + + hostNetwork
bool @@ -10800,6 +10814,38 @@ bool +

OpsLogSidecar +

+

+(Appears on:GatewaySpec) +

+
+

RGWLoggingSpec is intended to extend the s3/swift logging for client operations

+
+ + + + + + + + + + + + + +
FieldDescription
+resources
+ + +Kubernetes core/v1.ResourceRequirements + + +		 +(Optional) +

Resources represents the way to specify resource requirements for the ops-log sidecar

+

PeerRemoteSpec

diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index 5f8eddd66..0259c0851 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -609,6 +609,13 @@ cephObjectStores: # sslCertificateRef: instances: 1 priorityClassName: system-cluster-critical + # opsLogSidecar: + # resources: + # limits: + # memory: "100Mi" + # requests: + # cpu: "100m" + # memory: "40Mi" storageClass: enabled: true name: ceph-bucket diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 156fee79f..53a806f02 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -11290,6 +11290,70 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + opsLogSidecar: + description: Enable enhanced operation Logs for S3 in a sidecar named ops-log + nullable: true + properties: + resources: + description: Resources represents the way to specify resource requirements for the ops-log sidecar + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object placement: nullable: true properties: diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 3f557e0c8..25b576af4 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -11281,6 +11281,70 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + opsLogSidecar: + description: Enable enhanced operation Logs for S3 in a sidecar named ops-log + nullable: true + properties: + resources: + description: Resources represents the way to specify resource requirements for the ops-log sidecar + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object placement: nullable: true properties: diff --git a/deploy/examples/object.yaml b/deploy/examples/object.yaml index efc6114ab..321684f99 100644 --- a/deploy/examples/object.yaml +++ b/deploy/examples/object.yaml @@ -112,6 +112,13 @@ spec: #zone: #name: zone-a # service endpoint healthcheck + # opsLogSidecar: + # resources: + # limits: + # memory: "60Mi" + # requests: + # cpu: "100m" + # memory: "60Mi" healthCheck: # Configure the pod probes for the rgw daemon startupProbe: diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 2f5804082..de8c70cce 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1717,6 +1717,11 @@ type GatewaySpec struct { // +nullable Service *RGWServiceSpec `json:"service,omitempty"` + // Enable enhanced operation Logs for S3 in a sidecar named ops-log + // +optional + // +nullable + OpsLogSidecar *OpsLogSidecar `json:"opsLogSidecar,omitempty"` + // Whether host networking is enabled for the rgw daemon. If not set, the network settings from the cluster CR will be applied. // +kubebuilder:pruning:PreserveUnknownFields // +nullable @@ -1752,6 +1757,13 @@ type GatewaySpec struct { RgwCommandFlags map[string]string `json:"rgwCommandFlags,omitempty"` } +// RGWLoggingSpec is intended to extend the s3/swift logging for client operations +type OpsLogSidecar struct { + // Resources represents the way to specify resource requirements for the ops-log sidecar + // +optional + Resources v1.ResourceRequirements `json:"resources,omitempty"` +} + // EndpointAddress is a tuple that describes a single IP address or host name. This is a subset of // Kubernetes's v1.EndpointAddress. // +structType=atomic diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 38f4fbd25..90d36b973 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -2714,6 +2714,11 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { *out = new(RGWServiceSpec) (*in).DeepCopyInto(*out) } + if in.OpsLogSidecar != nil { + in, out := &in.OpsLogSidecar, &out.OpsLogSidecar + *out = new(OpsLogSidecar) + (*in).DeepCopyInto(*out) + } if in.HostNetwork != nil { in, out := &in.HostNetwork, &out.HostNetwork *out = new(bool) @@ -4049,6 +4054,23 @@ func (in *ObjectZoneSpec) DeepCopy() *ObjectZoneSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpsLogSidecar) DeepCopyInto(out *OpsLogSidecar) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpsLogSidecar. +func (in *OpsLogSidecar) DeepCopy() *OpsLogSidecar { + if in == nil { + return nil + } + out := new(OpsLogSidecar) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PeerRemoteSpec) DeepCopyInto(out *PeerRemoteSpec) { *out = *in diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index 5c818bf1a..37eb9c810 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -106,11 +106,12 @@ PERIODICITY=%s LOG_ROTATE_CEPH_FILE=/etc/logrotate.d/ceph LOG_MAX_SIZE=%s ROTATE=%s +ADDITIONAL_LOG_FILES=%s # edit the logrotate file to only rotate a specific daemon log # otherwise we will logrotate log files without reloading certain daemons # this might happen when multiple daemons run on the same machine -sed -i "s|*.log|$CEPH_CLIENT_ID.log|" "$LOG_ROTATE_CEPH_FILE" +sed -i "s|*.log|$CEPH_CLIENT_ID.log $ADDITIONAL_LOG_FILES|" "$LOG_ROTATE_CEPH_FILE" # replace default daily with given user input sed --in-place "s/daily/$PERIODICITY/g" "$LOG_ROTATE_CEPH_FILE" @@ -799,14 +800,17 @@ func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) { } // LogCollectorContainer rotate logs -func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container { +func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, additionalLogFiles ...string) *v1.Container { maxLogSize, periodicity := GetLogRotateConfig(c) - rotation := "7" + if strings.Contains(daemonID, "-client.rbd-mirror") { rotation = "28" } + // Convert the variadic string slice into a space-separated string + additionalLogs := strings.Join(additionalLogFiles, " ") + logger.Debugf("additional log file %q will be used for logCollector", additionalLogs) logger.Debugf("setting periodicity to %q. Supported periodicity are hourly, daily, weekly and monthly", periodicity) return &v1.Container{ @@ -817,7 +821,7 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai "-e", // Exit immediately if a command exits with a non-zero status. "-m", // Terminal job control, allows job to be terminated by SIGTERM "-c", // Command to run - fmt.Sprintf(cronLogRotate, daemonID, periodicity, maxLogSize.String(), rotation), + fmt.Sprintf(cronLogRotate, daemonID, periodicity, maxLogSize.String(), rotation, additionalLogs), }, Image: c.CephVersion.Image, ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy), @@ -829,6 +833,26 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai } } +// rgw operations will be logged in sidecar ops-log +func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, Resources v1.ResourceRequirements) *v1.Container { + return &v1.Container{ + Name: "ops-log", + Command: []string{ + "bash", + "-x", // Enable debugging mode + "-c", // Run the following command + fmt.Sprintf("tail -n+1 -F %s", path.Join(config.VarLogCephDir, opsLogFile)), + }, + Image: c.CephVersion.Image, + ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy), + VolumeMounts: DaemonVolumeMounts(config.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath), + SecurityContext: PodSecurityContext(), + Resources: Resources, + // We need a TTY for the bash job control (enabled by -m) + TTY: true, + } +} + // CreateExternalMetricsEndpoints creates external metric endpoint func createExternalMetricsEndpoints(namespace string, monitoringSpec cephv1.MonitoringSpec, ownerInfo *k8sutil.OwnerInfo) (*v1.Endpoints, error) { labels := AppLabels("rook-ceph-mgr", namespace) diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index ae741177a..8c825f36f 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -384,7 +384,7 @@ func TestLogCollectorContainer(t *testing.T) { t.Run("Periodicity 1d and no MaxlogSize", func(t *testing.T) { c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d"}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "0", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "0", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -392,7 +392,7 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("1M") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1h", MaxLogSize: &maxsize}} got := LogCollectorContainer("ceph-client.rbd-mirror.a", ns, c) - want := fmt.Sprintf(cronLogRotate, "ceph-client.rbd-mirror.a", "hourly", "1M", "28") + want := fmt.Sprintf(cronLogRotate, "ceph-client.rbd-mirror.a", "hourly", "1M", "28", "") assert.Equal(t, want, got.Command[5]) }) @@ -400,7 +400,7 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("1Gi") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "weekly", MaxLogSize: &maxsize}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "1073M", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "1073M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -408,7 +408,7 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("1Mi") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, MaxLogSize: &maxsize}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -416,7 +416,7 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("10G") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "10G", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "10G", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -424,7 +424,7 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("10Mi") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "weekly", MaxLogSize: &maxsize}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "10M", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "10M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -432,7 +432,7 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("1M") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "") assert.Equal(t, want, got.Command[5]) }) @@ -440,7 +440,17 @@ func TestLogCollectorContainer(t *testing.T) { maxsize, _ := resource.ParseQuantity("500K") c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} got := LogCollectorContainer(daemonId, ns, c) - want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7") + want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "") + assert.Equal(t, want, got.Command[5]) + }) + + t.Run("AdditionalLogFile, 500k MaxlogSize and Periodicity 1d", func(t *testing.T) { + additionalLogFile := "/tmp/ceph_test_log_500KB.log" + maxsize, _ := resource.ParseQuantity("500K") + c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}} + got := LogCollectorContainer(daemonId, ns, c, additionalLogFile) + + want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", additionalLogFile) assert.Equal(t, want, got.Command[5]) }) } diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 682aa5d2d..8c3608f2b 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -183,6 +183,12 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri configOptions["rgw_zone"] = rgwConfig.Zone configOptions["rgw_zonegroup"] = rgwConfig.ZoneGroup + if c.store.Spec.Gateway.OpsLogSidecar != nil { + configOptions["rgw_enable_ops_log"] = "true" + } else { + configOptions["rgw_enable_ops_log"] = "false" + } + configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions) if err != nil { return configOptions, err diff --git a/pkg/operator/ceph/object/config_test.go b/pkg/operator/ceph/object/config_test.go index d0ba361c2..effe400ed 100644 --- a/pkg/operator/ceph/object/config_test.go +++ b/pkg/operator/ceph/object/config_test.go @@ -102,6 +102,7 @@ func TestGenerateCephXUser(t *testing.T) { // RGW configs determined here are applied to the running RGW func Test_clusterConfig_generateMonConfigOptions(t *testing.T) { defaultConfigs := map[string]string{ + "rgw_enable_ops_log": "false", "rgw_enable_usage_log": "true", "rgw_log_nonexistent_bucket": "true", "rgw_log_object_name_utc": "true", diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go index e3f5e4f06..371d8ef10 100644 --- a/pkg/operator/ceph/object/spec.go +++ b/pkg/operator/ceph/object/spec.go @@ -158,11 +158,26 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec ServiceAccountName: serviceAccountName, } + opsLogFile := "" + if opsLogSidecar := c.store.Spec.Gateway.OpsLogSidecar; opsLogSidecar != nil { + // Generate ops log file name based on rgw daemon ID + opsLogName := fmt.Sprintf("ops-log-%s.log", getDaemonName(rgwConfig)) + opsLogFile = path.Join(cephconfig.VarLogCephDir, opsLogName) + + // Add the side-car container named ops-log + podSpec.Containers = append(podSpec.Containers, + *controller.RgwOpsLogSidecarContainer(opsLogName, + c.clusterInfo.Namespace, *c.clusterSpec, + opsLogSidecar.Resources)) + } + // If the log collector is enabled we add the side-car container if c.clusterSpec.LogCollector.Enabled { shareProcessNamespace := true podSpec.ShareProcessNamespace = &shareProcessNamespace - podSpec.Containers = append(podSpec.Containers, *controller.LogCollectorContainer(getDaemonName(rgwConfig), c.clusterInfo.Namespace, *c.clusterSpec)) + podSpec.Containers = append(podSpec.Containers, + *controller.LogCollectorContainer(getDaemonName(rgwConfig), + c.clusterInfo.Namespace, *c.clusterSpec, opsLogFile)) } // Replace default unreachable node toleration