feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #14646 from ideepika/wip-rgw-sidecar

Browse Source 
add rgw ops sidecar for op logs
This commit is contained in:
Travis Nielsen
2024-12-17 10:35:08 -07:00
committed by GitHub
parent c116748182 c5d27467d6
commit 1a44ab3fa3
13 changed files with 309 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
+18
View File
@@ -235,6 +235,24 @@ The gateway settings correspond to the RGW daemon settings.
service.beta.openshift.io/serving-cert-secret-name: <name of TLS secret for automatic generation>
```
* `opsLogSidecar`: By default, all RGW logs are included in the main RGW container logs.
For enhanced observability of the operations, the [RGW operations log](https://docs.ceph.com/en/latest/radosgw/config-ref/#confval-rgw_enable_ops_log) can be separated from the main RGW container logs by enabling a sidecar.
OpsLogSidecar can be enabled by the following example:
```yaml
gateway:
opsLogSidecar:
resources:
requests: {}
limits: {}
```
Once enabled, logs can be accessed in RGW pod `ops-log` sidecar containers. For example:
```sh
kubectl --namespace rook-ceph logs rook-ceph-rgw-my-store-a-59d48474d8-jv7ps --container ops-log
```
## Zone Settings
The [zone](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) settings allow the object store to join custom created [ceph-object-zone](ceph-object-zone-crd.md).
Documentation/CRDs/specification.md
+46
View File
@@ -7134,6 +7134,20 @@ RGWServiceSpec
</tr>
<tr>
<td>
<code>opsLogSidecar</code><br/>
<em>
<a href="#ceph.rook.io/v1.OpsLogSidecar">
OpsLogSidecar
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>Enable enhanced operation Logs for S3 in a sidecar named ops-log</p>
</td>
</tr>
<tr>
<td>
<code>hostNetwork</code><br/>
<em>
bool
@@ -10800,6 +10814,38 @@ bool
</tr>
</tbody>
</table>
<h3 id="ceph.rook.io/v1.OpsLogSidecar">OpsLogSidecar
</h3>
<p>
(<em>Appears on:</em><a href="#ceph.rook.io/v1.GatewaySpec">GatewaySpec</a>)
</p>
<div>
<p>RGWLoggingSpec is intended to extend the s3/swift logging for client operations</p>
</div>
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>resources</code><br/>
<em>
<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcerequirements-v1-core">
Kubernetes core/v1.ResourceRequirements
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>Resources represents the way to specify resource requirements for the ops-log sidecar</p>
</td>
</tr>
</tbody>
</table>
<h3 id="ceph.rook.io/v1.PeerRemoteSpec">PeerRemoteSpec
</h3>
<p>
deploy/charts/rook-ceph-cluster/values.yaml
+7
View File
@@ -609,6 +609,13 @@ cephObjectStores:
# sslCertificateRef:
instances: 1
priorityClassName: system-cluster-critical
# opsLogSidecar:
# resources:
# limits:
# memory: "100Mi"
# requests:
# cpu: "100m"
# memory: "40Mi"
storageClass:
enabled: true
name: ceph-bucket
deploy/charts/rook-ceph/templates/resources.yaml
+64
View File
@@ -11290,6 +11290,70 @@ spec:
nullable: true
type: object
x-kubernetes-preserve-unknown-fields: true
opsLogSidecar:
description: Enable enhanced operation Logs for S3 in a sidecar named ops-log
nullable: true
properties:
resources:
description: Resources represents the way to specify resource requirements for the ops-log sidecar
properties:
claims:
description: |-
Claims lists the names of resources, defined in spec.resourceClaims,
that are used by this container.
This is an alpha field and requires enabling the
DynamicResourceAllocation feature gate.
This field is immutable. It can only be set for containers.
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: |-
Name must match the name of one entry in pod.spec.resourceClaims of
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Limits describes the maximum amount of compute resources allowed.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Requests describes the minimum amount of compute resources required.
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
otherwise to an implementation-defined value. Requests cannot exceed Limits.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
type: object
type: object
placement:
nullable: true
properties:
deploy/examples/crds.yaml
+64
View File
@@ -11281,6 +11281,70 @@ spec:
nullable: true
type: object
x-kubernetes-preserve-unknown-fields: true
opsLogSidecar:
description: Enable enhanced operation Logs for S3 in a sidecar named ops-log
nullable: true
properties:
resources:
description: Resources represents the way to specify resource requirements for the ops-log sidecar
properties:
claims:
description: |-
Claims lists the names of resources, defined in spec.resourceClaims,
that are used by this container.
This is an alpha field and requires enabling the
DynamicResourceAllocation feature gate.
This field is immutable. It can only be set for containers.
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: |-
Name must match the name of one entry in pod.spec.resourceClaims of
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Limits describes the maximum amount of compute resources allowed.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Requests describes the minimum amount of compute resources required.
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
otherwise to an implementation-defined value. Requests cannot exceed Limits.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
type: object
type: object
placement:
nullable: true
properties:
deploy/examples/object.yaml
+7
View File
@@ -112,6 +112,13 @@ spec:
#zone:
#name: zone-a
# service endpoint healthcheck
# opsLogSidecar:
# resources:
# limits:
# memory: "60Mi"
# requests:
# cpu: "100m"
# memory: "60Mi"
healthCheck:
# Configure the pod probes for the rgw daemon
startupProbe:
pkg/apis/ceph.rook.io/v1/types.go
+12
View File
@@ -1717,6 +1717,11 @@ type GatewaySpec struct {
// +nullable
Service *RGWServiceSpec `json:"service,omitempty"`
// Enable enhanced operation Logs for S3 in a sidecar named ops-log
// +optional
// +nullable
OpsLogSidecar *OpsLogSidecar `json:"opsLogSidecar,omitempty"`
// Whether host networking is enabled for the rgw daemon. If not set, the network settings from the cluster CR will be applied.
// +kubebuilder:pruning:PreserveUnknownFields
// +nullable
@@ -1752,6 +1757,13 @@ type GatewaySpec struct {
RgwCommandFlags map[string]string `json:"rgwCommandFlags,omitempty"`
}
// RGWLoggingSpec is intended to extend the s3/swift logging for client operations
type OpsLogSidecar struct {
// Resources represents the way to specify resource requirements for the ops-log sidecar
// +optional
Resources v1.ResourceRequirements `json:"resources,omitempty"`
}
// EndpointAddress is a tuple that describes a single IP address or host name. This is a subset of
// Kubernetes's v1.EndpointAddress.
// +structType=atomic
pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+22
View File
@@ -2714,6 +2714,11 @@ func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) {
*out = new(RGWServiceSpec)
(*in).DeepCopyInto(*out)
}
if in.OpsLogSidecar != nil {
in, out := &in.OpsLogSidecar, &out.OpsLogSidecar
*out = new(OpsLogSidecar)
(*in).DeepCopyInto(*out)
}
if in.HostNetwork != nil {
in, out := &in.HostNetwork, &out.HostNetwork
*out = new(bool)
@@ -4049,6 +4054,23 @@ func (in *ObjectZoneSpec) DeepCopy() *ObjectZoneSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OpsLogSidecar) DeepCopyInto(out *OpsLogSidecar) {
*out = *in
in.Resources.DeepCopyInto(&out.Resources)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpsLogSidecar.
func (in *OpsLogSidecar) DeepCopy() *OpsLogSidecar {
if in == nil {
return nil
}
out := new(OpsLogSidecar)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PeerRemoteSpec) DeepCopyInto(out *PeerRemoteSpec) {
*out = *in
pkg/operator/ceph/controller/spec.go
+28 -4
View File
@@ -106,11 +106,12 @@ PERIODICITY=%s
LOG_ROTATE_CEPH_FILE=/etc/logrotate.d/ceph
LOG_MAX_SIZE=%s
ROTATE=%s
ADDITIONAL_LOG_FILES=%s
# edit the logrotate file to only rotate a specific daemon log
# otherwise we will logrotate log files without reloading certain daemons
# this might happen when multiple daemons run on the same machine
sed -i "s|*.log|$CEPH_CLIENT_ID.log|" "$LOG_ROTATE_CEPH_FILE"
sed -i "s|*.log|$CEPH_CLIENT_ID.log $ADDITIONAL_LOG_FILES|" "$LOG_ROTATE_CEPH_FILE"
# replace default daily with given user input
sed --in-place "s/daily/$PERIODICITY/g" "$LOG_ROTATE_CEPH_FILE"
@@ -799,14 +800,17 @@ func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) {
}
// LogCollectorContainer rotate logs
func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {
func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec, additionalLogFiles ...string) *v1.Container {
maxLogSize, periodicity := GetLogRotateConfig(c)
rotation := "7"
if strings.Contains(daemonID, "-client.rbd-mirror") {
rotation = "28"
}
// Convert the variadic string slice into a space-separated string
additionalLogs := strings.Join(additionalLogFiles, " ")
logger.Debugf("additional log file %q will be used for logCollector", additionalLogs)
logger.Debugf("setting periodicity to %q. Supported periodicity are hourly, daily, weekly and monthly", periodicity)
return &v1.Container{
@@ -817,7 +821,7 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai
"-e", // Exit immediately if a command exits with a non-zero status.
"-m", // Terminal job control, allows job to be terminated by SIGTERM
"-c", // Command to run
fmt.Sprintf(cronLogRotate, daemonID, periodicity, maxLogSize.String(), rotation),
fmt.Sprintf(cronLogRotate, daemonID, periodicity, maxLogSize.String(), rotation, additionalLogs),
},
Image: c.CephVersion.Image,
ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy),
@@ -829,6 +833,26 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai
}
}
// rgw operations will be logged in sidecar ops-log
func RgwOpsLogSidecarContainer(opsLogFile, ns string, c cephv1.ClusterSpec, Resources v1.ResourceRequirements) *v1.Container {
return &v1.Container{
Name: "ops-log",
Command: []string{
"bash",
"-x", // Enable debugging mode
"-c", // Run the following command
fmt.Sprintf("tail -n+1 -F %s", path.Join(config.VarLogCephDir, opsLogFile)),
},
Image: c.CephVersion.Image,
ImagePullPolicy: GetContainerImagePullPolicy(c.CephVersion.ImagePullPolicy),
VolumeMounts: DaemonVolumeMounts(config.NewDatalessDaemonDataPathMap(ns, c.DataDirHostPath), "", c.DataDirHostPath),
SecurityContext: PodSecurityContext(),
Resources: Resources,
// We need a TTY for the bash job control (enabled by -m)
TTY: true,
}
}
// CreateExternalMetricsEndpoints creates external metric endpoint
func createExternalMetricsEndpoints(namespace string, monitoringSpec cephv1.MonitoringSpec, ownerInfo *k8sutil.OwnerInfo) (*v1.Endpoints, error) {
labels := AppLabels("rook-ceph-mgr", namespace)
pkg/operator/ceph/controller/spec_test.go
+18 -8
View File
@@ -384,7 +384,7 @@ func TestLogCollectorContainer(t *testing.T) {
t.Run("Periodicity 1d and no MaxlogSize", func(t *testing.T) {
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d"}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "0", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "0", "7", "")
assert.Equal(t, want, got.Command[5])
})
@@ -392,7 +392,7 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("1M")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1h", MaxLogSize: &maxsize}}
got := LogCollectorContainer("ceph-client.rbd-mirror.a", ns, c)
want := fmt.Sprintf(cronLogRotate, "ceph-client.rbd-mirror.a", "hourly", "1M", "28")
want := fmt.Sprintf(cronLogRotate, "ceph-client.rbd-mirror.a", "hourly", "1M", "28", "")
assert.Equal(t, want, got.Command[5])
})
@@ -400,7 +400,7 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("1Gi")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "weekly", MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "1073M", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "1073M", "7", "")
assert.Equal(t, want, got.Command[5])
})
@@ -408,7 +408,7 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("1Mi")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "")
assert.Equal(t, want, got.Command[5])
})
@@ -416,7 +416,7 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("10G")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "10G", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "10G", "7", "")
assert.Equal(t, want, got.Command[5])
})
@@ -424,7 +424,7 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("10Mi")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "weekly", MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "10M", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "weekly", "10M", "7", "")
assert.Equal(t, want, got.Command[5])
})
@@ -432,7 +432,7 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("1M")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "")
assert.Equal(t, want, got.Command[5])
})
@@ -440,7 +440,17 @@ func TestLogCollectorContainer(t *testing.T) {
maxsize, _ := resource.ParseQuantity("500K")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c)
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7")
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", "")
assert.Equal(t, want, got.Command[5])
})
t.Run("AdditionalLogFile, 500k MaxlogSize and Periodicity 1d", func(t *testing.T) {
additionalLogFile := "/tmp/ceph_test_log_500KB.log"
maxsize, _ := resource.ParseQuantity("500K")
c := cephv1.ClusterSpec{LogCollector: cephv1.LogCollectorSpec{Enabled: true, Periodicity: "1d", MaxLogSize: &maxsize}}
got := LogCollectorContainer(daemonId, ns, c, additionalLogFile)
want := fmt.Sprintf(cronLogRotate, daemonId, "daily", "1M", "7", additionalLogFile)
assert.Equal(t, want, got.Command[5])
})
}
pkg/operator/ceph/object/config.go
+6
View File
@@ -183,6 +183,12 @@ func (c *clusterConfig) generateMonConfigOptions(rgwConfig *rgwConfig) (map[stri
configOptions["rgw_zone"] = rgwConfig.Zone
configOptions["rgw_zonegroup"] = rgwConfig.ZoneGroup
if c.store.Spec.Gateway.OpsLogSidecar != nil {
configOptions["rgw_enable_ops_log"] = "true"
} else {
configOptions["rgw_enable_ops_log"] = "false"
}
configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions)
if err != nil {
return configOptions, err
pkg/operator/ceph/object/config_test.go
+1
View File
@@ -102,6 +102,7 @@ func TestGenerateCephXUser(t *testing.T) {
// RGW configs determined here are applied to the running RGW
func Test_clusterConfig_generateMonConfigOptions(t *testing.T) {
defaultConfigs := map[string]string{
"rgw_enable_ops_log": "false",
"rgw_enable_usage_log": "true",
"rgw_log_nonexistent_bucket": "true",
"rgw_log_object_name_utc": "true",
pkg/operator/ceph/object/spec.go
+16 -1
View File
@@ -158,11 +158,26 @@ func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec
ServiceAccountName: serviceAccountName,
}
opsLogFile := ""
if opsLogSidecar := c.store.Spec.Gateway.OpsLogSidecar; opsLogSidecar != nil {
// Generate ops log file name based on rgw daemon ID
opsLogName := fmt.Sprintf("ops-log-%s.log", getDaemonName(rgwConfig))
opsLogFile = path.Join(cephconfig.VarLogCephDir, opsLogName)
// Add the side-car container named ops-log
podSpec.Containers = append(podSpec.Containers,
*controller.RgwOpsLogSidecarContainer(opsLogName,
c.clusterInfo.Namespace, *c.clusterSpec,
opsLogSidecar.Resources))
}
// If the log collector is enabled we add the side-car container
if c.clusterSpec.LogCollector.Enabled {
shareProcessNamespace := true
podSpec.ShareProcessNamespace = &shareProcessNamespace
podSpec.Containers = append(podSpec.Containers, *controller.LogCollectorContainer(getDaemonName(rgwConfig), c.clusterInfo.Namespace, *c.clusterSpec))
podSpec.Containers = append(podSpec.Containers,
*controller.LogCollectorContainer(getDaemonName(rgwConfig),
c.clusterInfo.Namespace, *c.clusterSpec, opsLogFile))
}
// Replace default unreachable node toleration