feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

csi: add log rotation for csi cephfs nfs pod containers

Browse Source 
Use the same logrotate flow of rbd, as merged in #14305

1) Make the csi cephfs and nfs container logs persisted in a file

2) Use the cephcluster api specs to configure the log rotate

3) Add log rotation to rotate the log file and
Add a sidecar log collector container

And some fixes on the implementation:

1) Add a operator namespace in the log file path

2) Only add volume to the container if logrotate is enabled

Closes: https://github.com/rook/rook/issues/12809, https://github.com/rook/rook/issues/14429

Signed-off-by: parth-gr <partharora1010@gmail.com>
This commit is contained in:
parth-gr committed 2024-07-12 12:50:35 +05:30
1 parent 28addd1301
commit 4ca8b12e12
10 files changed
+159 -27
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/operator/ceph/csi/controller.go
+4 -4
View File
@@ -224,7 +224,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
// if at least one cephcluster is present update the csi lograte sidecar
// with the first listed ceph cluster specs with logrotate enabled
setCSILogrotateParams(cephClusters.Items)
r.setCSILogrotateParams(cephClusters.Items)
err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
if err != nil {
@@ -309,7 +309,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
return reconcileResult, nil
}
func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
func (r *ReconcileCSI) setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
logger.Debug("set logrotate values in csi param")
spec := cephClustersItems[0].Spec
for _, cluster := range cephClustersItems {
@@ -318,9 +318,9 @@ func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
break
}
}
CSIParam.CsiLogDirPath = spec.DataDirHostPath
csiRootPath = spec.DataDirHostPath
if spec.DataDirHostPath == "" {
CSIParam.CsiLogDirPath = k8sutil.DataDir
csiRootPath = k8sutil.DataDir
}
CSIParam.CSILogRotation = spec.LogCollector.Enabled
pkg/operator/ceph/csi/spec.go
+33 -4
View File
@@ -20,6 +20,7 @@ import (
"context"
_ "embed"
"fmt"
"path"
"strconv"
"strings"
"time"
@@ -52,7 +53,7 @@ type Param struct {
ResizerImage string
DriverNamePrefix string
KubeletDirPath string
CsiLogDirPath string
CsiLogRootPath string
ForceCephFSKernelClient string
CephFSKernelMountOptions string
CephFSPluginUpdateStrategy string
@@ -99,9 +100,9 @@ type Param struct {
CSINFSPodLabels map[string]string
CSIRBDPodLabels map[string]string
CSILogRotation bool
CsiComponentName string
CSILogRotationMaxSize string
CSILogRotationPeriod string
CSILogFolder string
Privileged bool
}
@@ -188,6 +189,8 @@ var (
LogrotateTemplatePath string
holderEnabled bool
csiRootPath string
)
const (
@@ -288,6 +291,8 @@ const (
rbdDriverSuffix = "rbd.csi.ceph.com"
cephFSDriverSuffix = "cephfs.csi.ceph.com"
nfsDriverSuffix = "nfs.csi.ceph.com"
nodePlugin = "node-plugin"
controllerPlugin = "controller-plugin"
)
func CSIEnabled() bool {
@@ -359,7 +364,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
return errors.Wrap(err, "failed to load rbdplugin template")
}
if tp.CSILogRotation {
tp.CSILogFolder = "rbd-plugin"
tp.CsiComponentName = nodePlugin
tp.CsiLogRootPath = path.Join(csiRootPath, RBDDriverName)
applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp)
}
@@ -368,7 +374,8 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
return errors.Wrap(err, "failed to load rbd provisioner deployment template")
}
if tp.CSILogRotation {
tp.CSILogFolder = "rbd-provisioner"
tp.CsiComponentName = controllerPlugin
tp.CsiLogRootPath = path.Join(csiRootPath, RBDDriverName)
applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
}
@@ -394,11 +401,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if err != nil {
return errors.Wrap(err, "failed to load CephFS plugin template")
}
if tp.CSILogRotation {
tp.CsiComponentName = nodePlugin
tp.CsiLogRootPath = path.Join(csiRootPath, CephFSDriverName)
applyLogrotateSidecar(&cephfsPlugin.Spec.Template, "csi-cephfs-daemonset-log-collector", LogrotateTemplatePath, tp)
}
cephfsProvisionerDeployment, err = templateToDeployment("cephfs-provisioner", CephFSProvisionerDepTemplatePath, tp)
if err != nil {
return errors.Wrap(err, "failed to load rbd provisioner deployment template")
}
if tp.CSILogRotation {
tp.CsiComponentName = controllerPlugin
tp.CsiLogRootPath = path.Join(csiRootPath, CephFSDriverName)
applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp)
}
// Create service if either liveness or GRPC metrics are enabled.
if CSIParam.EnableLiveness {
cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp)
@@ -422,11 +440,22 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if err != nil {
return errors.Wrap(err, "failed to load nfs plugin template")
}
if tp.CSILogRotation {
tp.CsiComponentName = nodePlugin
tp.CsiLogRootPath = path.Join(csiRootPath, NFSDriverName)
applyLogrotateSidecar(&nfsPlugin.Spec.Template, "csi-nfs-daemonset-log-collector", LogrotateTemplatePath, tp)
}
nfsProvisionerDeployment, err = templateToDeployment("nfs-provisioner", NFSProvisionerDepTemplatePath, tp)
if err != nil {
return errors.Wrap(err, "failed to load nfs provisioner deployment template")
}
if tp.CSILogRotation {
tp.CsiComponentName = controllerPlugin
tp.CsiLogRootPath = path.Join(csiRootPath, NFSDriverName)
applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp)
}
enabledDrivers = append(enabledDrivers, driverDetails{
name: NFSDriverShortName,
fullName: NFSDriverName,
pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
+27
View File
@@ -119,6 +119,11 @@ spec:
- "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
- "--pidlimit=-1"
- "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-cephfsplugin.log"
{{ end }}
{{ if .CSIEnableMetadata }}
- "--setmetadata={{ .CSIEnableMetadata }}"
{{ end }}
@@ -166,6 +171,10 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
mountPath: /etc/ceph/ceph.conf
@@ -191,6 +200,11 @@ spec:
- "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}"
- "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
- "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log"
{{ end }}
ports:
- containerPort: {{ .CSIAddonsPort }}
env:
@@ -220,6 +234,10 @@ spec:
volumeMounts:
- name: socket-dir
mountPath: /csi
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ end }}
{{ if .EnableLiveness }}
- name: liveness-prometheus
@@ -267,6 +285,15 @@ spec:
emptyDir: {
medium: "Memory"
}
{{ if .CSILogRotation }}
- name: csi-log
hostPath:
path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
configMap:
pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
+18
View File
@@ -72,6 +72,11 @@ spec:
- "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com"
- "--pidlimit=-1"
- "--forcecephkernelclient={{ .ForceCephFSKernelClient }}"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-cephfsplugin.log"
{{ end }}
{{ if .CephFSKernelMountOptions }}
- "--kernelmountoptions={{ .CephFSKernelMountOptions }}"
{{ end }}
@@ -115,6 +120,10 @@ spec:
mountPath: /tmp/csi/keys
- name: host-run-mount
mountPath: /run/mount
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ if .EnablePluginSelinuxHostMount }}
- name: etc-selinux
mountPath: /etc/selinux
@@ -198,6 +207,15 @@ spec:
- name: host-run-mount
hostPath:
path: /run/mount
{{ if .CSILogRotation }}
- name: csi-log
hostPath:
path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
{{ end }}
{{ if .EnablePluginSelinuxHostMount }}
- name: etc-selinux
hostPath:
pkg/operator/ceph/csi/template/csi-logrotate-sidecar.yaml
+7 -7
View File
@@ -1,18 +1,18 @@
args:
- |
echo "Starting the csi-logrotate-sidecar"
mkdir -p {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }};
echo '{{ .CsiLogDirPath }}cephcsi/log/{{ .CSILogFolder }}/*.log {
mkdir -p {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}
echo '{{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/*.log {
{{ .CSILogRotationPeriod }}
missingok
rotate 7
compress
copytruncate
notifempty
}' > {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi;
echo "File creation container completed";
}' > {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}/csi
echo "File creation container completed"
LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi
LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}/csi
LOG_MAX_SIZE={{ .CSILogRotationMaxSize }}
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
@@ -29,7 +29,7 @@ image: {{ .CSIPluginImage }}
imagePullPolicy: IfNotPresent
name: log-collector
volumeMounts:
- mountPath: {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}
- mountPath: {{ .CsiLogRootPath }}/logrotate-config/{{ .CsiComponentName }}
name: csi-logs-logrotate
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/{{ .CSILogFolder }}
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
pkg/operator/ceph/csi/template/nfs/csi-nfsplugin-provisioner-dep.yaml
+18
View File
@@ -113,6 +113,11 @@ spec:
- "--controllerserver=true"
- "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com"
- "--pidlimit=-1"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-nfsplugin.log"
{{ end }}
env:
- name: POD_IP
valueFrom:
@@ -147,6 +152,10 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
mountPath: /etc/ceph/ceph.conf
@@ -176,6 +185,15 @@ spec:
emptyDir: {
medium: "Memory"
}
{{ if .CSILogRotation }}
- name: csi-log
hostPath:
path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
configMap:
pkg/operator/ceph/csi/template/nfs/csi-nfsplugin.yaml
+18
View File
@@ -66,6 +66,11 @@ spec:
- "--nodeserver=true"
- "--drivername={{ .DriverNamePrefix }}nfs.csi.ceph.com"
- "--pidlimit=-1"
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-nfsplugin.log"
{{ end }}
env:
- name: NODE_ID
valueFrom:
@@ -98,6 +103,10 @@ spec:
mountPath: /tmp/csi/keys
- name: host-run-mount
mountPath: /run/mount
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ if .EnablePluginSelinuxHostMount }}
- name: etc-selinux
mountPath: /etc/selinux
@@ -147,6 +156,15 @@ spec:
- name: host-run-mount
hostPath:
path: /run/mount
{{ if .CSILogRotation }}
- name: csi-log
hostPath:
path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
{{ end }}
{{ if .EnablePluginSelinuxHostMount }}
- name: etc-selinux
hostPath:
pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+7 -5
View File
@@ -158,7 +158,7 @@ spec:
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-addons.log"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log"
{{ end }}
ports:
- containerPort: {{ .CSIAddonsPort }}
@@ -190,7 +190,7 @@ spec:
- name: socket-dir
mountPath: /csi
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ end }}
@@ -207,7 +207,7 @@ spec:
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-rbdplugin.log"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-rbdplugin.log"
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
@@ -248,7 +248,7 @@ spec:
- mountPath: /dev
name: host-dev
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
- mountPath: /sys
@@ -298,13 +298,15 @@ spec:
- name: host-dev
hostPath:
path: /dev
{{ if .CSILogRotation }}
- name: csi-log
hostPath:
path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
{{ end }}
- name: host-sys
hostPath:
path: /sys
pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
+7 -5
View File
@@ -76,7 +76,7 @@ spec:
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-rbdplugin.log"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-rbdplugin.log"
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
@@ -127,7 +127,7 @@ spec:
- name: host-run-mount
mountPath: /run/mount
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ if .EnablePluginSelinuxHostMount }}
@@ -166,7 +166,7 @@ spec:
{{ if .CSILogRotation }}
- "--logtostderr=false"
- "--alsologtostderr=true"
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-addons.log"
- "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log"
{{ end }}
ports:
- containerPort: {{ .CSIAddonsPort }}
@@ -194,7 +194,7 @@ spec:
- name: plugin-dir
mountPath: /csi
{{ if .CSILogRotation }}
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
- mountPath: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
name: csi-log
{{ end }}
{{ end }}
@@ -229,13 +229,15 @@ spec:
hostPath:
path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com"
type: DirectoryOrCreate
{{ if .CSILogRotation }}
- name: csi-log
hostPath:
path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
path: {{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}
type: DirectoryOrCreate
- name: csi-logs-logrotate
emptyDir:
type: DirectoryOrCreate
{{ end }}
- name: plugin-mount-dir
hostPath:
path: "{{ .KubeletDirPath }}/plugins"
pkg/operator/ceph/csi/util_test.go
+20 -2
View File
@@ -120,13 +120,22 @@ func Test_applyVolumeToPodSpec(t *testing.T) {
Param: CSIParam,
Namespace: "foo",
}
// rbdplugin has 13 volumes by default
defaultVolumes := 13
// rbdplugin has 11 volumes by default
defaultVolumes := 11
ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes)
// enable csi logrotate, two more volume mounts get added
tp.CSILogRotation = true
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2)
tp.CSILogRotation = false
// add new volume
volumes := []corev1.Volume{
{
@@ -198,6 +207,15 @@ func Test_applyVolumeMountToContainer(t *testing.T) {
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes)
// enable csi logrotate, one more volumes get added
tp.CSILogRotation = true
ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
assert.Nil(t, err)
applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec)
assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1)
tp.CSILogRotation = false
// add new volume mount
volumeMounts := []corev1.VolumeMount{
{