Merge pull request #14305 from parth-gr/csi-logrotate
csi: add log rotation for csi rbd pod containers
This commit is contained in:
8 files changed
+166
-20
No files matched your search
@@ -757,9 +757,7 @@ func PrivilegedContext(runAsRoot bool) *v1.SecurityContext {
|
||||
return sec
|
||||
}
|
||||
|
||||
// LogCollectorContainer rotate logs
|
||||
func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {
|
||||
|
||||
func GetLogRotateConfig(c cephv1.ClusterSpec) (resource.Quantity, string) {
|
||||
var maxLogSize resource.Quantity
|
||||
if c.LogCollector.MaxLogSize != nil {
|
||||
size := c.LogCollector.MaxLogSize.Value() / 1000 / 1000
|
||||
@@ -771,20 +769,28 @@ func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Contai
|
||||
maxLogSize = resource.MustParse(fmt.Sprintf("%dM", size))
|
||||
}
|
||||
|
||||
var periodicity string
|
||||
switch c.LogCollector.Periodicity {
|
||||
case "1h", "hourly":
|
||||
periodicity = "hourly"
|
||||
case "weekly", "monthly":
|
||||
periodicity = c.LogCollector.Periodicity
|
||||
default:
|
||||
periodicity = "daily"
|
||||
}
|
||||
|
||||
return maxLogSize, periodicity
|
||||
}
|
||||
|
||||
// LogCollectorContainer rotate logs
|
||||
func LogCollectorContainer(daemonID, ns string, c cephv1.ClusterSpec) *v1.Container {
|
||||
maxLogSize, periodicity := GetLogRotateConfig(c)
|
||||
|
||||
rotation := "7"
|
||||
if strings.Contains(daemonID, "-client.rbd-mirror") {
|
||||
rotation = "28"
|
||||
}
|
||||
|
||||
var periodicity string
|
||||
if c.LogCollector.Periodicity == "1h" || c.LogCollector.Periodicity == "hourly" {
|
||||
periodicity = "hourly"
|
||||
} else if c.LogCollector.Periodicity == "weekly" || c.LogCollector.Periodicity == "monthly" {
|
||||
periodicity = c.LogCollector.Periodicity
|
||||
} else {
|
||||
periodicity = "daily"
|
||||
}
|
||||
|
||||
logger.Debugf("setting periodicity to %q. Supported periodicity are hourly, daily, weekly and monthly", periodicity)
|
||||
|
||||
return &v1.Container{
|
||||
|
||||
@@ -205,7 +205,7 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters")
|
||||
}
|
||||
|
||||
// // Do not nothing if no ceph cluster is present
|
||||
// Do nothing if no ceph cluster is present
|
||||
if len(cephClusters.Items) == 0 {
|
||||
logger.Debug("no ceph cluster found not deploying ceph csi driver")
|
||||
EnableRBD, EnableCephFS, EnableNFS = false, false, false
|
||||
@@ -217,6 +217,10 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
// if at least one cephcluster is present update the csi lograte sidecar
|
||||
// with the first listed ceph cluster specs with logrotate enabled
|
||||
setCSILogrotateParams(cephClusters.Items)
|
||||
|
||||
err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
|
||||
@@ -299,3 +303,24 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
|
||||
|
||||
return reconcileResult, nil
|
||||
}
|
||||
|
||||
func setCSILogrotateParams(cephClustersItems []cephv1.CephCluster) {
|
||||
logger.Debug("set logrotate values in csi param")
|
||||
spec := cephClustersItems[0].Spec
|
||||
for _, cluster := range cephClustersItems {
|
||||
if cluster.Spec.LogCollector.Enabled {
|
||||
spec = cluster.Spec
|
||||
break
|
||||
}
|
||||
}
|
||||
CSIParam.CsiLogDirPath = spec.DataDirHostPath
|
||||
if spec.DataDirHostPath == "" {
|
||||
CSIParam.CsiLogDirPath = k8sutil.DataDir
|
||||
}
|
||||
CSIParam.CSILogRotation = spec.LogCollector.Enabled
|
||||
if spec.LogCollector.Enabled {
|
||||
maxSize, period := opcontroller.GetLogRotateConfig(spec)
|
||||
CSIParam.CSILogRotationMaxSize = maxSize.String()
|
||||
CSIParam.CSILogRotationPeriod = period
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,7 @@ type Param struct {
|
||||
ResizerImage string
|
||||
DriverNamePrefix string
|
||||
KubeletDirPath string
|
||||
CsiLogDirPath string
|
||||
ForceCephFSKernelClient string
|
||||
CephFSKernelMountOptions string
|
||||
CephFSPluginUpdateStrategy string
|
||||
@@ -97,6 +98,10 @@ type Param struct {
|
||||
CSICephFSPodLabels map[string]string
|
||||
CSINFSPodLabels map[string]string
|
||||
CSIRBDPodLabels map[string]string
|
||||
CSILogRotation bool
|
||||
CSILogRotationMaxSize string
|
||||
CSILogRotationPeriod string
|
||||
CSILogFolder string
|
||||
}
|
||||
|
||||
type templateParam struct {
|
||||
@@ -178,6 +183,9 @@ var (
|
||||
//go:embed template/nfs/csi-nfsplugin-holder.yaml
|
||||
NFSPluginHolderTemplatePath string
|
||||
|
||||
//go:embed template/csi-logrotate-sidecar.yaml
|
||||
LogrotateTemplatePath string
|
||||
|
||||
holderEnabled bool
|
||||
)
|
||||
|
||||
@@ -349,11 +357,19 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to load rbdplugin template")
|
||||
}
|
||||
if tp.CSILogRotation {
|
||||
tp.CSILogFolder = "rbd-plugin"
|
||||
applyLogrotateSidecar(&rbdPlugin.Spec.Template, "csi-rbd-daemonset-log-collector", LogrotateTemplatePath, tp)
|
||||
}
|
||||
|
||||
rbdProvisionerDeployment, err = templateToDeployment("rbd-provisioner", RBDProvisionerDepTemplatePath, tp)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to load rbd provisioner deployment template")
|
||||
}
|
||||
if tp.CSILogRotation {
|
||||
tp.CSILogFolder = "rbd-provisioner"
|
||||
applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
|
||||
}
|
||||
|
||||
// Create service if either liveness or GRPC metrics are enabled.
|
||||
if CSIParam.EnableLiveness {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
args:
|
||||
- |
|
||||
echo "Starting the csi-logrotate-sidecar"
|
||||
mkdir -p {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }};
|
||||
echo '{{ .CsiLogDirPath }}cephcsi/log/{{ .CSILogFolder }}/*.log {
|
||||
{{ .CSILogRotationPeriod }}
|
||||
missingok
|
||||
rotate 7
|
||||
compress
|
||||
copytruncate
|
||||
notifempty
|
||||
}' > {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi;
|
||||
echo "File creation container completed";
|
||||
|
||||
LOG_ROTATE_CEPH_CSI_FILE={{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}/csi
|
||||
LOG_MAX_SIZE={{ .CSILogRotationMaxSize }}
|
||||
if [ "$LOG_MAX_SIZE" != "0" ]; then
|
||||
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
|
||||
fi
|
||||
|
||||
while true; do
|
||||
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
|
||||
sleep 15m
|
||||
done
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
image: {{ .CSIPluginImage }}
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: log-collector
|
||||
volumeMounts:
|
||||
- mountPath: {{ .CsiLogDirPath }}/cephcsi/logrotate-config/{{ .CSILogFolder }}
|
||||
name: csi-logs-logrotate
|
||||
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/{{ .CSILogFolder }}
|
||||
name: csi-log
|
||||
@@ -38,9 +38,9 @@ spec:
|
||||
- "--extra-create-metadata=true"
|
||||
- "--prevent-volume-mode-conversion=true"
|
||||
- "--feature-gates=HonorPVReclaimPolicy=true"
|
||||
{{- if .EnableCSITopology }}
|
||||
{{ if .EnableCSITopology }}
|
||||
- "--feature-gates=Topology=true"
|
||||
{{- end }}
|
||||
{{ end }}
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
@@ -116,7 +116,7 @@ spec:
|
||||
{{ if .EnableOMAPGenerator }}
|
||||
- name: csi-omap-generator
|
||||
image: {{ .CSIPluginImage }}
|
||||
args :
|
||||
args:
|
||||
- "--type=controller"
|
||||
- "--drivernamespace=$(DRIVER_NAMESPACE)"
|
||||
- "--v={{ .LogLevel }}"
|
||||
@@ -142,7 +142,7 @@ spec:
|
||||
{{ if .EnableCSIAddonsSideCar }}
|
||||
- name: csi-addons
|
||||
image: {{ .CSIAddonsImage }}
|
||||
args :
|
||||
args:
|
||||
- "--node-id=$(NODE_ID)"
|
||||
- "--v={{ .LogLevel }}"
|
||||
- "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
|
||||
@@ -155,6 +155,11 @@ spec:
|
||||
- "--leader-election-lease-duration={{ .LeaderElectionLeaseDuration }}"
|
||||
- "--leader-election-renew-deadline={{ .LeaderElectionRenewDeadline }}"
|
||||
- "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
|
||||
{{ if .CSILogRotation }}
|
||||
- "--logtostderr=false"
|
||||
- "--alsologtostderr=true"
|
||||
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-addons.log"
|
||||
{{ end }}
|
||||
ports:
|
||||
- containerPort: {{ .CSIAddonsPort }}
|
||||
env:
|
||||
@@ -180,10 +185,14 @@ spec:
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
{{ if .CSILogRotation }}
|
||||
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
|
||||
name: csi-log
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
- name: csi-rbdplugin
|
||||
image: {{ .CSIPluginImage }}
|
||||
args :
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v={{ .LogLevel }}"
|
||||
@@ -191,6 +200,11 @@ spec:
|
||||
- "--controllerserver=true"
|
||||
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
|
||||
- "--pidlimit=-1"
|
||||
{{ if .CSILogRotation }}
|
||||
- "--logtostderr=false"
|
||||
- "--alsologtostderr=true"
|
||||
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner/csi-rbdplugin.log"
|
||||
{{ end }}
|
||||
{{ if .EnableCSIAddonsSideCar }}
|
||||
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
|
||||
{{ end }}
|
||||
@@ -225,6 +239,10 @@ spec:
|
||||
mountPath: /csi
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
{{ if .CSILogRotation }}
|
||||
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
|
||||
name: csi-log
|
||||
{{ end }}
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
@@ -272,6 +290,13 @@ spec:
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: csi-log
|
||||
hostPath:
|
||||
path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-provisioner
|
||||
type: DirectoryOrCreate
|
||||
- name: csi-logs-logrotate
|
||||
emptyDir:
|
||||
type: DirectoryOrCreate
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
|
||||
@@ -73,6 +73,11 @@ spec:
|
||||
- "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com"
|
||||
- "--pidlimit=-1"
|
||||
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
|
||||
{{ if .CSILogRotation }}
|
||||
- "--logtostderr=false"
|
||||
- "--alsologtostderr=true"
|
||||
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-rbdplugin.log"
|
||||
{{ end }}
|
||||
{{ if .EnableCSIAddonsSideCar }}
|
||||
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
|
||||
{{ end }}
|
||||
@@ -121,6 +126,10 @@ spec:
|
||||
mountPath: /tmp/csi/keys
|
||||
- name: host-run-mount
|
||||
mountPath: /run/mount
|
||||
{{ if .CSILogRotation }}
|
||||
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
|
||||
name: csi-log
|
||||
{{ end }}
|
||||
{{ if .EnablePluginSelinuxHostMount }}
|
||||
- name: etc-selinux
|
||||
mountPath: /etc/selinux
|
||||
@@ -154,6 +163,11 @@ spec:
|
||||
- "--namespace=$(POD_NAMESPACE)"
|
||||
- "--pod-uid=$(POD_UID)"
|
||||
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/"
|
||||
{{ if .CSILogRotation }}
|
||||
- "--logtostderr=false"
|
||||
- "--alsologtostderr=true"
|
||||
- "--log_file={{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin/csi-addons.log"
|
||||
{{ end }}
|
||||
ports:
|
||||
- containerPort: {{ .CSIAddonsPort }}
|
||||
env:
|
||||
@@ -179,6 +193,10 @@ spec:
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
{{ if .CSILogRotation }}
|
||||
- mountPath: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
|
||||
name: csi-log
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
{{ if .EnableLiveness }}
|
||||
- name: liveness-prometheus
|
||||
@@ -211,6 +229,13 @@ spec:
|
||||
hostPath:
|
||||
path: "{{ .KubeletDirPath }}/plugins/{{ .DriverNamePrefix }}rbd.csi.ceph.com"
|
||||
type: DirectoryOrCreate
|
||||
- name: csi-log
|
||||
hostPath:
|
||||
path: {{ .CsiLogDirPath }}/cephcsi/log/rbd-plugin
|
||||
type: DirectoryOrCreate
|
||||
- name: csi-logs-logrotate
|
||||
emptyDir:
|
||||
type: DirectoryOrCreate
|
||||
- name: plugin-mount-dir
|
||||
hostPath:
|
||||
path: "{{ .KubeletDirPath }}/plugins"
|
||||
|
||||
@@ -84,6 +84,20 @@ func templateToDeployment(name, templateData string, p templateParam) (*apps.Dep
|
||||
return &dep, nil
|
||||
}
|
||||
|
||||
func applyLogrotateSidecar(specTemplate *corev1.PodTemplateSpec, name, templateData string, p templateParam) {
|
||||
var logrotateSidecarContainer corev1.Container
|
||||
t, err := loadTemplate(name, templateData, p)
|
||||
if err != nil {
|
||||
panic(errors.Wrap(err, "failed to load logrotate container template"))
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(t, &logrotateSidecarContainer)
|
||||
if err != nil {
|
||||
panic(errors.Wrap(err, "failed to unmarshal logrotate container template"))
|
||||
}
|
||||
specTemplate.Spec.Containers = append(specTemplate.Spec.Containers, logrotateSidecarContainer)
|
||||
}
|
||||
|
||||
func applyResourcesToContainers(opConfig map[string]string, key string, podspec *corev1.PodSpec) {
|
||||
resource := getComputeResource(opConfig, key)
|
||||
|
||||
|
||||
@@ -120,8 +120,8 @@ func Test_applyVolumeToPodSpec(t *testing.T) {
|
||||
Param: CSIParam,
|
||||
Namespace: "foo",
|
||||
}
|
||||
// rbdplugin has 11 volumes by default
|
||||
defaultVolumes := 11
|
||||
// rbdplugin has 13 volumes by default
|
||||
defaultVolumes := 13
|
||||
ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp)
|
||||
assert.Nil(t, err)
|
||||
applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec)
|
||||
|
||||
Reference in new issue
Block a user