From d3c3d25c60232554936e913f7efd7ca9f55f4a14 Mon Sep 17 00:00:00 2001 From: Skala Networks Date: Thu, 20 Feb 2025 10:36:06 -0500 Subject: [PATCH] csi: bind cephfs and rbd provisionners on non-colliding ports for hostNetwork setups Signed-off-by: Skala Networks --- Documentation/Helm-Charts/operator-chart.md | 2 ++ deploy/charts/rook-ceph/templates/configmap.yaml | 6 ++++++ deploy/charts/rook-ceph/values.yaml | 6 ++++++ deploy/examples/operator.yaml | 5 +++++ pkg/operator/ceph/csi/csi.go | 12 ++++++++++++ pkg/operator/ceph/csi/spec.go | 16 ++++++++++------ .../cephfs/csi-cephfsplugin-provisioner-dep.yaml | 4 ++-- .../rbd/csi-rbdplugin-provisioner-dep.yaml | 4 ++-- 8 files changed, 45 insertions(+), 10 deletions(-) diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index 6b97e20c9..4307fc083 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -67,7 +67,9 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.csiAddons.enabled` | Enable CSIAddons | `false` | | `csi.csiAddons.repository` | CSIAddons sidecar image repository | `"quay.io/csiaddons/k8s-sidecar"` | | `csi.csiAddons.tag` | CSIAddons sidecar image tag | `"v0.11.0"` | +| `csi.csiAddonsCephFSProvisionerPort` | CSI Addons server port for the Ceph FS provisioner | `9070` | | `csi.csiAddonsPort` | CSI Addons server port | `9070` | +| `csi.csiAddonsRBDProvisionerPort` | CSI Addons server port for the RBD provisioner | `9070` | | `csi.csiCephFSPluginResource` | CEPH CSI CephFS plugin resource requirement list | see values.yaml | | `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `nil` | | `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `nil` | diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 576764acc..03e79640f 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -208,6 +208,12 @@ data: {{- if .Values.csi.csiAddonsPort }} CSIADDONS_PORT: {{ .Values.csi.csiAddonsPort | quote }} {{- end }} +{{- if .Values.csi.csiAddonsRBDProvisionerPort }} + CSIADDONS_RBD_PROVISIONER_PORT: {{ .Values.csi.csiAddonsRBDProvisionerPort | quote }} +{{- end }} +{{- if .Values.csi.csiAddonsCephFSProvisionerPort }} + CSIADDONS_CEPHFS_PROVISIONER_PORT: {{ .Values.csi.csiAddonsCephFSProvisionerPort | quote }} +{{- end }} {{- if .Values.csi.forceCephFSKernelClient }} CSI_FORCE_CEPHFS_KERNEL_CLIENT: {{ .Values.csi.forceCephFSKernelClient | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 16efe55a2..4ce764357 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -443,6 +443,12 @@ csi: # -- CSI Addons server port # @default -- `9070` csiAddonsPort: + # -- CSI Addons server port for the RBD provisioner + # @default -- `9070` + csiAddonsRBDProvisionerPort: + # -- CSI Addons server port for the Ceph FS provisioner + # @default -- `9070` + csiAddonsCephFSProvisionerPort: # -- Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS # you may want to disable this setting. However, this will cause an issue during upgrades diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 5af25f6da..8a44c36c4 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -470,7 +470,12 @@ data: # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" # Configure CSI RBD liveness metrics port # CSI_RBD_LIVENESS_METRICS_PORT: "9080" + + # We can override the ports for each individual component that uses the CSIADDONS sidecar + # This is useful if we're running in hostNetwork, where ports may conflict on the same host # CSIADDONS_PORT: "9070" + # CSIADDONS_RBD_PROVISIONER_PORT: "9070" + # CSIADDONS_CEPHFS_PROVISIONER_PORT: "9070" # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 9769a519f..1e66cf459 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -97,10 +97,22 @@ func (r *ReconcileCSI) setParams() error { if err != nil { return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") } + CSIParam.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port") } + + CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort) + if err != nil { + return errors.Wrap(err, "failed to get CSI Addons port for RBD provisioner") + } + + CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort) + if err != nil { + return errors.Wrap(err, "failed to get CSI Addons port for Ceph FS provisioner") + } + CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 9381619cc..72797b166 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -83,6 +83,8 @@ type Param struct { SidecarLogLevel uint8 CephFSLivenessMetricsPort uint16 CSIAddonsPort uint16 + CSIAddonsRBDProvisionerPort uint16 + CSIAddonsCephFSProvisionerPort uint16 RBDLivenessMetricsPort uint16 KubeApiBurst uint16 KubeApiQPS float32 @@ -219,12 +221,14 @@ const ( // kubelet directory path DefaultKubeletDirPath = "/var/lib/kubelet" - // grpc metrics and liveness port for cephfs and rbd - DefaultCephFSGRPCMerticsPort uint16 = 9091 - DefaultCephFSLivenessMerticsPort uint16 = 9081 - DefaultRBDGRPCMerticsPort uint16 = 9090 - DefaultRBDLivenessMerticsPort uint16 = 9080 - DefaultCSIAddonsPort uint16 = 9070 + // gRPC metrics and liveness port for CephFS and RBD + DefaultCephFSGRPCMerticsPort uint16 = 9091 + DefaultCephFSLivenessMerticsPort uint16 = 9081 + DefaultRBDGRPCMerticsPort uint16 = 9090 + DefaultRBDLivenessMerticsPort uint16 = 9080 + DefaultCSIAddonsPort uint16 = 9070 + DefaultCSIAddonsRBDProvisionerPort uint16 = 9070 + DefaultCSIAddonsCephFSProvisionerPort uint16 = 9070 // default log level for csi containers defaultLogLevel uint8 = 0 diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index 5ed29f704..ff9554ad7 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -216,7 +216,7 @@ spec: - "--node-id=$(NODE_ID)" - "--v={{ .LogLevel }}" - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-port={{ .CSIAddonsPort }}" + - "--controller-port={{ .CSIAddonsCephFSProvisionerPort }}" - "--pod=$(POD_NAME)" - "--namespace=$(POD_NAMESPACE)" - "--pod-uid=$(POD_UID)" @@ -231,7 +231,7 @@ spec: - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log" {{ end }} ports: - - containerPort: {{ .CSIAddonsPort }} + - containerPort: {{ .CSIAddonsCephFSProvisionerPort }} env: - name: NODE_ID valueFrom: diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 135338322..bacd76682 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -169,7 +169,7 @@ spec: - "--node-id=$(NODE_ID)" - "--v={{ .LogLevel }}" - "--csi-addons-address=$(CSIADDONS_ENDPOINT)" - - "--controller-port={{ .CSIAddonsPort }}" + - "--controller-port={{ .CSIAddonsRBDProvisionerPort }}" - "--pod=$(POD_NAME)" - "--namespace=$(POD_NAMESPACE)" - "--pod-uid=$(POD_UID)" @@ -184,7 +184,7 @@ spec: - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log" {{ end }} ports: - - containerPort: {{ .CSIAddonsPort }} + - containerPort: {{ .CSIAddonsRBDProvisionerPort }} env: - name: NODE_ID valueFrom: