feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #9534 from yati1998/csi-addon

Browse Source 
csi: implement csi-addons sidecar
This commit is contained in:
Sébastien Han
2022-01-18 17:55:07 +01:00
committed by GitHub
parent 46ef24b5c9 2b92dc812c
commit c6ca35eea1
16 changed files with 217 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/ceph-upgrade.md
+2
View File
@@ -552,6 +552,7 @@ ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0"
ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
```
### **Use default images**
@@ -577,4 +578,5 @@ k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
quay.io/cephcsi/cephcsi:v3.5.0
quay.io/csiaddons/volumereplication-operator:v0.1.0
quay.io/csiaddons/k8s-sidecar:v0.1.0
```
Documentation/helm-operator.md
+3
View File
@@ -130,6 +130,7 @@ The following tables lists the configurable parameters of the rook-operator char
| `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port. | `9091` |
| `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port. | `9081` |
| `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port. | `9090` |
| `csi.csiAddonsPort` | CSI Addons server port. | `9070` |
| `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port. | `8080` |
| `csi.forceCephFSKernelClient` | Enable Ceph Kernel clients on kernel < 4.17 which support quotas for Cephfs. | `true` |
| `csi.kubeletDirPath` | Kubelet root directory path (if the Kubelet uses a different path for the `--root-dir` flag) | `/var/lib/kubelet` |
@@ -145,6 +146,8 @@ The following tables lists the configurable parameters of the rook-operator char
| `csi.rbdPodLabels` | Labels to add to the CSI RBD Pods. | <none> |
| `csi.volumeReplication.enabled` | Enable Volume Replication. | `false` |
| `csi.volumeReplication.image` | Volume Replication Controller image. | `quay.io/csiaddons/volumereplication-operator:v0.1.0` |
| `csi.csiAddons.enabled` | Enable CSIAddons | `false` |
| `csi.csiAddons.image` | CSIAddons Sidecar image. | `quay.io/csiaddons/k8s-sidecar:v0.1.0` |
| `admissionController.tolerations` | Array of tolerations in YAML format which will be added to admission controller deployment. | <none> |
| `admissionController.nodeAffinity` | The node labels for affinity of the admission controller deployment (***) | <none> |
| `allowMultipleFilesystems` | **(experimental in Octopus (v15))** Allows multiple filesystems to be deployed to a Ceph cluster. | `false` |
build/rbac/rbac.yaml
+3
View File
@@ -757,6 +757,9 @@ spec:
# Ceph mgr Prometheus Metrics
- min: 9283
max: 9283
# port for CSIAddons
- min: 9061
max: 9079
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
deploy/charts/rook-ceph/templates/deployment.yaml
+12
View File
@@ -91,6 +91,8 @@ spec:
value: {{ .Values.csi.enableOMAPGenerator | quote }}
- name: CSI_ENABLE_VOLUME_REPLICATION
value: {{ .Values.csi.volumeReplication.enabled | quote }}
- name: CSI_ENABLE_CSIADDONS
value: {{ .Values.csi.csiAddons.enabled | quote }}
- name: CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT
value: {{ .Values.csi.enablePluginSelinuxHostMount | quote }}
{{- if .Values.csi.enableCSIHostNetwork }}
@@ -161,6 +163,16 @@ spec:
value: {{ .Values.csi.volumeReplication.image | quote }}
{{- end }}
{{- end }}
{{- if .Values.csi.csiAddons }}
{{- if .Values.csi.csiAddons.image }}
- name: ROOK_CSIADDONS_IMAGE
value: {{ .Values.csi.csiAddons.image | quote }}
{{- end }}
{{- if .Values.csi.csiAddonsPort }}
- name: CSIADDONS_PORT
value: {{ .Values.csi.csiAddonsPort | quote }}
{{- end }}
{{- end }}
{{- if .Values.csi.cephfsPodLabels }}
- name: ROOK_CSI_CEPHFS_POD_LABELS
value: {{ .Values.csi.cephfsPodLabels | quote }}
deploy/charts/rook-ceph/templates/psp.yaml
+3
View File
@@ -76,6 +76,9 @@ spec:
# Ceph mgr Prometheus Metrics
- min: 9283
max: 9283
# port for CSIAddons
- min: 9061
max: 9079
{{- if .Values.rbacEnable }}
---
apiVersion: rbac.authorization.k8s.io/v1
deploy/charts/rook-ceph/templates/role.yaml
+17
View File
@@ -61,6 +61,18 @@ rules:
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
{{- if .Values.csi.csiAddons }}
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
namespace: {{ .Release.Namespace }} # namespace:operator
rules:
- apiGroups: ["csiaddons.openshift.io"]
resources: ["csiaddonsnodes"]
verbs: ["create"]
---
{{- end }}
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
@@ -76,4 +88,9 @@ rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
{{- if .Values.csi.csiAddons }}
- apiGroups: ["csiaddons.openshift.io"]
resources: ["csiaddonsnodes"]
verbs: ["create"]
{{- end }}
{{- end }}
deploy/charts/rook-ceph/templates/rolebinding.yaml
+16
View File
@@ -32,6 +32,22 @@ roleRef:
name: cephfs-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io
---
{{- if .Values.csi.csiAddons }}
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin-role-cfg
namespace: {{ .Release.Namespace }} # namespace:operator
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: {{ .Release.Namespace }} # namespace:operator
roleRef:
kind: Role
name: rbd-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
---
{{- end -}}
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
deploy/charts/rook-ceph/values.yaml
+5 -1
View File
@@ -268,6 +268,7 @@ csi:
#cephfsGrpcMetricsPort: 9091
#cephfsLivenessMetricsPort: 9081
#rbdGrpcMetricsPort: 9090
#csiAddonsPort: 9070
# Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS
# you may want to disable this setting. However, this will cause an issue during upgrades
# with the FUSE client. See the upgrade guide: https://rook.io/docs/rook/v1.2/ceph-upgrade.html
@@ -296,7 +297,10 @@ csi:
volumeReplication:
enabled: false
#image: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
# Enable the CSIAddons sidecar.
csiAddons:
enabled: false
#image: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
enableDiscoveryDaemon: false
cephCommandsTimeoutSeconds: "15"
deploy/examples/common.yaml
+30
View File
@@ -841,6 +841,9 @@ spec:
# Ceph mgr Prometheus Metrics
- min: 9283
max: 9283
# port for CSIAddons
- min: 9061
max: 9079
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
@@ -860,6 +863,16 @@ rules:
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
namespace: rook-ceph # namespace:operator
rules:
- apiGroups: ["csiaddons.openshift.io"]
resources: ["csiaddonsnodes"]
verbs: ["create"]
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-provisioner-cfg
namespace: rook-ceph # namespace:operator
@@ -873,6 +886,9 @@ rules:
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: ["csiaddons.openshift.io"]
resources: ["csiaddonsnodes"]
verbs: ["create"]
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
@@ -1045,6 +1061,20 @@ roleRef:
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin-role-cfg
namespace: rook-ceph # namespace:operator
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: rook-ceph # namespace:operator
roleRef:
kind: Role
name: rbd-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role-cfg
namespace: rook-ceph # namespace:operator
deploy/examples/images.txt
+1
View File
@@ -5,5 +5,6 @@
k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
quay.io/ceph/ceph:v16.2.7
quay.io/cephcsi/cephcsi:v3.5.0
quay.io/csiaddons/k8s-sidecar:v0.1.0
quay.io/csiaddons/volumereplication-operator:v0.1.0
rook/ceph:master
deploy/examples/operator-openshift.yaml
+4
View File
@@ -403,6 +403,7 @@ data:
# Configure CSI RBD grpc and liveness metrics port
# CSI_RBD_GRPC_METRICS_PORT: "9090"
# CSI_RBD_LIVENESS_METRICS_PORT: "9080"
# CSIADDONS_PORT: "9070"
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
@@ -417,6 +418,9 @@ data:
# The timeout value (in seconds) of Ceph commands. It should be >= 1. If this variable is not set or is an invalid value, it's default to 15.
ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15"
# CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
# Enable the csi addons sidecar.
CSI_ENABLE_CSIADDONS: "false"
# ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
---
# The deployment for the rook operator
# OLM: BEGIN OPERATOR DEPLOYMENT
deploy/examples/operator.yaml
+4
View File
@@ -321,6 +321,7 @@ data:
# Configure CSI RBD grpc and liveness metrics port
# CSI_RBD_GRPC_METRICS_PORT: "9090"
# CSI_RBD_LIVENESS_METRICS_PORT: "9080"
# CSIADDONS_PORT: "9070"
# Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
@@ -335,6 +336,9 @@ data:
# See https://rook.io/docs/rook/latest/ceph-csi-drivers.html#rbd-mirroring
CSI_ENABLE_VOLUME_REPLICATION: "false"
# CSI_VOLUME_REPLICATION_IMAGE: "quay.io/csiaddons/volumereplication-operator:v0.1.0"
# Enable the csi addons sidecar.
CSI_ENABLE_CSIADDONS: "false"
# ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.1.0"
---
# OLM: BEGIN OPERATOR DEPLOYMENT
apiVersion: apps/v1
pkg/operator/ceph/csi/csi.go
+1
View File
@@ -94,6 +94,7 @@ func (r *ReconcileCSI) setParams() error {
CSIParam.SnapshotterImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage)
CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath)
CSIParam.VolumeReplicationImage = k8sutil.GetValue(r.opConfig.Parameters, "CSI_VOLUME_REPLICATION_IMAGE", DefaultVolumeReplicationImage)
CSIParam.CSIAddonsImage = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage)
csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "")
CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels)
csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "")
pkg/operator/ceph/csi/spec.go
+14
View File
@@ -51,16 +51,19 @@ type Param struct {
PluginPriorityClassName string
ProvisionerPriorityClassName string
VolumeReplicationImage string
CSIAddonsImage string
EnablePluginSelinuxHostMount bool
EnableCSIHostNetwork bool
EnableOMAPGenerator bool
EnableRBDSnapshotter bool
EnableCephFSSnapshotter bool
EnableVolumeReplicationSideCar bool
EnableCSIAddonsSideCar bool
LogLevel uint8
CephFSGRPCMetricsPort uint16
CephFSLivenessMetricsPort uint16
RBDGRPCMetricsPort uint16
CSIAddonsPort uint16
RBDLivenessMetricsPort uint16
ProvisionerReplicas int32
CSICephFSPodLabels map[string]string
@@ -105,6 +108,7 @@ var (
DefaultSnapshotterImage = "k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0"
DefaultResizerImage = "k8s.gcr.io/sig-storage/csi-resizer:v1.3.0"
DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0"
DefaultCSIAddonsImage = "quay.io/csiaddons/k8s-sidecar:v0.1.0"
// Local package template path for RBD
//go:embed template/rbd/csi-rbdplugin.yaml
@@ -162,6 +166,7 @@ const (
DefaultCephFSLivenessMerticsPort uint16 = 9081
DefaultRBDGRPCMerticsPort uint16 = 9090
DefaultRBDLivenessMerticsPort uint16 = 9080
DefaultCSIAddonsPort uint16 = 9070
detectCSIVersionName = "rook-ceph-csi-detect-version"
// default log level for csi containers
@@ -270,6 +275,10 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if err != nil {
return errors.Wrap(err, "error getting CSI RBD GRPC metrics port.")
}
tp.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort)
if err != nil {
return errors.Wrap(err, "failed to get CSI Addons port")
}
tp.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort)
if err != nil {
return errors.Wrap(err, "error getting CSI RBD liveness metrics port.")
@@ -305,6 +314,11 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
tp.EnableVolumeReplicationSideCar = true
}
tp.EnableCSIAddonsSideCar = false
if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", "false"), "true") {
tp.EnableCSIAddonsSideCar = true
}
if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) {
tp.CephFSPluginUpdateStrategy = onDelete
} else {
pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+50
View File
@@ -127,6 +127,49 @@ spec:
- name: socket-dir
mountPath: /csi
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- name: csi-addons
image: {{ .CSIAddonsImage }}
args :
- "--node-id=$(NODE_ID)"
- "--v={{ .LogLevel }}"
- "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
- "--controller-ip=$(POD_IP)"
- "--controller-port={{ .CSIAddonsPort }}"
- "--pod=$(POD_NAME)"
- "--namespace=$(POD_NAMESPACE)"
- "--pod-uid=$(POD_UID)"
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/pv/"
ports:
- containerPort: {{ .CSIAddonsPort }}
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSIADDONS_ENDPOINT
value: unix:///csi/csi-addons.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
{{ end }}
- name: csi-rbdplugin
image: {{ .CSIPluginImage }}
args :
@@ -140,6 +183,9 @@ spec:
- "--metricsport={{ .RBDGRPCMetricsPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
{{ if .EnableCSIAddonsSideCar }}
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
{{ end }}
env:
- name: POD_IP
valueFrom:
@@ -155,6 +201,10 @@ spec:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
{{ if .EnableCSIAddonsSideCar }}
- name: CSIADDONS_ENDPOINT
value: unix:///csi/csi-addons.sock
{{ end }}
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
+52
View File
@@ -68,6 +68,9 @@ spec:
- "--metricspath=/metrics"
- "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}"
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/pv/"
{{ if .EnableCSIAddonsSideCar }}
- "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)"
{{ end }}
env:
- name: POD_IP
valueFrom:
@@ -83,6 +86,10 @@ spec:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
{{ if .EnableCSIAddonsSideCar }}
- name: CSIADDONS_ENDPOINT
value: unix:///csi/csi-addons.sock
{{ end }}
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
@@ -111,6 +118,51 @@ spec:
mountPath: /etc/selinux
readOnly: true
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- name: csi-addons
securityContext:
privileged: true
image: {{ .CSIAddonsImage }}
args :
- "--node-id=$(NODE_ID)"
- "--v={{ .LogLevel }}"
- "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
- "--controller-ip=$(POD_IP)"
- "--controller-port={{ .CSIAddonsPort }}"
- "--pod=$(POD_NAME)"
- "--namespace=$(POD_NAMESPACE)"
- "--pod-uid=$(POD_UID)"
- "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/pv/"
ports:
- containerPort: {{ .CSIAddonsPort }}
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSIADDONS_ENDPOINT
value: unix:///csi/csi-addons.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /csi
{{ end }}
- name: liveness-prometheus
securityContext:
privileged: true