diff --git a/.github/workflows/yaml-lint.yaml b/.github/workflows/yaml-lint.yaml new file mode 100644 index 000000000..80318ff1f --- /dev/null +++ b/.github/workflows/yaml-lint.yaml @@ -0,0 +1,30 @@ +name: Yaml Lint +on: + pull_request: + paths: + - "**.yaml" + +jobs: + yaml-linter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: yaml-lint + uses: ibiqlik/action-yamllint@v3 + with: + config_data: | + extends: default + ignore: | + .mergify.yml + cluster/charts/ + cluster/examples/kubernetes/ceph/csi/template + .github/workflows/ + cluster/examples/kubernetes/ceph/crds.yaml + cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml + tests/ + rules: + line-length: disable + new-lines: disable + document-start: disable + comments: disable + truthy: disable diff --git a/cluster/examples/kubernetes/cassandra/cluster.yaml b/cluster/examples/kubernetes/cassandra/cluster.yaml index 37f8869fb..0cef61e3f 100644 --- a/cluster/examples/kubernetes/cassandra/cluster.yaml +++ b/cluster/examples/kubernetes/cassandra/cluster.yaml @@ -5,7 +5,6 @@ metadata: name: rook-cassandra --- - # Role for cassandra members. apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -36,7 +35,6 @@ rules: - get --- - # ServiceAccount for cassandra members. apiVersion: v1 kind: ServiceAccount @@ -45,7 +43,6 @@ metadata: namespace: rook-cassandra --- - # RoleBinding for cassandra members. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -57,12 +54,11 @@ roleRef: kind: Role name: rook-cassandra-member subjects: -- kind: ServiceAccount - name: rook-cassandra-member - namespace: rook-cassandra + - kind: ServiceAccount + name: rook-cassandra-member + namespace: rook-cassandra --- - # Cassandra Cluster apiVersion: cassandra.rook.io/v1alpha1 kind: Cluster @@ -82,12 +78,12 @@ spec: members: 3 storage: volumeClaimTemplates: - - metadata: - name: rook-cassandra-data - spec: - resources: - requests: - storage: 5Gi + - metadata: + name: rook-cassandra-data + spec: + resources: + requests: + storage: 5Gi resources: requests: cpu: 1 diff --git a/cluster/examples/kubernetes/cassandra/operator.yaml b/cluster/examples/kubernetes/cassandra/operator.yaml index 3773ddf92..807352870 100644 --- a/cluster/examples/kubernetes/cassandra/operator.yaml +++ b/cluster/examples/kubernetes/cassandra/operator.yaml @@ -1,4 +1,3 @@ - # Namespace where Cassandra Operator will live apiVersion: v1 kind: Namespace @@ -6,7 +5,6 @@ metadata: name: rook-cassandra-system --- - # Cassandra Cluster CRD apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -54,7 +52,7 @@ spec: type: object # TODO: Check if we can ref the already existing schema required: - - "volumeClaimTemplates" + - "volumeClaimTemplates" placement: type: object resources: @@ -82,7 +80,6 @@ spec: - "datacenter" --- - # ClusterRole for cassandra-operator. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -162,43 +159,41 @@ roleRef: kind: ClusterRole name: rook-cassandra-operator subjects: -- kind: ServiceAccount - name: rook-cassandra-operator - namespace: rook-cassandra-system + - kind: ServiceAccount + name: rook-cassandra-operator + namespace: rook-cassandra-system --- # cassandra-operator StatefulSet. - apiVersion: apps/v1 - kind: StatefulSet - metadata: - name: rook-cassandra-operator - namespace: rook-cassandra-system - labels: - app: rook-cassandra-operator - spec: - replicas: 1 - serviceName: "non-existent-service" - selector: - matchLabels: - app: rook-cassandra-operator - template: - metadata: - labels: - app: rook-cassandra-operator - spec: - serviceAccountName: rook-cassandra-operator - containers: - - name: rook-cassandra-operator - image: rook/cassandra:master - imagePullPolicy: "Always" - args: ["cassandra", "operator"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rook-cassandra-operator + namespace: rook-cassandra-system + labels: + app: rook-cassandra-operator +spec: + replicas: 1 + serviceName: "non-existent-service" + selector: + matchLabels: + app: rook-cassandra-operator + template: + metadata: + labels: + app: rook-cassandra-operator + spec: + serviceAccountName: rook-cassandra-operator + containers: + - name: rook-cassandra-operator + image: rook/cassandra:master + imagePullPolicy: "Always" + args: ["cassandra", "operator"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml index d4ae74e3e..8701982a6 100644 --- a/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-on-pvc.yaml @@ -40,8 +40,8 @@ spec: mgr: count: 1 modules: - - name: pg_autoscaler - enabled: true + - name: pg_autoscaler + enabled: true dashboard: enabled: true ssl: true @@ -49,115 +49,115 @@ spec: disable: false storage: storageClassDeviceSets: - - name: set1 - # The number of OSDs to create from this device set - count: 3 - # IMPORTANT: If volumes specified by the storageClassName are not portable across nodes - # this needs to be set to false. For example, if using the local storage provisioner - # this should be false. - portable: true - # Certain storage class in the Cloud are slow - # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal - # Currently, "gp2" has been identified as such - tuneDeviceClass: true - # Certain storage class in the Cloud are fast - # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal - # Currently, "managed-premium" has been identified as such - tuneFastDeviceClass: false - # whether to encrypt the deviceSet or not - encrypted: false - # Since the OSDs could end up on any node, an effort needs to be made to spread the OSDs - # across nodes as much as possible. Unfortunately the pod anti-affinity breaks down - # as soon as you have more than one OSD per node. The topology spread constraints will - # give us an even spread on K8s 1.18 or newer. - placement: - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - rook-ceph-osd - preparePlacement: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: + - name: set1 + # The number of OSDs to create from this device set + count: 3 + # IMPORTANT: If volumes specified by the storageClassName are not portable across nodes + # this needs to be set to false. For example, if using the local storage provisioner + # this should be false. + portable: true + # Certain storage class in the Cloud are slow + # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal + # Currently, "gp2" has been identified as such + tuneDeviceClass: true + # Certain storage class in the Cloud are fast + # Rook can configure the OSD running on PVC to accommodate that by tuning some of the Ceph internal + # Currently, "managed-premium" has been identified as such + tuneFastDeviceClass: false + # whether to encrypt the deviceSet or not + encrypted: false + # Since the OSDs could end up on any node, an effort needs to be made to spread the OSDs + # across nodes as much as possible. Unfortunately the pod anti-affinity breaks down + # as soon as you have more than one OSD per node. The topology spread constraints will + # give us an even spread on K8s 1.18 or newer. + placement: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway labelSelector: matchExpressions: - - key: app - operator: In - values: - - rook-ceph-osd - - key: app - operator: In - values: - - rook-ceph-osd-prepare - topologyKey: kubernetes.io/hostname - topologySpreadConstraints: - - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - rook-ceph-osd-prepare - resources: - # These are the OSD daemon limits. For OSD prepare limits, see the separate section below for "prepareosd" resources - # limits: - # cpu: "500m" - # memory: "4Gi" - # requests: - # cpu: "500m" - # memory: "4Gi" - volumeClaimTemplates: - - metadata: - name: data - # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph - # annotations: - # crushDeviceClass: hybrid - spec: - resources: - requests: - storage: 10Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) - storageClassName: gp2 - volumeMode: Block - accessModes: - - ReadWriteOnce - # dedicated block device to store bluestore database (block.db) - # - metadata: - # name: metadata - # spec: - # resources: - # requests: - # # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing - # storage: 5Gi - # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) - # storageClassName: io1 - # volumeMode: Block - # accessModes: - # - ReadWriteOnce - # dedicated block device to store bluestore wal (block.wal) - # - metadata: - # name: wal - # spec: - # resources: - # requests: - # # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing - # storage: 5Gi - # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) - # storageClassName: io1 - # volumeMode: Block - # accessModes: - # - ReadWriteOnce - # Scheduler name for OSD pod placement - # schedulerName: osd-scheduler + - key: app + operator: In + values: + - rook-ceph-osd + preparePlacement: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-osd + - key: app + operator: In + values: + - rook-ceph-osd-prepare + topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-osd-prepare + resources: + # These are the OSD daemon limits. For OSD prepare limits, see the separate section below for "prepareosd" resources + # limits: + # cpu: "500m" + # memory: "4Gi" + # requests: + # cpu: "500m" + # memory: "4Gi" + volumeClaimTemplates: + - metadata: + name: data + # if you are looking at giving your OSD a different CRUSH device class than the one detected by Ceph + # annotations: + # crushDeviceClass: hybrid + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + # dedicated block device to store bluestore database (block.db) + # - metadata: + # name: metadata + # spec: + # resources: + # requests: + # # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + # storage: 5Gi + # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + # storageClassName: io1 + # volumeMode: Block + # accessModes: + # - ReadWriteOnce + # dedicated block device to store bluestore wal (block.wal) + # - metadata: + # name: wal + # spec: + # resources: + # requests: + # # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + # storage: 5Gi + # # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + # storageClassName: io1 + # volumeMode: Block + # accessModes: + # - ReadWriteOnce + # Scheduler name for OSD pod placement + # schedulerName: osd-scheduler resources: # prepareosd: # limits: @@ -178,10 +178,10 @@ spec: # kms: # # name of the config map containing all the kms connection details # connectionDetails: - #KMS_PROVIDER: "vault" - #VAULT_ADDR: VAULT_ADDR_CHANGE_ME # e,g: https://vault.my-domain.com:8200 - #VAULT_BACKEND_PATH: "rook" - #VAULT_SECRET_ENGINE: "kv" + # KMS_PROVIDER: "vault" + # VAULT_ADDR: VAULT_ADDR_CHANGE_ME # e,g: https://vault.my-domain.com:8200 + # VAULT_BACKEND_PATH: "rook" + # VAULT_SECRET_ENGINE: "kv" # # name of the secret containing the kms authentication token # tokenSecretName: rook-vault-token # UNCOMMENT THIS TO ENABLE A KMS CONNECTION diff --git a/cluster/examples/kubernetes/ceph/cluster-stretched.yaml b/cluster/examples/kubernetes/ceph/cluster-stretched.yaml index 90e4f63fb..6d3048f1b 100644 --- a/cluster/examples/kubernetes/ceph/cluster-stretched.yaml +++ b/cluster/examples/kubernetes/ceph/cluster-stretched.yaml @@ -31,10 +31,10 @@ spec: # the osds are started. subFailureDomain: host zones: - - name: a - arbiter: true - - name: b - - name: c + - name: a + arbiter: true + - name: b + - name: c mgr: count: 2 cephVersion: @@ -57,18 +57,18 @@ spec: # settings as other mons. In this example, the arbiter has a toleration to run on a master node. arbiter: tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule osd: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: - - b - - c + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - b + - c disruptionManagement: managePodBudgets: true diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index d88f3176d..432d50950 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -55,10 +55,10 @@ spec: # mgr is active, Rook will update the mgr services to match the active mgr. count: 1 modules: - # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules - # are already enabled by other settings in the cluster CR. - - name: pg_autoscaler - enabled: true + # Several modules should not need to be included in this list. The "dashboard" and "monitoring" modules + # are already enabled by other settings in the cluster CR. + - name: pg_autoscaler + enabled: true # enable the ceph dashboard for viewing cluster status dashboard: enabled: true diff --git a/cluster/examples/kubernetes/ceph/common-external.yaml b/cluster/examples/kubernetes/ceph/common-external.yaml index 6a3091843..04fbd5b24 100644 --- a/cluster/examples/kubernetes/ceph/common-external.yaml +++ b/cluster/examples/kubernetes/ceph/common-external.yaml @@ -33,9 +33,9 @@ roleRef: kind: ClusterRole name: rook-ceph-cluster-mgmt subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -47,9 +47,9 @@ roleRef: kind: Role name: rook-ceph-cmd-reporter subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph-external # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph-external # namespace:cluster --- apiVersion: v1 kind: ServiceAccount @@ -63,15 +63,15 @@ metadata: name: rook-ceph-cmd-reporter namespace: rook-ceph-external # namespace:cluster rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete + - apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index ee19a1b80..66e23ceb4 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -70,26 +70,26 @@ metadata: operator: rook storage-backend: ceph rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - secrets - - pods - - pods/log - - services - - configmaps - - deployments - - daemonsets - verbs: - - get - - list - - watch - - patch - - create - - update - - delete + - apiGroups: + - "" + - apps + - extensions + resources: + - secrets + - pods + - pods/log + - services + - configmaps + - deployments + - daemonsets + verbs: + - get + - list + - watch + - patch + - create + - update + - delete --- # The role for the operator to manage resources in its own namespace apiVersion: rbac.authorization.k8s.io/v1 @@ -101,40 +101,40 @@ metadata: operator: rook storage-backend: ceph rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - - services - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - apps - - extensions - resources: - - daemonsets - - statefulsets - - deployments - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - batch - resources: - - cronjobs - verbs: - - delete + - apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - apps + - extensions + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - cronjobs + verbs: + - delete --- # The cluster role for managing the Rook CRDs apiVersion: rbac.authorization.k8s.io/v1 @@ -145,116 +145,116 @@ metadata: operator: rook storage-backend: ceph rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - - cronjobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete - - get - - update -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get + - apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + - services + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + - endpoints + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" + - apiGroups: + - policy + - apps + - extensions + resources: + # This is for the clusterdisruption controller + - poddisruptionbudgets + # This is for both clusterdisruption and nodedrain controllers + - deployments + - replicasets + verbs: + - "*" + - apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - update + - apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get --- # Aspects of ceph-mgr that require cluster-wide access kind: ClusterRole @@ -265,26 +265,26 @@ metadata: operator: rook storage-backend: ceph rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch + - apiGroups: + - "" + resources: + - configmaps + - nodes + - nodes/proxy + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - list + - get + - watch --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -294,27 +294,27 @@ metadata: operator: rook storage-backend: ceph rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" + - apiGroups: + - "" + verbs: + - "*" + resources: + - secrets + - configmaps + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "objectbucket.io" + verbs: + - "*" + resources: + - "*" # OLM: END OPERATOR ROLE # OLM: BEGIN SERVICE ACCOUNT SYSTEM --- @@ -347,9 +347,9 @@ roleRef: kind: Role name: rook-ceph-system subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator --- # Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes kind: ClusterRoleBinding @@ -364,9 +364,9 @@ roleRef: kind: ClusterRole name: rook-ceph-global subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator # OLM: END OPERATOR ROLEBINDING ################################################################################################################# # Beginning of cluster-specific resources. The example will assume the cluster will be created in the "rook-ceph" @@ -413,25 +413,25 @@ metadata: name: rook-ceph-osd namespace: rook-ceph # namespace:cluster rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: [ "get", "list", "watch", "create", "update", "delete" ] -- apiGroups: ["ceph.rook.io"] - resources: ["cephclusters", "cephclusters/finalizers"] - verbs: [ "get", "list", "create", "update", "delete" ] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["ceph.rook.io"] + resources: ["cephclusters", "cephclusters/finalizers"] + verbs: ["get", "list", "create", "update", "delete"] --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-osd rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list --- # Aspects of ceph-mgr that require access to the system namespace kind: ClusterRole @@ -439,14 +439,14 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-mgr-system rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch --- # Aspects of ceph-mgr that operate within the cluster's namespace kind: Role @@ -455,36 +455,36 @@ metadata: name: rook-ceph-mgr namespace: rook-ceph # namespace:cluster rules: -- apiGroups: - - "" - resources: - - pods - - services - - pods/log - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" + - apiGroups: + - "" + resources: + - pods + - services + - pods/log + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" # OLM: END CLUSTER ROLE # OLM: BEGIN CMD REPORTER ROLE --- @@ -494,18 +494,18 @@ metadata: name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete + - apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete # OLM: END CMD REPORTER ROLE # OLM: BEGIN CLUSTER ROLEBINDING --- @@ -520,9 +520,9 @@ roleRef: kind: ClusterRole name: rook-ceph-cluster-mgmt subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator --- # Allow the osd pods in this namespace to work with configmaps kind: RoleBinding @@ -535,9 +535,9 @@ roleRef: kind: Role name: rook-ceph-osd subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster --- # Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules kind: RoleBinding @@ -550,24 +550,24 @@ roleRef: kind: Role name: rook-ceph-mgr subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster --- # Allow the ceph mgr to access the rook system resources necessary for the mgr modules kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rook-ceph-mgr-system - namespace: rook-ceph # namespace:operator + namespace: rook-ceph # namespace:operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: rook-ceph-mgr-system subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster --- # Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules kind: ClusterRoleBinding @@ -579,9 +579,9 @@ roleRef: kind: ClusterRole name: rook-ceph-mgr-cluster subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster --- # Allow the ceph osd to access cluster-wide resources necessary for determining their topology location @@ -594,9 +594,9 @@ roleRef: kind: ClusterRole name: rook-ceph-osd subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster # OLM: END CLUSTER ROLEBINDING # OLM: BEGIN CMD REPORTER ROLEBINDING @@ -611,9 +611,9 @@ roleRef: kind: Role name: rook-ceph-cmd-reporter subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster # OLM: END CMD REPORTER ROLEBINDING ################################################################################################################# # Beginning of pod security policy resources. The example will assume the cluster will be created in the @@ -629,8 +629,8 @@ metadata: # need to be renamed with a value that will match before others. name: 00-rook-privileged annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' + seccomp.security.alpha.kubernetes.io/allowedProfileNames: "runtime/default" + seccomp.security.alpha.kubernetes.io/defaultProfileName: "runtime/default" spec: privileged: true allowedCapabilities: @@ -698,7 +698,7 @@ spec: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: 'psp:rook' + name: "psp:rook" rules: - apiGroups: - policy @@ -716,7 +716,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: 'psp:rook' + name: "psp:rook" subjects: - kind: ServiceAccount name: rook-ceph-system @@ -732,9 +732,9 @@ roleRef: kind: ClusterRole name: psp:rook subjects: -- kind: ServiceAccount - name: default - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: default + namespace: rook-ceph # namespace:cluster --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -746,9 +746,9 @@ roleRef: kind: ClusterRole name: psp:rook subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -760,9 +760,9 @@ roleRef: kind: ClusterRole name: psp:rook subjects: -- kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -774,9 +774,9 @@ roleRef: kind: ClusterRole name: psp:rook subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster # OLM: END CLUSTER POD SECURITY POLICY BINDINGS # OLM: BEGIN CSI CEPHFS SERVICE ACCOUNT --- @@ -909,7 +909,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: 'psp:rook' + name: "psp:rook" subjects: - kind: ServiceAccount name: rook-csi-cephfs-plugin-sa @@ -922,7 +922,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: 'psp:rook' + name: "psp:rook" subjects: - kind: ServiceAccount name: rook-csi-cephfs-provisioner-sa @@ -1081,7 +1081,7 @@ rules: verbs: ["update", "patch"] - apiGroups: [""] resources: ["configmaps"] - verbs: [ "get"] + verbs: ["get"] # OLM: END CSI RBD CLUSTER ROLE # OLM: BEGIN CSI RBD CLUSTER ROLEBINDING --- @@ -1092,7 +1092,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: 'psp:rook' + name: "psp:rook" subjects: - kind: ServiceAccount name: rook-csi-rbd-plugin-sa @@ -1105,7 +1105,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: 'psp:rook' + name: "psp:rook" subjects: - kind: ServiceAccount name: rook-csi-rbd-provisioner-sa diff --git a/cluster/examples/kubernetes/ceph/crds.yaml b/cluster/examples/kubernetes/ceph/crds.yaml index 3b2859340..87a1f301d 100644 --- a/cluster/examples/kubernetes/ceph/crds.yaml +++ b/cluster/examples/kubernetes/ceph/crds.yaml @@ -1447,4 +1447,4 @@ spec: x-kubernetes-preserve-unknown-fields: true subresources: status: {} -# OLM: END OBJECTBUCKETCLAIM CRD +# OLM: END OBJECTBUCKETCLAIM CRD \ No newline at end of file diff --git a/cluster/examples/kubernetes/ceph/csi/cephfs/kube-registry.yaml b/cluster/examples/kubernetes/ceph/csi/cephfs/kube-registry.yaml index 308fce3e3..22ca0df82 100644 --- a/cluster/examples/kubernetes/ceph/csi/cephfs/kube-registry.yaml +++ b/cluster/examples/kubernetes/ceph/csi/cephfs/kube-registry.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system spec: accessModes: - - ReadWriteMany + - ReadWriteMany resources: requests: storage: 1Gi @@ -31,38 +31,38 @@ spec: kubernetes.io/cluster-service: "true" spec: containers: - - name: registry - image: registry:2 - imagePullPolicy: Always - resources: - limits: - cpu: 100m - memory: 100Mi - env: - # Configuration reference: https://docs.docker.com/registry/configuration/ - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_HTTP_SECRET - value: "Ple4seCh4ngeThisN0tAVerySecretV4lue" - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - ports: - - containerPort: 5000 - name: registry - protocol: TCP - livenessProbe: - httpGet: - path: / - port: registry - readinessProbe: - httpGet: - path: / - port: registry + - name: registry + image: registry:2 + imagePullPolicy: Always + resources: + limits: + cpu: 100m + memory: 100Mi + env: + # Configuration reference: https://docs.docker.com/registry/configuration/ + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_HTTP_SECRET + value: "Ple4seCh4ngeThisN0tAVerySecretV4lue" + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + ports: + - containerPort: 5000 + name: registry + protocol: TCP + livenessProbe: + httpGet: + path: / + port: registry + readinessProbe: + httpGet: + path: / + port: registry volumes: - - name: image-store - persistentVolumeClaim: - claimName: cephfs-pvc - readOnly: false + - name: image-store + persistentVolumeClaim: + claimName: cephfs-pvc + readOnly: false diff --git a/cluster/examples/kubernetes/ceph/csi/cephfs/pod.yaml b/cluster/examples/kubernetes/ceph/csi/cephfs/pod.yaml index e805c8bcb..ff208263a 100644 --- a/cluster/examples/kubernetes/ceph/csi/cephfs/pod.yaml +++ b/cluster/examples/kubernetes/ceph/csi/cephfs/pod.yaml @@ -5,13 +5,13 @@ metadata: name: csicephfs-demo-pod spec: containers: - - name: web-server - image: nginx - volumeMounts: - - name: mypvc - mountPath: /var/lib/www/html + - name: web-server + image: nginx + volumeMounts: + - name: mypvc + mountPath: /var/lib/www/html volumes: - - name: mypvc - persistentVolumeClaim: - claimName: cephfs-pvc - readOnly: false + - name: mypvc + persistentVolumeClaim: + claimName: cephfs-pvc + readOnly: false diff --git a/cluster/examples/kubernetes/ceph/csi/cephfs/pvc.yaml b/cluster/examples/kubernetes/ceph/csi/cephfs/pvc.yaml index a11dc59a7..0f6addb69 100644 --- a/cluster/examples/kubernetes/ceph/csi/cephfs/pvc.yaml +++ b/cluster/examples/kubernetes/ceph/csi/cephfs/pvc.yaml @@ -5,7 +5,7 @@ metadata: name: cephfs-pvc spec: accessModes: - - ReadWriteOnce + - ReadWriteOnce resources: requests: storage: 1Gi diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/pod.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/pod.yaml index 3a75fb981..504ba7896 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/pod.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/pod.yaml @@ -5,13 +5,13 @@ metadata: name: csirbd-demo-pod spec: containers: - - name: web-server - image: nginx - volumeMounts: - - name: mypvc - mountPath: /var/lib/www/html + - name: web-server + image: nginx + volumeMounts: + - name: mypvc + mountPath: /var/lib/www/html volumes: - - name: mypvc - persistentVolumeClaim: - claimName: rbd-pvc - readOnly: false + - name: mypvc + persistentVolumeClaim: + claimName: rbd-pvc + readOnly: false diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/pvc.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/pvc.yaml index 734dcbf65..516a5aa87 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/pvc.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/pvc.yaml @@ -5,7 +5,7 @@ metadata: name: rbd-pvc spec: accessModes: - - ReadWriteOnce + - ReadWriteOnce resources: requests: storage: 1Gi diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-ec.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-ec.yaml index 504d3bbb5..f49f2fa89 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-ec.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-ec.yaml @@ -29,53 +29,53 @@ spec: apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-ceph-block + name: rook-ceph-block # Change "rook-ceph" provisioner prefix to match the operator namespace if needed -provisioner: rook-ceph.rbd.csi.ceph.com # driver:namespace:operator +provisioner: rook-ceph.rbd.csi.ceph.com # driver:namespace:operator parameters: - # clusterID is the namespace where the rook cluster is running - # If you change this namespace, also change the namespace below where the secret namespaces are defined - clusterID: rook-ceph # namespace:cluster + # clusterID is the namespace where the rook cluster is running + # If you change this namespace, also change the namespace below where the secret namespaces are defined + clusterID: rook-ceph # namespace:cluster - # If you want to use erasure coded pool with RBD, you need to create - # two pools. one erasure coded and one replicated. - # You need to specify the replicated pool here in the `pool` parameter, it is - # used for the metadata of the images. - # The erasure coded pool must be set as the `dataPool` parameter below. - dataPool: ec-data-pool - pool: replicated-metadata-pool + # If you want to use erasure coded pool with RBD, you need to create + # two pools. one erasure coded and one replicated. + # You need to specify the replicated pool here in the `pool` parameter, it is + # used for the metadata of the images. + # The erasure coded pool must be set as the `dataPool` parameter below. + dataPool: ec-data-pool + pool: replicated-metadata-pool - # (optional) mapOptions is a comma-separated list of map options. - # For krbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options - # For nbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options - # mapOptions: lock_on_read,queue_depth=1024 + # (optional) mapOptions is a comma-separated list of map options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # mapOptions: lock_on_read,queue_depth=1024 - # (optional) unmapOptions is a comma-separated list of unmap options. - # For krbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options - # For nbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options - # unmapOptions: force + # (optional) unmapOptions is a comma-separated list of unmap options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # unmapOptions: force - # RBD image format. Defaults to "2". - imageFormat: "2" + # RBD image format. Defaults to "2". + imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. - imageFeatures: layering + # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + imageFeatures: layering - # The secrets contain Ceph admin credentials. These are generated automatically by the operator - # in the same namespace as the cluster. - csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster - # Specify the filesystem type of the volume. If not specified, csi-provisioner - # will set default as `ext4`. - csi.storage.k8s.io/fstype: ext4 + # The secrets contain Ceph admin credentials. These are generated automatically by the operator + # in the same namespace as the cluster. + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster + # Specify the filesystem type of the volume. If not specified, csi-provisioner + # will set default as `ext4`. + csi.storage.k8s.io/fstype: ext4 # uncomment the following to use rbd-nbd as mounter on supported nodes # **IMPORTANT**: If you are using rbd-nbd as the mounter, during upgrade you will be hit a ceph-csi # issue that causes the mount to be disconnected. You will need to follow special upgrade steps diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml index 764af76b2..fbe52e3f3 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass-test.yaml @@ -17,39 +17,39 @@ spec: apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-ceph-block + name: rook-ceph-block # Change "rook-ceph" provisioner prefix to match the operator namespace if needed -provisioner: rook-ceph.rbd.csi.ceph.com # driver:namespace:operator +provisioner: rook-ceph.rbd.csi.ceph.com # driver:namespace:operator parameters: - # clusterID is the namespace where the rook cluster is running - # If you change this namespace, also change the namespace below where the secret namespaces are defined - clusterID: rook-ceph # namespace:cluster + # clusterID is the namespace where the rook cluster is running + # If you change this namespace, also change the namespace below where the secret namespaces are defined + clusterID: rook-ceph # namespace:cluster - # If you want to use erasure coded pool with RBD, you need to create - # two pools. one erasure coded and one replicated. - # You need to specify the replicated pool here in the `pool` parameter, it is - # used for the metadata of the images. - # The erasure coded pool must be set as the `dataPool` parameter below. - #dataPool: ec-data-pool - pool: replicapool + # If you want to use erasure coded pool with RBD, you need to create + # two pools. one erasure coded and one replicated. + # You need to specify the replicated pool here in the `pool` parameter, it is + # used for the metadata of the images. + # The erasure coded pool must be set as the `dataPool` parameter below. + #dataPool: ec-data-pool + pool: replicapool - # RBD image format. Defaults to "2". - imageFormat: "2" + # RBD image format. Defaults to "2". + imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. - imageFeatures: layering + # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + imageFeatures: layering - # The secrets contain Ceph admin credentials. These are generated automatically by the operator - # in the same namespace as the cluster. - csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster - # Specify the filesystem type of the volume. If not specified, csi-provisioner - # will set default as `ext4`. - csi.storage.k8s.io/fstype: ext4 + # The secrets contain Ceph admin credentials. These are generated automatically by the operator + # in the same namespace as the cluster. + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster + # Specify the filesystem type of the volume. If not specified, csi-provisioner + # will set default as `ext4`. + csi.storage.k8s.io/fstype: ext4 # uncomment the following to use rbd-nbd as mounter on supported nodes #mounter: rbd-nbd allowVolumeExpansion: true diff --git a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml index f90735cca..8077139fc 100644 --- a/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml +++ b/cluster/examples/kubernetes/ceph/csi/rbd/storageclass.yaml @@ -17,54 +17,54 @@ spec: apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-ceph-block + name: rook-ceph-block # Change "rook-ceph" provisioner prefix to match the operator namespace if needed provisioner: rook-ceph.rbd.csi.ceph.com parameters: - # clusterID is the namespace where the rook cluster is running - # If you change this namespace, also change the namespace below where the secret namespaces are defined - clusterID: rook-ceph # namespace:cluster + # clusterID is the namespace where the rook cluster is running + # If you change this namespace, also change the namespace below where the secret namespaces are defined + clusterID: rook-ceph # namespace:cluster - # If you want to use erasure coded pool with RBD, you need to create - # two pools. one erasure coded and one replicated. - # You need to specify the replicated pool here in the `pool` parameter, it is - # used for the metadata of the images. - # The erasure coded pool must be set as the `dataPool` parameter below. - #dataPool: ec-data-pool - pool: replicapool + # If you want to use erasure coded pool with RBD, you need to create + # two pools. one erasure coded and one replicated. + # You need to specify the replicated pool here in the `pool` parameter, it is + # used for the metadata of the images. + # The erasure coded pool must be set as the `dataPool` parameter below. + #dataPool: ec-data-pool + pool: replicapool - # (optional) mapOptions is a comma-separated list of map options. - # For krbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options - # For nbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options - # mapOptions: lock_on_read,queue_depth=1024 + # (optional) mapOptions is a comma-separated list of map options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # mapOptions: lock_on_read,queue_depth=1024 - # (optional) unmapOptions is a comma-separated list of unmap options. - # For krbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options - # For nbd options refer - # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options - # unmapOptions: force + # (optional) unmapOptions is a comma-separated list of unmap options. + # For krbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options + # For nbd options refer + # https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options + # unmapOptions: force - # RBD image format. Defaults to "2". - imageFormat: "2" + # RBD image format. Defaults to "2". + imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. - imageFeatures: layering + # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + imageFeatures: layering - # The secrets contain Ceph admin credentials. These are generated automatically by the operator - # in the same namespace as the cluster. - csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node - csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster - # Specify the filesystem type of the volume. If not specified, csi-provisioner - # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock - # in hyperconverged settings where the volume is mounted on the same node as the osds. - csi.storage.k8s.io/fstype: ext4 + # The secrets contain Ceph admin credentials. These are generated automatically by the operator + # in the same namespace as the cluster. + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster + # Specify the filesystem type of the volume. If not specified, csi-provisioner + # will set default as `ext4`. Note that `xfs` is not recommended due to potential deadlock + # in hyperconverged settings where the volume is mounted on the same node as the osds. + csi.storage.k8s.io/fstype: ext4 # uncomment the following to use rbd-nbd as mounter on supported nodes # **IMPORTANT**: If you are using rbd-nbd as the mounter, during upgrade you will be hit a ceph-csi # issue that causes the mount to be disconnected. You will need to follow special upgrade steps diff --git a/cluster/examples/kubernetes/ceph/dashboard-external-http.yaml b/cluster/examples/kubernetes/ceph/dashboard-external-http.yaml index 332b032a3..27fc59866 100644 --- a/cluster/examples/kubernetes/ceph/dashboard-external-http.yaml +++ b/cluster/examples/kubernetes/ceph/dashboard-external-http.yaml @@ -8,10 +8,10 @@ metadata: rook_cluster: rook-ceph # namespace:cluster spec: ports: - - name: dashboard - port: 7000 - protocol: TCP - targetPort: 7000 + - name: dashboard + port: 7000 + protocol: TCP + targetPort: 7000 selector: app: rook-ceph-mgr rook_cluster: rook-ceph diff --git a/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml b/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml index c9c3a0146..9d30fadca 100644 --- a/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml +++ b/cluster/examples/kubernetes/ceph/dashboard-external-https.yaml @@ -8,10 +8,10 @@ metadata: rook_cluster: rook-ceph # namespace:cluster spec: ports: - - name: dashboard - port: 8443 - protocol: TCP - targetPort: 8443 + - name: dashboard + port: 8443 + protocol: TCP + targetPort: 8443 selector: app: rook-ceph-mgr rook_cluster: rook-ceph diff --git a/cluster/examples/kubernetes/ceph/dashboard-ingress-https.yaml b/cluster/examples/kubernetes/ceph/dashboard-ingress-https.yaml index 5e497f8c8..3028a8856 100644 --- a/cluster/examples/kubernetes/ceph/dashboard-ingress-https.yaml +++ b/cluster/examples/kubernetes/ceph/dashboard-ingress-https.yaml @@ -18,14 +18,14 @@ metadata: proxy_ssl_verify off; spec: tls: - - hosts: - - rook-ceph.example.com - secretName: rook-ceph.example.com + - hosts: + - rook-ceph.example.com + secretName: rook-ceph.example.com rules: - - host: rook-ceph.example.com - http: - paths: - - path: / - backend: - serviceName: rook-ceph-mgr-dashboard - servicePort: https-dashboard + - host: rook-ceph.example.com + http: + paths: + - path: / + backend: + serviceName: rook-ceph-mgr-dashboard + servicePort: https-dashboard diff --git a/cluster/examples/kubernetes/ceph/dashboard-loadbalancer.yaml b/cluster/examples/kubernetes/ceph/dashboard-loadbalancer.yaml index 69b732d8a..98c0b9075 100644 --- a/cluster/examples/kubernetes/ceph/dashboard-loadbalancer.yaml +++ b/cluster/examples/kubernetes/ceph/dashboard-loadbalancer.yaml @@ -8,10 +8,10 @@ metadata: rook_cluster: rook-ceph # namespace:cluster spec: ports: - - name: dashboard - port: 8443 - protocol: TCP - targetPort: 8443 + - name: dashboard + port: 8443 + protocol: TCP + targetPort: 8443 selector: app: rook-ceph-mgr rook_cluster: rook-ceph diff --git a/cluster/examples/kubernetes/ceph/direct-mount.yaml b/cluster/examples/kubernetes/ceph/direct-mount.yaml index 1ed86da64..ad0af7779 100644 --- a/cluster/examples/kubernetes/ceph/direct-mount.yaml +++ b/cluster/examples/kubernetes/ceph/direct-mount.yaml @@ -17,33 +17,33 @@ spec: spec: dnsPolicy: ClusterFirstWithHostNet containers: - - name: rook-direct-mount - image: rook/ceph:master - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - imagePullPolicy: IfNotPresent - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - securityContext: - privileged: true - volumeMounts: - - mountPath: /dev - name: dev - - mountPath: /sys/bus - name: sysbus - - mountPath: /lib/modules - name: libmodules - - name: mon-endpoint-volume - mountPath: /etc/rook + - name: rook-direct-mount + image: rook/ceph:master + command: ["/tini"] + args: ["-g", "--", "/usr/local/bin/toolbox.sh"] + imagePullPolicy: IfNotPresent + env: + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + securityContext: + privileged: true + volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules + - name: mon-endpoint-volume + mountPath: /etc/rook # if hostNetwork: false, the "rbd map" command hangs, see https://github.com/rook/rook/issues/2021 hostNetwork: true volumes: @@ -60,5 +60,5 @@ spec: configMap: name: rook-ceph-mon-endpoints items: - - key: data - path: mon-endpoints + - key: data + path: mon-endpoints diff --git a/cluster/examples/kubernetes/ceph/filesystem-ec.yaml b/cluster/examples/kubernetes/ceph/filesystem-ec.yaml index ca436ee2e..205adfd13 100644 --- a/cluster/examples/kubernetes/ceph/filesystem-ec.yaml +++ b/cluster/examples/kubernetes/ceph/filesystem-ec.yaml @@ -35,42 +35,42 @@ spec: activeStandby: true # The affinity rules to apply to the mds deployment placement: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: role - # operator: In - # values: - # - mds-node - # topologySpreadConstraints: - # tolerations: - # - key: mds-node - # operator: Exists - # podAffinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: role + # operator: In + # values: + # - mds-node + # topologySpreadConstraints: + # tolerations: + # - key: mds-node + # operator: Exists + # podAffinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: app - operator: In - values: - - rook-ceph-mds + - key: app + operator: In + values: + - rook-ceph-mds # topologyKey: kubernetes.io/hostname will place MDS across different hosts topologyKey: kubernetes.io/hostname - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: - key: app operator: In values: - - rook-ceph-mds - # topologyKey: */zone can be used to spread MDS across different AZ - # Use in k8s cluster if your cluster is v1.16 or lower - # Use in k8s cluster is v1.17 or upper - topologyKey: topology.kubernetes.io/zone + - rook-ceph-mds + # topologyKey: */zone can be used to spread MDS across different AZ + # Use in k8s cluster if your cluster is v1.16 or lower + # Use in k8s cluster is v1.17 or upper + topologyKey: topology.kubernetes.io/zone # A key/value list of annotations annotations: # key: value diff --git a/cluster/examples/kubernetes/ceph/filesystem.yaml b/cluster/examples/kubernetes/ceph/filesystem.yaml index 70b0b8edd..548cb7845 100644 --- a/cluster/examples/kubernetes/ceph/filesystem.yaml +++ b/cluster/examples/kubernetes/ceph/filesystem.yaml @@ -18,7 +18,8 @@ spec: parameters: # Inline compression mode for the data pool # Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression - compression_mode: none + compression_mode: + none # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size #target_size_ratio: ".5" @@ -33,7 +34,8 @@ spec: parameters: # Inline compression mode for the data pool # Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression - compression_mode: none + compression_mode: + none # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size #target_size_ratio: ".5" @@ -48,38 +50,38 @@ spec: activeStandby: true # The affinity rules to apply to the mds deployment placement: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: role - # operator: In - # values: - # - mds-node - # topologySpreadConstraints: - # tolerations: - # - key: mds-node - # operator: Exists - # podAffinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: role + # operator: In + # values: + # - mds-node + # topologySpreadConstraints: + # tolerations: + # - key: mds-node + # operator: Exists + # podAffinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - - key: app - operator: In - values: - - rook-ceph-mds + - key: app + operator: In + values: + - rook-ceph-mds # topologyKey: kubernetes.io/hostname will place MDS across different hosts topologyKey: kubernetes.io/hostname - preferredDuringSchedulingIgnoredDuringExecution: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - - key: app - operator: In - values: - - rook-ceph-mds + - key: app + operator: In + values: + - rook-ceph-mds # topologyKey: */zone can be used to spread MDS across different AZ # Use in k8s cluster if your cluster is v1.16 or lower # Use in k8s cluster is v1.17 or upper diff --git a/cluster/examples/kubernetes/ceph/flex/kube-registry.yaml b/cluster/examples/kubernetes/ceph/flex/kube-registry.yaml index 60a774481..344fefe73 100644 --- a/cluster/examples/kubernetes/ceph/flex/kube-registry.yaml +++ b/cluster/examples/kubernetes/ceph/flex/kube-registry.yaml @@ -18,49 +18,49 @@ spec: kubernetes.io/cluster-service: "true" spec: containers: - - name: registry - image: registry:2 - imagePullPolicy: Always - resources: - limits: - cpu: 100m - memory: 100Mi - env: - # Configuration reference: https://docs.docker.com/registry/configuration/ - - name: REGISTRY_HTTP_ADDR - value: :5000 - - name: REGISTRY_HTTP_SECRET - value: "Ple4seCh4ngeThisN0tAVerySecretV4lue" - - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY - value: /var/lib/registry - volumeMounts: - - name: image-store - mountPath: /var/lib/registry - ports: - - containerPort: 5000 - name: registry - protocol: TCP - livenessProbe: - httpGet: - path: / - port: registry - readinessProbe: - httpGet: - path: / - port: registry + - name: registry + image: registry:2 + imagePullPolicy: Always + resources: + limits: + cpu: 100m + memory: 100Mi + env: + # Configuration reference: https://docs.docker.com/registry/configuration/ + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_HTTP_SECRET + value: "Ple4seCh4ngeThisN0tAVerySecretV4lue" + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + ports: + - containerPort: 5000 + name: registry + protocol: TCP + livenessProbe: + httpGet: + path: / + port: registry + readinessProbe: + httpGet: + path: / + port: registry volumes: - - name: image-store - flexVolume: - driver: ceph.rook.io/rook - fsType: ceph - options: - fsName: myfs # name of the filesystem specified in the filesystem CRD. - clusterNamespace: rook-ceph # namespace where the Rook cluster is deployed - # by default the path is /, but you can override and mount a specific path of the filesystem by using the path attribute - # the path must exist on the filesystem, otherwise mounting the filesystem at that path will fail - # path: /some/path/inside/cephfs - # (Optional) Specify an existing Ceph user that will be used for mounting storage with this StorageClass. - #mountUser: user1 - # (Optional) Specify an existing Kubernetes secret name containing just one key holding the Ceph user secret. - # The secret must exist in each namespace(s) where the storage will be consumed. - #mountSecret: ceph-user1-secret + - name: image-store + flexVolume: + driver: ceph.rook.io/rook + fsType: ceph + options: + fsName: myfs # name of the filesystem specified in the filesystem CRD. + clusterNamespace: rook-ceph # namespace where the Rook cluster is deployed + # by default the path is /, but you can override and mount a specific path of the filesystem by using the path attribute + # the path must exist on the filesystem, otherwise mounting the filesystem at that path will fail + # path: /some/path/inside/cephfs + # (Optional) Specify an existing Ceph user that will be used for mounting storage with this StorageClass. + #mountUser: user1 + # (Optional) Specify an existing Kubernetes secret name containing just one key holding the Ceph user secret. + # The secret must exist in each namespace(s) where the storage will be consumed. + #mountSecret: ceph-user1-secret diff --git a/cluster/examples/kubernetes/ceph/flex/storageclass-ec.yaml b/cluster/examples/kubernetes/ceph/flex/storageclass-ec.yaml index a2509f9e3..7678aa95b 100644 --- a/cluster/examples/kubernetes/ceph/flex/storageclass-ec.yaml +++ b/cluster/examples/kubernetes/ceph/flex/storageclass-ec.yaml @@ -30,7 +30,7 @@ spec: apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-ceph-block + name: rook-ceph-block provisioner: ceph.rook.io/block # Works for Kubernetes 1.14+ allowVolumeExpansion: true diff --git a/cluster/examples/kubernetes/ceph/flex/storageclass-test.yaml b/cluster/examples/kubernetes/ceph/flex/storageclass-test.yaml index b41765090..a1a4a9d2b 100644 --- a/cluster/examples/kubernetes/ceph/flex/storageclass-test.yaml +++ b/cluster/examples/kubernetes/ceph/flex/storageclass-test.yaml @@ -15,7 +15,7 @@ spec: apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-ceph-block + name: rook-ceph-block provisioner: ceph.rook.io/block # Works for Kubernetes 1.14+ allowVolumeExpansion: true diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules-external.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules-external.yaml index cf8488efb..02c1d37c0 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules-external.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules-external.yaml @@ -8,31 +8,35 @@ metadata: namespace: rook-ceph spec: groups: - - name: persistent-volume-alert.rules - rules: - - alert: PersistentVolumeUsageNearFull - annotations: - description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed - 75%. Free up some space. - message: PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion - is required. - severity_level: warning - storage_type: ceph - expr: | - (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75 - for: 5s - labels: - severity: warning - - alert: PersistentVolumeUsageCritical - annotations: - description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed - 85%. Free up some space immediately. - message: PVC {{ $labels.persistentvolumeclaim }} is critically full. Data - deletion is required. - severity_level: error - storage_type: ceph - expr: | - (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85 - for: 5s - labels: - severity: critical + - name: persistent-volume-alert.rules + rules: + - alert: PersistentVolumeUsageNearFull + annotations: + description: + PVC {{ $labels.persistentvolumeclaim }} utilization has crossed + 75%. Free up some space. + message: + PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion + is required. + severity_level: warning + storage_type: ceph + expr: | + (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75 + for: 5s + labels: + severity: warning + - alert: PersistentVolumeUsageCritical + annotations: + description: + PVC {{ $labels.persistentvolumeclaim }} utilization has crossed + 85%. Free up some space immediately. + message: + PVC {{ $labels.persistentvolumeclaim }} is critically full. Data + deletion is required. + severity_level: error + storage_type: ceph + expr: | + (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85 + for: 5s + labels: + severity: critical diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml index fdb8baf50..17b78954c 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus-ceph-v14-rules.yaml @@ -8,309 +8,329 @@ metadata: namespace: rook-ceph spec: groups: - - name: ceph.rules - rules: - - expr: | - kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} * on (node) group_right() max(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"},"node","$1","exported_instance","(.*)")) by (node) - record: cluster:ceph_node_down:join_kube - - expr: | - avg(topk by (ceph_daemon) (1, label_replace(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"}, "instance", "$1", "exported_instance", "(.*)"), "device", "$1", "device", "/dev/(.*)")) * on(instance, device) group_right(ceph_daemon) topk by (instance,device) (1,(irate(node_disk_read_time_seconds_total[1m]) + irate(node_disk_write_time_seconds_total[1m]) / (clamp_min(irate(node_disk_reads_completed_total[1m]), 1) + irate(node_disk_writes_completed_total[1m]))))) - record: cluster:ceph_disk_latency:join_ceph_node_disk_irate1m - - name: telemeter.rules - rules: - - expr: | - count(ceph_osd_metadata{job="rook-ceph-mgr"}) - record: job:ceph_osd_metadata:count - - expr: | - count(kube_persistentvolume_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"}) - record: job:kube_pv:count - - expr: | - sum(ceph_pool_rd{job="rook-ceph-mgr"}+ ceph_pool_wr{job="rook-ceph-mgr"}) - record: job:ceph_pools_iops:total - - expr: | - sum(ceph_pool_rd_bytes{job="rook-ceph-mgr"}+ ceph_pool_wr_bytes{job="rook-ceph-mgr"}) - record: job:ceph_pools_iops_bytes:total - - expr: | - count(count(ceph_mon_metadata{job="rook-ceph-mgr"} or ceph_osd_metadata{job="rook-ceph-mgr"} or ceph_rgw_metadata{job="rook-ceph-mgr"} or ceph_mds_metadata{job="rook-ceph-mgr"} or ceph_mgr_metadata{job="rook-ceph-mgr"}) by(ceph_version)) - record: job:ceph_versions_running:count - - name: ceph-mgr-status - rules: - - alert: CephMgrIsAbsent - annotations: - description: Ceph Manager has disappeared from Prometheus target discovery. - message: Storage metrics collector service not available anymore. - severity_level: critical - storage_type: ceph - expr: | - absent(up{job="rook-ceph-mgr"} == 1) - for: 5m - labels: - severity: critical - - alert: CephMgrIsMissingReplicas - annotations: - description: Ceph Manager is missing replicas. - message: Storage metrics collector service doesn't have required no of replicas. - severity_level: warning - storage_type: ceph - expr: | - sum(up{job="rook-ceph-mgr"}) < 1 - for: 5m - labels: - severity: warning - - name: ceph-mds-status - rules: - - alert: CephMdsMissingReplicas - annotations: - description: Minimum required replicas for storage metadata service not available. - Might affect the working of storage cluster. - message: Insufficient replicas for storage metadata service. - severity_level: warning - storage_type: ceph - expr: | - sum(ceph_mds_metadata{job="rook-ceph-mgr"} == 1) < 2 - for: 5m - labels: - severity: warning - - name: quorum-alert.rules - rules: - - alert: CephMonQuorumAtRisk - annotations: - description: Storage cluster quorum is low. Contact Support. - message: Storage quorum at risk - severity_level: error - storage_type: ceph - expr: | - count(ceph_mon_quorum_status{job="rook-ceph-mgr"} == 1) <= ((count(ceph_mon_metadata{job="rook-ceph-mgr"}) % 2) + 1) - for: 15m - labels: - severity: critical - - alert: CephMonHighNumberOfLeaderChanges - annotations: - description: Ceph Monitor {{ $labels.ceph_daemon }} on host {{ $labels.hostname - }} has seen {{ $value | printf "%.2f" }} leader changes per minute recently. - message: Storage Cluster has seen many leader changes recently. - severity_level: warning - storage_type: ceph - expr: | - (ceph_mon_metadata{job="rook-ceph-mgr"} * on (ceph_daemon) group_left() (rate(ceph_mon_num_elections{job="rook-ceph-mgr"}[5m]) * 60)) > 0.95 - for: 5m - labels: - severity: warning - - name: ceph-node-alert.rules - rules: - - alert: CephNodeDown - annotations: - description: Storage node {{ $labels.node }} went down. Please check the node - immediately. - message: Storage node {{ $labels.node }} went down - severity_level: error - storage_type: ceph - expr: | - cluster:ceph_node_down:join_kube == 0 - for: 30s - labels: - severity: critical - - name: osd-alert.rules - rules: - - alert: CephOSDCriticallyFull - annotations: - description: Utilization of storage device {{ $labels.ceph_daemon }} of device_class - type {{$labels.device_class}} has crossed 80% on host {{ $labels.hostname - }}. Immediately free up some space or add capacity of type {{$labels.device_class}}. - message: Back-end storage device is critically full. - severity_level: error - storage_type: ceph - expr: | - (ceph_osd_metadata * on (ceph_daemon) group_right(device_class) (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)) >= 0.80 - for: 40s - labels: - severity: critical - - alert: CephOSDFlapping - annotations: - description: Storage daemon {{ $labels.ceph_daemon }} has restarted 5 times - in last 5 minutes. Please check the pod events or ceph status to find out - the cause. - message: Ceph storage osd flapping. - severity_level: error - storage_type: ceph - expr: | - changes(ceph_osd_up[5m]) >= 10 - for: 0s - labels: - severity: critical - - alert: CephOSDNearFull - annotations: - description: Utilization of storage device {{ $labels.ceph_daemon }} of device_class - type {{$labels.device_class}} has crossed 75% on host {{ $labels.hostname - }}. Immediately free up some space or add capacity of type {{$labels.device_class}}. - message: Back-end storage device is nearing full. - severity_level: warning - storage_type: ceph - expr: | - (ceph_osd_metadata * on (ceph_daemon) group_right(device_class) (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)) >= 0.75 - for: 40s - labels: - severity: warning - - alert: CephOSDDiskNotResponding - annotations: - description: Disk device {{ $labels.device }} not responding, on host {{ $labels.host - }}. - message: Disk not responding - severity_level: error - storage_type: ceph - expr: | - label_replace((ceph_osd_in == 1 and ceph_osd_up == 0),"disk","$1","ceph_daemon","osd.(.*)") + on(ceph_daemon) group_left(host, device) label_replace(ceph_disk_occupation,"host","$1","exported_instance","(.*)") - for: 1m - labels: - severity: critical - - alert: CephOSDDiskUnavailable - annotations: - description: Disk device {{ $labels.device }} not accessible on host {{ $labels.host - }}. - message: Disk not accessible - severity_level: error - storage_type: ceph - expr: | - label_replace((ceph_osd_in == 0 and ceph_osd_up == 0),"disk","$1","ceph_daemon","osd.(.*)") + on(ceph_daemon) group_left(host, device) label_replace(ceph_disk_occupation,"host","$1","exported_instance","(.*)") - for: 1m - labels: - severity: critical - - alert: CephDataRecoveryTakingTooLong - annotations: - description: Data recovery has been active for too long. Contact Support. - message: Data recovery is slow - severity_level: warning - storage_type: ceph - expr: | - ceph_pg_undersized > 0 - for: 2h - labels: - severity: warning - - alert: CephPGRepairTakingTooLong - annotations: - description: Self heal operations taking too long. Contact Support. - message: Self heal problems detected - severity_level: warning - storage_type: ceph - expr: | - ceph_pg_inconsistent > 0 - for: 1h - labels: - severity: warning - - name: persistent-volume-alert.rules - rules: - - alert: PersistentVolumeUsageNearFull - annotations: - description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed - 75%. Free up some space or expand the PVC. - message: PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion - or PVC expansion is required. - severity_level: warning - storage_type: ceph - expr: | - (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75 - for: 5s - labels: - severity: warning - - alert: PersistentVolumeUsageCritical - annotations: - description: PVC {{ $labels.persistentvolumeclaim }} utilization has crossed - 85%. Free up some space or expand the PVC immediately. - message: PVC {{ $labels.persistentvolumeclaim }} is critically full. Data - deletion or PVC expansion is required. - severity_level: error - storage_type: ceph - expr: | - (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85 - for: 5s - labels: - severity: critical - - name: cluster-state-alert.rules - rules: - - alert: CephClusterErrorState - annotations: - description: Storage cluster is in error state for more than 10m. - message: Storage cluster is in error state - severity_level: error - storage_type: ceph - expr: | - ceph_health_status{job="rook-ceph-mgr"} > 1 - for: 10m - labels: - severity: critical - - alert: CephClusterWarningState - annotations: - description: Storage cluster is in warning state for more than 10m. - message: Storage cluster is in degraded state - severity_level: warning - storage_type: ceph - expr: | - ceph_health_status{job="rook-ceph-mgr"} == 1 - for: 10m - labels: - severity: warning - - alert: CephOSDVersionMismatch - annotations: - description: There are {{ $value }} different versions of Ceph OSD components - running. - message: There are multiple versions of storage services running. - severity_level: warning - storage_type: ceph - expr: | - count(count(ceph_osd_metadata{job="rook-ceph-mgr"}) by (ceph_version)) > 1 - for: 10m - labels: - severity: warning - - alert: CephMonVersionMismatch - annotations: - description: There are {{ $value }} different versions of Ceph Mon components - running. - message: There are multiple versions of storage services running. - severity_level: warning - storage_type: ceph - expr: | - count(count(ceph_mon_metadata{job="rook-ceph-mgr"}) by (ceph_version)) > 1 - for: 10m - labels: - severity: warning - - name: cluster-utilization-alert.rules - rules: - - alert: CephClusterNearFull - annotations: - description: Storage cluster utilization has crossed 75% and will become read-only - at 85%. Free up some space or expand the storage cluster. - message: Storage cluster is nearing full. Data deletion or cluster expansion - is required. - severity_level: warning - storage_type: ceph - expr: | - ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes > 0.75 - for: 5s - labels: - severity: warning - - alert: CephClusterCriticallyFull - annotations: - description: Storage cluster utilization has crossed 80% and will become read-only - at 85%. Free up some space or expand the storage cluster immediately. - message: Storage cluster is critically full and needs immediate data deletion - or cluster expansion. - severity_level: error - storage_type: ceph - expr: | - ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes > 0.80 - for: 5s - labels: - severity: critical - - alert: CephClusterReadOnly - annotations: - description: Storage cluster utilization has crossed 85% and will become read-only - now. Free up some space or expand the storage cluster immediately. - message: Storage cluster is read-only now and needs immediate data deletion - or cluster expansion. - severity_level: error - storage_type: ceph - expr: | - ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes >= 0.85 - for: 0s - labels: - severity: critical + - name: ceph.rules + rules: + - expr: | + kube_node_status_condition{condition="Ready",job="kube-state-metrics",status="true"} * on (node) group_right() max(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"},"node","$1","exported_instance","(.*)")) by (node) + record: cluster:ceph_node_down:join_kube + - expr: | + avg(topk by (ceph_daemon) (1, label_replace(label_replace(ceph_disk_occupation{job="rook-ceph-mgr"}, "instance", "$1", "exported_instance", "(.*)"), "device", "$1", "device", "/dev/(.*)")) * on(instance, device) group_right(ceph_daemon) topk by (instance,device) (1,(irate(node_disk_read_time_seconds_total[1m]) + irate(node_disk_write_time_seconds_total[1m]) / (clamp_min(irate(node_disk_reads_completed_total[1m]), 1) + irate(node_disk_writes_completed_total[1m]))))) + record: cluster:ceph_disk_latency:join_ceph_node_disk_irate1m + - name: telemeter.rules + rules: + - expr: | + count(ceph_osd_metadata{job="rook-ceph-mgr"}) + record: job:ceph_osd_metadata:count + - expr: | + count(kube_persistentvolume_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"}) + record: job:kube_pv:count + - expr: | + sum(ceph_pool_rd{job="rook-ceph-mgr"}+ ceph_pool_wr{job="rook-ceph-mgr"}) + record: job:ceph_pools_iops:total + - expr: | + sum(ceph_pool_rd_bytes{job="rook-ceph-mgr"}+ ceph_pool_wr_bytes{job="rook-ceph-mgr"}) + record: job:ceph_pools_iops_bytes:total + - expr: | + count(count(ceph_mon_metadata{job="rook-ceph-mgr"} or ceph_osd_metadata{job="rook-ceph-mgr"} or ceph_rgw_metadata{job="rook-ceph-mgr"} or ceph_mds_metadata{job="rook-ceph-mgr"} or ceph_mgr_metadata{job="rook-ceph-mgr"}) by(ceph_version)) + record: job:ceph_versions_running:count + - name: ceph-mgr-status + rules: + - alert: CephMgrIsAbsent + annotations: + description: Ceph Manager has disappeared from Prometheus target discovery. + message: Storage metrics collector service not available anymore. + severity_level: critical + storage_type: ceph + expr: | + absent(up{job="rook-ceph-mgr"} == 1) + for: 5m + labels: + severity: critical + - alert: CephMgrIsMissingReplicas + annotations: + description: Ceph Manager is missing replicas. + message: Storage metrics collector service doesn't have required no of replicas. + severity_level: warning + storage_type: ceph + expr: | + sum(up{job="rook-ceph-mgr"}) < 1 + for: 5m + labels: + severity: warning + - name: ceph-mds-status + rules: + - alert: CephMdsMissingReplicas + annotations: + description: + Minimum required replicas for storage metadata service not available. + Might affect the working of storage cluster. + message: Insufficient replicas for storage metadata service. + severity_level: warning + storage_type: ceph + expr: | + sum(ceph_mds_metadata{job="rook-ceph-mgr"} == 1) < 2 + for: 5m + labels: + severity: warning + - name: quorum-alert.rules + rules: + - alert: CephMonQuorumAtRisk + annotations: + description: Storage cluster quorum is low. Contact Support. + message: Storage quorum at risk + severity_level: error + storage_type: ceph + expr: | + count(ceph_mon_quorum_status{job="rook-ceph-mgr"} == 1) <= ((count(ceph_mon_metadata{job="rook-ceph-mgr"}) % 2) + 1) + for: 15m + labels: + severity: critical + - alert: CephMonHighNumberOfLeaderChanges + annotations: + description: + Ceph Monitor {{ $labels.ceph_daemon }} on host {{ $labels.hostname + }} has seen {{ $value | printf "%.2f" }} leader changes per minute recently. + message: Storage Cluster has seen many leader changes recently. + severity_level: warning + storage_type: ceph + expr: | + (ceph_mon_metadata{job="rook-ceph-mgr"} * on (ceph_daemon) group_left() (rate(ceph_mon_num_elections{job="rook-ceph-mgr"}[5m]) * 60)) > 0.95 + for: 5m + labels: + severity: warning + - name: ceph-node-alert.rules + rules: + - alert: CephNodeDown + annotations: + description: + Storage node {{ $labels.node }} went down. Please check the node + immediately. + message: Storage node {{ $labels.node }} went down + severity_level: error + storage_type: ceph + expr: | + cluster:ceph_node_down:join_kube == 0 + for: 30s + labels: + severity: critical + - name: osd-alert.rules + rules: + - alert: CephOSDCriticallyFull + annotations: + description: + Utilization of storage device {{ $labels.ceph_daemon }} of device_class + type {{$labels.device_class}} has crossed 80% on host {{ $labels.hostname + }}. Immediately free up some space or add capacity of type {{$labels.device_class}}. + message: Back-end storage device is critically full. + severity_level: error + storage_type: ceph + expr: | + (ceph_osd_metadata * on (ceph_daemon) group_right(device_class) (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)) >= 0.80 + for: 40s + labels: + severity: critical + - alert: CephOSDFlapping + annotations: + description: + Storage daemon {{ $labels.ceph_daemon }} has restarted 5 times + in last 5 minutes. Please check the pod events or ceph status to find out + the cause. + message: Ceph storage osd flapping. + severity_level: error + storage_type: ceph + expr: | + changes(ceph_osd_up[5m]) >= 10 + for: 0s + labels: + severity: critical + - alert: CephOSDNearFull + annotations: + description: + Utilization of storage device {{ $labels.ceph_daemon }} of device_class + type {{$labels.device_class}} has crossed 75% on host {{ $labels.hostname + }}. Immediately free up some space or add capacity of type {{$labels.device_class}}. + message: Back-end storage device is nearing full. + severity_level: warning + storage_type: ceph + expr: | + (ceph_osd_metadata * on (ceph_daemon) group_right(device_class) (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)) >= 0.75 + for: 40s + labels: + severity: warning + - alert: CephOSDDiskNotResponding + annotations: + description: + Disk device {{ $labels.device }} not responding, on host {{ $labels.host + }}. + message: Disk not responding + severity_level: error + storage_type: ceph + expr: | + label_replace((ceph_osd_in == 1 and ceph_osd_up == 0),"disk","$1","ceph_daemon","osd.(.*)") + on(ceph_daemon) group_left(host, device) label_replace(ceph_disk_occupation,"host","$1","exported_instance","(.*)") + for: 1m + labels: + severity: critical + - alert: CephOSDDiskUnavailable + annotations: + description: + Disk device {{ $labels.device }} not accessible on host {{ $labels.host + }}. + message: Disk not accessible + severity_level: error + storage_type: ceph + expr: | + label_replace((ceph_osd_in == 0 and ceph_osd_up == 0),"disk","$1","ceph_daemon","osd.(.*)") + on(ceph_daemon) group_left(host, device) label_replace(ceph_disk_occupation,"host","$1","exported_instance","(.*)") + for: 1m + labels: + severity: critical + - alert: CephDataRecoveryTakingTooLong + annotations: + description: Data recovery has been active for too long. Contact Support. + message: Data recovery is slow + severity_level: warning + storage_type: ceph + expr: | + ceph_pg_undersized > 0 + for: 2h + labels: + severity: warning + - alert: CephPGRepairTakingTooLong + annotations: + description: Self heal operations taking too long. Contact Support. + message: Self heal problems detected + severity_level: warning + storage_type: ceph + expr: | + ceph_pg_inconsistent > 0 + for: 1h + labels: + severity: warning + - name: persistent-volume-alert.rules + rules: + - alert: PersistentVolumeUsageNearFull + annotations: + description: + PVC {{ $labels.persistentvolumeclaim }} utilization has crossed + 75%. Free up some space or expand the PVC. + message: + PVC {{ $labels.persistentvolumeclaim }} is nearing full. Data deletion + or PVC expansion is required. + severity_level: warning + storage_type: ceph + expr: | + (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.75 + for: 5s + labels: + severity: warning + - alert: PersistentVolumeUsageCritical + annotations: + description: + PVC {{ $labels.persistentvolumeclaim }} utilization has crossed + 85%. Free up some space or expand the PVC immediately. + message: + PVC {{ $labels.persistentvolumeclaim }} is critically full. Data + deletion or PVC expansion is required. + severity_level: error + storage_type: ceph + expr: | + (kubelet_volume_stats_used_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) / (kubelet_volume_stats_capacity_bytes * on (namespace,persistentvolumeclaim) group_left(storageclass, provisioner) (kube_persistentvolumeclaim_info * on (storageclass) group_left(provisioner) kube_storageclass_info {provisioner=~"(.*rbd.csi.ceph.com)|(.*cephfs.csi.ceph.com)"})) > 0.85 + for: 5s + labels: + severity: critical + - name: cluster-state-alert.rules + rules: + - alert: CephClusterErrorState + annotations: + description: Storage cluster is in error state for more than 10m. + message: Storage cluster is in error state + severity_level: error + storage_type: ceph + expr: | + ceph_health_status{job="rook-ceph-mgr"} > 1 + for: 10m + labels: + severity: critical + - alert: CephClusterWarningState + annotations: + description: Storage cluster is in warning state for more than 10m. + message: Storage cluster is in degraded state + severity_level: warning + storage_type: ceph + expr: | + ceph_health_status{job="rook-ceph-mgr"} == 1 + for: 10m + labels: + severity: warning + - alert: CephOSDVersionMismatch + annotations: + description: + There are {{ $value }} different versions of Ceph OSD components + running. + message: There are multiple versions of storage services running. + severity_level: warning + storage_type: ceph + expr: | + count(count(ceph_osd_metadata{job="rook-ceph-mgr"}) by (ceph_version)) > 1 + for: 10m + labels: + severity: warning + - alert: CephMonVersionMismatch + annotations: + description: + There are {{ $value }} different versions of Ceph Mon components + running. + message: There are multiple versions of storage services running. + severity_level: warning + storage_type: ceph + expr: | + count(count(ceph_mon_metadata{job="rook-ceph-mgr"}) by (ceph_version)) > 1 + for: 10m + labels: + severity: warning + - name: cluster-utilization-alert.rules + rules: + - alert: CephClusterNearFull + annotations: + description: + Storage cluster utilization has crossed 75% and will become read-only + at 85%. Free up some space or expand the storage cluster. + message: + Storage cluster is nearing full. Data deletion or cluster expansion + is required. + severity_level: warning + storage_type: ceph + expr: | + ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes > 0.75 + for: 5s + labels: + severity: warning + - alert: CephClusterCriticallyFull + annotations: + description: + Storage cluster utilization has crossed 80% and will become read-only + at 85%. Free up some space or expand the storage cluster immediately. + message: + Storage cluster is critically full and needs immediate data deletion + or cluster expansion. + severity_level: error + storage_type: ceph + expr: | + ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes > 0.80 + for: 5s + labels: + severity: critical + - alert: CephClusterReadOnly + annotations: + description: + Storage cluster utilization has crossed 85% and will become read-only + now. Free up some space or expand the storage cluster immediately. + message: + Storage cluster is read-only now and needs immediate data deletion + or cluster expansion. + severity_level: error + storage_type: ceph + expr: | + ceph_cluster_total_used_raw_bytes / ceph_cluster_total_bytes >= 0.85 + for: 0s + labels: + severity: critical diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus-service.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus-service.yaml index 4a8d35d27..7ad7d0d74 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/prometheus-service.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus-service.yaml @@ -6,10 +6,10 @@ metadata: spec: type: NodePort ports: - - name: web - nodePort: 30900 - port: 9090 - protocol: TCP - targetPort: web + - name: web + nodePort: 30900 + port: 9090 + protocol: TCP + targetPort: web selector: prometheus: rook-prometheus diff --git a/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml b/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml index df25ad975..aff1285c5 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/prometheus.yaml @@ -10,8 +10,8 @@ metadata: name: prometheus aggregationRule: clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-prometheus: "true" + - matchLabels: + rbac.ceph.rook.io/aggregate-to-prometheus: "true" rules: [] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -21,19 +21,19 @@ metadata: labels: rbac.ceph.rook.io/aggregate-to-prometheus: "true" rules: -- apiGroups: [""] - resources: - - nodes - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -44,9 +44,9 @@ roleRef: kind: ClusterRole name: prometheus subjects: -- kind: ServiceAccount - name: prometheus - namespace: rook-ceph + - kind: ServiceAccount + name: prometheus + namespace: rook-ceph --- apiVersion: monitoring.coreos.com/v1 kind: Prometheus diff --git a/cluster/examples/kubernetes/ceph/monitoring/rbac.yaml b/cluster/examples/kubernetes/ceph/monitoring/rbac.yaml index 0a9d80537..f959571b5 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/rbac.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/rbac.yaml @@ -7,12 +7,12 @@ metadata: name: rook-ceph-monitor namespace: rook-ceph rules: -- apiGroups: - - monitoring.coreos.com - resources: - - '*' - verbs: - - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - "*" + verbs: + - "*" # OLM: END ROLE --- # OLM: BEGIN ROLE BINDING @@ -27,9 +27,9 @@ roleRef: kind: Role name: rook-ceph-monitor subjects: -- kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # OLM: END ROLE BINDING --- # OLM: BEGIN ROLE @@ -40,16 +40,16 @@ metadata: name: rook-ceph-metrics namespace: rook-ceph rules: - - apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch + - apiGroups: + - "" + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch # OLM: END ROLE --- # OLM: BEGIN ROLE BINDING @@ -64,9 +64,9 @@ roleRef: kind: Role name: rook-ceph-metrics subjects: -- kind: ServiceAccount - # change to the serviceaccount and namespace to use for monitoring - name: prometheus-k8s - namespace: rook-ceph + - kind: ServiceAccount + # change to the serviceaccount and namespace to use for monitoring + name: prometheus-k8s + namespace: rook-ceph # OLM: END ROLE BINDING --- diff --git a/cluster/examples/kubernetes/ceph/monitoring/service-monitor.yaml b/cluster/examples/kubernetes/ceph/monitoring/service-monitor.yaml index 930d0dc34..9bf062687 100644 --- a/cluster/examples/kubernetes/ceph/monitoring/service-monitor.yaml +++ b/cluster/examples/kubernetes/ceph/monitoring/service-monitor.yaml @@ -14,6 +14,6 @@ spec: app: rook-ceph-mgr rook_cluster: rook-ceph endpoints: - - port: http-metrics - path: /metrics - interval: 5s + - port: http-metrics + path: /metrics + interval: 5s diff --git a/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml b/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml index 601f84fe0..1b58b8f0f 100644 --- a/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml +++ b/cluster/examples/kubernetes/ceph/object-bucket-claim-delete.yaml @@ -3,12 +3,12 @@ kind: ObjectBucketClaim metadata: name: ceph-delete-bucket spec: - # To create a new bucket specify either `bucketName` or + # To create a new bucket specify either `bucketName` or # `generateBucketName` here. Both cannot be used. To access # an existing bucket the bucket name needs to be defined in # the StorageClass referenced here, and both `bucketName` and # `generateBucketName` must be omitted in the OBC. - #bucketName: + #bucketName: generateBucketName: ceph-bkt storageClassName: rook-ceph-delete-bucket additionalConfig: diff --git a/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml b/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml index e5aaa426c..b4de5c734 100644 --- a/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml +++ b/cluster/examples/kubernetes/ceph/object-bucket-claim-retain.yaml @@ -3,12 +3,12 @@ kind: ObjectBucketClaim metadata: name: ceph-retain-bucket spec: - # To create a new bucket specify either `bucketName` or + # To create a new bucket specify either `bucketName` or # `generateBucketName` here. Both cannot be used. To access # an existing bucket the bucket name needs to be defined in # the StorageClass referenced here, and both `bucketName` and # `generateBucketName` must be omitted in the OBC. - #bucketName: + #bucketName: generateBucketName: ceph-bkt storageClassName: rook-ceph-retain-bucket additionalConfig: diff --git a/cluster/examples/kubernetes/ceph/operator-openshift.yaml b/cluster/examples/kubernetes/ceph/operator-openshift.yaml index 6f9cfe267..971c875fa 100644 --- a/cluster/examples/kubernetes/ceph/operator-openshift.yaml +++ b/cluster/examples/kubernetes/ceph/operator-openshift.yaml @@ -59,7 +59,7 @@ allowPrivilegedContainer: true allowHostNetwork: true allowHostDirVolumePlugin: true priority: -allowedCapabilities: ['*'] +allowedCapabilities: ["*"] allowHostPorts: true allowHostPID: true allowHostIPC: true @@ -77,7 +77,7 @@ supplementalGroups: allowedFlexVolumes: - driver: "ceph.rook.io/rook" - driver: "ceph.rook.io/rook-ceph" -volumes: ['*'] +volumes: ["*"] users: # A user needs to be added for each rook service account. # This assumes running in the default sample "rook-ceph" namespace. @@ -384,148 +384,148 @@ spec: spec: serviceAccountName: rook-ceph-system containers: - - name: rook-ceph-operator - image: rook/ceph:master - args: ["ceph", "operator"] - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - env: - - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false" - # To disable RBAC, uncomment the following: - # - name: RBAC_ENABLED - # value: "false" - # Rook Agent toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: AGENT_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate - # - name: AGENT_TOLERATION_KEY - # value: "" - # (Optional) Rook Agent priority class name to set on the pod(s) - # - name: AGENT_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Rook Agent NodeAffinity. - # - name: AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook,ceph" - # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. - # `Any` uses Ceph admin credentials by default/fallback. - # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and - # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. - # to the namespace in which the `mountSecret` Kubernetes secret namespace. - # - name: AGENT_MOUNT_SECURITY_MODE - # value: "Any" - # Set the path where the Rook agent can find the flex volumes - - name: FLEXVOLUME_DIR_PATH - value: "/etc/kubernetes/kubelet-plugins/volume/exec" - # Set the path where kernel modules can be found - # - name: LIB_MODULES_DIR_PATH - # value: "" - # Mount any extra directories into the agent container - # - name: AGENT_MOUNTS - # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" - # Rook Discover toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: DISCOVER_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate - # - name: DISCOVER_TOLERATION_KEY - # value: "" - # (Optional) Rook Discover priority class name to set on the pod(s) - # - name: DISCOVER_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Discover Agent NodeAffinity. - # - name: DISCOVER_AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook, ceph" - # (Optional) Discover Agent Pod Labels. - # - name: DISCOVER_AGENT_POD_LABELS - # value: "key1=value1,key2=value2" + - name: rook-ceph-operator + image: rook/ceph:master + args: ["ceph", "operator"] + volumeMounts: + - mountPath: /var/lib/rook + name: rook-config + - mountPath: /etc/ceph + name: default-config-dir + env: + - name: ROOK_CURRENT_NAMESPACE_ONLY + value: "false" + # To disable RBAC, uncomment the following: + # - name: RBAC_ENABLED + # value: "false" + # Rook Agent toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: AGENT_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate + # - name: AGENT_TOLERATION_KEY + # value: "" + # (Optional) Rook Agent priority class name to set on the pod(s) + # - name: AGENT_PRIORITY_CLASS_NAME + # value: "" + # (Optional) Rook Agent NodeAffinity. + # - name: AGENT_NODE_AFFINITY + # value: "role=storage-node; storage=rook,ceph" + # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. + # `Any` uses Ceph admin credentials by default/fallback. + # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and + # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. + # to the namespace in which the `mountSecret` Kubernetes secret namespace. + # - name: AGENT_MOUNT_SECURITY_MODE + # value: "Any" + # Set the path where the Rook agent can find the flex volumes + - name: FLEXVOLUME_DIR_PATH + value: "/etc/kubernetes/kubelet-plugins/volume/exec" + # Set the path where kernel modules can be found + # - name: LIB_MODULES_DIR_PATH + # value: "" + # Mount any extra directories into the agent container + # - name: AGENT_MOUNTS + # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" + # Rook Discover toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: DISCOVER_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate + # - name: DISCOVER_TOLERATION_KEY + # value: "" + # (Optional) Rook Discover priority class name to set on the pod(s) + # - name: DISCOVER_PRIORITY_CLASS_NAME + # value: "" + # (Optional) Discover Agent NodeAffinity. + # - name: DISCOVER_AGENT_NODE_AFFINITY + # value: "role=storage-node; storage=rook, ceph" + # (Optional) Discover Agent Pod Labels. + # - name: DISCOVER_AGENT_POD_LABELS + # value: "key1=value1,key2=value2" - # The logging level for the operator: INFO | DEBUG - - name: ROOK_LOG_LEVEL - value: "INFO" - # The duration between discovering devices in the rook-discover daemonset. - - name: ROOK_DISCOVER_DEVICES_INTERVAL - value: "60m" - # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. - # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues. - # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "true" - # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). - # Disable it here if you have similar issues. - # For more details see https://github.com/rook/rook/issues/2417 - - name: ROOK_ENABLE_SELINUX_RELABELING - value: "true" - # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues. - # For more details see https://github.com/rook/rook/issues/2254 - - name: ROOK_ENABLE_FSGROUP - value: "true" - # Disable automatic orchestration when new devices are discovered - - name: ROOK_DISABLE_DEVICE_HOTPLUG - value: "false" - # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". - # In case of more than one regex, use comma to separate between them. - # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # add regex expression after putting a comma to blacklist a disk - # If value is empty, the default regex will be used. - - name: DISCOVER_DAEMON_UDEV_BLACKLIST - value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" + # The logging level for the operator: INFO | DEBUG + - name: ROOK_LOG_LEVEL + value: "INFO" + # The duration between discovering devices in the rook-discover daemonset. + - name: ROOK_DISCOVER_DEVICES_INTERVAL + value: "60m" + # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. + # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues. + # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 + - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED + value: "true" + # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). + # Disable it here if you have similar issues. + # For more details see https://github.com/rook/rook/issues/2417 + - name: ROOK_ENABLE_SELINUX_RELABELING + value: "true" + # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues. + # For more details see https://github.com/rook/rook/issues/2254 + - name: ROOK_ENABLE_FSGROUP + value: "true" + # Disable automatic orchestration when new devices are discovered + - name: ROOK_DISABLE_DEVICE_HOTPLUG + value: "false" + # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". + # In case of more than one regex, use comma to separate between them. + # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" + # add regex expression after putting a comma to blacklist a disk + # If value is empty, the default regex will be used. + - name: DISCOVER_DAEMON_UDEV_BLACKLIST + value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release - # in favor of the CSI driver. - - name: ROOK_ENABLE_FLEX_DRIVER - value: "false" + # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release + # in favor of the CSI driver. + - name: ROOK_ENABLE_FLEX_DRIVER + value: "false" - # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. - # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "false" + # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. + # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. + - name: ROOK_ENABLE_DISCOVERY_DAEMON + value: "false" - # Whether to start machineDisruptionBudget and machineLabel controller to watch for the osd pods and MDBs. - - name: ROOK_ENABLE_MACHINE_DISRUPTION_BUDGET - value: "false" + # Whether to start machineDisruptionBudget and machineLabel controller to watch for the osd pods and MDBs. + - name: ROOK_ENABLE_MACHINE_DISRUPTION_BUDGET + value: "false" - # Time to wait until the node controller will move Rook pods to other - # nodes after detecting an unreachable node. - # Pods affected by this setting are: - # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox - # The value used in this variable replaces the default value of 300 secs - # added automatically by k8s as Toleration for - # - # The total amount of time to reschedule Rook pods in healthy nodes - # before detecting a condition will be the sum of: - # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) - # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds - - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS - value: "5" + # Time to wait until the node controller will move Rook pods to other + # nodes after detecting an unreachable node. + # Pods affected by this setting are: + # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox + # The value used in this variable replaces the default value of 300 secs + # added automatically by k8s as Toleration for + # + # The total amount of time to reschedule Rook pods in healthy nodes + # before detecting a condition will be the sum of: + # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) + # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds + - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS + value: "5" - # The name of the node to pass with the downward API - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # The pod name to pass with the downward API - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # The pod namespace to pass with the downward API - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + # The name of the node to pass with the downward API + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # The pod name to pass with the downward API + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # The pod namespace to pass with the downward API + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - # Uncomment it to run lib bucket provisioner in multithreaded mode - #- name: LIB_BUCKET_PROVISIONER_THREADS - # value: "5" + # Uncomment it to run lib bucket provisioner in multithreaded mode + #- name: LIB_BUCKET_PROVISIONER_THREADS + # value: "5" volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} + - name: rook-config + emptyDir: {} + - name: default-config-dir + emptyDir: {} # OLM: END OPERATOR DEPLOYMENT diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index e69486c43..959561825 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -307,172 +307,172 @@ spec: spec: serviceAccountName: rook-ceph-system containers: - - name: rook-ceph-operator - image: rook/ceph:master - args: ["ceph", "operator"] - volumeMounts: - - mountPath: /var/lib/rook - name: rook-config - - mountPath: /etc/ceph - name: default-config-dir - env: - # If the operator should only watch for cluster CRDs in the same namespace, set this to "true". - # If this is not set to true, the operator will watch for cluster CRDs in all namespaces. - - name: ROOK_CURRENT_NAMESPACE_ONLY - value: "false" - # To disable RBAC, uncomment the following: - # - name: RBAC_ENABLED - # value: "false" - # Rook Agent toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: AGENT_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate - # - name: AGENT_TOLERATION_KEY - # value: "" - # (Optional) Rook Agent tolerations list. Put here list of taints you want to tolerate in YAML format. - # - name: AGENT_TOLERATIONS - # value: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) Rook Agent priority class name to set on the pod(s) - # - name: AGENT_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Rook Agent NodeAffinity. - # - name: AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook,ceph" - # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. - # `Any` uses Ceph admin credentials by default/fallback. - # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and - # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. - # to the namespace in which the `mountSecret` Kubernetes secret namespace. - # - name: AGENT_MOUNT_SECURITY_MODE - # value: "Any" - # Set the path where the Rook agent can find the flex volumes - # - name: FLEXVOLUME_DIR_PATH - # value: "" - # Set the path where kernel modules can be found - # - name: LIB_MODULES_DIR_PATH - # value: "" - # Mount any extra directories into the agent container - # - name: AGENT_MOUNTS - # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" - # Rook Discover toleration. Will tolerate all taints with all keys. - # Choose between NoSchedule, PreferNoSchedule and NoExecute: - # - name: DISCOVER_TOLERATION - # value: "NoSchedule" - # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate - # - name: DISCOVER_TOLERATION_KEY - # value: "" - # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. - # - name: DISCOVER_TOLERATIONS - # value: | - # - effect: NoSchedule - # key: node-role.kubernetes.io/controlplane - # operator: Exists - # - effect: NoExecute - # key: node-role.kubernetes.io/etcd - # operator: Exists - # (Optional) Rook Discover priority class name to set on the pod(s) - # - name: DISCOVER_PRIORITY_CLASS_NAME - # value: "" - # (Optional) Discover Agent NodeAffinity. - # - name: DISCOVER_AGENT_NODE_AFFINITY - # value: "role=storage-node; storage=rook, ceph" - # (Optional) Discover Agent Pod Labels. - # - name: DISCOVER_AGENT_POD_LABELS - # value: "key1=value1,key2=value2" + - name: rook-ceph-operator + image: rook/ceph:master + args: ["ceph", "operator"] + volumeMounts: + - mountPath: /var/lib/rook + name: rook-config + - mountPath: /etc/ceph + name: default-config-dir + env: + # If the operator should only watch for cluster CRDs in the same namespace, set this to "true". + # If this is not set to true, the operator will watch for cluster CRDs in all namespaces. + - name: ROOK_CURRENT_NAMESPACE_ONLY + value: "false" + # To disable RBAC, uncomment the following: + # - name: RBAC_ENABLED + # value: "false" + # Rook Agent toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: AGENT_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate + # - name: AGENT_TOLERATION_KEY + # value: "" + # (Optional) Rook Agent tolerations list. Put here list of taints you want to tolerate in YAML format. + # - name: AGENT_TOLERATIONS + # value: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/controlplane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists + # (Optional) Rook Agent priority class name to set on the pod(s) + # - name: AGENT_PRIORITY_CLASS_NAME + # value: "" + # (Optional) Rook Agent NodeAffinity. + # - name: AGENT_NODE_AFFINITY + # value: "role=storage-node; storage=rook,ceph" + # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`. + # `Any` uses Ceph admin credentials by default/fallback. + # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and + # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name. + # to the namespace in which the `mountSecret` Kubernetes secret namespace. + # - name: AGENT_MOUNT_SECURITY_MODE + # value: "Any" + # Set the path where the Rook agent can find the flex volumes + # - name: FLEXVOLUME_DIR_PATH + # value: "" + # Set the path where kernel modules can be found + # - name: LIB_MODULES_DIR_PATH + # value: "" + # Mount any extra directories into the agent container + # - name: AGENT_MOUNTS + # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2" + # Rook Discover toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: DISCOVER_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate + # - name: DISCOVER_TOLERATION_KEY + # value: "" + # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. + # - name: DISCOVER_TOLERATIONS + # value: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/controlplane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists + # (Optional) Rook Discover priority class name to set on the pod(s) + # - name: DISCOVER_PRIORITY_CLASS_NAME + # value: "" + # (Optional) Discover Agent NodeAffinity. + # - name: DISCOVER_AGENT_NODE_AFFINITY + # value: "role=storage-node; storage=rook, ceph" + # (Optional) Discover Agent Pod Labels. + # - name: DISCOVER_AGENT_POD_LABELS + # value: "key1=value1,key2=value2" - # The logging level for the operator: INFO | DEBUG - - name: ROOK_LOG_LEVEL - value: "INFO" + # The logging level for the operator: INFO | DEBUG + - name: ROOK_LOG_LEVEL + value: "INFO" - # The duration between discovering devices in the rook-discover daemonset. - - name: ROOK_DISCOVER_DEVICES_INTERVAL - value: "60m" + # The duration between discovering devices in the rook-discover daemonset. + - name: ROOK_DISCOVER_DEVICES_INTERVAL + value: "60m" - # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. - # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues. - # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 - - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED - value: "false" + # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. + # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues. + # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 + - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED + value: "false" - # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). - # Disable it here if you have similar issues. - # For more details see https://github.com/rook/rook/issues/2417 - - name: ROOK_ENABLE_SELINUX_RELABELING - value: "true" + # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins). + # Disable it here if you have similar issues. + # For more details see https://github.com/rook/rook/issues/2417 + - name: ROOK_ENABLE_SELINUX_RELABELING + value: "true" - # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues. - # For more details see https://github.com/rook/rook/issues/2254 - - name: ROOK_ENABLE_FSGROUP - value: "true" + # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues. + # For more details see https://github.com/rook/rook/issues/2254 + - name: ROOK_ENABLE_FSGROUP + value: "true" - # Disable automatic orchestration when new devices are discovered - - name: ROOK_DISABLE_DEVICE_HOTPLUG - value: "false" + # Disable automatic orchestration when new devices are discovered + - name: ROOK_DISABLE_DEVICE_HOTPLUG + value: "false" - # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". - # In case of more than one regex, use comma to separate between them. - # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # Add regex expression after putting a comma to blacklist a disk - # If value is empty, the default regex will be used. - - name: DISCOVER_DAEMON_UDEV_BLACKLIST - value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" + # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". + # In case of more than one regex, use comma to separate between them. + # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" + # Add regex expression after putting a comma to blacklist a disk + # If value is empty, the default regex will be used. + - name: DISCOVER_DAEMON_UDEV_BLACKLIST + value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" - # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release - # in favor of the CSI driver. - - name: ROOK_ENABLE_FLEX_DRIVER - value: "false" + # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release + # in favor of the CSI driver. + - name: ROOK_ENABLE_FLEX_DRIVER + value: "false" - # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. - # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. - - name: ROOK_ENABLE_DISCOVERY_DAEMON - value: "false" + # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster. + # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs. + - name: ROOK_ENABLE_DISCOVERY_DAEMON + value: "false" - # Time to wait until the node controller will move Rook pods to other - # nodes after detecting an unreachable node. - # Pods affected by this setting are: - # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox - # The value used in this variable replaces the default value of 300 secs - # added automatically by k8s as Toleration for - # - # The total amount of time to reschedule Rook pods in healthy nodes - # before detecting a condition will be the sum of: - # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) - # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds - - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS - value: "5" + # Time to wait until the node controller will move Rook pods to other + # nodes after detecting an unreachable node. + # Pods affected by this setting are: + # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox + # The value used in this variable replaces the default value of 300 secs + # added automatically by k8s as Toleration for + # + # The total amount of time to reschedule Rook pods in healthy nodes + # before detecting a condition will be the sum of: + # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) + # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds + - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS + value: "5" - # The name of the node to pass with the downward API - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # The pod name to pass with the downward API - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # The pod namespace to pass with the downward API - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + # The name of the node to pass with the downward API + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # The pod name to pass with the downward API + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # The pod namespace to pass with the downward API + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - # Uncomment it to run lib bucket provisioner in multithreaded mode - #- name: LIB_BUCKET_PROVISIONER_THREADS - # value: "5" + # Uncomment it to run lib bucket provisioner in multithreaded mode + #- name: LIB_BUCKET_PROVISIONER_THREADS + # value: "5" # Uncomment it to run rook operator on the host network #hostNetwork: true volumes: - - name: rook-config - emptyDir: {} - - name: default-config-dir - emptyDir: {} + - name: rook-config + emptyDir: {} + - name: default-config-dir + emptyDir: {} # OLM: END OPERATOR DEPLOYMENT diff --git a/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml b/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml index 261f08848..e55755c18 100644 --- a/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml +++ b/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml @@ -780,4 +780,4 @@ spec: scope: Namespaced version: v1 subresources: - status: {} + status: {} \ No newline at end of file diff --git a/cluster/examples/kubernetes/ceph/rgw-external.yaml b/cluster/examples/kubernetes/ceph/rgw-external.yaml index 57b0b0de3..8897c0fd6 100644 --- a/cluster/examples/kubernetes/ceph/rgw-external.yaml +++ b/cluster/examples/kubernetes/ceph/rgw-external.yaml @@ -9,10 +9,10 @@ metadata: rook_object_store: my-store spec: ports: - - name: rgw - port: 80 # service port mentioned in object store crd - protocol: TCP - targetPort: 8080 + - name: rgw + port: 80 # service port mentioned in object store crd + protocol: TCP + targetPort: 8080 selector: app: rook-ceph-rgw rook_cluster: rook-ceph # namespace:cluster diff --git a/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml b/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml index 8d155635e..63b25fabd 100644 --- a/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml +++ b/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml @@ -7,11 +7,11 @@ provisioner: rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster # when its OBC is deleted. reclaimPolicy: Delete parameters: - objectStoreName: my-store - objectStoreNamespace: rook-ceph # namespace:cluster - region: us-east-1 - # To accommodate brownfield cases reference the existing bucket name here instead - # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant - # access to the bucket by creating a new user, attaching it to the bucket, and - # providing the credentials via a Secret in the namespace of the requesting OBC. - #bucketName: + objectStoreName: my-store + objectStoreNamespace: rook-ceph # namespace:cluster + region: us-east-1 + # To accommodate brownfield cases reference the existing bucket name here instead + # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant + # access to the bucket by creating a new user, attaching it to the bucket, and + # providing the credentials via a Secret in the namespace of the requesting OBC. + #bucketName: diff --git a/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml b/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml index 31a606a43..37361a846 100644 --- a/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml +++ b/cluster/examples/kubernetes/ceph/storageclass-bucket-retain.yaml @@ -6,11 +6,11 @@ provisioner: rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster # set the reclaim policy to retain the bucket when its OBC is deleted reclaimPolicy: Retain parameters: - objectStoreName: my-store # port 80 assumed - objectStoreNamespace: rook-ceph # namespace:cluster - region: us-east-1 - # To accommodate brownfield cases reference the existing bucket name here instead - # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant - # access to the bucket by creating a new user, attaching it to the bucket, and - # providing the credentials via a Secret in the namespace of the requesting OBC. - #bucketName: + objectStoreName: my-store # port 80 assumed + objectStoreNamespace: rook-ceph # namespace:cluster + region: us-east-1 + # To accommodate brownfield cases reference the existing bucket name here instead + # of in the ObjectBucketClaim (OBC). In this case the provisioner will grant + # access to the bucket by creating a new user, attaching it to the bucket, and + # providing the credentials via a Secret in the namespace of the requesting OBC. + #bucketName: diff --git a/cluster/examples/kubernetes/ceph/toolbox-job.yaml b/cluster/examples/kubernetes/ceph/toolbox-job.yaml index c6e11d876..3fee97919 100644 --- a/cluster/examples/kubernetes/ceph/toolbox-job.yaml +++ b/cluster/examples/kubernetes/ceph/toolbox-job.yaml @@ -9,50 +9,50 @@ spec: template: spec: initContainers: - - name: config-init - image: rook/ceph:master - command: ["/usr/local/bin/toolbox.sh"] - args: ["--skip-watch"] - imagePullPolicy: IfNotPresent - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - volumeMounts: - - mountPath: /etc/ceph - name: ceph-config - - name: mon-endpoint-volume - mountPath: /etc/rook + - name: config-init + image: rook/ceph:master + command: ["/usr/local/bin/toolbox.sh"] + args: ["--skip-watch"] + imagePullPolicy: IfNotPresent + env: + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + volumeMounts: + - mountPath: /etc/ceph + name: ceph-config + - name: mon-endpoint-volume + mountPath: /etc/rook containers: - - name: script - image: rook/ceph:master - volumeMounts: - - mountPath: /etc/ceph - name: ceph-config - readOnly: true - command: - - "bash" - - "-c" - - | - # Modify this script to run any ceph, rbd, radosgw-admin, or other commands that could - # be run in the toolbox pod. The output of the commands can be seen by getting the pod log. - # - # example: print the ceph status - ceph status + - name: script + image: rook/ceph:master + volumeMounts: + - mountPath: /etc/ceph + name: ceph-config + readOnly: true + command: + - "bash" + - "-c" + - | + # Modify this script to run any ceph, rbd, radosgw-admin, or other commands that could + # be run in the toolbox pod. The output of the commands can be seen by getting the pod log. + # + # example: print the ceph status + ceph status volumes: - - name: mon-endpoint-volume - configMap: - name: rook-ceph-mon-endpoints - items: - - key: data - path: mon-endpoints - - name: ceph-config - emptyDir: {} + - name: mon-endpoint-volume + configMap: + name: rook-ceph-mon-endpoints + items: + - key: data + path: mon-endpoints + - name: ceph-config + emptyDir: {} restartPolicy: Never diff --git a/cluster/examples/kubernetes/ceph/toolbox.yaml b/cluster/examples/kubernetes/ceph/toolbox.yaml index 9b6ca5a06..28edc958b 100644 --- a/cluster/examples/kubernetes/ceph/toolbox.yaml +++ b/cluster/examples/kubernetes/ceph/toolbox.yaml @@ -17,34 +17,34 @@ spec: spec: dnsPolicy: ClusterFirstWithHostNet containers: - - name: rook-ceph-tools - image: rook/ceph:master - command: ["/tini"] - args: ["-g", "--", "/usr/local/bin/toolbox.sh"] - imagePullPolicy: IfNotPresent - env: - - name: ROOK_CEPH_USERNAME - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-username - - name: ROOK_CEPH_SECRET - valueFrom: - secretKeyRef: - name: rook-ceph-mon - key: ceph-secret - volumeMounts: - - mountPath: /etc/ceph - name: ceph-config - - name: mon-endpoint-volume - mountPath: /etc/rook + - name: rook-ceph-tools + image: rook/ceph:master + command: ["/tini"] + args: ["-g", "--", "/usr/local/bin/toolbox.sh"] + imagePullPolicy: IfNotPresent + env: + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + volumeMounts: + - mountPath: /etc/ceph + name: ceph-config + - name: mon-endpoint-volume + mountPath: /etc/rook volumes: - name: mon-endpoint-volume configMap: name: rook-ceph-mon-endpoints items: - - key: data - path: mon-endpoints + - key: data + path: mon-endpoints - name: ceph-config emptyDir: {} tolerations: diff --git a/cluster/examples/kubernetes/mysql.yaml b/cluster/examples/kubernetes/mysql.yaml index 0b6d9f67b..c3ac4ce1f 100644 --- a/cluster/examples/kubernetes/mysql.yaml +++ b/cluster/examples/kubernetes/mysql.yaml @@ -21,7 +21,7 @@ metadata: spec: storageClassName: rook-ceph-block accessModes: - - ReadWriteOnce + - ReadWriteOnce resources: requests: storage: 20Gi @@ -47,18 +47,18 @@ spec: tier: mysql spec: containers: - - image: mysql:5.6 - name: mysql - env: - - name: MYSQL_ROOT_PASSWORD - value: changeme - ports: - - containerPort: 3306 + - image: mysql:5.6 name: mysql - volumeMounts: - - name: mysql-persistent-storage - mountPath: /var/lib/mysql + env: + - name: MYSQL_ROOT_PASSWORD + value: changeme + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql volumes: - - name: mysql-persistent-storage - persistentVolumeClaim: - claimName: mysql-pv-claim + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim diff --git a/cluster/examples/kubernetes/nfs/busybox-rc.yaml b/cluster/examples/kubernetes/nfs/busybox-rc.yaml index 01d79f79a..4b5c8fc24 100644 --- a/cluster/examples/kubernetes/nfs/busybox-rc.yaml +++ b/cluster/examples/kubernetes/nfs/busybox-rc.yaml @@ -18,18 +18,18 @@ spec: role: busybox spec: containers: - - image: busybox - command: - - sh - - -c - - 'while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done' - imagePullPolicy: IfNotPresent - name: busybox - volumeMounts: - # name must match the volume name below - - name: rook-nfs-vol - mountPath: "/mnt" + - image: busybox + command: + - sh + - -c + - "while true; do date > /mnt/index.html; hostname >> /mnt/index.html; sleep $(($RANDOM % 5 + 5)); done" + imagePullPolicy: IfNotPresent + name: busybox + volumeMounts: + # name must match the volume name below + - name: rook-nfs-vol + mountPath: "/mnt" volumes: - - name: rook-nfs-vol - persistentVolumeClaim: - claimName: rook-nfs-pv-claim + - name: rook-nfs-vol + persistentVolumeClaim: + claimName: rook-nfs-pv-claim diff --git a/cluster/examples/kubernetes/nfs/common.yaml b/cluster/examples/kubernetes/nfs/common.yaml index 2d5aac4d6..619ca7ee4 100644 --- a/cluster/examples/kubernetes/nfs/common.yaml +++ b/cluster/examples/kubernetes/nfs/common.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Namespace metadata: - name: rook-nfs-system + name: rook-nfs-system --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -12,13 +12,13 @@ metadata: name: nfsservers.nfs.rook.io spec: additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - name: AGE - type: date - - JSONPath: .status.state - description: NFS Server instance state - name: State - type: string + - JSONPath: .metadata.creationTimestamp + name: AGE + type: date + - JSONPath: .status.state + description: NFS Server instance state + name: State + type: string group: nfs.rook.io names: kind: NFSServer @@ -33,14 +33,16 @@ spec: description: NFSServer is the Schema for the nfsservers API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation + description: + "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" type: string kind: - description: 'Kind is a string value representing the REST resource this + description: + "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" type: string metadata: type: object @@ -50,7 +52,8 @@ spec: annotations: additionalProperties: type: string - description: The annotations-related configuration to add/set on each + description: + The annotations-related configuration to add/set on each Pod related object. type: object exports: @@ -65,46 +68,52 @@ spec: description: PVC from which the NFS daemon gets storage for sharing properties: claimName: - description: 'ClaimName is the name of a PersistentVolumeClaim + description: + "ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims" type: string readOnly: - description: Will force the ReadOnly setting in VolumeMounts. + description: + Will force the ReadOnly setting in VolumeMounts. Default false. type: boolean required: - - claimName + - claimName type: object server: description: The NFS server configuration properties: accessMode: - description: Reading and Writing permissions on the export + description: + Reading and Writing permissions on the export Valid values are "ReadOnly", "ReadWrite" and "none" enum: - - ReadOnly - - ReadWrite - - none + - ReadOnly + - ReadWrite + - none type: string allowedClients: description: The clients allowed to access the NFS export items: - description: AllowedClientsSpec represents the client specs + description: + AllowedClientsSpec represents the client specs for accessing the NFS export properties: accessMode: - description: Reading and Writing permissions for the + description: + Reading and Writing permissions for the client to access the NFS export Valid values are "ReadOnly", "ReadWrite" and "none" Gets overridden when ServerSpec.accessMode is specified enum: - - ReadOnly - - ReadWrite - - none + - ReadOnly + - ReadWrite + - none type: string clients: - description: The clients that can access the share Values + description: + The clients that can access the share Values can be hostname, ip address, netgroup, CIDR network address, or all items: @@ -114,26 +123,28 @@ spec: description: Name of the clients group type: string squash: - description: Squash options for clients Valid values + description: + Squash options for clients Valid values are "none", "rootid", "root", and "all" Gets overridden when ServerSpec.squash is specified enum: - - none - - rootid - - root - - all + - none + - rootid + - root + - all type: string type: object type: array squash: - description: This prevents the root users connected remotely + description: + This prevents the root users connected remotely from having root privileges Valid values are "none", "rootid", "root", and "all" enum: - - none - - rootid - - root - - all + - none + - rootid + - root + - all type: string type: object type: object @@ -155,9 +166,9 @@ spec: type: object version: v1alpha1 versions: - - name: v1alpha1 - served: true - storage: true + - name: v1alpha1 + served: true + storage: true status: acceptedNames: kind: "" diff --git a/cluster/examples/kubernetes/nfs/nfs-ceph.yaml b/cluster/examples/kubernetes/nfs/nfs-ceph.yaml index 0afe33be5..fbdc51dab 100644 --- a/cluster/examples/kubernetes/nfs/nfs-ceph.yaml +++ b/cluster/examples/kubernetes/nfs/nfs-ceph.yaml @@ -10,7 +10,7 @@ metadata: spec: storageClassName: rook-ceph-block accessModes: - - ReadWriteMany + - ReadWriteMany resources: requests: storage: 2Gi @@ -23,15 +23,15 @@ metadata: spec: replicas: 1 exports: - - name: share1 - server: - accessMode: ReadWrite - squash: "none" - # A Persistent Volume Claim must be created before creating NFS CRD instance. - # Create a Ceph cluster for using this example - # Create a ceph PVC after creating the rook ceph cluster using ceph-pvc.yaml - persistentVolumeClaim: - claimName: nfs-ceph-claim + - name: share1 + server: + accessMode: ReadWrite + squash: "none" + # A Persistent Volume Claim must be created before creating NFS CRD instance. + # Create a Ceph cluster for using this example + # Create a ceph PVC after creating the rook ceph cluster using ceph-pvc.yaml + persistentVolumeClaim: + claimName: nfs-ceph-claim # A key/value list of annotations annotations: rook: nfs diff --git a/cluster/examples/kubernetes/nfs/nfs-xfs.yaml b/cluster/examples/kubernetes/nfs/nfs-xfs.yaml index 36f5a490b..2a85ff032 100644 --- a/cluster/examples/kubernetes/nfs/nfs-xfs.yaml +++ b/cluster/examples/kubernetes/nfs/nfs-xfs.yaml @@ -25,7 +25,7 @@ metadata: spec: storageClassName: "standard-xfs" accessModes: - - ReadWriteOnce + - ReadWriteOnce resources: requests: storage: 1Gi @@ -38,13 +38,13 @@ metadata: spec: replicas: 1 exports: - - name: share1 - server: - accessMode: ReadWrite - squash: "none" - # A Persistent Volume Claim must be created before creating NFS CRD instance. - persistentVolumeClaim: - claimName: nfs-xfs-claim + - name: share1 + server: + accessMode: ReadWrite + squash: "none" + # A Persistent Volume Claim must be created before creating NFS CRD instance. + persistentVolumeClaim: + claimName: nfs-xfs-claim # A key/value list of annotations annotations: rook: nfs diff --git a/cluster/examples/kubernetes/nfs/nfs.yaml b/cluster/examples/kubernetes/nfs/nfs.yaml index 742fa340d..742fcf9de 100644 --- a/cluster/examples/kubernetes/nfs/nfs.yaml +++ b/cluster/examples/kubernetes/nfs/nfs.yaml @@ -7,7 +7,7 @@ metadata: namespace: rook-nfs spec: accessModes: - - ReadWriteMany + - ReadWriteMany resources: requests: storage: 1Gi @@ -20,13 +20,13 @@ metadata: spec: replicas: 1 exports: - - name: share1 - server: - accessMode: ReadWrite - squash: "none" - # A Persistent Volume Claim must be created before creating NFS CRD instance. - persistentVolumeClaim: - claimName: nfs-default-claim + - name: share1 + server: + accessMode: ReadWrite + squash: "none" + # A Persistent Volume Claim must be created before creating NFS CRD instance. + persistentVolumeClaim: + claimName: nfs-default-claim # A key/value list of annotations annotations: rook: nfs diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index 1ca65a757..f515e0ad5 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -13,89 +13,89 @@ roleRef: kind: ClusterRole name: rook-nfs-operator subjects: -- kind: ServiceAccount - name: rook-nfs-operator - namespace: rook-nfs-system + - kind: ServiceAccount + name: rook-nfs-operator + namespace: rook-nfs-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: rook-nfs-operator rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - get - - watch - - create -- apiGroups: - - "" - resources: - - services - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - nfs.rook.io - resources: - - nfsservers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - nfs.rook.io - resources: - - nfsservers/status - - nfsservers/finalizers - verbs: - - get - - patch - - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - watch + - create + - apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - list + - patch + - update + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - patch + - update + - watch + - apiGroups: + - nfs.rook.io + resources: + - nfsservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - nfs.rook.io + resources: + - nfsservers/status + - nfsservers/finalizers + verbs: + - get + - patch + - update --- apiVersion: apps/v1 kind: Deployment @@ -116,16 +116,16 @@ spec: spec: serviceAccountName: rook-nfs-operator containers: - - name: rook-nfs-operator - image: rook/nfs:master - imagePullPolicy: IfNotPresent - args: ["nfs", "operator"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + - name: rook-nfs-operator + image: rook/nfs:master + imagePullPolicy: IfNotPresent + args: ["nfs", "operator"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/cluster/examples/kubernetes/nfs/psp.yaml b/cluster/examples/kubernetes/nfs/psp.yaml index a500e682b..c61051110 100644 --- a/cluster/examples/kubernetes/nfs/psp.yaml +++ b/cluster/examples/kubernetes/nfs/psp.yaml @@ -7,8 +7,8 @@ spec: fsGroup: rule: RunAsAny allowedCapabilities: - - DAC_READ_SEARCH - - SYS_RESOURCE + - DAC_READ_SEARCH + - SYS_RESOURCE runAsUser: rule: RunAsAny seLinux: @@ -16,9 +16,9 @@ spec: supplementalGroups: rule: RunAsAny volumes: - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - hostPath + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret + - hostPath diff --git a/cluster/examples/kubernetes/nfs/rbac.yaml b/cluster/examples/kubernetes/nfs/rbac.yaml index 8e3d9f774..3f1224d0f 100644 --- a/cluster/examples/kubernetes/nfs/rbac.yaml +++ b/cluster/examples/kubernetes/nfs/rbac.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Namespace metadata: - name: rook-nfs + name: rook-nfs --- apiVersion: v1 kind: ServiceAccount @@ -38,11 +38,11 @@ rules: resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: - - nfs.rook.io + - nfs.rook.io resources: - - "*" + - "*" verbs: - - "*" + - "*" --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -50,8 +50,9 @@ metadata: name: rook-nfs-provisioner-runner subjects: - kind: ServiceAccount - name: rook-nfs-server - # replace with namespace where provisioner is deployed + name: + rook-nfs-server + # replace with namespace where provisioner is deployed namespace: rook-nfs roleRef: kind: ClusterRole diff --git a/cluster/examples/kubernetes/nfs/scc.yaml b/cluster/examples/kubernetes/nfs/scc.yaml index 6f9a236ad..4c939ddcd 100644 --- a/cluster/examples/kubernetes/nfs/scc.yaml +++ b/cluster/examples/kubernetes/nfs/scc.yaml @@ -9,17 +9,17 @@ allowHostPID: false allowHostPorts: false allowPrivilegedContainer: false allowedCapabilities: -- SYS_ADMIN -- DAC_READ_SEARCH + - SYS_ADMIN + - DAC_READ_SEARCH defaultAddCapabilities: null fsGroup: type: MustRunAs priority: null readOnlyRootFilesystem: false requiredDropCapabilities: -- KILL -- MKNOD -- SYS_CHROOT + - KILL + - MKNOD + - SYS_CHROOT runAsUser: type: RunAsAny seLinuxContext: @@ -27,10 +27,10 @@ seLinuxContext: supplementalGroups: type: RunAsAny volumes: -- configMap -- downwardAPI -- emptyDir -- persistentVolumeClaim -- secret + - configMap + - downwardAPI + - emptyDir + - persistentVolumeClaim + - secret users: - system:serviceaccount:rook-nfs:rook-nfs-server diff --git a/cluster/examples/kubernetes/nfs/web-rc.yaml b/cluster/examples/kubernetes/nfs/web-rc.yaml index 946496eaf..92987c8c1 100644 --- a/cluster/examples/kubernetes/nfs/web-rc.yaml +++ b/cluster/examples/kubernetes/nfs/web-rc.yaml @@ -18,16 +18,16 @@ spec: role: web-frontend spec: containers: - - name: web - image: nginx - ports: - - name: web - containerPort: 80 - volumeMounts: + - name: web + image: nginx + ports: + - name: web + containerPort: 80 + volumeMounts: # name must match the volume name below - name: rook-nfs-vol mountPath: "/usr/share/nginx/html" volumes: - - name: rook-nfs-vol - persistentVolumeClaim: - claimName: rook-nfs-pv-claim + - name: rook-nfs-vol + persistentVolumeClaim: + claimName: rook-nfs-pv-claim diff --git a/cluster/examples/kubernetes/nfs/webhook.yaml b/cluster/examples/kubernetes/nfs/webhook.yaml index ddd6cddf9..af0a918c8 100644 --- a/cluster/examples/kubernetes/nfs/webhook.yaml +++ b/cluster/examples/kubernetes/nfs/webhook.yaml @@ -10,11 +10,11 @@ metadata: name: rook-nfs-webhook namespace: rook-nfs-system rules: -- apiGroups: [""] - resources: ["secrets"] - resourceNames: - - 'rook-nfs-webhook-cert' - verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: + - "rook-nfs-webhook-cert" + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -26,10 +26,10 @@ roleRef: kind: Role name: rook-nfs-webhook subjects: -- apiGroup: "" - kind: ServiceAccount - name: rook-nfs-webhook - namespace: rook-nfs-system + - apiGroup: "" + kind: ServiceAccount + name: rook-nfs-webhook + namespace: rook-nfs-system --- apiVersion: cert-manager.io/v1alpha2 kind: Certificate @@ -38,8 +38,8 @@ metadata: namespace: rook-nfs-system spec: dnsNames: - - rook-nfs-webhook.rook-nfs-system.svc - - rook-nfs-webhook.rook-nfs-system.svc.cluster.local + - rook-nfs-webhook.rook-nfs-system.svc + - rook-nfs-webhook.rook-nfs-system.svc.cluster.local issuerRef: kind: Issuer name: rook-nfs-selfsigned-issuer @@ -61,24 +61,24 @@ metadata: creationTimestamp: null name: rook-nfs-validating-webhook-configuration webhooks: -- clientConfig: - caBundle: Cg== - service: - name: rook-nfs-webhook - namespace: rook-nfs-system - path: /validate-nfs-rook-io-v1alpha1-nfsserver - failurePolicy: Fail - name: validation.nfsserver.nfs.rook.io - rules: - - apiGroups: - - nfs.rook.io - apiVersions: - - v1alpha1 - operations: - - CREATE - - UPDATE - resources: - - nfsservers + - clientConfig: + caBundle: Cg== + service: + name: rook-nfs-webhook + namespace: rook-nfs-system + path: /validate-nfs-rook-io-v1alpha1-nfsserver + failurePolicy: Fail + name: validation.nfsserver.nfs.rook.io + rules: + - apiGroups: + - nfs.rook.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - nfsservers --- kind: Service apiVersion: v1 @@ -89,8 +89,8 @@ spec: selector: app: rook-nfs-webhook ports: - - port: 443 - targetPort: webhook-server + - port: 443 + targetPort: webhook-server --- apiVersion: apps/v1 kind: Deployment @@ -110,19 +110,19 @@ spec: app: rook-nfs-webhook spec: containers: - - name: rook-nfs-webhook - image: rook/nfs:master - imagePullPolicy: IfNotPresent - args: ["nfs", "webhook"] - ports: - - containerPort: 9443 - name: webhook-server - volumeMounts: - - mountPath: /tmp/k8s-webhook-server/serving-certs - name: cert - readOnly: true + - name: rook-nfs-webhook + image: rook/nfs:master + imagePullPolicy: IfNotPresent + args: ["nfs", "webhook"] + ports: + - containerPort: 9443 + name: webhook-server + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true volumes: - - name: cert - secret: - defaultMode: 420 - secretName: rook-nfs-webhook-cert \ No newline at end of file + - name: cert + secret: + defaultMode: 420 + secretName: rook-nfs-webhook-cert diff --git a/cluster/examples/kubernetes/wordpress.yaml b/cluster/examples/kubernetes/wordpress.yaml index f400abc1b..e1d88c390 100644 --- a/cluster/examples/kubernetes/wordpress.yaml +++ b/cluster/examples/kubernetes/wordpress.yaml @@ -6,7 +6,7 @@ metadata: app: wordpress spec: ports: - - port: 80 + - port: 80 selector: app: wordpress tier: frontend @@ -21,7 +21,7 @@ metadata: spec: storageClassName: rook-ceph-block accessModes: - - ReadWriteOnce + - ReadWriteOnce resources: requests: storage: 20Gi @@ -47,20 +47,20 @@ spec: tier: frontend spec: containers: - - image: wordpress:4.6.1-apache - name: wordpress - env: - - name: WORDPRESS_DB_HOST - value: wordpress-mysql - - name: WORDPRESS_DB_PASSWORD - value: changeme - ports: - - containerPort: 80 + - image: wordpress:4.6.1-apache name: wordpress - volumeMounts: - - name: wordpress-persistent-storage - mountPath: /var/www/html + env: + - name: WORDPRESS_DB_HOST + value: wordpress-mysql + - name: WORDPRESS_DB_PASSWORD + value: changeme + ports: + - containerPort: 80 + name: wordpress + volumeMounts: + - name: wordpress-persistent-storage + mountPath: /var/www/html volumes: - - name: wordpress-persistent-storage - persistentVolumeClaim: - claimName: wp-pv-claim + - name: wordpress-persistent-storage + persistentVolumeClaim: + claimName: wp-pv-claim diff --git a/cluster/olm/ceph/assemble/metadata-common.yaml b/cluster/olm/ceph/assemble/metadata-common.yaml index 8a8d82fa4..568c36d65 100644 --- a/cluster/olm/ceph/assemble/metadata-common.yaml +++ b/cluster/olm/ceph/assemble/metadata-common.yaml @@ -1,5 +1,5 @@ spec: - replaces: rook-ceph.v1.1.1 + replaces: rook-ceph.v1.1.1 customresourcedefinitions: owned: - kind: CephCluster @@ -22,92 +22,92 @@ spec: displayName: Coding Chunks path: dataPool.erasureCoded.codingChunks x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool" + - "urn:alm:descriptor:com.tectonic.ui:number" - description: Data Chunks displayName: Data Chunks path: dataPool.erasureCoded.dataChunks x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool" + - "urn:alm:descriptor:com.tectonic.ui:number" - description: failureDomain displayName: failureDomain path: dataPool.failureDomain x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool' - - 'urn:alm:descriptor:com.tectonic.ui:text' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool" + - "urn:alm:descriptor:com.tectonic.ui:text" - description: Size displayName: Size path: dataPool.replicated.size x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:dataPool" + - "urn:alm:descriptor:com.tectonic.ui:number" - description: Annotations displayName: Annotations path: gateway.annotations x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:io.kubernetes:annotations' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:io.kubernetes:annotations" - description: Instances displayName: Instances path: gateway.instances x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:com.tectonic.ui:number" - description: Resources displayName: Resources path: gateway.resources x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:com.tectonic.ui:resourceRequirements' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:com.tectonic.ui:resourceRequirements" - description: placement displayName: placement path: gateway.placement x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:io.kubernetes:placement' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:io.kubernetes:placement" - description: securePort displayName: securePort path: gateway.securePort x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:io.kubernetes:securePort' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:io.kubernetes:securePort" - description: sslCertificateRef displayName: sslCertificateRef path: gateway.sslCertificateRef x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:io.kubernetes:sslCertificateRef' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:io.kubernetes:sslCertificateRef" - description: Type displayName: Type path: gateway.type x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway' - - 'urn:alm:descriptor:com.tectonic.ui:text' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:gateway" + - "urn:alm:descriptor:com.tectonic.ui:text" - description: Coding Chunks displayName: Coding Chunks path: metadataPool.erasureCoded.codingChunks x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool" + - "urn:alm:descriptor:com.tectonic.ui:number" - description: Data Chunks displayName: Data Chunks path: metadataPool.erasureCoded.dataChunks x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool" + - "urn:alm:descriptor:com.tectonic.ui:number" - description: failureDomain displayName: failureDomain path: metadataPool.failureDomain x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool' - - 'urn:alm:descriptor:com.tectonic.ui:text' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool" + - "urn:alm:descriptor:com.tectonic.ui:text" - description: Size displayName: Size path: metadataPool.replicated.size x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool' - - 'urn:alm:descriptor:com.tectonic.ui:number' + - "urn:alm:descriptor:com.tectonic.ui:fieldGroup:metadataPool" + - "urn:alm:descriptor:com.tectonic.ui:number" - kind: CephObjectStoreUser name: cephobjectstoreusers.ceph.rook.io version: v1 @@ -178,7 +178,16 @@ spec: ## Before you start https://rook.io/docs/rook/v1.0/k8s-pre-reqs.html - keywords: ["rook", "ceph", "storage", "object storage", "open source", "block storage", "shared filesystem"] + keywords: + [ + "rook", + "ceph", + "storage", + "object storage", + "open source", + "block storage", + "shared filesystem", + ] minKubeVersion: 1.10.0 labels: alm-owner-etcd: rookoperator @@ -193,8 +202,8 @@ spec: - name: Documentation url: https://rook.github.io/docs/rook/v1.0/ icon: - - base64data: PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDIzLjAuMiwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCA3MCA3MCIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgNzAgNzA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4KPHN0eWxlIHR5cGU9InRleHQvY3NzIj4KCS5zdDB7ZmlsbDojMkIyQjJCO30KPC9zdHlsZT4KPGc+Cgk8Zz4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTUwLjUsNjcuNkgxOS45Yy04LDAtMTQuNS02LjUtMTQuNS0xNC41VjI5LjJjMC0xLjEsMC45LTIuMSwyLjEtMi4xaDU1LjRjMS4xLDAsMi4xLDAuOSwyLjEsMi4xdjIzLjkKCQkJCUM2NSw2MS4xLDU4LjUsNjcuNiw1MC41LDY3LjZ6IE05LjYsMzEuMnYyMS45YzAsNS43LDQuNiwxMC4zLDEwLjMsMTAuM2gzMC42YzUuNywwLDEwLjMtNC42LDEwLjMtMTAuM1YzMS4ySDkuNnoiLz4KCQk8L2c+CgkJPGc+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik00Mi40LDU2LjdIMjhjLTEuMSwwLTIuMS0wLjktMi4xLTIuMXYtNy4yYzAtNS4xLDQuMi05LjMsOS4zLTkuM3M5LjMsNC4yLDkuMyw5LjN2Ny4yCgkJCQlDNDQuNSw1NS43LDQzLjYsNTYuNyw0Mi40LDU2Ljd6IE0zMCw1Mi41aDEwLjN2LTUuMmMwLTIuOS0yLjMtNS4yLTUuMi01LjJjLTIuOSwwLTUuMiwyLjMtNS4yLDUuMlY1Mi41eiIvPgoJCTwvZz4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTYyLjksMjMuMkM2Mi45LDIzLjIsNjIuOSwyMy4yLDYyLjksMjMuMmwtMTEuMSwwYy0xLjEsMC0yLjEtMC45LTIuMS0yLjFjMC0xLjEsMC45LTIuMSwyLjEtMi4xCgkJCQljMCwwLDAsMCwwLDBsOS4xLDBWNi43aC02Ljl2My41YzAsMC41LTAuMiwxLjEtMC42LDEuNWMtMC40LDAuNC0wLjksMC42LTEuNSwwLjZsMCwwbC0xMS4xLDBjLTEuMSwwLTIuMS0wLjktMi4xLTIuMVY2LjdoLTYuOQoJCQkJdjMuNWMwLDEuMS0wLjksMi4xLTIuMSwyLjFsLTExLjEsMGMtMC41LDAtMS4xLTAuMi0xLjUtMC42Yy0wLjQtMC40LTAuNi0wLjktMC42LTEuNVY2LjdIOS42djEyLjRoOWMxLjEsMCwyLjEsMC45LDIuMSwyLjEKCQkJCXMtMC45LDIuMS0yLjEsMi4xaC0xMWMtMS4xLDAtMi4xLTAuOS0yLjEtMi4xVjQuNmMwLTEuMSwwLjktMi4xLDIuMS0yLjFoMTEuMWMxLjEsMCwyLjEsMC45LDIuMSwyLjF2My41bDcsMFY0LjYKCQkJCWMwLTEuMSwwLjktMi4xLDIuMS0yLjFoMTEuMWMxLjEsMCwyLjEsMC45LDIuMSwyLjF2My41bDYuOSwwVjQuNmMwLTEuMSwwLjktMi4xLDIuMS0yLjFoMTEuMUM2NCwyLjYsNjUsMy41LDY1LDQuNnYxNi41CgkJCQljMCwwLjUtMC4yLDEuMS0wLjYsMS41QzY0LDIzLDYzLjQsMjMuMiw2Mi45LDIzLjJ6Ii8+CgkJPC9nPgoJPC9nPgo8L2c+Cjwvc3ZnPg== - mediatype: image/svg+xml + - base64data: PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDIzLjAuMiwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCA3MCA3MCIgc3R5bGU9ImVuYWJsZS1iYWNrZ3JvdW5kOm5ldyAwIDAgNzAgNzA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4KPHN0eWxlIHR5cGU9InRleHQvY3NzIj4KCS5zdDB7ZmlsbDojMkIyQjJCO30KPC9zdHlsZT4KPGc+Cgk8Zz4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTUwLjUsNjcuNkgxOS45Yy04LDAtMTQuNS02LjUtMTQuNS0xNC41VjI5LjJjMC0xLjEsMC45LTIuMSwyLjEtMi4xaDU1LjRjMS4xLDAsMi4xLDAuOSwyLjEsMi4xdjIzLjkKCQkJCUM2NSw2MS4xLDU4LjUsNjcuNiw1MC41LDY3LjZ6IE05LjYsMzEuMnYyMS45YzAsNS43LDQuNiwxMC4zLDEwLjMsMTAuM2gzMC42YzUuNywwLDEwLjMtNC42LDEwLjMtMTAuM1YzMS4ySDkuNnoiLz4KCQk8L2c+CgkJPGc+CgkJCTxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik00Mi40LDU2LjdIMjhjLTEuMSwwLTIuMS0wLjktMi4xLTIuMXYtNy4yYzAtNS4xLDQuMi05LjMsOS4zLTkuM3M5LjMsNC4yLDkuMyw5LjN2Ny4yCgkJCQlDNDQuNSw1NS43LDQzLjYsNTYuNyw0Mi40LDU2Ljd6IE0zMCw1Mi41aDEwLjN2LTUuMmMwLTIuOS0yLjMtNS4yLTUuMi01LjJjLTIuOSwwLTUuMiwyLjMtNS4yLDUuMlY1Mi41eiIvPgoJCTwvZz4KCQk8Zz4KCQkJPHBhdGggY2xhc3M9InN0MCIgZD0iTTYyLjksMjMuMkM2Mi45LDIzLjIsNjIuOSwyMy4yLDYyLjksMjMuMmwtMTEuMSwwYy0xLjEsMC0yLjEtMC45LTIuMS0yLjFjMC0xLjEsMC45LTIuMSwyLjEtMi4xCgkJCQljMCwwLDAsMCwwLDBsOS4xLDBWNi43aC02Ljl2My41YzAsMC41LTAuMiwxLjEtMC42LDEuNWMtMC40LDAuNC0wLjksMC42LTEuNSwwLjZsMCwwbC0xMS4xLDBjLTEuMSwwLTIuMS0wLjktMi4xLTIuMVY2LjdoLTYuOQoJCQkJdjMuNWMwLDEuMS0wLjksMi4xLTIuMSwyLjFsLTExLjEsMGMtMC41LDAtMS4xLTAuMi0xLjUtMC42Yy0wLjQtMC40LTAuNi0wLjktMC42LTEuNVY2LjdIOS42djEyLjRoOWMxLjEsMCwyLjEsMC45LDIuMSwyLjEKCQkJCXMtMC45LDIuMS0yLjEsMi4xaC0xMWMtMS4xLDAtMi4xLTAuOS0yLjEtMi4xVjQuNmMwLTEuMSwwLjktMi4xLDIuMS0yLjFoMTEuMWMxLjEsMCwyLjEsMC45LDIuMSwyLjF2My41bDcsMFY0LjYKCQkJCWMwLTEuMSwwLjktMi4xLDIuMS0yLjFoMTEuMWMxLjEsMCwyLjEsMC45LDIuMSwyLjF2My41bDYuOSwwVjQuNmMwLTEuMSwwLjktMi4xLDIuMS0yLjFoMTEuMUM2NCwyLjYsNjUsMy41LDY1LDQuNnYxNi41CgkJCQljMCwwLjUtMC4yLDEuMS0wLjYsMS41QzY0LDIzLDYzLjQsMjMuMiw2Mi45LDIzLjJ6Ii8+CgkJPC9nPgoJPC9nPgo8L2c+Cjwvc3ZnPg== + mediatype: image/svg+xml installModes: - type: OwnNamespace supported: true diff --git a/cluster/olm/ceph/assemble/metadata-ocp.yaml b/cluster/olm/ceph/assemble/metadata-ocp.yaml index f2af0a667..97472c923 100644 --- a/cluster/olm/ceph/assemble/metadata-ocp.yaml +++ b/cluster/olm/ceph/assemble/metadata-ocp.yaml @@ -16,4 +16,4 @@ spec: - name: Red Hat, Inc. email: customerservice@redhat.com provider: - name: Red Hat, Inc. \ No newline at end of file + name: Red Hat, Inc. diff --git a/cluster/olm/ceph/assemble/rook-ceph.package.yaml b/cluster/olm/ceph/assemble/rook-ceph.package.yaml index d31d9d695..b0c145f4a 100644 --- a/cluster/olm/ceph/assemble/rook-ceph.package.yaml +++ b/cluster/olm/ceph/assemble/rook-ceph.package.yaml @@ -1,4 +1,4 @@ packageName: rook-ceph channels: -- name: beta - currentCSV: rook-ceph.v1.2.2 + - name: beta + currentCSV: rook-ceph.v1.2.2