diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 4736c1176..b8b11e19e 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -73,7 +73,7 @@ spec: # serve the dashboard at the given port. # port: 8443 # serve the dashboard using SSL - ssl: true + ssl: false # The url of the Prometheus instance # prometheusEndpoint: ://: # Whether SSL should be verified if the Prometheus server is using https @@ -263,8 +263,8 @@ spec: #crashcollector: rook-ceph-crashcollector-priority-class storage: # cluster level storage configuration and selection useAllNodes: true - useAllDevices: true - #deviceFilter: + useAllDevices: false + deviceFilter: "^nvme[0-9]+n[0-9]+$" config: # crushRoot: "custom-root" # specify a non-default root label for the CRUSH map # metadataDevice: "md0" # specify a non-rotational storage so ceph-volume will use it as block db device of bluestore. diff --git a/deploy/examples/csi/cephfs/storageclass-ec.yaml b/deploy/examples/csi/cephfs/storageclass-ec.yaml index 7c39ea70b..1ad5f5787 100644 --- a/deploy/examples/csi/cephfs/storageclass-ec.yaml +++ b/deploy/examples/csi/cephfs/storageclass-ec.yaml @@ -1,7 +1,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-cephfs + name: rook-cephfs-ec provisioner: rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name parameters: # clusterID is the namespace where the rook cluster is running @@ -9,13 +9,13 @@ parameters: clusterID: rook-ceph # namespace:cluster # CephFS filesystem name into which the volume shall be created - fsName: myfs-ec + fsName: myfs # Ceph pool into which the volume shall be created # Required for provisionVolume: "true" # For erasure coded pools, we have to create a replicated pool as the default data pool and an erasure-coded # pool as a secondary pool. - pool: myfs-ec-erasurecoded + pool: myfs-erasurecoded # The secrets contain Ceph admin credentials. These are generated automatically by the operator # in the same namespace as the cluster. @@ -32,7 +32,7 @@ parameters: # If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse # or by setting the default mounter explicitly via --volumemounter command-line argument. # mounter: kernel -reclaimPolicy: Delete +reclaimPolicy: Retain allowVolumeExpansion: true mountOptions: # uncomment the following line for debugging diff --git a/deploy/examples/csi/cephfs/storageclass.yaml b/deploy/examples/csi/cephfs/storageclass.yaml index 01dec94d0..6e3450bb8 100644 --- a/deploy/examples/csi/cephfs/storageclass.yaml +++ b/deploy/examples/csi/cephfs/storageclass.yaml @@ -21,8 +21,11 @@ parameters: csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/controller-publish-secret-name: rook-csi-cephfs-provisioner - csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph # namespace:cluster + # Keep these disabled for the existing StorageClass. StorageClass parameters + # are immutable, and this cluster was created before these parameters were + # added to the example manifest. + # csi.storage.k8s.io/controller-publish-secret-name: rook-csi-cephfs-provisioner + # csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph # namespace:cluster csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster diff --git a/deploy/examples/csi/rbd/storageclass-ec.yaml b/deploy/examples/csi/rbd/storageclass-ec.yaml index 5a401d233..2c8f3e6fa 100644 --- a/deploy/examples/csi/rbd/storageclass-ec.yaml +++ b/deploy/examples/csi/rbd/storageclass-ec.yaml @@ -8,28 +8,24 @@ apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: - name: replicated-metadata-pool + name: rbd-ec-data namespace: rook-ceph # namespace:cluster spec: - replicated: - size: 2 ---- -apiVersion: ceph.rook.io/v1 -kind: CephBlockPool -metadata: - name: ec-data-pool - namespace: rook-ceph # namespace:cluster -spec: - # Make sure you have enough nodes and OSDs running bluestore to support the replica size or erasure code chunks. - # For the below settings, you need at least 3 OSDs on different nodes (because the `failureDomain` is `host` by default). + application: rbd + failureDomain: host erasureCoded: - dataChunks: 2 - codingChunks: 1 + dataChunks: 3 + codingChunks: 2 + algorithm: isa + stripeUnit: 16Ki + parameters: + compression_mode: none + allow_ec_optimizations: "true" --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rook-ceph-block + name: rook-ceph-block-ec provisioner: rook-ceph.rbd.csi.ceph.com # csi-provisioner-name parameters: # clusterID is the namespace where the rook cluster is running @@ -41,8 +37,8 @@ parameters: # You need to specify the replicated pool here in the `pool` parameter, it is # used for the metadata of the images. # The erasure coded pool must be set as the `dataPool` parameter below. - dataPool: ec-data-pool - pool: replicated-metadata-pool + dataPool: rbd-ec-data + pool: replicapool # (optional) mapOptions is a comma-separated list of map options. # For krbd options refer @@ -87,4 +83,4 @@ parameters: # Its still in Alpha support. Therefore, this option is not recommended for production use. #mounter: rbd-nbd allowVolumeExpansion: true -reclaimPolicy: Delete +reclaimPolicy: Retain diff --git a/deploy/examples/csi/rbd/storageclass.yaml b/deploy/examples/csi/rbd/storageclass.yaml index f810cf6ae..a02893299 100644 --- a/deploy/examples/csi/rbd/storageclass.yaml +++ b/deploy/examples/csi/rbd/storageclass.yaml @@ -77,8 +77,11 @@ parameters: csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph # namespace:cluster csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph # namespace:cluster - csi.storage.k8s.io/controller-publish-secret-name: rook-csi-rbd-provisioner - csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph # namespace:cluster + # Keep these disabled for the existing StorageClass. StorageClass parameters + # are immutable, and this cluster was created before these parameters were + # added to the example manifest. + # csi.storage.k8s.io/controller-publish-secret-name: rook-csi-rbd-provisioner + # csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph # namespace:cluster csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph # namespace:cluster # Specify the filesystem type of the volume. If not specified, csi-provisioner diff --git a/deploy/examples/dashboard-ingress-https.yaml b/deploy/examples/dashboard-ingress-https.yaml index 72da62873..03999e3a5 100644 --- a/deploy/examples/dashboard-ingress-https.yaml +++ b/deploy/examples/dashboard-ingress-https.yaml @@ -12,17 +12,14 @@ metadata: namespace: rook-ceph # namespace:cluster annotations: kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/server-snippet: | - proxy_ssl_verify off; + traefik.ingress.kubernetes.io/router.entrypoints: websecure spec: - ingressClassName: "nginx" + ingressClassName: "traefik" tls: - hosts: - - rook-ceph.example.com - secretName: rook-ceph.example.com + - ceph.brunner.ninja rules: - - host: rook-ceph.example.com + - host: ceph.brunner.ninja http: paths: - path: / @@ -31,4 +28,4 @@ spec: service: name: rook-ceph-mgr-dashboard port: - name: https-dashboard + number: 7000 diff --git a/deploy/examples/filesystem-ec.yaml b/deploy/examples/filesystem-ec.yaml index f5dd0dfb5..679a5abc6 100644 --- a/deploy/examples/filesystem-ec.yaml +++ b/deploy/examples/filesystem-ec.yaml @@ -7,7 +7,7 @@ apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: - name: myfs-ec + name: myfs namespace: rook-ceph # namespace:cluster spec: # The metadata pool spec @@ -24,8 +24,8 @@ spec: # You need at least three `bluestore` OSDs on different nodes for this config to work - name: erasurecoded erasureCoded: - dataChunks: 2 - codingChunks: 1 + dataChunks: 3 + codingChunks: 2 # Inline compression mode for the data pool parameters: compression_mode: none diff --git a/deploy/examples/filesystem.yaml b/deploy/examples/filesystem.yaml index d43af0075..500f70362 100644 --- a/deploy/examples/filesystem.yaml +++ b/deploy/examples/filesystem.yaml @@ -41,6 +41,18 @@ spec: # gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool # for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size #target_size_ratio: ".5" + # Bulk file contents are stored here. The replicated pool above remains the + # required default pool for inode backtraces and other small metadata. + - name: erasurecoded + failureDomain: host + erasureCoded: + dataChunks: 3 + codingChunks: 2 + algorithm: isa + stripeUnit: 16Ki + parameters: + compression_mode: none + allow_ec_optimizations: "true" # Whether to preserve filesystem after CephFilesystem CRD deletion preserveFilesystemOnDelete: true # The metadata service (mds) configuration diff --git a/deploy/examples/nfs-games.yaml b/deploy/examples/nfs-games.yaml new file mode 100644 index 000000000..f7bd3e7c4 --- /dev/null +++ b/deploy/examples/nfs-games.yaml @@ -0,0 +1,32 @@ +apiVersion: ceph.rook.io/v1 +kind: CephNFS +metadata: + name: games + namespace: rook-ceph +spec: + server: + active: 1 + priorityClassName: system-cluster-critical + logLevel: NIV_INFO +--- +apiVersion: v1 +kind: Service +metadata: + name: rook-ceph-nfs-games-load-balancer + namespace: rook-ceph + annotations: + metallb.io/address-pool: default +spec: + type: LoadBalancer + externalTrafficPolicy: Local + loadBalancerSourceRanges: + - 192.168.0.170/32 + ports: + - name: nfs + port: 2049 + targetPort: 2049 + protocol: TCP + selector: + app: rook-ceph-nfs + ceph_nfs: games + instance: a diff --git a/deploy/examples/object-ec-data-pool.yaml b/deploy/examples/object-ec-data-pool.yaml new file mode 100644 index 000000000..d32dfb85b --- /dev/null +++ b/deploy/examples/object-ec-data-pool.yaml @@ -0,0 +1,25 @@ +################################################################################################################# +# EC 3+2 data pool for the existing my-store RGW realm. +# RGW metadata, indexes, logs, and control data remain in replicated pools. +################################################################################################################# + +apiVersion: ceph.rook.io/v1 +kind: CephBlockPool +metadata: + name: my-store.rgw.buckets.ec + namespace: rook-ceph +spec: + application: rgw + failureDomain: host + erasureCoded: + dataChunks: 3 + codingChunks: 2 + algorithm: isa + stripeUnit: 16Ki + parameters: + # Keep autoscaling non-destructive during the migration. The old 256-PG + # bucket pool still exists, so revisit bulk/autoscaling after the cutover. + pg_num: "32" + pg_autoscale_mode: warn + compression_mode: none + allow_ec_optimizations: "true" diff --git a/deploy/examples/object-ec.yaml b/deploy/examples/object-ec.yaml index a897f009e..d6ffd99a3 100644 --- a/deploy/examples/object-ec.yaml +++ b/deploy/examples/object-ec.yaml @@ -30,7 +30,7 @@ spec: failureDomain: host erasureCoded: dataChunks: 2 - codingChunks: 1 + codingChunks: 2 parameters: # RGW data pools are expected to be large, so mark them as "bulk" so that the PG autoscaler will treat them accordingly # (see https://docs.ceph.com/en/reef/rados/operations/placement-groups/) diff --git a/deploy/examples/object.yaml b/deploy/examples/object.yaml index 3eca91c04..a8cac7d12 100644 --- a/deploy/examples/object.yaml +++ b/deploy/examples/object.yaml @@ -29,7 +29,10 @@ spec: dataPool: failureDomain: host replicated: - size: 3 + # Keep the existing large bucket-data pool at its current size during the + # EC migration. After its payload has been verified in EC, raise this + # small residual/default pool to size 3. + size: 2 # Disallow setting pool with replica 1, this could lead to data loss without recovery. # Make sure you're *ABSOLUTELY CERTAIN* that is what you want requireSafeReplicaSize: true diff --git a/install.sh b/install.sh new file mode 100755 index 000000000..03073eb06 --- /dev/null +++ b/install.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +#set -e + +kubectl apply -f deploy/examples/crds.yaml +kubectl apply -f deploy/examples/common.yaml +kubectl apply -f deploy/examples/csi-operator.yaml +kubectl apply -f deploy/examples/operator.yaml +kubectl apply -f deploy/examples/cluster.yaml +kubectl apply -f deploy/examples/csi/rbd/storageclass.yaml +kubectl apply -f deploy/examples/csi/rbd/storageclass-ec.yaml +kubectl apply -f deploy/examples/filesystem.yaml +kubectl apply -f deploy/examples/csi/cephfs/storageclass.yaml +kubectl apply -f deploy/examples/csi/cephfs/storageclass-ec.yaml +kubectl apply -f deploy/examples/object.yaml +kubectl apply -f deploy/examples/object-ec-data-pool.yaml +kubectl apply -f deploy/examples/dashboard-ingress-https.yaml +kubectl apply -f deploy/examples/toolbox.yaml +# Enable after the Ceph upgrade has finished: +# kubectl apply -f deploy/examples/nfs-games.yaml