diff --git a/Documentation/CRDs/Cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster.md index 7f1155946..417ff4400 100644 --- a/Documentation/CRDs/Cluster/external-cluster.md +++ b/Documentation/CRDs/Cluster/external-cluster.md @@ -60,6 +60,9 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name -- * `--upgrade`: (optional) Upgrades the cephCSIKeyrings(For example: client.csi-cephfs-provisioner) and client.healthchecker ceph users with new permissions needed for the new cluster version and older permission will still be applied. * `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--k8s-cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem. * `--v2-port-enable`: (optional) Enables the v2 mon port (3300) for mons. +* `--topology-pools`: (optional) comma-separated list of topology-constrained rbd pools +* `--topology-failure-domain-label`: (optional) k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain +* `--topology-failure-domain-values`: (optional) comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list ### Multi-tenancy @@ -84,6 +87,17 @@ See the [Multisite doc](https://docs.ceph.com/en/quincy/radosgw/multisite/#confi python3 create-external-cluster-resources.py --rbd-data-pool-name --format bash --rgw-endpoint --rgw-realm-name > --rgw-zonegroup-name --rgw-zone-name > ``` +### Topology Based Provisioning + +Enable Topology Based Provisioning for RBD pools by passing `--topology-pools`, `--topology-failure-domain-label` and `--topology-failure-domain-values` flags. +A new storageclass will be created by the import script named `ceph-rbd-topology` with `volumeBindingMode: WaitForFirstConsumer` +and will configure topologyConstrainedPools according the input provided. +Later use the storageclass to create a volume in the pool matching the topology of the pod scheduling. + +```console +python3 create-external-cluster-resources.py --rbd-data-pool-name pool_name --topology-pools p,q,r --topology-failure-domain-label labelName --topology-failure-domain-values x,y,z --format bash +``` + ### Upgrade Example 1) If consumer cluster doesn't have restricted caps, this will upgrade all the default csi-users (non-restricted): diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index 61039c9eb..a27da02f5 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -474,6 +474,24 @@ class RadosJSON: required=False, help="provides the name of the rgw-zonegroup", ) + output_group.add_argument( + "--topology-pools", + default="", + required=False, + help="comma-separated list of topology-constrained rbd pools", + ) + output_group.add_argument( + "--topology-failure-domain-label", + default="", + required=False, + help="k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain", + ) + output_group.add_argument( + "--topology-failure-domain-values", + default="", + required=False, + help="comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list", + ) upgrade_group = argP.add_argument_group("upgrade") upgrade_group.add_argument( @@ -1321,16 +1339,15 @@ class RadosJSON: "", ) - def validate_rbd_pool(self): - if not self.cluster.pool_exists(self._arg_parser.rbd_data_pool_name): + def validate_rbd_pool(self, pool_name): + if not self.cluster.pool_exists(pool_name): raise ExecutionFailureException( - f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist" + f"The provided pool, '{pool_name}', does not exist" ) - def init_rbd_pool(self): + def init_rbd_pool(self, rbd_pool_name): if isinstance(self.cluster, DummyRados): return - rbd_pool_name = self._arg_parser.rbd_data_pool_name ioctx = self.cluster.open_ioctx(rbd_pool_name) rbd_inst = rbd.RBD() rbd_inst.pool_init(ioctx, True) @@ -1501,6 +1518,54 @@ class RadosJSON: return "-1" return "" + def convert_comma_seprated_to_array(self, value): + return value.split(",") + + def raise_exception_if_any_topology_flag_is_missing(self): + if ( + ( + self._arg_parser.topology_pools != "" + and ( + self._arg_parser.topology_failure_domain_label == "" + or self._arg_parser.topology_failure_domain_values == "" + ) + ) + or ( + self._arg_parser.topology_failure_domain_label != "" + and ( + self._arg_parser.topology_pools == "" + or self._arg_parser.topology_failure_domain_values == "" + ) + ) + or ( + self._arg_parser.topology_failure_domain_values != "" + and ( + self._arg_parser.topology_pools == "" + or self._arg_parser.topology_failure_domain_label == "" + ) + ) + ): + raise ExecutionFailureException( + "provide all the topology flags --topology-pools, --topology-failure-domain-label, --topology-failure-domain-values" + ) + + def validate_topology_values(self, topology_pools, topology_fd): + if len(topology_pools) != len(topology_fd): + raise ExecutionFailureException( + f"The provided topology pools, '{topology_pools}', and " + f"topology failure domain, '{topology_fd}'," + f"are of different length, '{len(topology_pools)}' and '{len(topology_fd)}' respctively" + ) + return + + def validate_topology_rbd_pools(self, topology_rbd_pools): + for pool in topology_rbd_pools: + self.validate_rbd_pool(pool) + + def init_topology_rbd_pools(self, topology_rbd_pools): + for pool in topology_rbd_pools: + self.init_rbd_pool(pool) + def _gen_output_map(self): if self.out_map: return @@ -1510,8 +1575,8 @@ class RadosJSON: self._arg_parser.k8s_cluster_name = ( self._arg_parser.k8s_cluster_name.lower() ) # always convert cluster name to lowercase characters - self.validate_rbd_pool() - self.init_rbd_pool() + self.validate_rbd_pool(self._arg_parser.rbd_data_pool_name) + self.init_rbd_pool(self._arg_parser.rbd_data_pool_name) self.validate_rados_namespace() self._excluded_keys.add("K8S_CLUSTER_NAME") self.get_cephfs_data_pool_details() @@ -1585,6 +1650,33 @@ class RadosJSON: self.out_map["RBD_METADATA_EC_POOL_NAME"] = ( self.validate_rbd_metadata_ec_pool_name() ) + self.out_map["TOPOLOGY_POOLS"] = self._arg_parser.topology_pools + self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] = ( + self._arg_parser.topology_failure_domain_label + ) + self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] = ( + self._arg_parser.topology_failure_domain_values + ) + if ( + self._arg_parser.topology_pools != "" + and self._arg_parser.topology_failure_domain_label != "" + and self._arg_parser.topology_failure_domain_values != "" + ): + self.validate_topology_values( + self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"]), + self.convert_comma_seprated_to_array( + self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] + ), + ) + self.validate_topology_rbd_pools( + self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"]) + ) + self.init_topology_rbd_pools( + self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"]) + ) + else: + self.raise_exception_if_any_topology_flag_is_missing() + self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix self.out_map["RGW_ENDPOINT"] = "" if self._arg_parser.rgw_endpoint: @@ -1821,6 +1913,34 @@ class RadosJSON: } ) + # if 'TOPOLOGY_POOLS', 'TOPOLOGY_FAILURE_DOMAIN_LABEL', 'TOPOLOGY_FAILURE_DOMAIN_VALUES' exists, + # then only add 'topology' StorageClass + if ( + self.out_map["TOPOLOGY_POOLS"] + and self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] + and self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] + ): + json_out.append( + { + "name": "ceph-rbd-topology", + "kind": "StorageClass", + "data": { + "topologyFailureDomainLabel": self.out_map[ + "TOPOLOGY_FAILURE_DOMAIN_LABEL" + ], + "topologyFailureDomainValues": self.convert_comma_seprated_to_array( + self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] + ), + "topologyPools": self.convert_comma_seprated_to_array( + self.out_map["TOPOLOGY_POOLS"] + ), + "csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", + "csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", + "csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", + }, + } + ) + # if 'CEPHFS_FS_NAME' exists, then only add 'cephfs' StorageClass if self.out_map["CEPHFS_FS_NAME"]: json_out.append( diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh index 77381e715..ed17a7abc 100644 --- a/deploy/examples/import-external-cluster.sh +++ b/deploy/examples/import-external-cluster.sh @@ -19,9 +19,10 @@ ROOK_RBD_FEATURES=${ROOK_RBD_FEATURES:-"layering"} ROOK_EXTERNAL_MAX_MON_ID=2 ROOK_EXTERNAL_MAPPING={} RBD_STORAGE_CLASS_NAME=ceph-rbd +RBD_TOPOLOGY_STORAGE_CLASS_NAME=ceph-rbd-topology CEPHFS_STORAGE_CLASS_NAME=cephfs ROOK_EXTERNAL_MONITOR_SECRET=mon-secret -OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph +OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph CSI_DRIVER_NAME_PREFIX=${CSI_DRIVER_NAME_PREFIX:-$OPERATOR_NAMESPACE} RBD_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".rbd.csi.ceph.com" # csi-provisioner-name CEPHFS_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".cephfs.csi.ceph.com" # csi-provisioner-name @@ -298,6 +299,62 @@ eof fi } +function getTopologyTemplate() { + topology=$( + cat <<-END + {"poolName":"$1", + "domainSegments":[ + {"domainLabel":"$2","value":"$3"}]}, +END + ) +} + +function createTopology() { + TOPOLOGY="" + declare -a topology_failure_domain_values_array=() + declare -a topology_pools_array=() + topology_pools=("$(echo "$TOPOLOGY_POOLS" | tr "," "\n")") + for i in ${topology_pools[0]}; do topology_pools_array+=("$i"); done + topology_failure_domain_values=("$(echo "$TOPOLOGY_FAILURE_DOMAIN_VALUES" | tr "," "\n")") + for i in ${topology_failure_domain_values[0]}; do topology_failure_domain_values_array+=("$i"); done + for ((i = 0; i < ${#topology_failure_domain_values_array[@]}; i++)); do + getTopologyTemplate "${topology_pools_array[$i]}" "$TOPOLOGY_FAILURE_DOMAIN_LABEL" "${topology_failure_domain_values_array[$i]}" + TOPOLOGY="$TOPOLOGY"$'\n'"$topology" + topology="" + done +} + +function createRBDTopologyStorageClass() { + if ! kubectl -n "$NAMESPACE" get storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME &>/dev/null; then + cat </dev/null; then cat <