external: add toplogyconstraintpool support for rbd storageclass

with topologyconstrain pool we can enable any
replica pool usage and also can store data at any
topology

Signed-off-by: parth-gr <partharora1010@gmail.com>
This commit is contained in:
parth-gr committed 2024-03-11 12:47:41 +05:30
1 parent 571f47f73f
commit 04b71cf63e
3 files changed
+203 -8

No files matched your search

@@ -60,6 +60,9 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --
* `--upgrade`: (optional) Upgrades the cephCSIKeyrings(For example: client.csi-cephfs-provisioner) and client.healthchecker ceph users with new permissions needed for the new cluster version and older permission will still be applied.
* `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--k8s-cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem.
* `--v2-port-enable`: (optional) Enables the v2 mon port (3300) for mons.
* `--topology-pools`: (optional) comma-separated list of topology-constrained rbd pools
* `--topology-failure-domain-label`: (optional) k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain
* `--topology-failure-domain-values`: (optional) comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list
### Multi-tenancy
@@ -84,6 +87,17 @@ See the [Multisite doc](https://docs.ceph.com/en/quincy/radosgw/multisite/#confi
python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --format bash --rgw-endpoint <rgw_endpoint> --rgw-realm-name <rgw_realm_name>> --rgw-zonegroup-name <rgw_zonegroup_name> --rgw-zone-name <rgw_zone_name>>
```
### Topology Based Provisioning
Enable Topology Based Provisioning for RBD pools by passing `--topology-pools`, `--topology-failure-domain-label` and `--topology-failure-domain-values` flags.
A new storageclass will be created by the import script named `ceph-rbd-topology` with `volumeBindingMode: WaitForFirstConsumer`
and will configure topologyConstrainedPools according the input provided.
Later use the storageclass to create a volume in the pool matching the topology of the pod scheduling.
```console
python3 create-external-cluster-resources.py --rbd-data-pool-name pool_name --topology-pools p,q,r --topology-failure-domain-label labelName --topology-failure-domain-values x,y,z --format bash
```
### Upgrade Example
1) If consumer cluster doesn't have restricted caps, this will upgrade all the default csi-users (non-restricted):
@@ -474,6 +474,24 @@ class RadosJSON:
required=False,
help="provides the name of the rgw-zonegroup",
)
output_group.add_argument(
"--topology-pools",
default="",
required=False,
help="comma-separated list of topology-constrained rbd pools",
)
output_group.add_argument(
"--topology-failure-domain-label",
default="",
required=False,
help="k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain",
)
output_group.add_argument(
"--topology-failure-domain-values",
default="",
required=False,
help="comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list",
)
upgrade_group = argP.add_argument_group("upgrade")
upgrade_group.add_argument(
@@ -1321,16 +1339,15 @@ class RadosJSON:
"",
)
def validate_rbd_pool(self):
if not self.cluster.pool_exists(self._arg_parser.rbd_data_pool_name):
def validate_rbd_pool(self, pool_name):
if not self.cluster.pool_exists(pool_name):
raise ExecutionFailureException(
f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist"
f"The provided pool, '{pool_name}', does not exist"
)
def init_rbd_pool(self):
def init_rbd_pool(self, rbd_pool_name):
if isinstance(self.cluster, DummyRados):
return
rbd_pool_name = self._arg_parser.rbd_data_pool_name
ioctx = self.cluster.open_ioctx(rbd_pool_name)
rbd_inst = rbd.RBD()
rbd_inst.pool_init(ioctx, True)
@@ -1501,6 +1518,54 @@ class RadosJSON:
return "-1"
return ""
def convert_comma_seprated_to_array(self, value):
return value.split(",")
def raise_exception_if_any_topology_flag_is_missing(self):
if (
(
self._arg_parser.topology_pools != ""
and (
self._arg_parser.topology_failure_domain_label == ""
or self._arg_parser.topology_failure_domain_values == ""
)
)
or (
self._arg_parser.topology_failure_domain_label != ""
and (
self._arg_parser.topology_pools == ""
or self._arg_parser.topology_failure_domain_values == ""
)
)
or (
self._arg_parser.topology_failure_domain_values != ""
and (
self._arg_parser.topology_pools == ""
or self._arg_parser.topology_failure_domain_label == ""
)
)
):
raise ExecutionFailureException(
"provide all the topology flags --topology-pools, --topology-failure-domain-label, --topology-failure-domain-values"
)
def validate_topology_values(self, topology_pools, topology_fd):
if len(topology_pools) != len(topology_fd):
raise ExecutionFailureException(
f"The provided topology pools, '{topology_pools}', and "
f"topology failure domain, '{topology_fd}',"
f"are of different length, '{len(topology_pools)}' and '{len(topology_fd)}' respctively"
)
return
def validate_topology_rbd_pools(self, topology_rbd_pools):
for pool in topology_rbd_pools:
self.validate_rbd_pool(pool)
def init_topology_rbd_pools(self, topology_rbd_pools):
for pool in topology_rbd_pools:
self.init_rbd_pool(pool)
def _gen_output_map(self):
if self.out_map:
return
@@ -1510,8 +1575,8 @@ class RadosJSON:
self._arg_parser.k8s_cluster_name = (
self._arg_parser.k8s_cluster_name.lower()
) # always convert cluster name to lowercase characters
self.validate_rbd_pool()
self.init_rbd_pool()
self.validate_rbd_pool(self._arg_parser.rbd_data_pool_name)
self.init_rbd_pool(self._arg_parser.rbd_data_pool_name)
self.validate_rados_namespace()
self._excluded_keys.add("K8S_CLUSTER_NAME")
self.get_cephfs_data_pool_details()
@@ -1585,6 +1650,33 @@ class RadosJSON:
self.out_map["RBD_METADATA_EC_POOL_NAME"] = (
self.validate_rbd_metadata_ec_pool_name()
)
self.out_map["TOPOLOGY_POOLS"] = self._arg_parser.topology_pools
self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] = (
self._arg_parser.topology_failure_domain_label
)
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] = (
self._arg_parser.topology_failure_domain_values
)
if (
self._arg_parser.topology_pools != ""
and self._arg_parser.topology_failure_domain_label != ""
and self._arg_parser.topology_failure_domain_values != ""
):
self.validate_topology_values(
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"]),
self.convert_comma_seprated_to_array(
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
),
)
self.validate_topology_rbd_pools(
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"])
)
self.init_topology_rbd_pools(
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"])
)
else:
self.raise_exception_if_any_topology_flag_is_missing()
self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix
self.out_map["RGW_ENDPOINT"] = ""
if self._arg_parser.rgw_endpoint:
@@ -1821,6 +1913,34 @@ class RadosJSON:
}
)
# if 'TOPOLOGY_POOLS', 'TOPOLOGY_FAILURE_DOMAIN_LABEL', 'TOPOLOGY_FAILURE_DOMAIN_VALUES' exists,
# then only add 'topology' StorageClass
if (
self.out_map["TOPOLOGY_POOLS"]
and self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"]
and self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
):
json_out.append(
{
"name": "ceph-rbd-topology",
"kind": "StorageClass",
"data": {
"topologyFailureDomainLabel": self.out_map[
"TOPOLOGY_FAILURE_DOMAIN_LABEL"
],
"topologyFailureDomainValues": self.convert_comma_seprated_to_array(
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
),
"topologyPools": self.convert_comma_seprated_to_array(
self.out_map["TOPOLOGY_POOLS"]
),
"csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}",
"csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}",
"csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}",
},
}
)
# if 'CEPHFS_FS_NAME' exists, then only add 'cephfs' StorageClass
if self.out_map["CEPHFS_FS_NAME"]:
json_out.append(
+62 -1
View File
@@ -19,9 +19,10 @@ ROOK_RBD_FEATURES=${ROOK_RBD_FEATURES:-"layering"}
ROOK_EXTERNAL_MAX_MON_ID=2
ROOK_EXTERNAL_MAPPING={}
RBD_STORAGE_CLASS_NAME=ceph-rbd
RBD_TOPOLOGY_STORAGE_CLASS_NAME=ceph-rbd-topology
CEPHFS_STORAGE_CLASS_NAME=cephfs
ROOK_EXTERNAL_MONITOR_SECRET=mon-secret
OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph
OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph
CSI_DRIVER_NAME_PREFIX=${CSI_DRIVER_NAME_PREFIX:-$OPERATOR_NAMESPACE}
RBD_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".rbd.csi.ceph.com" # csi-provisioner-name
CEPHFS_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".cephfs.csi.ceph.com" # csi-provisioner-name
@@ -298,6 +299,62 @@ eof
fi
}
function getTopologyTemplate() {
topology=$(
cat <<-END
{"poolName":"$1",
"domainSegments":[
{"domainLabel":"$2","value":"$3"}]},
END
)
}
function createTopology() {
TOPOLOGY=""
declare -a topology_failure_domain_values_array=()
declare -a topology_pools_array=()
topology_pools=("$(echo "$TOPOLOGY_POOLS" | tr "," "\n")")
for i in ${topology_pools[0]}; do topology_pools_array+=("$i"); done
topology_failure_domain_values=("$(echo "$TOPOLOGY_FAILURE_DOMAIN_VALUES" | tr "," "\n")")
for i in ${topology_failure_domain_values[0]}; do topology_failure_domain_values_array+=("$i"); done
for ((i = 0; i < ${#topology_failure_domain_values_array[@]}; i++)); do
getTopologyTemplate "${topology_pools_array[$i]}" "$TOPOLOGY_FAILURE_DOMAIN_LABEL" "${topology_failure_domain_values_array[$i]}"
TOPOLOGY="$TOPOLOGY"$'\n'"$topology"
topology=""
done
}
function createRBDTopologyStorageClass() {
if ! kubectl -n "$NAMESPACE" get storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME &>/dev/null; then
cat <<eof | kubectl create -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: $RBD_TOPOLOGY_STORAGE_CLASS_NAME
provisioner: $RBD_PROVISIONER
parameters:
clusterID: $CLUSTER_ID_RBD
imageFormat: "2"
imageFeatures: $ROOK_RBD_FEATURES
topologyConstrainedPools: |
[$TOPOLOGY
]
csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME"
csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE
csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME"
csi.storage.k8s.io/controller-expand-secret-namespace: $NAMESPACE
csi.storage.k8s.io/node-stage-secret-name: "rook-$CSI_RBD_NODE_SECRET_NAME"
csi.storage.k8s.io/node-stage-secret-namespace: $NAMESPACE
csi.storage.k8s.io/fstype: ext4
allowVolumeExpansion: true
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
eof
else
echo "storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME already exists"
fi
}
function createCephFSStorageClass() {
if ! $KUBECTL -n "$NAMESPACE" get storageclass $CEPHFS_STORAGE_CLASS_NAME &>/dev/null; then
cat <<eof | $KUBECTL create -f -
@@ -357,3 +414,7 @@ fi
if [ -n "$CEPHFS_FS_NAME" ] && [ -n "$CEPHFS_POOL_NAME" ]; then
createCephFSStorageClass
fi
if [ -n "$TOPOLOGY_POOLS" ] && [ -n "$TOPOLOGY_FAILURE_DOMAIN_LABEL" ] && [ -n "$TOPOLOGY_FAILURE_DOMAIN_VALUES" ]; then
createTopology
createRBDTopologyStorageClass
fi