external: add toplogyconstraintpool support for rbd storageclass
with topologyconstrain pool we can enable any replica pool usage and also can store data at any topology Signed-off-by: parth-gr <partharora1010@gmail.com>
This commit is contained in:
1 parent
571f47f73f
commit
04b71cf63e
3 files changed
+203
-8
No files matched your search
@@ -60,6 +60,9 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --
|
||||
* `--upgrade`: (optional) Upgrades the cephCSIKeyrings(For example: client.csi-cephfs-provisioner) and client.healthchecker ceph users with new permissions needed for the new cluster version and older permission will still be applied.
|
||||
* `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--k8s-cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem.
|
||||
* `--v2-port-enable`: (optional) Enables the v2 mon port (3300) for mons.
|
||||
* `--topology-pools`: (optional) comma-separated list of topology-constrained rbd pools
|
||||
* `--topology-failure-domain-label`: (optional) k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain
|
||||
* `--topology-failure-domain-values`: (optional) comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list
|
||||
|
||||
### Multi-tenancy
|
||||
|
||||
@@ -84,6 +87,17 @@ See the [Multisite doc](https://docs.ceph.com/en/quincy/radosgw/multisite/#confi
|
||||
python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --format bash --rgw-endpoint <rgw_endpoint> --rgw-realm-name <rgw_realm_name>> --rgw-zonegroup-name <rgw_zonegroup_name> --rgw-zone-name <rgw_zone_name>>
|
||||
```
|
||||
|
||||
### Topology Based Provisioning
|
||||
|
||||
Enable Topology Based Provisioning for RBD pools by passing `--topology-pools`, `--topology-failure-domain-label` and `--topology-failure-domain-values` flags.
|
||||
A new storageclass will be created by the import script named `ceph-rbd-topology` with `volumeBindingMode: WaitForFirstConsumer`
|
||||
and will configure topologyConstrainedPools according the input provided.
|
||||
Later use the storageclass to create a volume in the pool matching the topology of the pod scheduling.
|
||||
|
||||
```console
|
||||
python3 create-external-cluster-resources.py --rbd-data-pool-name pool_name --topology-pools p,q,r --topology-failure-domain-label labelName --topology-failure-domain-values x,y,z --format bash
|
||||
```
|
||||
|
||||
### Upgrade Example
|
||||
|
||||
1) If consumer cluster doesn't have restricted caps, this will upgrade all the default csi-users (non-restricted):
|
||||
|
||||
@@ -474,6 +474,24 @@ class RadosJSON:
|
||||
required=False,
|
||||
help="provides the name of the rgw-zonegroup",
|
||||
)
|
||||
output_group.add_argument(
|
||||
"--topology-pools",
|
||||
default="",
|
||||
required=False,
|
||||
help="comma-separated list of topology-constrained rbd pools",
|
||||
)
|
||||
output_group.add_argument(
|
||||
"--topology-failure-domain-label",
|
||||
default="",
|
||||
required=False,
|
||||
help="k8s cluster failure domain label (example: zone,rack,host,etc) for the topology-pools that are matching the ceph domain",
|
||||
)
|
||||
output_group.add_argument(
|
||||
"--topology-failure-domain-values",
|
||||
default="",
|
||||
required=False,
|
||||
help="comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list",
|
||||
)
|
||||
|
||||
upgrade_group = argP.add_argument_group("upgrade")
|
||||
upgrade_group.add_argument(
|
||||
@@ -1321,16 +1339,15 @@ class RadosJSON:
|
||||
"",
|
||||
)
|
||||
|
||||
def validate_rbd_pool(self):
|
||||
if not self.cluster.pool_exists(self._arg_parser.rbd_data_pool_name):
|
||||
def validate_rbd_pool(self, pool_name):
|
||||
if not self.cluster.pool_exists(pool_name):
|
||||
raise ExecutionFailureException(
|
||||
f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist"
|
||||
f"The provided pool, '{pool_name}', does not exist"
|
||||
)
|
||||
|
||||
def init_rbd_pool(self):
|
||||
def init_rbd_pool(self, rbd_pool_name):
|
||||
if isinstance(self.cluster, DummyRados):
|
||||
return
|
||||
rbd_pool_name = self._arg_parser.rbd_data_pool_name
|
||||
ioctx = self.cluster.open_ioctx(rbd_pool_name)
|
||||
rbd_inst = rbd.RBD()
|
||||
rbd_inst.pool_init(ioctx, True)
|
||||
@@ -1501,6 +1518,54 @@ class RadosJSON:
|
||||
return "-1"
|
||||
return ""
|
||||
|
||||
def convert_comma_seprated_to_array(self, value):
|
||||
return value.split(",")
|
||||
|
||||
def raise_exception_if_any_topology_flag_is_missing(self):
|
||||
if (
|
||||
(
|
||||
self._arg_parser.topology_pools != ""
|
||||
and (
|
||||
self._arg_parser.topology_failure_domain_label == ""
|
||||
or self._arg_parser.topology_failure_domain_values == ""
|
||||
)
|
||||
)
|
||||
or (
|
||||
self._arg_parser.topology_failure_domain_label != ""
|
||||
and (
|
||||
self._arg_parser.topology_pools == ""
|
||||
or self._arg_parser.topology_failure_domain_values == ""
|
||||
)
|
||||
)
|
||||
or (
|
||||
self._arg_parser.topology_failure_domain_values != ""
|
||||
and (
|
||||
self._arg_parser.topology_pools == ""
|
||||
or self._arg_parser.topology_failure_domain_label == ""
|
||||
)
|
||||
)
|
||||
):
|
||||
raise ExecutionFailureException(
|
||||
"provide all the topology flags --topology-pools, --topology-failure-domain-label, --topology-failure-domain-values"
|
||||
)
|
||||
|
||||
def validate_topology_values(self, topology_pools, topology_fd):
|
||||
if len(topology_pools) != len(topology_fd):
|
||||
raise ExecutionFailureException(
|
||||
f"The provided topology pools, '{topology_pools}', and "
|
||||
f"topology failure domain, '{topology_fd}',"
|
||||
f"are of different length, '{len(topology_pools)}' and '{len(topology_fd)}' respctively"
|
||||
)
|
||||
return
|
||||
|
||||
def validate_topology_rbd_pools(self, topology_rbd_pools):
|
||||
for pool in topology_rbd_pools:
|
||||
self.validate_rbd_pool(pool)
|
||||
|
||||
def init_topology_rbd_pools(self, topology_rbd_pools):
|
||||
for pool in topology_rbd_pools:
|
||||
self.init_rbd_pool(pool)
|
||||
|
||||
def _gen_output_map(self):
|
||||
if self.out_map:
|
||||
return
|
||||
@@ -1510,8 +1575,8 @@ class RadosJSON:
|
||||
self._arg_parser.k8s_cluster_name = (
|
||||
self._arg_parser.k8s_cluster_name.lower()
|
||||
) # always convert cluster name to lowercase characters
|
||||
self.validate_rbd_pool()
|
||||
self.init_rbd_pool()
|
||||
self.validate_rbd_pool(self._arg_parser.rbd_data_pool_name)
|
||||
self.init_rbd_pool(self._arg_parser.rbd_data_pool_name)
|
||||
self.validate_rados_namespace()
|
||||
self._excluded_keys.add("K8S_CLUSTER_NAME")
|
||||
self.get_cephfs_data_pool_details()
|
||||
@@ -1585,6 +1650,33 @@ class RadosJSON:
|
||||
self.out_map["RBD_METADATA_EC_POOL_NAME"] = (
|
||||
self.validate_rbd_metadata_ec_pool_name()
|
||||
)
|
||||
self.out_map["TOPOLOGY_POOLS"] = self._arg_parser.topology_pools
|
||||
self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"] = (
|
||||
self._arg_parser.topology_failure_domain_label
|
||||
)
|
||||
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"] = (
|
||||
self._arg_parser.topology_failure_domain_values
|
||||
)
|
||||
if (
|
||||
self._arg_parser.topology_pools != ""
|
||||
and self._arg_parser.topology_failure_domain_label != ""
|
||||
and self._arg_parser.topology_failure_domain_values != ""
|
||||
):
|
||||
self.validate_topology_values(
|
||||
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"]),
|
||||
self.convert_comma_seprated_to_array(
|
||||
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
|
||||
),
|
||||
)
|
||||
self.validate_topology_rbd_pools(
|
||||
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"])
|
||||
)
|
||||
self.init_topology_rbd_pools(
|
||||
self.convert_comma_seprated_to_array(self.out_map["TOPOLOGY_POOLS"])
|
||||
)
|
||||
else:
|
||||
self.raise_exception_if_any_topology_flag_is_missing()
|
||||
|
||||
self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix
|
||||
self.out_map["RGW_ENDPOINT"] = ""
|
||||
if self._arg_parser.rgw_endpoint:
|
||||
@@ -1821,6 +1913,34 @@ class RadosJSON:
|
||||
}
|
||||
)
|
||||
|
||||
# if 'TOPOLOGY_POOLS', 'TOPOLOGY_FAILURE_DOMAIN_LABEL', 'TOPOLOGY_FAILURE_DOMAIN_VALUES' exists,
|
||||
# then only add 'topology' StorageClass
|
||||
if (
|
||||
self.out_map["TOPOLOGY_POOLS"]
|
||||
and self.out_map["TOPOLOGY_FAILURE_DOMAIN_LABEL"]
|
||||
and self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
|
||||
):
|
||||
json_out.append(
|
||||
{
|
||||
"name": "ceph-rbd-topology",
|
||||
"kind": "StorageClass",
|
||||
"data": {
|
||||
"topologyFailureDomainLabel": self.out_map[
|
||||
"TOPOLOGY_FAILURE_DOMAIN_LABEL"
|
||||
],
|
||||
"topologyFailureDomainValues": self.convert_comma_seprated_to_array(
|
||||
self.out_map["TOPOLOGY_FAILURE_DOMAIN_VALUES"]
|
||||
),
|
||||
"topologyPools": self.convert_comma_seprated_to_array(
|
||||
self.out_map["TOPOLOGY_POOLS"]
|
||||
),
|
||||
"csi.storage.k8s.io/provisioner-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}",
|
||||
"csi.storage.k8s.io/controller-expand-secret-name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}",
|
||||
"csi.storage.k8s.io/node-stage-secret-name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
# if 'CEPHFS_FS_NAME' exists, then only add 'cephfs' StorageClass
|
||||
if self.out_map["CEPHFS_FS_NAME"]:
|
||||
json_out.append(
|
||||
|
||||
@@ -19,9 +19,10 @@ ROOK_RBD_FEATURES=${ROOK_RBD_FEATURES:-"layering"}
|
||||
ROOK_EXTERNAL_MAX_MON_ID=2
|
||||
ROOK_EXTERNAL_MAPPING={}
|
||||
RBD_STORAGE_CLASS_NAME=ceph-rbd
|
||||
RBD_TOPOLOGY_STORAGE_CLASS_NAME=ceph-rbd-topology
|
||||
CEPHFS_STORAGE_CLASS_NAME=cephfs
|
||||
ROOK_EXTERNAL_MONITOR_SECRET=mon-secret
|
||||
OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph
|
||||
OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph
|
||||
CSI_DRIVER_NAME_PREFIX=${CSI_DRIVER_NAME_PREFIX:-$OPERATOR_NAMESPACE}
|
||||
RBD_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".rbd.csi.ceph.com" # csi-provisioner-name
|
||||
CEPHFS_PROVISIONER=$CSI_DRIVER_NAME_PREFIX".cephfs.csi.ceph.com" # csi-provisioner-name
|
||||
@@ -298,6 +299,62 @@ eof
|
||||
fi
|
||||
}
|
||||
|
||||
function getTopologyTemplate() {
|
||||
topology=$(
|
||||
cat <<-END
|
||||
{"poolName":"$1",
|
||||
"domainSegments":[
|
||||
{"domainLabel":"$2","value":"$3"}]},
|
||||
END
|
||||
)
|
||||
}
|
||||
|
||||
function createTopology() {
|
||||
TOPOLOGY=""
|
||||
declare -a topology_failure_domain_values_array=()
|
||||
declare -a topology_pools_array=()
|
||||
topology_pools=("$(echo "$TOPOLOGY_POOLS" | tr "," "\n")")
|
||||
for i in ${topology_pools[0]}; do topology_pools_array+=("$i"); done
|
||||
topology_failure_domain_values=("$(echo "$TOPOLOGY_FAILURE_DOMAIN_VALUES" | tr "," "\n")")
|
||||
for i in ${topology_failure_domain_values[0]}; do topology_failure_domain_values_array+=("$i"); done
|
||||
for ((i = 0; i < ${#topology_failure_domain_values_array[@]}; i++)); do
|
||||
getTopologyTemplate "${topology_pools_array[$i]}" "$TOPOLOGY_FAILURE_DOMAIN_LABEL" "${topology_failure_domain_values_array[$i]}"
|
||||
TOPOLOGY="$TOPOLOGY"$'\n'"$topology"
|
||||
topology=""
|
||||
done
|
||||
}
|
||||
|
||||
function createRBDTopologyStorageClass() {
|
||||
if ! kubectl -n "$NAMESPACE" get storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME &>/dev/null; then
|
||||
cat <<eof | kubectl create -f -
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: $RBD_TOPOLOGY_STORAGE_CLASS_NAME
|
||||
provisioner: $RBD_PROVISIONER
|
||||
parameters:
|
||||
clusterID: $CLUSTER_ID_RBD
|
||||
imageFormat: "2"
|
||||
imageFeatures: $ROOK_RBD_FEATURES
|
||||
topologyConstrainedPools: |
|
||||
[$TOPOLOGY
|
||||
]
|
||||
csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME"
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE
|
||||
csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME"
|
||||
csi.storage.k8s.io/controller-expand-secret-namespace: $NAMESPACE
|
||||
csi.storage.k8s.io/node-stage-secret-name: "rook-$CSI_RBD_NODE_SECRET_NAME"
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: $NAMESPACE
|
||||
csi.storage.k8s.io/fstype: ext4
|
||||
allowVolumeExpansion: true
|
||||
reclaimPolicy: Delete
|
||||
volumeBindingMode: WaitForFirstConsumer
|
||||
eof
|
||||
else
|
||||
echo "storageclass $RBD_TOPOLOGY_STORAGE_CLASS_NAME already exists"
|
||||
fi
|
||||
}
|
||||
|
||||
function createCephFSStorageClass() {
|
||||
if ! $KUBECTL -n "$NAMESPACE" get storageclass $CEPHFS_STORAGE_CLASS_NAME &>/dev/null; then
|
||||
cat <<eof | $KUBECTL create -f -
|
||||
@@ -357,3 +414,7 @@ fi
|
||||
if [ -n "$CEPHFS_FS_NAME" ] && [ -n "$CEPHFS_POOL_NAME" ]; then
|
||||
createCephFSStorageClass
|
||||
fi
|
||||
if [ -n "$TOPOLOGY_POOLS" ] && [ -n "$TOPOLOGY_FAILURE_DOMAIN_LABEL" ] && [ -n "$TOPOLOGY_FAILURE_DOMAIN_VALUES" ]; then
|
||||
createTopology
|
||||
createRBDTopologyStorageClass
|
||||
fi
|
||||
Reference in new issue
Block a user