From c9a864ccdb683f45e231e10250e2b72eef44a654 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Tue, 28 Jun 2022 19:33:19 +0530 Subject: [PATCH] external: add support for subvolumegroup and rados namespace Using subvolume group and rados namespace for external deployment was complicated and not documented, Automating the process for the users, so they just need to pass the respective names by cli flag for creation Closes: rook#10427 Signed-off-by: parth-gr --- .github/workflows/canary-integration-test.yml | 34 +++++++++++++++++-- .../CRDs/Cluster/ceph-cluster-crd.md | 4 +-- .../CRDs/Cluster/external-cluster.md | 14 ++++---- .../create-external-cluster-resources.py | 33 ++++++++++++++++++ deploy/examples/import-external-cluster.sh | 18 ++++++++-- 5 files changed, 87 insertions(+), 16 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 57954b503..955f841a7 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -94,13 +94,40 @@ jobs: toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done" - - name: test rados namespace + - name: test subvolumegroup validation + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + # pass the correct subvolumegroup and cephfs_filesystem flag name + kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group group-a --cephfs-filesystem-name myfs + # pass the wrong subvolumegroup name + if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group false-test-subvolume-group); then + echo "unexpectedly succeeded after passing the wrong subvolumegroup name: $output" + exit 1 + else + echo "script failed because wrong subvolumegroup name was passed" + fi + + - name: test of rados namespace run: | kubectl create -f deploy/examples/radosnamespace.yaml toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- rbd namespace ls replicapool --format=json|jq .[0].name|grep -q "namespace-a"; do sleep 1 && echo 'waiting for the rados namespace to be created'; done" kubectl delete -f deploy/examples/radosnamespace.yaml + - name: test rados namespace validation + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + # create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool + kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosNamespace1 + kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosNamespace1 + # test the rados namespace which not exit for replicapool(false testing) + if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace false-test-namespace); then + echo "unexpectedly succeeded after passing the wrong rados namespace: $output" + exit 1 + else + echo "script failed because wrong rados namespace was passed" + fi + - name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag run: | toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') @@ -140,14 +167,15 @@ jobs: timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" # pass the invalid rgw-endpoint of different ceph cluster if output=$(timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"); then - echo $output + echo "unexpectedly succeeded after passing the wrong rgw-endpoint: $output" + exit 1 else echo "validation failed because wrong endpoint was provided" fi # pass the valid rgw-endpoint of same ceph cluster with --rgw-tls-cert-path timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-tls-cert-path my-cert; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" # pass the valid rgw-endpoint of same ceph cluster with --rgw-skip-tls - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" + timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" - name: check-ownerreferences run: tests/scripts/github-action-helper.sh check_ownerreferences diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 1ce873fda..cc43de3b3 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -100,10 +100,10 @@ These are general purpose Ceph container with all necessary daemons and dependen | TAG | MEANING | | -------------------- | --------------------------------------------------------- | -| vRELNUM | Latest release in this series (e.g., *v17* = Quincy) | +| vRELNUM | Latest release in this series (e.g., *v17* = Quincy) | | vRELNUM.Y | Latest stable release in this stable series (e.g., v17.2) | | vRELNUM.Y.Z | A specific release (e.g., v17.2.1) | -| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v17.2.1-20220623) | +| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v17.2.1-20220623) | A specific will contain a specific release of Ceph as well as security fixes from the Operating System. diff --git a/Documentation/CRDs/Cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster.md index 6396e09ad..5e9412c98 100644 --- a/Documentation/CRDs/Cluster/external-cluster.md +++ b/Documentation/CRDs/Cluster/external-cluster.md @@ -37,15 +37,11 @@ In order to configure an external Ceph cluster with Rook, we need to extract som - `--namespace`: Namespace where CephCluster will run, for example `rook-ceph-external` - `--format bash`: The format of the output - `--rbd-data-pool-name`: The name of the RBD data pool -- `--rbd-metadata-ec-pool-name`: (optional) Provides the name of erasure coded RBD metadata pool -- `--rados-namespace`: (optional) Divides a pool into separate logical namespaces -- `--cephfs-filesystem-name`: (optional) The name of the filesystem -- `--cephfs-metadata-pool-name`: (optional) Provides the name of the cephfs metadata pool -- `--cephfs-data-pool-name`: (optional) Provides the name of the CephFS data pool - `--rgw-endpoint`: (optional) The RADOS Gateway endpoint in the format `:` - `--rgw-pool-prefix`: (optional) The prefix of the RGW pools. If not specified, the default prefix is `default` - `--rgw-tls-cert-path`: (optional) RADOS Gateway endpoint TLS certificate file path - `--rgw-skip-tls`: (optional) Ignore TLS certification validation when a self-signed certificate is provided (NOT RECOMMENDED) +- `--rbd-metadata-ec-pool-name`: (optional) Provides the name of erasure coded RBD metadata pool, used for creating ECRBDStorageClass. - `--monitoring-endpoint`: (optional) Ceph Manager prometheus exporter endpoints (comma separated list of entries of active and standby mgrs) - `--monitoring-endpoint-port`: (optional) Ceph Manager prometheus exporter port - `--ceph-conf`: (optional) Provide a Ceph conf file @@ -53,6 +49,11 @@ In order to configure an external Ceph cluster with Rook, we need to extract som - `--output`: (optional) Output will be stored into the provided file - `--dry-run`: (optional) Prints the executed commands without running them - `--run-as-user`: (optional) Provides a user name to check the cluster's health status, must be prefixed by `client`. +- `--cephfs-metadata-pool-name`: (optional) Provides the name of the cephfs metadata pool +- `--cephfs-filesystem-name`: (optional) The name of the filesystem, used for creating CephFS StorageClass +- `--cephfs-data-pool-name`: (optional) Provides the name of the CephFS data pool, used for creating CephFS StorageClass +- `--rados-namespace`: (optional) Divides a pool into separate logical namespaces, used for creating RBD PVC in a RadosNamespaces +- `--subvolume-group`: (optional) Provides the name of the subvolume group, used for creating CephFS PVC in a subvolumeGroup - `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem. !!! note @@ -126,9 +127,6 @@ In order to configure an external Ceph cluster with Rook, we need to extract som kubectl -n rook-ceph-external get sc ``` - !!! note - For CephFS StorageClass you also need to export `CEPHFS_FS_NAME`, `CEPHFS_POOL_NAME` or you can pass these parameters with a CLI flag (--cephfs-data-pool-name,--cephfs-filesystem-name) while running the python script. For creating ECRBDStorageClass you need to pass --rbd-metadata-ec-pool-name CLI flag or export `RBD_METADATA_EC_POOL_NAME`. - 7. Then you can now create a [persistent volume](/deploy/examples/csi) based on these StorageClass. ### CephCluster example (management) diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index d53ca1267..b086cc242 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -423,6 +423,12 @@ class RadosJSON: required=False, help="divides a pool into separate logical namespaces", ) + output_group.add_argument( + "--subvolume-group", + default="", + required=False, + help="provides the name of the subvolume group", + ) upgrade_group = argP.add_argument_group("upgrade") upgrade_group.add_argument( @@ -1208,6 +1214,31 @@ class RadosJSON: ).format(rados_namespace, rbd_pool_name) ) + def validate_subvolume_group(self): + cephfs_filesystem_name = self._arg_parser.cephfs_filesystem_name + subvolume_group = self._arg_parser.subvolume_group + if subvolume_group == "": + return + if cephfs_filesystem_name == "": + raise ExecutionFailureException( + "if subvolume group is passed cephfs filesystem name is mandatory to pass" + ) + + cmd = [ + "ceph", + "fs", + "subvolumegroup", + "getpath", + cephfs_filesystem_name, + subvolume_group, + ] + try: + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as execErr: + raise ExecutionFailureException( + "subvolume group {} passed doesn't exist".format(subvolume_group) + ) + def get_rgw_fsid(self): access_key = self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"] secret_key = self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"] @@ -1269,6 +1300,7 @@ class RadosJSON: ) self.validate_pool() self.validate_rados_namespace() + self.validate_subvolume_group() self._excluded_keys.add("CLUSTER_NAME") self.get_cephfs_data_pool_details() self.out_map["NAMESPACE"] = self._arg_parser.namespace @@ -1295,6 +1327,7 @@ class RadosJSON: "RESTRICTED_AUTH_PERMISSION" ] = self._arg_parser.restricted_auth_permission self.out_map["RADOS_NAMESPACE"] = self._arg_parser.rados_namespace + self.out_map["SUBVOLUME_GROUP"] = self._arg_parser.subvolume_group self.out_map["CSI_CEPHFS_NODE_SECRET"] = "" self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"] = "" # create CephFS node and provisioner keyring only when MDS exists diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh index bccfabc2f..cce0a0b03 100644 --- a/deploy/examples/import-external-cluster.sh +++ b/deploy/examples/import-external-cluster.sh @@ -22,6 +22,8 @@ ROOK_EXTERNAL_MONITOR_SECRET=mon-secret OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph RBD_PROVISIONER=$OPERATOR_NAMESPACE".rbd.csi.ceph.com" # driver:namespace:operator CEPHFS_PROVISIONER=$OPERATOR_NAMESPACE".cephfs.csi.ceph.com" # driver:namespace:operator +CLUSTER_ID_RBD=$NAMESPACE +CLUSTER_ID_CEPHFS=$NAMESPACE : "${ROOK_EXTERNAL_ADMIN_SECRET:=admin-secret}" ############# @@ -93,6 +95,15 @@ function checkEnvVars() { fi } +function importClusterID() { + if [ -n "$RADOS_NAMESPACE_CLUSTER_ID" ]; then + CLUSTER_ID_RBD=$(kubectl -n "$NAMESPACE" get cephblockpoolradosnamespace.ceph.rook.io/"$RADOS_NAMESPACE_CLUSTER_ID" -o jsonpath='{.status.info.clusterID}') + fi + if [ -n "$SUBVOLUME_GROUP_CLUSTER_ID" ]; then + CLUSTER_ID_CEPHFS=$(kubectl -n "$NAMESPACE" get cephfilesystemsubvolumegroup.ceph.rook.io/"$SUBVOLUME_GROUP_CLUSTER_ID" -o jsonpath='{.status.info.clusterID}') + fi +} + function importSecret() { kubectl -n "$NAMESPACE" \ create \ @@ -181,7 +192,7 @@ metadata: name: $RBD_STORAGE_CLASS_NAME provisioner: $RBD_PROVISIONER parameters: - clusterID: $NAMESPACE + clusterID: $CLUSTER_ID_RBD pool: $RBD_POOL_NAME dataPool: $RBD_METADATA_EC_POOL_NAME imageFormat: "2" @@ -206,7 +217,7 @@ metadata: name: $RBD_STORAGE_CLASS_NAME provisioner: $RBD_PROVISIONER parameters: - clusterID: $NAMESPACE + clusterID: $CLUSTER_ID_RBD pool: $RBD_POOL_NAME imageFormat: "2" imageFeatures: layering @@ -230,7 +241,7 @@ metadata: name: $CEPHFS_STORAGE_CLASS_NAME provisioner: $CEPHFS_PROVISIONER parameters: - clusterID: $NAMESPACE + clusterID: $CLUSTER_ID_CEPHFS fsName: $CEPHFS_FS_NAME pool: $CEPHFS_POOL_NAME csi.storage.k8s.io/provisioner-secret-name: $CSI_CEPHFS_PROVISIONER_SECRET_NAME @@ -248,6 +259,7 @@ eof # MAIN # ######## checkEnvVars +importClusterID importSecret importConfigMap importCsiRBDNodeSecret