diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 57954b503..955f841a7 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -94,13 +94,40 @@ jobs: toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph fs subvolumegroup ls myfs|jq .[0].name|grep -q "group-a"; do sleep 1 && echo 'waiting for the subvolumegroup to be created'; done" - - name: test rados namespace + - name: test subvolumegroup validation + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + # pass the correct subvolumegroup and cephfs_filesystem flag name + kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group group-a --cephfs-filesystem-name myfs + # pass the wrong subvolumegroup name + if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --subvolume-group false-test-subvolume-group); then + echo "unexpectedly succeeded after passing the wrong subvolumegroup name: $output" + exit 1 + else + echo "script failed because wrong subvolumegroup name was passed" + fi + + - name: test of rados namespace run: | kubectl create -f deploy/examples/radosnamespace.yaml toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- rbd namespace ls replicapool --format=json|jq .[0].name|grep -q "namespace-a"; do sleep 1 && echo 'waiting for the rados namespace to be created'; done" kubectl delete -f deploy/examples/radosnamespace.yaml + - name: test rados namespace validation + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + # create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool + kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/radosNamespace1 + kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace radosNamespace1 + # test the rados namespace which not exit for replicapool(false testing) + if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace false-test-namespace); then + echo "unexpectedly succeeded after passing the wrong rados namespace: $output" + exit 1 + else + echo "script failed because wrong rados namespace was passed" + fi + - name: test external script with restricted_auth_permission flag and without having cephfs_filesystem flag run: | toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') @@ -140,14 +167,15 @@ jobs: timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" # pass the invalid rgw-endpoint of different ceph cluster if output=$(timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"); then - echo $output + echo "unexpectedly succeeded after passing the wrong rgw-endpoint: $output" + exit 1 else echo "validation failed because wrong endpoint was provided" fi # pass the valid rgw-endpoint of same ceph cluster with --rgw-tls-cert-path timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-tls-cert-path my-cert; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" # pass the valid rgw-endpoint of same ceph cluster with --rgw-skip-tls - timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" + timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint --rgw-skip-tls true; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" - name: check-ownerreferences run: tests/scripts/github-action-helper.sh check_ownerreferences diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 1ce873fda..cc43de3b3 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -100,10 +100,10 @@ These are general purpose Ceph container with all necessary daemons and dependen | TAG | MEANING | | -------------------- | --------------------------------------------------------- | -| vRELNUM | Latest release in this series (e.g., *v17* = Quincy) | +| vRELNUM | Latest release in this series (e.g., *v17* = Quincy) | | vRELNUM.Y | Latest stable release in this stable series (e.g., v17.2) | | vRELNUM.Y.Z | A specific release (e.g., v17.2.1) | -| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v17.2.1-20220623) | +| vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v17.2.1-20220623) | A specific will contain a specific release of Ceph as well as security fixes from the Operating System. diff --git a/Documentation/CRDs/Cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster.md index 6396e09ad..5e9412c98 100644 --- a/Documentation/CRDs/Cluster/external-cluster.md +++ b/Documentation/CRDs/Cluster/external-cluster.md @@ -37,15 +37,11 @@ In order to configure an external Ceph cluster with Rook, we need to extract som - `--namespace`: Namespace where CephCluster will run, for example `rook-ceph-external` - `--format bash`: The format of the output - `--rbd-data-pool-name`: The name of the RBD data pool -- `--rbd-metadata-ec-pool-name`: (optional) Provides the name of erasure coded RBD metadata pool -- `--rados-namespace`: (optional) Divides a pool into separate logical namespaces -- `--cephfs-filesystem-name`: (optional) The name of the filesystem -- `--cephfs-metadata-pool-name`: (optional) Provides the name of the cephfs metadata pool -- `--cephfs-data-pool-name`: (optional) Provides the name of the CephFS data pool - `--rgw-endpoint`: (optional) The RADOS Gateway endpoint in the format `:` - `--rgw-pool-prefix`: (optional) The prefix of the RGW pools. If not specified, the default prefix is `default` - `--rgw-tls-cert-path`: (optional) RADOS Gateway endpoint TLS certificate file path - `--rgw-skip-tls`: (optional) Ignore TLS certification validation when a self-signed certificate is provided (NOT RECOMMENDED) +- `--rbd-metadata-ec-pool-name`: (optional) Provides the name of erasure coded RBD metadata pool, used for creating ECRBDStorageClass. - `--monitoring-endpoint`: (optional) Ceph Manager prometheus exporter endpoints (comma separated list of entries of active and standby mgrs) - `--monitoring-endpoint-port`: (optional) Ceph Manager prometheus exporter port - `--ceph-conf`: (optional) Provide a Ceph conf file @@ -53,6 +49,11 @@ In order to configure an external Ceph cluster with Rook, we need to extract som - `--output`: (optional) Output will be stored into the provided file - `--dry-run`: (optional) Prints the executed commands without running them - `--run-as-user`: (optional) Provides a user name to check the cluster's health status, must be prefixed by `client`. +- `--cephfs-metadata-pool-name`: (optional) Provides the name of the cephfs metadata pool +- `--cephfs-filesystem-name`: (optional) The name of the filesystem, used for creating CephFS StorageClass +- `--cephfs-data-pool-name`: (optional) Provides the name of the CephFS data pool, used for creating CephFS StorageClass +- `--rados-namespace`: (optional) Divides a pool into separate logical namespaces, used for creating RBD PVC in a RadosNamespaces +- `--subvolume-group`: (optional) Provides the name of the subvolume group, used for creating CephFS PVC in a subvolumeGroup - `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem. !!! note @@ -126,9 +127,6 @@ In order to configure an external Ceph cluster with Rook, we need to extract som kubectl -n rook-ceph-external get sc ``` - !!! note - For CephFS StorageClass you also need to export `CEPHFS_FS_NAME`, `CEPHFS_POOL_NAME` or you can pass these parameters with a CLI flag (--cephfs-data-pool-name,--cephfs-filesystem-name) while running the python script. For creating ECRBDStorageClass you need to pass --rbd-metadata-ec-pool-name CLI flag or export `RBD_METADATA_EC_POOL_NAME`. - 7. Then you can now create a [persistent volume](/deploy/examples/csi) based on these StorageClass. ### CephCluster example (management) diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index d53ca1267..b086cc242 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -423,6 +423,12 @@ class RadosJSON: required=False, help="divides a pool into separate logical namespaces", ) + output_group.add_argument( + "--subvolume-group", + default="", + required=False, + help="provides the name of the subvolume group", + ) upgrade_group = argP.add_argument_group("upgrade") upgrade_group.add_argument( @@ -1208,6 +1214,31 @@ class RadosJSON: ).format(rados_namespace, rbd_pool_name) ) + def validate_subvolume_group(self): + cephfs_filesystem_name = self._arg_parser.cephfs_filesystem_name + subvolume_group = self._arg_parser.subvolume_group + if subvolume_group == "": + return + if cephfs_filesystem_name == "": + raise ExecutionFailureException( + "if subvolume group is passed cephfs filesystem name is mandatory to pass" + ) + + cmd = [ + "ceph", + "fs", + "subvolumegroup", + "getpath", + cephfs_filesystem_name, + subvolume_group, + ] + try: + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as execErr: + raise ExecutionFailureException( + "subvolume group {} passed doesn't exist".format(subvolume_group) + ) + def get_rgw_fsid(self): access_key = self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"] secret_key = self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"] @@ -1269,6 +1300,7 @@ class RadosJSON: ) self.validate_pool() self.validate_rados_namespace() + self.validate_subvolume_group() self._excluded_keys.add("CLUSTER_NAME") self.get_cephfs_data_pool_details() self.out_map["NAMESPACE"] = self._arg_parser.namespace @@ -1295,6 +1327,7 @@ class RadosJSON: "RESTRICTED_AUTH_PERMISSION" ] = self._arg_parser.restricted_auth_permission self.out_map["RADOS_NAMESPACE"] = self._arg_parser.rados_namespace + self.out_map["SUBVOLUME_GROUP"] = self._arg_parser.subvolume_group self.out_map["CSI_CEPHFS_NODE_SECRET"] = "" self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"] = "" # create CephFS node and provisioner keyring only when MDS exists diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh index bccfabc2f..cce0a0b03 100644 --- a/deploy/examples/import-external-cluster.sh +++ b/deploy/examples/import-external-cluster.sh @@ -22,6 +22,8 @@ ROOK_EXTERNAL_MONITOR_SECRET=mon-secret OPERATOR_NAMESPACE=rook-ceph # default set to rook-ceph RBD_PROVISIONER=$OPERATOR_NAMESPACE".rbd.csi.ceph.com" # driver:namespace:operator CEPHFS_PROVISIONER=$OPERATOR_NAMESPACE".cephfs.csi.ceph.com" # driver:namespace:operator +CLUSTER_ID_RBD=$NAMESPACE +CLUSTER_ID_CEPHFS=$NAMESPACE : "${ROOK_EXTERNAL_ADMIN_SECRET:=admin-secret}" ############# @@ -93,6 +95,15 @@ function checkEnvVars() { fi } +function importClusterID() { + if [ -n "$RADOS_NAMESPACE_CLUSTER_ID" ]; then + CLUSTER_ID_RBD=$(kubectl -n "$NAMESPACE" get cephblockpoolradosnamespace.ceph.rook.io/"$RADOS_NAMESPACE_CLUSTER_ID" -o jsonpath='{.status.info.clusterID}') + fi + if [ -n "$SUBVOLUME_GROUP_CLUSTER_ID" ]; then + CLUSTER_ID_CEPHFS=$(kubectl -n "$NAMESPACE" get cephfilesystemsubvolumegroup.ceph.rook.io/"$SUBVOLUME_GROUP_CLUSTER_ID" -o jsonpath='{.status.info.clusterID}') + fi +} + function importSecret() { kubectl -n "$NAMESPACE" \ create \ @@ -181,7 +192,7 @@ metadata: name: $RBD_STORAGE_CLASS_NAME provisioner: $RBD_PROVISIONER parameters: - clusterID: $NAMESPACE + clusterID: $CLUSTER_ID_RBD pool: $RBD_POOL_NAME dataPool: $RBD_METADATA_EC_POOL_NAME imageFormat: "2" @@ -206,7 +217,7 @@ metadata: name: $RBD_STORAGE_CLASS_NAME provisioner: $RBD_PROVISIONER parameters: - clusterID: $NAMESPACE + clusterID: $CLUSTER_ID_RBD pool: $RBD_POOL_NAME imageFormat: "2" imageFeatures: layering @@ -230,7 +241,7 @@ metadata: name: $CEPHFS_STORAGE_CLASS_NAME provisioner: $CEPHFS_PROVISIONER parameters: - clusterID: $NAMESPACE + clusterID: $CLUSTER_ID_CEPHFS fsName: $CEPHFS_FS_NAME pool: $CEPHFS_POOL_NAME csi.storage.k8s.io/provisioner-secret-name: $CSI_CEPHFS_PROVISIONER_SECRET_NAME @@ -248,6 +259,7 @@ eof # MAIN # ######## checkEnvVars +importClusterID importSecret importConfigMap importCsiRBDNodeSecret