From 7fa4e5eac1d058986bdd9ec1ebf9e9039cc9831f Mon Sep 17 00:00:00 2001 From: parth-gr Date: Thu, 6 Apr 2023 20:10:07 +0530 Subject: [PATCH] external: add support for multisite in external cluster Add support for realm, zone and zonegroup Signed-off-by: parth-gr (cherry picked from commit b60da0b8c1170f1e5691cf39e100750aef1f67ea) --- .github/workflows/canary-integration-test.yml | 2 +- .../CRDs/Cluster/external-cluster.md | 2 + .../create-external-cluster-resources.py | 68 +++++++++++++------ 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 177d1dfb0..18ad3dcc4 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -189,7 +189,7 @@ jobs: kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm1 # pass wrong realm if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-realm-name realm3); then - exit 1 + echo "script run completed with stderr error after passing the wrong realm: $output" else echo "script failed because wrong realm was passed" fi diff --git a/Documentation/CRDs/Cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster.md index 4c2ed4d41..7f1ad0425 100644 --- a/Documentation/CRDs/Cluster/external-cluster.md +++ b/Documentation/CRDs/Cluster/external-cluster.md @@ -58,6 +58,8 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name -- - `--rados-namespace`: (optional) Divides a pool into separate logical namespaces, used for creating RBD PVC in a RadosNamespaces - `--subvolume-group`: (optional) Provides the name of the subvolume group, used for creating CephFS PVC in a subvolumeGroup - `--rgw-realm-name`: (optional) Provides the name of the rgw-realm +- `--rgw-zone-name`: (optional) Provides the name of the rgw-zone +- `--rgw-zonegroup-name`: (optional) Provides the name of the rgw-zone-group - `--upgrade`: (optional) Upgrades the 'Ceph CSI keyrings (For example: client.csi-cephfs-provisioner) with new permissions needed for the new cluster version and older permission will still be applied. - `--restricted-auth-permission`: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are `--rbd-data-pool-name`, and `--cluster-name`. `--cephfs-filesystem-name` flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem. diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index 730fd5072..a18012a35 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -435,6 +435,18 @@ class RadosJSON: required=False, help="provides the name of the rgw-realm", ) + output_group.add_argument( + "--rgw-zone-name", + default="", + required=False, + help="provides the name of the rgw-zone", + ) + output_group.add_argument( + "--rgw-zonegroup-name", + default="", + required=False, + help="provides the name of the rgw-zonegroup", + ) upgrade_group = argP.add_argument_group("upgrade") upgrade_group.add_argument( @@ -1106,6 +1118,10 @@ class RadosJSON: "buckets=*;users=*;usage=read;metadata=read;zone=read", "--rgw-realm", self._arg_parser.rgw_realm_name, + "--rgw-zonegroup", + self._arg_parser.rgw_zonegroup_name, + "--rgw-zone", + self._arg_parser.rgw_zone_name, ] if self._arg_parser.dry_run: return self.dry_run("ceph " + " ".join(cmd)) @@ -1273,7 +1289,7 @@ class RadosJSON: return r1["info"]["storage_backends"][0]["cluster_id"], "" - def validate_rgw_endpoint(self): + def validate_rgw_endpoint(self, info_cap_supported): # if the 'cluster' instance is a dummy one, # don't try to reach out to the endpoint if isinstance(self.cluster, DummyRados): @@ -1289,15 +1305,17 @@ class RadosJSON: return "-1" # check if the rgw endpoint is valid and belongs to the same cluster - fsid = self.get_fsid() - rgw_fsid, err = self.get_rgw_fsid(base_url, verify) - if err == "-1": - return "-1" - if fsid != rgw_fsid: - sys.stderr.write( - f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'" - ) - return "-1" + # only check if info cap is supported + if info_cap_supported: + fsid = self.get_fsid() + rgw_fsid, err = self.get_rgw_fsid(base_url, verify) + if err == "-1": + return "-1" + if fsid != rgw_fsid: + sys.stderr.write( + f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'" + ) + return "-1" # check if the rgw endpoint exist # only validate if rgw_pool_prefix is passed else it will take default value and we don't create these default pools @@ -1308,22 +1326,22 @@ class RadosJSON: f"{self._arg_parser.rgw_pool_prefix}.rgw.control", f"{self._arg_parser.rgw_pool_prefix}.rgw.log", ] - if not self.cluster.pool_exists(self._arg_parser.rgw_pool_to_validate): + if not self.cluster.pool_exists(rgw_pool_to_validate): sys.stderr.write( - f"The provided pool, '{self._arg_parser.rgw_pool_to_validate}', does not exist" + f"The provided pool, '{rgw_pool_to_validate}', does not exist" ) return "-1" return "" - def validate_rgw_realm_name(self): - if self._arg_parser.rgw_realm_name != "": + def validate_rgw_multisite(self, rgw_multisite_config, rgw_multisite_config_flag): + if rgw_multisite_config != "": cmd = [ "radosgw-admin", "realm", "get", - "--rgw-realm", - self._arg_parser.rgw_realm_name, + rgw_multisite_config_flag, + rgw_multisite_config, ] try: _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) @@ -1332,7 +1350,9 @@ class RadosJSON: f"failed to execute command {cmd}. Output: {execErr.output}. " f"Code: {execErr.returncode}. Error: {execErr.stderr}" ) - raise Exception(err_msg) + sys.stderr.write(err_msg) + return "-1" + return "" def _gen_output_map(self): if self.out_map: @@ -1347,7 +1367,6 @@ class RadosJSON: self.validate_rbd_pool() self.validate_rados_namespace() self.validate_subvolume_group() - self.validate_rgw_realm_name() self._excluded_keys.add("CLUSTER_NAME") self.get_cephfs_data_pool_details() self.out_map["NAMESPACE"] = self._arg_parser.namespace @@ -1396,21 +1415,28 @@ class RadosJSON: self.out_map[ "RBD_METADATA_EC_POOL_NAME" ] = self.validate_rbd_metadata_ec_pool_name() - self.out_map["RGW_REALM_NAME"] = self._arg_parser.rgw_realm_name self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix self.out_map["RGW_ENDPOINT"] = "" if self._arg_parser.rgw_endpoint: if self._arg_parser.dry_run: self.create_rgw_admin_ops_user() else: + err = self.validate_rgw_multisite( + self._arg_parser.rgw_realm_name, "--rgw-realm" + ) + err = self.validate_rgw_multisite( + self._arg_parser.rgw_zonegroup_name, "--rgw-zonegroup" + ) + err = self.validate_rgw_multisite( + self._arg_parser.rgw_zone_name, "--rgw-zone" + ) ( self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"], self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"], info_cap_supported, err, ) = self.create_rgw_admin_ops_user() - if info_cap_supported: - err = self.validate_rgw_endpoint() + err = self.validate_rgw_endpoint(info_cap_supported) if self._arg_parser.rgw_tls_cert_path: self.out_map["RGW_TLS_CERT"] = self.validate_rgw_endpoint_tls_cert() # if there is no error, set the RGW_ENDPOINT