From e44df846871fe048eab6599a5895912ba69f9856 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Tue, 21 Mar 2023 19:46:42 +0530 Subject: [PATCH] external: make rgw call separate from cephfs and rbd if the rge endint is not reachable don't block the creation of rbd and cephfs sc, just print the error instead of raising exception Signed-off-by: parth-gr --- .github/workflows/canary-integration-test.yml | 3 +- .../create-external-cluster-resources.py | 135 ++++++++++-------- 2 files changed, 80 insertions(+), 58 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index e2df76b13..3fddb13cd 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -167,8 +167,7 @@ jobs: timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done" # pass the invalid rgw-endpoint of different ceph cluster if output=$(timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"); then - echo "unexpectedly succeeded after passing the wrong rgw-endpoint: $output" - exit 1 + echo "script run completed with stderr error after passing the wrong rgw-endpoint: $output" else echo "validation failed because wrong endpoint was provided" fi diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index a9ac2afda..518620c6f 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -544,7 +544,7 @@ class RadosJSON: # if the 'cluster' instance is a dummy one, # don't try to reach out to the endpoint if isinstance(self.cluster, DummyRados): - return + return "", "", "" protocols = ["http", "https"] response_error = None for prefix in protocols: @@ -562,13 +562,18 @@ class RadosJSON: else: r = requests.head(ep, timeout=timeout) if r.status_code == 200: - return prefix, verify + return prefix, verify, "" except Exception as err: response_error = err continue - raise ExecutionFailureException( + sys.stderr.write( f"unable to connect to endpoint: {endpoint_str}, failed error: {response_error}" ) + return ( + "", + "", + ("-1"), + ) def __init__(self, arg_list=None): self.out_map = {} @@ -751,8 +756,9 @@ class RadosJSON: monitoring_endpoint_ip, monitoring_endpoint_port ) self._invalid_endpoint(monitoring_endpoint) - self.endpoint_dial(monitoring_endpoint) - + _, _, err = self.endpoint_dial(monitoring_endpoint) + if err == "-1": + raise ExecutionFailureException(err) # add the validated active mgr IP into the first index mgr_ips.insert(0, monitoring_endpoint_ip) all_mgr_ips_str = ",".join(mgr_ips) @@ -1090,14 +1096,15 @@ class RadosJSON: f"failed to execute command {cmd}. Output: {execErr.output}. " f"Code: {execErr.returncode}. Error: {execErr.stderr}" ) - raise Exception(err_msg) + sys.stderr.write(err_msg) + return None, None, False, "-1" else: err_msg = ( f"failed to execute command {cmd}. Output: {execErr.output}. " f"Code: {execErr.returncode}. Error: {execErr.stderr}" ) - - raise Exception(err_msg) + sys.stderr.write(err_msg) + return None, None, False, "-1" # if it is python2, don't check for ceph version for adding `info=read` cap(rgw_validation) if sys.version_info.major < 3: @@ -1106,6 +1113,7 @@ class RadosJSON: jsonoutput["keys"][0]["access_key"], jsonoutput["keys"][0]["secret_key"], False, + "", ) # separately add info=read caps for rgw-endpoint ip validation @@ -1134,13 +1142,15 @@ class RadosJSON: f"failed to execute command {cmd}. Output: {execErr.output}. " f"Code: {execErr.returncode}. Error: {execErr.stderr}" ) - raise Exception(err_msg) + sys.stderr.write(err_msg) + return None, None, False, "-1" jsonoutput = json.loads(output) return ( jsonoutput["keys"][0]["access_key"], jsonoutput["keys"][0]["secret_key"], info_cap_supported, + "", ) def convert_fqdn_rgw_endpoint_to_ip(self, fqdn_rgw_endpoint): @@ -1156,34 +1166,11 @@ class RadosJSON: rgw_endpoint = self._join_host_port(rgw_endpoint_ip, rgw_endpoint_port) return rgw_endpoint - def validate_pool(self): - pools_to_validate = [self._arg_parser.rbd_data_pool_name] - # if rgw_endpoint is provided, validate it - if self._arg_parser.rgw_endpoint: - rgw_endpoint = self._arg_parser.rgw_endpoint - self._invalid_endpoint(rgw_endpoint) - cert = None - if ( - not self._arg_parser.rgw_skip_tls - and self.validate_rgw_endpoint_tls_cert() - ): - cert = self._arg_parser.rgw_tls_cert_path - self.endpoint_dial(rgw_endpoint, cert=cert) - # only validate if rgw_pool_prefix is passed else it will take default value and we don't create these default pools - if self._arg_parser.rgw_pool_prefix != "default": - rgw_pool_to_validate = [ - f"{self._arg_parser.rgw_pool_prefix}.rgw.meta", - ".rgw.root", - f"{self._arg_parser.rgw_pool_prefix}.rgw.control", - f"{self._arg_parser.rgw_pool_prefix}.rgw.log", - ] - pools_to_validate.extend(rgw_pool_to_validate) - - for pool in pools_to_validate: - if not self.cluster.pool_exists(pool): - raise ExecutionFailureException( - f"The provided pool, '{pool}', does not exist" - ) + def validate_rbd_pool(self): + if not self.cluster.pool_exists(self._arg_parser.rbd_data_pool_name): + raise ExecutionFailureException( + f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist" + ) def validate_rados_namespace(self): rbd_pool_name = self._arg_parser.rbd_data_pool_name @@ -1223,14 +1210,10 @@ class RadosJSON: f"subvolume group {subvolume_group} passed doesn't exist" ) - def get_rgw_fsid(self): + def get_rgw_fsid(self, base_url, verify): access_key = self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"] secret_key = self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"] rgw_endpoint = self._arg_parser.rgw_endpoint - cert = None - if not self._arg_parser.rgw_skip_tls and self.validate_rgw_endpoint_tls_cert(): - cert = self._arg_parser.rgw_tls_cert_path - base_url, verify = self.endpoint_dial(rgw_endpoint, cert=cert) base_url = base_url + "://" + rgw_endpoint + "/admin/info?" params = {"format": "json"} request_url = base_url + urllib.parse.urlencode(params) @@ -1242,28 +1225,64 @@ class RadosJSON: verify=verify, ) except requests.exceptions.Timeout: - raise ExecutionFailureException( + sys.stderr.write( f"invalid endpoint:, not able to call admin-ops api{rgw_endpoint}" ) + return "", "-1" r1 = r.json() if r1 is None or r1.get("info") is None: - return "" # Invalid rgw-endpoint exception will returned by validate_rgw_endpoint() + sys.stderr.write( + f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid." + ) + return ( + "", + "-1", + ) - return r1["info"]["storage_backends"][0]["cluster_id"] + return r1["info"]["storage_backends"][0]["cluster_id"], "" def validate_rgw_endpoint(self): # if the 'cluster' instance is a dummy one, # don't try to reach out to the endpoint if isinstance(self.cluster, DummyRados): return + + # check if the rgw endpoint is reachable + rgw_endpoint = self._arg_parser.rgw_endpoint + cert = None + if not self._arg_parser.rgw_skip_tls and self.validate_rgw_endpoint_tls_cert(): + cert = self._arg_parser.rgw_tls_cert_path + base_url, verify, err = self.endpoint_dial(rgw_endpoint, cert=cert) + if err != "": + return "-1" + + # check if the rgw endpoint is valid and belongs to the same cluster fsid = self.get_fsid() - rgw_fsid = self.get_rgw_fsid() + rgw_fsid, err = self.get_rgw_fsid(base_url, verify) + if err == "-1": + return "-1" if fsid != rgw_fsid: - raise ExecutionFailureException( - ( - f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'" - ) + sys.stderr.write( + f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'" ) + return "-1" + + # check if the rgw endpoint exist + # only validate if rgw_pool_prefix is passed else it will take default value and we don't create these default pools + if self._arg_parser.rgw_pool_prefix != "default": + rgw_pool_to_validate = [ + f"{self._arg_parser.rgw_pool_prefix}.rgw.meta", + ".rgw.root", + f"{self._arg_parser.rgw_pool_prefix}.rgw.control", + f"{self._arg_parser.rgw_pool_prefix}.rgw.log", + ] + if not self.cluster.pool_exists(self._arg_parser.rgw_pool_to_validate): + sys.stderr.write( + f"The provided pool, '{self._arg_parser.rgw_pool_to_validate}', does not exist" + ) + return "-1" + + return "" def validate_rgw_realm_name(self): if self._arg_parser.rgw_realm_name != "": @@ -1275,7 +1294,7 @@ class RadosJSON: self._arg_parser.rgw_realm_name, ] try: - output = subprocess.check_output(cmd, stderr=subprocess.PIPE) + _ = subprocess.check_output(cmd, stderr=subprocess.PIPE) except subprocess.CalledProcessError as execErr: err_msg = ( f"failed to execute command {cmd}. Output: {execErr.output}. " @@ -1293,7 +1312,7 @@ class RadosJSON: self._arg_parser.rgw_endpoint = self.convert_fqdn_rgw_endpoint_to_ip( self._arg_parser.rgw_endpoint ) - self.validate_pool() + self.validate_rbd_pool() self.validate_rados_namespace() self.validate_subvolume_group() self.validate_rgw_realm_name() @@ -1336,7 +1355,6 @@ class RadosJSON: self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"], self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"], ) = self.create_cephCSIKeyring_user("client.csi-cephfs-provisioner") - self.out_map["RGW_ENDPOINT"] = self._arg_parser.rgw_endpoint self.out_map["RGW_TLS_CERT"] = "" ( self.out_map["MONITORING_ENDPOINT"], @@ -1348,6 +1366,7 @@ class RadosJSON: ] = self.validate_rgw_metadata_ec_pool_name() self.out_map["RGW_REALM_NAME"] = self._arg_parser.rgw_realm_name self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix + self.out_map["RGW_ENDPOINT"] = "" if self._arg_parser.rgw_endpoint: if self._arg_parser.dry_run: self.create_rgw_admin_ops_user() @@ -1356,11 +1375,15 @@ class RadosJSON: self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"], self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"], info_cap_supported, + err, ) = self.create_rgw_admin_ops_user() if info_cap_supported: - self.validate_rgw_endpoint() - if self._arg_parser.rgw_tls_cert_path: - self.out_map["RGW_TLS_CERT"] = self.validate_rgw_endpoint_tls_cert() + err = self.validate_rgw_endpoint() + if self._arg_parser.rgw_tls_cert_path: + self.out_map["RGW_TLS_CERT"] = self.validate_rgw_endpoint_tls_cert() + # if there is no error, set the RGW_ENDPOINT + if err != "-1": + self.out_map["RGW_ENDPOINT"] = self._arg_parser.rgw_endpoint def gen_shell_out(self): self._gen_output_map()