feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #11947 from parth-gr/external-seprate-rgw

Browse Source 
external: make rgw call separate from cephfs and rbd
This commit is contained in:
Travis Nielsen authored and GitHub committed 2023-03-31 09:35:26 -06:00
2 parents 87e836424e e44df84687
commit 12badbe4c3
2 files changed
+80 -58
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/canary-integration-test.yml
+1 -2
View File
@@ -167,8 +167,7 @@ jobs:
timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
# pass the invalid rgw-endpoint of different ceph cluster
if output=$(timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint 10.108.96.128:80; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"); then
echo "unexpectedly succeeded after passing the wrong rgw-endpoint: $output"
exit 1
echo "script run completed with stderr error after passing the wrong rgw-endpoint: $output"
else
echo "validation failed because wrong endpoint was provided"
fi
deploy/examples/create-external-cluster-resources.py
+79 -56
View File
@@ -544,7 +544,7 @@ class RadosJSON:
# if the 'cluster' instance is a dummy one,
# don't try to reach out to the endpoint
if isinstance(self.cluster, DummyRados):
return
return "", "", ""
protocols = ["http", "https"]
response_error = None
for prefix in protocols:
@@ -562,13 +562,18 @@ class RadosJSON:
else:
r = requests.head(ep, timeout=timeout)
if r.status_code == 200:
return prefix, verify
return prefix, verify, ""
except Exception as err:
response_error = err
continue
raise ExecutionFailureException(
sys.stderr.write(
f"unable to connect to endpoint: {endpoint_str}, failed error: {response_error}"
)
return (
"",
"",
("-1"),
)
def __init__(self, arg_list=None):
self.out_map = {}
@@ -751,8 +756,9 @@ class RadosJSON:
monitoring_endpoint_ip, monitoring_endpoint_port
)
self._invalid_endpoint(monitoring_endpoint)
self.endpoint_dial(monitoring_endpoint)
_, _, err = self.endpoint_dial(monitoring_endpoint)
if err == "-1":
raise ExecutionFailureException(err)
# add the validated active mgr IP into the first index
mgr_ips.insert(0, monitoring_endpoint_ip)
all_mgr_ips_str = ",".join(mgr_ips)
@@ -1090,14 +1096,15 @@ class RadosJSON:
f"failed to execute command {cmd}. Output: {execErr.output}. "
f"Code: {execErr.returncode}. Error: {execErr.stderr}"
)
raise Exception(err_msg)
sys.stderr.write(err_msg)
return None, None, False, "-1"
else:
err_msg = (
f"failed to execute command {cmd}. Output: {execErr.output}. "
f"Code: {execErr.returncode}. Error: {execErr.stderr}"
)
raise Exception(err_msg)
sys.stderr.write(err_msg)
return None, None, False, "-1"
# if it is python2, don't check for ceph version for adding `info=read` cap(rgw_validation)
if sys.version_info.major < 3:
@@ -1106,6 +1113,7 @@ class RadosJSON:
jsonoutput["keys"][0]["access_key"],
jsonoutput["keys"][0]["secret_key"],
False,
"",
)
# separately add info=read caps for rgw-endpoint ip validation
@@ -1134,13 +1142,15 @@ class RadosJSON:
f"failed to execute command {cmd}. Output: {execErr.output}. "
f"Code: {execErr.returncode}. Error: {execErr.stderr}"
)
raise Exception(err_msg)
sys.stderr.write(err_msg)
return None, None, False, "-1"
jsonoutput = json.loads(output)
return (
jsonoutput["keys"][0]["access_key"],
jsonoutput["keys"][0]["secret_key"],
info_cap_supported,
"",
)
def convert_fqdn_rgw_endpoint_to_ip(self, fqdn_rgw_endpoint):
@@ -1156,34 +1166,11 @@ class RadosJSON:
rgw_endpoint = self._join_host_port(rgw_endpoint_ip, rgw_endpoint_port)
return rgw_endpoint
def validate_pool(self):
pools_to_validate = [self._arg_parser.rbd_data_pool_name]
# if rgw_endpoint is provided, validate it
if self._arg_parser.rgw_endpoint:
rgw_endpoint = self._arg_parser.rgw_endpoint
self._invalid_endpoint(rgw_endpoint)
cert = None
if (
not self._arg_parser.rgw_skip_tls
and self.validate_rgw_endpoint_tls_cert()
):
cert = self._arg_parser.rgw_tls_cert_path
self.endpoint_dial(rgw_endpoint, cert=cert)
# only validate if rgw_pool_prefix is passed else it will take default value and we don't create these default pools
if self._arg_parser.rgw_pool_prefix != "default":
rgw_pool_to_validate = [
f"{self._arg_parser.rgw_pool_prefix}.rgw.meta",
".rgw.root",
f"{self._arg_parser.rgw_pool_prefix}.rgw.control",
f"{self._arg_parser.rgw_pool_prefix}.rgw.log",
]
pools_to_validate.extend(rgw_pool_to_validate)
for pool in pools_to_validate:
if not self.cluster.pool_exists(pool):
raise ExecutionFailureException(
f"The provided pool, '{pool}', does not exist"
)
def validate_rbd_pool(self):
if not self.cluster.pool_exists(self._arg_parser.rbd_data_pool_name):
raise ExecutionFailureException(
f"The provided pool, '{self._arg_parser.rbd_data_pool_name}', does not exist"
)
def validate_rados_namespace(self):
rbd_pool_name = self._arg_parser.rbd_data_pool_name
@@ -1223,14 +1210,10 @@ class RadosJSON:
f"subvolume group {subvolume_group} passed doesn't exist"
)
def get_rgw_fsid(self):
def get_rgw_fsid(self, base_url, verify):
access_key = self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"]
secret_key = self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"]
rgw_endpoint = self._arg_parser.rgw_endpoint
cert = None
if not self._arg_parser.rgw_skip_tls and self.validate_rgw_endpoint_tls_cert():
cert = self._arg_parser.rgw_tls_cert_path
base_url, verify = self.endpoint_dial(rgw_endpoint, cert=cert)
base_url = base_url + "://" + rgw_endpoint + "/admin/info?"
params = {"format": "json"}
request_url = base_url + urllib.parse.urlencode(params)
@@ -1242,28 +1225,64 @@ class RadosJSON:
verify=verify,
)
except requests.exceptions.Timeout:
raise ExecutionFailureException(
sys.stderr.write(
f"invalid endpoint:, not able to call admin-ops api{rgw_endpoint}"
)
return "", "-1"
r1 = r.json()
if r1 is None or r1.get("info") is None:
return "" # Invalid rgw-endpoint exception will returned by validate_rgw_endpoint()
sys.stderr.write(
f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid."
)
return (
"",
"-1",
)
return r1["info"]["storage_backends"][0]["cluster_id"]
return r1["info"]["storage_backends"][0]["cluster_id"], ""
def validate_rgw_endpoint(self):
# if the 'cluster' instance is a dummy one,
# don't try to reach out to the endpoint
if isinstance(self.cluster, DummyRados):
return
# check if the rgw endpoint is reachable
rgw_endpoint = self._arg_parser.rgw_endpoint
cert = None
if not self._arg_parser.rgw_skip_tls and self.validate_rgw_endpoint_tls_cert():
cert = self._arg_parser.rgw_tls_cert_path
base_url, verify, err = self.endpoint_dial(rgw_endpoint, cert=cert)
if err != "":
return "-1"
# check if the rgw endpoint is valid and belongs to the same cluster
fsid = self.get_fsid()
rgw_fsid = self.get_rgw_fsid()
rgw_fsid, err = self.get_rgw_fsid(base_url, verify)
if err == "-1":
return "-1"
if fsid != rgw_fsid:
raise ExecutionFailureException(
(
f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'"
)
sys.stderr.write(
f"The provided rgw Endpoint, '{self._arg_parser.rgw_endpoint}', is invalid. We are validating by calling the adminops api through rgw-endpoint and validating the cluster_id '{rgw_fsid}' is equal to the ceph cluster fsid '{fsid}'"
)
return "-1"
# check if the rgw endpoint exist
# only validate if rgw_pool_prefix is passed else it will take default value and we don't create these default pools
if self._arg_parser.rgw_pool_prefix != "default":
rgw_pool_to_validate = [
f"{self._arg_parser.rgw_pool_prefix}.rgw.meta",
".rgw.root",
f"{self._arg_parser.rgw_pool_prefix}.rgw.control",
f"{self._arg_parser.rgw_pool_prefix}.rgw.log",
]
if not self.cluster.pool_exists(self._arg_parser.rgw_pool_to_validate):
sys.stderr.write(
f"The provided pool, '{self._arg_parser.rgw_pool_to_validate}', does not exist"
)
return "-1"
return ""
def validate_rgw_realm_name(self):
if self._arg_parser.rgw_realm_name != "":
@@ -1275,7 +1294,7 @@ class RadosJSON:
self._arg_parser.rgw_realm_name,
]
try:
output = subprocess.check_output(cmd, stderr=subprocess.PIPE)
_ = subprocess.check_output(cmd, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as execErr:
err_msg = (
f"failed to execute command {cmd}. Output: {execErr.output}. "
@@ -1293,7 +1312,7 @@ class RadosJSON:
self._arg_parser.rgw_endpoint = self.convert_fqdn_rgw_endpoint_to_ip(
self._arg_parser.rgw_endpoint
)
self.validate_pool()
self.validate_rbd_pool()
self.validate_rados_namespace()
self.validate_subvolume_group()
self.validate_rgw_realm_name()
@@ -1336,7 +1355,6 @@ class RadosJSON:
self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"],
self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"],
) = self.create_cephCSIKeyring_user("client.csi-cephfs-provisioner")
self.out_map["RGW_ENDPOINT"] = self._arg_parser.rgw_endpoint
self.out_map["RGW_TLS_CERT"] = ""
(
self.out_map["MONITORING_ENDPOINT"],
@@ -1348,6 +1366,7 @@ class RadosJSON:
] = self.validate_rgw_metadata_ec_pool_name()
self.out_map["RGW_REALM_NAME"] = self._arg_parser.rgw_realm_name
self.out_map["RGW_POOL_PREFIX"] = self._arg_parser.rgw_pool_prefix
self.out_map["RGW_ENDPOINT"] = ""
if self._arg_parser.rgw_endpoint:
if self._arg_parser.dry_run:
self.create_rgw_admin_ops_user()
@@ -1356,11 +1375,15 @@ class RadosJSON:
self.out_map["RGW_ADMIN_OPS_USER_ACCESS_KEY"],
self.out_map["RGW_ADMIN_OPS_USER_SECRET_KEY"],
info_cap_supported,
err,
) = self.create_rgw_admin_ops_user()
if info_cap_supported:
self.validate_rgw_endpoint()
if self._arg_parser.rgw_tls_cert_path:
self.out_map["RGW_TLS_CERT"] = self.validate_rgw_endpoint_tls_cert()
err = self.validate_rgw_endpoint()
if self._arg_parser.rgw_tls_cert_path:
self.out_map["RGW_TLS_CERT"] = self.validate_rgw_endpoint_tls_cert()
# if there is no error, set the RGW_ENDPOINT
if err != "-1":
self.out_map["RGW_ENDPOINT"] = self._arg_parser.rgw_endpoint
def gen_shell_out(self):
self._gen_output_map()