From c592d36fc78a611f2ce27d6ad1cea6b2cb90d2b8 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Wed, 9 Jul 2025 18:14:35 +0530 Subject: [PATCH] external: automate external users cephx key rotation Ceph has a new ceph auth rotate command currently present in ceph:main Add a new flag `--cephx-key-rotate` to rotate the cephx keys genrated by external python script, If we enable it, it will create a new user with suffix `.{x}` Signed-off-by: parth-gr --- .github/workflows/canary-integration-test.yml | 105 ++++++++ .../external-cluster/provider-export.md | 2 + .../external-cluster/upgrade-external.md | 1 + ...create-external-cluster-resources-tests.py | 8 +- .../create-external-cluster-resources.py | 231 +++++++++++++----- deploy/examples/import-external-cluster.sh | 60 ++++- tests/scripts/github-action-helper.sh | 24 ++ 7 files changed, 357 insertions(+), 74 deletions(-) diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 6315385f0..1e14a8f4e 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -312,6 +312,111 @@ jobs: # default value should be used kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --config-file /etc/ceph/external-config.ini + - name: test key rotation with --cephx-key-rotate + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + output1=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs) + + # check if it create a csi rbd, cephfs Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node client.csi-rbd-provisioner client.csi-cephfs-node client.csi-cephfs-provisioner client.healthchecker + + # rotate the key + output2=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs --cephx-key-rotate rotate) + + # check if it create a csi rbd, cephfs and Healthchecker rotated keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node.1 client.csi-rbd-provisioner.1 client.csi-cephfs-node.1 client.csi-cephfs-provisioner.1 client.healthchecker.1 + + # check the content of the secret, it should be changed + if [ "$output1" != "$output2" ]; then + echo "csi rbd and cephfs key rotated successfully" + else + echo "csi rbd and cephfs key not rotated" + exit 1 + fi + + # revert the key + output3=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs --cephx-key-rotate revert) + # check if it revert the key, it should be same as output1 + if [ "$output1" == "$output3" ]; then + echo "csi rbd and cephfs key reverted successfully" + else + echo "csi rbd and cephfs key not reverted" + exit 1 + fi + + # if --cephx-key-rotate is not set provide the latest key + output4=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --cephfs-filesystem-name myfs) + if [ "$output2" == "$output4" ]; then + echo "csi rbd and cephfs key uses latest generation" + else + echo "csi rbd and cephfs key doesn't uses latest generation" + exit 1 + fi + + - name: test key rotation with --cephx-key-rotate in multi tentat case, where --restricted-auth-permission is set + run: | + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + # create tenant1 and verify + # create `radosNamespace1` rados-namespace for `replicapool` rbd data-pool + kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/rn1 + output1=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true) + + # check if it create a csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker + + # rotate tenant1 and verify + output2=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate) + # check if it create rotated csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1.1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1.1 client.csi-cephfs-provisioner-rookstorage-myfs.1 client.csi-cephfs-node-rookstorage-myfs.1 client.healthchecker.1 + + # revert tenant1 and verify + output3=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate revert) + # check if it create a revert csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1 client.csi-rbd-provisioner-rookstorage-replicapool-rn1 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker + + # check if it revert the key, it should be same as output1 + if [ "$output1" == "$output3" ]; then + echo "csi rbd and cephfs key reverted successfully" + else + echo "csi rbd and cephfs key not reverted" + exit 1 + fi + + # create tenant2 and verify + # create `radosNamespace2` rados-namespace for `replicapool` rbd data-pool + kubectl -n rook-ceph exec $toolbox -- rbd namespace create replicapool/rn2 + output4=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true) + + # check if it create a csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker + + # rotate tenant2 and verify + output5=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate) + # check if it create rotated csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2.1 client.csi-rbd-provisioner-rookstorage-replicapool-rn2.1 client.csi-cephfs-provisioner-rookstorage-myfs.1 client.csi-cephfs-node-rookstorage-myfs.1 client.healthchecker.1 + + # rotate tenant1 (again) and verify + output6=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn1 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate rotate) + # check if it create rotated csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn1.2 client.csi-rbd-provisioner-rookstorage-replicapool-rn1.2 client.csi-cephfs-provisioner-rookstorage-myfs.2 client.csi-cephfs-node-rookstorage-myfs.2 client.healthchecker.2 + + # verify if tenant1 is rotated, tenant2 keys should not be rotated + # only rbd keys have separation of tenant1 and tenant2, and cephfs keys are shared + tests/scripts/github-action-helper.sh check_keys_not_exists client.csi-rbd-node-rookstorage-replicapool-rn2.2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2.2 + + # revert tenant2 and verify + output7=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rados-namespace rn2 --cephfs-filesystem-name myfs --k8s-cluster-name rookstorage --restricted-auth-permission true --cephx-key-rotate revert) + # check if it create a reverted csi rbd, cephfs and Healthchecker keys + tests/scripts/github-action-helper.sh check_keys_exists client.csi-rbd-node-rookstorage-replicapool-rn2 client.csi-rbd-provisioner-rookstorage-replicapool-rn2 client.csi-cephfs-provisioner-rookstorage-myfs client.csi-cephfs-node-rookstorage-myfs client.healthchecker + + # check if it revert the key, it should be same as output1 + if [ "$output4" == "$output7" ]; then + echo "csi rbd and cephfs key reverted successfully" + else + echo "csi rbd and cephfs key not reverted" + exit 1 + fi + - name: check-ownerreferences run: tests/scripts/github-action-helper.sh check_ownerreferences diff --git a/Documentation/CRDs/Cluster/external-cluster/provider-export.md b/Documentation/CRDs/Cluster/external-cluster/provider-export.md index 765552ac5..0d4be4972 100644 --- a/Documentation/CRDs/Cluster/external-cluster/provider-export.md +++ b/Documentation/CRDs/Cluster/external-cluster/provider-export.md @@ -43,6 +43,8 @@ python3 create-external-cluster-resources.py --rbd-data-pool-name -- * `--topology-failure-domain-label`: (optional) K8s cluster failure domain label (example: zone, rack, or host) for the topology-pools that match the ceph domain * `--topology-failure-domain-values`: (optional) Comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the `topology-pools` list * `--config-file`: Path to the configuration file, Priority: command-line-args > config.ini values > default values +* `--cephx-key-rotate`: Enable cephx key rotation for the users created by this script. This will create a new user with suffix `.{x}` and update the secrets with the new key. Set `--cephx-key-rotation rotate` to initiate rotation. To revert keys to the prior generation, set `--cephx-key-rotation revert` +Note: If user are reverting to prior generation, then the user should manually delete the prior used user keys from the cluster after reverting. ## 2. Copy the bash output diff --git a/Documentation/CRDs/Cluster/external-cluster/upgrade-external.md b/Documentation/CRDs/Cluster/external-cluster/upgrade-external.md index eddcb8283..ac525c47a 100644 --- a/Documentation/CRDs/Cluster/external-cluster/upgrade-external.md +++ b/Documentation/CRDs/Cluster/external-cluster/upgrade-external.md @@ -15,6 +15,7 @@ Upgrading the cluster would be different for restricted caps and non-restricted 2. If the consumer cluster has restricted caps Restricted users created using `--restricted-auth-permission` flag need to pass mandatory flags: '`--rbd-data-pool-name`(if it is a rbd user), `--k8s-cluster-name` and `--run-as-user`' flags while upgrading, in case of cephfs users if you have passed `--cephfs-filesystem-name` flag while creating CSI users then while upgrading it will be mandatory too. In this example the user would be `client.csi-rbd-node-rookstorage-replicapool` (following the pattern `csi-user-clusterName-poolName`) + If --cephx-key-rotate was set, it adds `.{x}` suffix to the user name, for example: `client.csi-rbd-node-rookstorage-replicapool.1` ```console python3 create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool diff --git a/deploy/examples/create-external-cluster-resources-tests.py b/deploy/examples/create-external-cluster-resources-tests.py index 8c1feecf9..13275ed66 100644 --- a/deploy/examples/create-external-cluster-resources-tests.py +++ b/deploy/examples/create-external-cluster-resources-tests.py @@ -66,18 +66,18 @@ class TestRadosJSON(unittest.TestCase): def test_method_create_cephCSIKeyring_cephFSProvisioner(self): csiKeyring = self.rjObj.create_cephCSIKeyring_user( - "client.csi-cephfs-provisioner" + "client.csi-cephfs-provisioner", 0 ) print(f"cephCSIKeyring without restricting it to a metadata pool. {csiKeyring}") self.rjObj._arg_parser.restricted_auth_permission = True self.rjObj._arg_parser.k8s_cluster_name = "openshift-storage" csiKeyring = self.rjObj.create_cephCSIKeyring_user( - "client.csi-cephfs-provisioner" + "client.csi-cephfs-provisioner", 0 ) print(f"cephCSIKeyring for a specific cluster. {csiKeyring}") self.rjObj._arg_parser.cephfs_filesystem_name = "myfs" csiKeyring = self.rjObj.create_cephCSIKeyring_user( - "client.csi-cephfs-provisioner" + "client.csi-cephfs-provisioner", 0 ) print(f"cephCSIKeyring for a specific metadata pool and cluster. {csiKeyring}") @@ -85,7 +85,7 @@ class TestRadosJSON(unittest.TestCase): self.rjObj.cluster.return_val = 1 self.rjObj.cluster.err_message = "Dummy Error" try: - self.rjObj.create_checkerKey("client.healthchecker") + self.rjObj.create_checkerKey("client.healthchecker", 0) self.fail("Failed to raise an exception, 'ext.ExecutionFailureException'") except ext.ExecutionFailureException as err: print(f"Successfully thrown error.\nError: {err}") diff --git a/deploy/examples/create-external-cluster-resources.py b/deploy/examples/create-external-cluster-resources.py index ecb5caf49..a1affa07a 100644 --- a/deploy/examples/create-external-cluster-resources.py +++ b/deploy/examples/create-external-cluster-resources.py @@ -155,6 +155,9 @@ class DummyRados(object): self.cmd_output_map[ """{"caps": ["mon", "allow r, allow command 'osd blocklist'", "mgr", "allow rw", "osd", "allow rw tag cephfs metadata=*", "mds", "allow *"], "entity": "client.csi-cephfs-provisioner", "format": "json", "prefix": "auth caps"}""" ] = """[{"entity":"client.csi-cephfs-provisioner","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow r, allow command 'osd blocklist'", "mgr":"allow rw", "osd":"allow rw tag cephfs metadata=*","mds":"allow *"}}]""" + self.cmd_output_map["""{"format": "json", "prefix": "auth ls"}"""] = ( + """{"auth_dump":[{"entity":"mds.ocs-storagecluster-cephfilesystem-a","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mds":"allow","mon":"allow profile mds","osd":"allow *"}},{"entity":"mds.ocs-storagecluster-cephfilesystem-b","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mds":"allow","mon":"allow profile mds","osd":"allow *"}},{"entity":"osd.0","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mgr":"allow profile osd","mon":"allow profile osd","osd":"allow *"}},{"entity":"osd.1","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mgr":"allow profile osd","mon":"allow profile osd","osd":"allow *"}},{"entity":"osd.2","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mgr":"allow profile osd","mon":"allow profile osd","osd":"allow *"}},{"entity":"client.admin","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mds":"allow *","mgr":"allow *","mon":"allow *","osd":"allow *"}},{"entity":"client.bootstrap-mds","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow profile bootstrap-mds"}},{"entity":"client.bootstrap-mgr","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow profile bootstrap-mgr"}},{"entity":"client.bootstrap-osd","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow profile bootstrap-osd"}},{"entity":"client.bootstrap-rbd","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow profile bootstrap-rbd"}},{"entity":"client.bootstrap-rbd-mirror","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow profile bootstrap-rbd-mirror"}},{"entity":"client.bootstrap-rgw","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow profile bootstrap-rgw"}},{"entity":"client.ceph-exporter","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mds":"allow r","mgr":"allow r","mon":"allow profile ceph-exporter","osd":"allow r"}},{"entity":"client.crash","key":"AQCphXdoVxHKDxAAPGh6L0GSogOXF30LxWmJuA==","caps":{"mgr":"allow rw","mon":"allow profile crash"}},{"entity":"client.csi-cephfs-node","key":"AQCphXdoo/07ARAA7nhj//Rssdn8eTquX6ZIbg==","caps":{"mds":"allow rw","mgr":"allow rw","mon":"allow r","osd":"allow rw tag cephfs *=*"}},{"entity":"client.csi-cephfs-provisioner","key":"AQCohXdo1muwMhAAJLFwpGfXVTP406q/LKKxNw==","caps":{"mds":"allow *","mgr":"allow rw","mon":"allow r, allow command 'osd blocklist'","osd":"allow rw tag cephfs metadata=*"}},{"entity":"client.csi-rbd-node","key":"AQCohXdoA42ZKBAA73DopBeUNOKBjDpHn900Aw==","caps":{"mgr":"allow rw","mon":"profile rbd","osd":"profile rbd"}},{"entity":"client.csi-rbd-provisioner","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mgr":"allow rw","mon":"profile rbd, allow command 'osd blocklist'","osd":"profile rbd"}},{"entity":"client.rbd-mirror-peer","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"profile rbd-mirror-peer","osd":"profile rbd"}},{"entity":"client.rgw.ocs.storagecluster.cephobjectstore.a","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mon":"allow rw","osd":"allow rwx"}},{"entity":"mgr.a","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mds":"allow *","mon":"allow profile mgr","osd":"allow *"}},{"entity":"mgr.b","key":"AQDFkbNeft5bFRAATndLNUSEKruozxiZi3lrdA==","caps":{"mds":"allow *","mon":"allow profile mgr","osd":"allow *"}}]}""" + ) self.cmd_output_map['{"format": "json", "prefix": "status"}'] = ceph_status_str def shutdown(self): @@ -463,6 +466,15 @@ class RadosJSON: required=False, help="comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the `topology-pools` list", ) + output_group.add_argument( + "--cephx-key-rotate", + default="", + required=False, + help="Enable cephx key rotation for the users created by this script. This will create a new user with suffix `.{x}` and update the secrets with the new key." + + "Set `--cephx-key-rotation rotate` to initiate rotation." + + "To revert keys to the prior generation, set `--cephx-key-rotation revert`." + + "Note: If user are reverting to prior generation, then the user should manually delete the prior used user keys from the cluster after reverting", + ) upgrade_group = argP.add_argument_group("upgrade") upgrade_group.add_argument( @@ -476,7 +488,8 @@ class RadosJSON: + "in case of cephfs users if you have passed --cephfs-filesystem-name flag while creating user then while upgrading it will be mandatory too" + "Sample run: `python3 /etc/ceph/create-external-cluster-resources.py --upgrade --rbd-data-pool-name replicapool --k8s-cluster-name rookstorage --run-as-user client.csi-rbd-node-rookstorage-replicapool`" + "PS: An existing non-restricted user cannot be converted to a restricted user by upgrading." - + "Upgrade flag should only be used to append new permissions to users, it shouldn't be used for changing user already applied permission, for example you shouldn't change in which pool user has access", + + "Upgrade flag should only be used to append new permissions to users, it shouldn't be used for changing user already applied permission, for example you shouldn't change in which pool user has access" + + "If --cephx-key-rotate was set, it adds `.{x}` suffix to the user name, for example: `client.csi-rbd-node-rookstorage-replicapool.1`", ) # Add command-line arguments @@ -855,7 +868,7 @@ class RadosJSON: return "" return str(json_out[0]["key"]) - def get_cephfs_provisioner_caps_and_entity(self): + def get_cephfs_provisioner_caps_and_entity(self, generation): entity = "client.csi-cephfs-provisioner" caps = { "mon": "allow r, allow command 'osd blocklist'", @@ -875,10 +888,12 @@ class RadosJSON: else: entity = f"{entity}-{k8s_cluster_name}-{cephfs_filesystem}" caps["osd"] = f"allow rw tag cephfs metadata={cephfs_filesystem}" + if generation != 0: + entity = f"{entity}.{generation}" return caps, entity - def get_cephfs_node_caps_and_entity(self): + def get_cephfs_node_caps_and_entity(self, generation): entity = "client.csi-cephfs-node" caps = { "mon": "allow r, allow command 'osd blocklist'", @@ -898,6 +913,8 @@ class RadosJSON: else: entity = f"{entity}-{k8s_cluster_name}-{cephfs_filesystem}" caps["osd"] = f"allow rw tag cephfs *={cephfs_filesystem}" + if generation != 0: + entity = f"{entity}.{generation}" return caps, entity @@ -934,7 +951,7 @@ class RadosJSON: entity = f"{entity}-{rados_namespace}" return entity - def get_rbd_provisioner_caps_and_entity(self): + def get_rbd_provisioner_caps_and_entity(self, generation): entity = "client.csi-rbd-provisioner" caps = { "mon": "profile rbd, allow command 'osd blocklist'", @@ -967,10 +984,12 @@ class RadosJSON: ) else: caps["osd"] = f"profile rbd pool={rbd_pool_name}" + if generation != 0: + entity = f"{entity}.{generation}" return caps, entity - def get_rbd_node_caps_and_entity(self): + def get_rbd_node_caps_and_entity(self, generation): entity = "client.csi-rbd-node" caps = { "mon": "profile rbd, allow command 'osd blocklist'", @@ -1002,10 +1021,12 @@ class RadosJSON: ) else: caps["osd"] = f"profile rbd pool={rbd_pool_name}" + if generation != 0: + entity = f"{entity}.{generation}" return caps, entity - def get_defaultUser_caps_and_entity(self): + def get_defaultUser_caps_and_entity(self, generation): entity = self.run_as_user caps = { "mon": "allow r, allow command quorum_status, allow command version", @@ -1014,40 +1035,34 @@ class RadosJSON: "mds": "allow *", } + if entity == self.EXTERNAL_USER_NAME: + if generation != 0: + entity = f"{entity}.{generation}" + return caps, entity - def get_caps_and_entity(self, user_name): + def get_caps_and_entity(self, user_name, generation): if "client.csi-cephfs-provisioner" in user_name: - if "client.csi-cephfs-provisioner" != user_name: - self._arg_parser.restricted_auth_permission = True - return self.get_cephfs_provisioner_caps_and_entity() + return self.get_cephfs_provisioner_caps_and_entity(generation) if "client.csi-cephfs-node" in user_name: - if "client.csi-cephfs-node" != user_name: - self._arg_parser.restricted_auth_permission = True - return self.get_cephfs_node_caps_and_entity() + return self.get_cephfs_node_caps_and_entity(generation) if "client.csi-rbd-provisioner" in user_name: - if "client.csi-rbd-provisioner" != user_name: - self._arg_parser.restricted_auth_permission = True - return self.get_rbd_provisioner_caps_and_entity() + return self.get_rbd_provisioner_caps_and_entity(generation) if "client.csi-rbd-node" in user_name: - if "client.csi-rbd-node" != user_name: - self._arg_parser.restricted_auth_permission = True - return self.get_rbd_node_caps_and_entity() + return self.get_rbd_node_caps_and_entity(generation) if "client.healthchecker" in user_name: - if "client.healthchecker" != user_name: - self._arg_parser.restricted_auth_permission = True - return self.get_defaultUser_caps_and_entity() + return self.get_defaultUser_caps_and_entity(generation) raise ExecutionFailureException( f"no user found with user_name: {user_name}, " "get_caps_and_entity command failed.\n" ) - def create_cephCSIKeyring_user(self, user): + def create_cephCSIKeyring_user(self, user, generation): """ command: ceph auth get-or-create client.csi-cephfs-provisioner mon 'allow r' mgr 'allow rw' osd 'allow rw tag cephfs metadata=*' """ - caps, entity = self.get_caps_and_entity(user) + caps, entity = self.get_caps_and_entity(user, generation) cmd_json = { "prefix": "auth get-or-create", "entity": entity, @@ -1070,8 +1085,9 @@ class RadosJSON: # check if user already exist user_key = self.check_user_exist(entity) if user_key != "": - return user_key, f"{entity.split('.', 1)[1]}" - # entity.split('.',1)[1] to rename entity(client.csi-rbd-node) as csi-rbd-node + # entity.split('.')[1] to rename entity(client.csi-rbd-node) as csi-rbd-node + # entity.split('.')[1] to rename entity(client.csi-rbd-node.5) as csi-rbd-node + return user_key, f"{entity.split('.')[1]}" ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) # if there is an unsuccessful attempt, @@ -1080,8 +1096,10 @@ class RadosJSON: f"'auth get-or-create {user}' command failed.\n" f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" ) - return str(json_out[0]["key"]), f"{entity.split('.', 1)[1]}" - # entity.split('.',1)[1] to rename entity(client.csi-rbd-node) as csi-rbd-node + + # entity.split('.')[1] to rename entity(client.csi-rbd-node) as csi-rbd-node + # entity.split('.')[1] to rename entity(client.csi-rbd-node.5) as csi-rbd-node + return str(json_out[0]["key"]), f"{entity.split('.')[1]}" def get_cephfs_data_pool_details(self): cmd_json = {"prefix": "fs ls", "format": "json"} @@ -1182,8 +1200,8 @@ class RadosJSON: f"Using the data-pool: '{self._arg_parser.cephfs_data_pool_name}'\n" ) - def create_checkerKey(self, user): - caps, entity = self.get_caps_and_entity(user) + def create_checkerKey(self, user, generation): + caps, entity = self.get_caps_and_entity(user, generation) cmd_json = { "prefix": "auth get-or-create", "entity": entity, @@ -1212,6 +1230,7 @@ class RadosJSON: f"'auth get-or-create {self.run_as_user}' command failed\n" f"Error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" ) + return str(json_out[0]["key"]) def get_ceph_dashboard_link(self): @@ -1563,24 +1582,65 @@ class RadosJSON: # python treats flag-name as flag_name internally, so converting back to flag-name, # so we can get those values from config file argValue = arg.replace("_", "-") - # do not add the config-file flag and also not add the boolean flags which are set to False - # because config.ini file treats boolean flags as True always + # do not add the `config-file`, `cephx-key-rotate` flags + # and also not add the boolean flags which are set to False, because config.ini file treats boolean flags as True always if ( argValue != "config-file" + and argValue != "cephx-key-rotate" and getattr(self._arg_parser, arg) != False ): argument += f"{argValue} = {str(getattr(self._arg_parser, arg))}\n" return argument - def _gen_output_map(self): + def get_cephx_latest_key_generation(self): + # starting with -1, to support cases where no key exists + generation = -1 + + # run ceph auth ls to list all the keys + cmd_json = {"prefix": "auth ls", "format": "json"} + if self._arg_parser.dry_run: + return self.dry_run("ceph " + cmd_json["prefix"]) + ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) + # if there is an unsuccessful attempt, + if ret_val != 0 or len(json_out) == 0: + raise ExecutionFailureException( + f"'auth ls' ceph call failed with error: {err_msg if ret_val != 0 else self.EMPTY_OUTPUT_LIST}" + ) + + # get a sample csi key based on user flags that will be created by the script + # needed to get the key type and get its highest generation + # setting the generation 0 as we just need the prefix type + # For example, client.csi-rbd-node, client.csi-rbd-node-cluster1-pool1-rn1 + _, csi_entity = self.get_rbd_node_caps_and_entity(0) + # remove "client." prefix from the csi_entity + csi_entity = f"{csi_entity.split('.')[1]}" + + # find the latest key generation + # by looking at the suffix after '.' in the ceph user name list + for eachJ in json_out["auth_dump"]: + words = eachJ["entity"].split(".") + if words[1].startswith(csi_entity) == False: + continue + + if len(words) == 2: + generation = max(0, generation) + elif len(words) == 3: + try: + generation = max(generation, int(words[2])) + except ValueError: + raise ExecutionFailureException( + f"Invalid csi ceph user name found in the auth ls output: '{eachJ}'" + ) + else: + raise ExecutionFailureException( + f"Invalid csi ceph user name found in the auth ls output: '{eachJ}'" + ) + + return generation + + def _gen_output_map(self, generation): if self.out_map: return - # support legacy flag with upgrades - if self._arg_parser.cluster_name: - self._arg_parser.k8s_cluster_name = self._arg_parser.cluster_name - self._arg_parser.k8s_cluster_name = ( - self._arg_parser.k8s_cluster_name.lower() - ) # always convert cluster name to lowercase characters self.validate_rbd_pool(self._arg_parser.rbd_data_pool_name) self.init_rbd_pool(self._arg_parser.rbd_data_pool_name) self.validate_rados_namespace() @@ -1588,23 +1648,24 @@ class RadosJSON: self.get_cephfs_data_pool_details() # double string needed for upstream exports of flags self.out_map["ARGS"] = f'"{self.getFinalUsedArgs()}"' + self.out_map["CEPHX_KEY_GENERATION"] = generation self.out_map["NAMESPACE"] = self._arg_parser.namespace self.out_map["K8S_CLUSTER_NAME"] = self._arg_parser.k8s_cluster_name self.out_map["ROOK_EXTERNAL_FSID"] = self.get_fsid() self.out_map["ROOK_EXTERNAL_USERNAME"] = self.run_as_user self.out_map["ROOK_EXTERNAL_CEPH_MON_DATA"] = self.get_ceph_external_mon_data() self.out_map["ROOK_EXTERNAL_USER_SECRET"] = self.create_checkerKey( - "client.healthchecker" + "client.healthchecker", generation ) self.out_map["ROOK_EXTERNAL_DASHBOARD_LINK"] = self.get_ceph_dashboard_link() ( self.out_map["CSI_RBD_NODE_SECRET"], self.out_map["CSI_RBD_NODE_SECRET_NAME"], - ) = self.create_cephCSIKeyring_user("client.csi-rbd-node") + ) = self.create_cephCSIKeyring_user("client.csi-rbd-node", generation) ( self.out_map["CSI_RBD_PROVISIONER_SECRET"], self.out_map["CSI_RBD_PROVISIONER_SECRET_NAME"], - ) = self.create_cephCSIKeyring_user("client.csi-rbd-provisioner") + ) = self.create_cephCSIKeyring_user("client.csi-rbd-provisioner", generation) self.out_map["CEPHFS_POOL_NAME"] = self._arg_parser.cephfs_data_pool_name self.out_map["CEPHFS_METADATA_POOL_NAME"] = ( self._arg_parser.cephfs_metadata_pool_name @@ -1622,11 +1683,13 @@ class RadosJSON: ( self.out_map["CSI_CEPHFS_NODE_SECRET"], self.out_map["CSI_CEPHFS_NODE_SECRET_NAME"], - ) = self.create_cephCSIKeyring_user("client.csi-cephfs-node") + ) = self.create_cephCSIKeyring_user("client.csi-cephfs-node", generation) ( self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"], self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"], - ) = self.create_cephCSIKeyring_user("client.csi-cephfs-provisioner") + ) = self.create_cephCSIKeyring_user( + "client.csi-cephfs-provisioner", generation + ) # create the default "csi" subvolumegroup self.get_or_create_subvolume_group( "csi", self._arg_parser.cephfs_filesystem_name @@ -1733,8 +1796,8 @@ class RadosJSON: err = "Please provide all the RGW multisite parameters or none of them" sys.stderr.write(err) - def gen_shell_out(self): - self._gen_output_map() + def gen_shell_out(self, generation): + self._gen_output_map(generation) shOutIO = StringIO() for k, v in self.out_map.items(): if v and k not in self._excluded_keys: @@ -1743,8 +1806,8 @@ class RadosJSON: shOutIO.close() return shOut - def gen_json_out(self): - self._gen_output_map() + def gen_json_out(self, generation): + self._gen_output_map(generation) if self._arg_parser.dry_run: return "" json_out = [ @@ -1777,7 +1840,9 @@ class RadosJSON: "name": "rook-ceph-operator-creds", "kind": "Secret", "data": { - "userID": self.out_map["ROOK_EXTERNAL_USERNAME"], + "userID": self.get_user_id( + self.out_map["ROOK_EXTERNAL_USERNAME"], generation + ), "userKey": self.out_map["ROOK_EXTERNAL_USER_SECRET"], }, }, @@ -1809,7 +1874,9 @@ class RadosJSON: "name": f"rook-{self.out_map['CSI_RBD_NODE_SECRET_NAME']}", "kind": "Secret", "data": { - "userID": self.out_map["CSI_RBD_NODE_SECRET_NAME"], + "userID": self.get_user_id( + self.out_map["CSI_RBD_NODE_SECRET_NAME"], generation + ), "userKey": self.out_map["CSI_RBD_NODE_SECRET"], }, } @@ -1824,7 +1891,9 @@ class RadosJSON: "name": f"rook-{self.out_map['CSI_RBD_PROVISIONER_SECRET_NAME']}", "kind": "Secret", "data": { - "userID": self.out_map["CSI_RBD_PROVISIONER_SECRET_NAME"], + "userID": self.get_user_id( + self.out_map["CSI_RBD_PROVISIONER_SECRET_NAME"], generation + ), "userKey": self.out_map["CSI_RBD_PROVISIONER_SECRET"], }, } @@ -1839,7 +1908,10 @@ class RadosJSON: "name": f"rook-{self.out_map['CSI_CEPHFS_PROVISIONER_SECRET_NAME']}", "kind": "Secret", "data": { - "adminID": self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"], + "adminID": self.get_user_id( + self.out_map["CSI_CEPHFS_PROVISIONER_SECRET_NAME"], + generation, + ), "adminKey": self.out_map["CSI_CEPHFS_PROVISIONER_SECRET"], }, } @@ -1854,7 +1926,9 @@ class RadosJSON: "name": f"rook-{self.out_map['CSI_CEPHFS_NODE_SECRET_NAME']}", "kind": "Secret", "data": { - "adminID": self.out_map["CSI_CEPHFS_NODE_SECRET_NAME"], + "adminID": self.get_user_id( + self.out_map["CSI_CEPHFS_NODE_SECRET_NAME"], generation + ), "adminKey": self.out_map["CSI_CEPHFS_NODE_SECRET"], }, } @@ -2005,7 +2079,15 @@ class RadosJSON: return json.dumps(json_out) + LINESEP - def upgrade_users_permissions(self): + def get_user_id(self, user_name, generation): + """ + The user ID is constructed as 'user_name.generation'. + """ + if generation == 0: + return user_name + return f"{user_name}.{generation}" + + def upgrade_users_permissions(self, generation): users = [ "client.csi-cephfs-node", "client.csi-cephfs-provisioner", @@ -2013,10 +2095,20 @@ class RadosJSON: "client.csi-rbd-provisioner", "client.healthchecker", ] + + if generation != 0: + users += [ + f"client.csi-cephfs-node.{generation}", + f"client.csi-cephfs-provisioner.{generation}", + f"client.csi-rbd-node.{generation}", + f"client.csi-rbd-provisioner.{generation}", + f"client.healthchecker.{generation}", + ] + if self.run_as_user != "" and self.run_as_user not in users: users.append(self.run_as_user) for user in users: - self.upgrade_user_permissions(user) + self.upgrade_user_permissions(user, generation) def get_rgw_pool_name_during_upgrade(self, user, caps): if user == "client.healthchecker": @@ -2033,7 +2125,7 @@ class RadosJSON: "failed to get rgw pool name for upgrade" ) - def upgrade_user_permissions(self, user): + def upgrade_user_permissions(self, user, generation): # check whether the given user exists or not cmd_json = {"prefix": "auth get", "entity": f"{user}", "format": "json"} ret_val, json_out, err_msg = self._common_cmd_json_gen(cmd_json) @@ -2042,7 +2134,7 @@ class RadosJSON: return existing_caps = json_out[0]["caps"] self.get_rgw_pool_name_during_upgrade(user, str(existing_caps)) - new_cap, _ = self.get_caps_and_entity(user) + new_cap, _ = self.get_caps_and_entity(user, generation) cap_keys = ["mon", "mgr", "osd", "mds"] caps = [] for eachCap in cap_keys: @@ -2078,13 +2170,32 @@ class RadosJSON: print(f"Updated user {user} successfully.") def main(self): + # support legacy flag with upgrades + if self._arg_parser.cluster_name: + self._arg_parser.k8s_cluster_name = self._arg_parser.cluster_name + self._arg_parser.k8s_cluster_name = ( + self._arg_parser.k8s_cluster_name.lower() + ) # always convert cluster name to lowercase characters + generation = 0 + current_generation = self.get_cephx_latest_key_generation() + if self._arg_parser.cephx_key_rotate == "rotate": + generation = current_generation + 1 + elif self._arg_parser.cephx_key_rotate == "revert" and current_generation > 0: + generation = current_generation - 1 + else: + if current_generation == -1: + # create the keys with generation 0 as no keys exist + generation = 0 + else: + generation = current_generation + generated_output = "" if self._arg_parser.upgrade: - self.upgrade_users_permissions() + self.upgrade_users_permissions(generation) elif self._arg_parser.format == "json": - generated_output = self.gen_json_out() + generated_output = self.gen_json_out(generation) elif self._arg_parser.format == "bash": - generated_output = self.gen_shell_out() + generated_output = self.gen_shell_out(generation) else: raise ExecutionFailureException( f"Unsupported format: {self._arg_parser.format}" diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh index 5435ccaa6..748c03ba8 100644 --- a/deploy/examples/import-external-cluster.sh +++ b/deploy/examples/import-external-cluster.sh @@ -138,6 +138,7 @@ function importClusterID() { } function importSecret() { + userID=$(getUserId "$ROOK_EXTERNAL_USERNAME") if ! $KUBECTL -n "$NAMESPACE" get secret "$MON_SECRET_NAME" &>/dev/null; then $KUBECTL -n "$NAMESPACE" \ create \ @@ -149,10 +150,15 @@ function importSecret() { --from-literal="$MON_SECRET_FSID_KEYNAME"="$ROOK_EXTERNAL_FSID" \ --from-literal="$MON_SECRET_ADMIN_KEYRING_KEYNAME"="$ROOK_EXTERNAL_ADMIN_SECRET" \ --from-literal="$MON_SECRET_MON_KEYRING_KEYNAME"="$ROOK_EXTERNAL_MONITOR_SECRET" \ - --from-literal="$MON_SECRET_CEPH_USERNAME_KEYNAME"="$ROOK_EXTERNAL_USERNAME" \ + --from-literal="$MON_SECRET_CEPH_USERNAME_KEYNAME"="$userID" \ --from-literal="$MON_SECRET_CEPH_SECRET_KEYNAME"="$ROOK_EXTERNAL_USER_SECRET" else echo "secret $MON_SECRET_NAME already exists" + $KUBECTL -n "$NAMESPACE" \ + patch \ + secret \ + "$MON_SECRET_NAME" \ + -p "{\"stringData\":{\"$MON_SECRET_CEPH_USERNAME_KEYNAME\":\"$userID\"}}" fi } @@ -187,7 +193,18 @@ function createInputCommadConfigMap() { fi } +function getUserId(){ + id="$1" + if [ -z "$CEPHX_KEY_GENERATION" ] || [ "$CEPHX_KEY_GENERATION" == 0 ]; then + echo "$id" + else + result="$id.$CEPHX_KEY_GENERATION" + echo "$result" + fi +} + function importCsiRBDNodeSecret() { + userID=$(getUserId "$CSI_RBD_NODE_SECRET_NAME") if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_RBD_NODE_SECRET_NAME" &>/dev/null; then $KUBECTL -n "$NAMESPACE" \ create \ @@ -195,14 +212,20 @@ function importCsiRBDNodeSecret() { generic \ --type="kubernetes.io/rook" \ "rook-""$CSI_RBD_NODE_SECRET_NAME" \ - --from-literal=userID="$CSI_RBD_NODE_SECRET_NAME" \ + --from-literal=userID="$userID" \ --from-literal=userKey="$CSI_RBD_NODE_SECRET" else - echo "secret rook-$CSI_RBD_NODE_SECRET_NAME already exists" + echo "secret 'rook-$CSI_RBD_NODE_SECRET_NAME' already exists" + $KUBECTL -n "$NAMESPACE" \ + patch \ + secret \ + "rook-$CSI_RBD_NODE_SECRET_NAME" \ + -p "{\"stringData\":{\"adminID\":\"$userID\",\"adminKey\":\"$CSI_CEPHFS_NODE_SECRET\"}}" fi } function importCsiRBDProvisionerSecret() { + userID=$(getUserId "$CSI_RBD_PROVISIONER_SECRET_NAME") if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" &>/dev/null; then $KUBECTL -n "$NAMESPACE" \ create \ @@ -210,14 +233,20 @@ function importCsiRBDProvisionerSecret() { generic \ --type="kubernetes.io/rook" \ "rook-""$CSI_RBD_PROVISIONER_SECRET_NAME" \ - --from-literal=userID="$CSI_RBD_PROVISIONER_SECRET_NAME" \ + --from-literal=userID="$userID" \ --from-literal=userKey="$CSI_RBD_PROVISIONER_SECRET" else - echo "secret $CSI_RBD_PROVISIONER_SECRET_NAME already exists" + echo "secret 'rook-$CSI_RBD_PROVISIONER_SECRET_NAME' already exists" + $KUBECTL -n "$NAMESPACE" \ + patch \ + secret \ + "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" \ + -p "{\"stringData\":{\"adminID\":\"$userID\",\"adminKey\":\"$CSI_CEPHFS_NODE_SECRET\"}}" fi } function importCsiCephFSNodeSecret() { + userID=$(getUserId "$CSI_CEPHFS_NODE_SECRET_NAME") if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_CEPHFS_NODE_SECRET_NAME" &>/dev/null; then $KUBECTL -n "$NAMESPACE" \ create \ @@ -225,14 +254,20 @@ function importCsiCephFSNodeSecret() { generic \ --type="kubernetes.io/rook" \ "rook-""$CSI_CEPHFS_NODE_SECRET_NAME" \ - --from-literal=adminID="$CSI_CEPHFS_NODE_SECRET_NAME" \ + --from-literal=adminID="$userID" \ --from-literal=adminKey="$CSI_CEPHFS_NODE_SECRET" else - echo "secret $CSI_CEPHFS_NODE_SECRET_NAME already exists" - fi + echo "secret 'rook-$CSI_CEPHFS_NODE_SECRET_NAME' already exists" + $KUBECTL -n "$NAMESPACE" \ + patch \ + secret \ + "rook-$CSI_CEPHFS_NODE_SECRET_NAME" \ + -p "{\"stringData\":{\"adminID\":\"$userID\",\"adminKey\":\"$CSI_CEPHFS_NODE_SECRET\"}}" + fi } function importCsiCephFSProvisionerSecret() { + userID=$(getUserId "$CSI_CEPHFS_PROVISIONER_SECRET_NAME") if ! $KUBECTL -n "$NAMESPACE" get secret "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" &>/dev/null; then $KUBECTL -n "$NAMESPACE" \ create \ @@ -240,10 +275,15 @@ function importCsiCephFSProvisionerSecret() { generic \ --type="kubernetes.io/rook" \ "rook-""$CSI_CEPHFS_PROVISIONER_SECRET_NAME" \ - --from-literal=adminID="$CSI_CEPHFS_PROVISIONER_SECRET_NAME" \ + --from-literal=adminID="$userID" \ --from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET" else - echo "secret $CSI_CEPHFS_PROVISIONER_SECRET_NAME already exists" + echo "secret 'rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME' already exists" + $KUBECTL -n "$NAMESPACE" \ + patch \ + secret \ + "rook-$CSI_CEPHFS_PROVISIONER_SECRET_NAME" \ + -p "{\"stringData\":{\"adminID\":\"$userID\",\"adminKey\":\"$CSI_CEPHFS_NODE_SECRET\"}}" fi } diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 8a77f3242..ae7fd09a6 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -805,6 +805,30 @@ function delete_cluster() { remove_cluster_prerequisites } +function check_keys_exists() { + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + for key in "${@}"; do + if kubectl -n rook-ceph exec "$toolbox" -- ceph auth get "$key"; then + echo "key '$key' exists" + else + echo "key '$key' not exists" + exit 1 + fi + done +} + +function check_keys_not_exists() { + toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}') + for key in "${@}"; do + if kubectl -n rook-ceph exec "$toolbox" -- ceph auth get "$key"; then + echo "key '$key' exists" + exit 1 + else + echo "key '$key' not exists" + fi + done +} + FUNCTION="$1" shift # remove function arg now that we've recorded it # call the function with the remainder of the user-provided args