diff --git a/.github/workflows/integration-test-keystone-auth-suite.yaml b/.github/workflows/integration-test-keystone-auth-suite.yaml new file mode 100644 index 000000000..a10fb2c55 --- /dev/null +++ b/.github/workflows/integration-test-keystone-auth-suite.yaml @@ -0,0 +1,66 @@ +name: Integration test CephKeystoneAuthSuite +on: + pull_request: + branches: + - master + - release-* + paths-ignore: + - "Documentation/**" + - "design/**" + +defaults: + run: + # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell + shell: bash --noprofile --norc -eo pipefail -x {0} + +# cancel the in-progress workflow when PR is refreshed. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + TestCephKeystoneAuthSuite: + if: ${{ github.event_name == 'pull_request' && github.ref != 'refs/heads/master' && !contains(github.event.pull_request.labels.*.name, 'skip-ci') }} + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + kubernetes-versions: ["v1.25.16", "v1.28.4"] + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: consider debugging + uses: ./.github/workflows/tmate_debug + with: + use-tmate: ${{ secrets.USE_TMATE }} + + - name: setup latest cluster resources + uses: ./.github/workflows/integration-test-config-latest-k8s + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + kubernetes-version: ${{ matrix.kubernetes-versions }} + + - name: TestCephKeystoneAuthSuite + run: | + tests/scripts/github-action-helper.sh collect_udev_logs_in_background + tests/scripts/helm.sh up + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) + SKIP_CLEANUP_POLICY=false go test -v -timeout 3600s -failfast -run CephKeystoneAuthSuite github.com/rook/rook/tests/integration + + - name: collect common logs + if: always() + run: | + export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" + export CLUSTER_NAMESPACE="keystoneauth-ns" + export OPERATOR_NAMESPACE="keystoneauth-ns-system" + tests/scripts/collect-logs.sh + + - name: Artifact + uses: actions/upload-artifact@v4 + if: failure() + with: + name: ceph-keystone-auth-suite-artifact-${{ matrix.kubernetes-versions }} + path: /home/runner/work/rook/rook/tests/integration/_output/tests/ diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index 645277adb..9ef7ca235 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -96,6 +96,68 @@ When the `zone` section is set pools with the object stores name will not be cre This is useful for applications that need object store credentials to be created in their own namespace, where neither OBCs nor COSI is being used to create buckets. The default is empty. +## Auth Settings + +The `auth`-section allows the configuration of authentication providers in addition to the regular authentication mechanism. + +Currently only OpenStack Keystone is supported. + +### Keystone Settings + +The keystone authentication can be configured in the `spec.auth.keystone` section of the CRD: + +```yaml +spec: + [...] + auth: + keystone: + acceptedRoles: + - admin + - member + - service + implicitTenants: "swift" + revocationInterval: 1200 + serviceUserSecretName: usersecret + tokenCacheSize: 1000 + url: https://keystone.example-namespace.svc/ + protocols: + swift: + accountInUrl: true + urlPrefix: /swift + [...] +``` + +Note: With this example configuration S3 is implicitly enabled even though it is not enabled in the `protocols` section. + +The following options can be configured in the `keystone`-section: + +* `acceptedRoles`: The OpenStack Keystone [roles](https://docs.openstack.org/keystone/latest/admin/cli-manage-projects-users-and-roles.html#roles-and-role-assignments) accepted by RGW when authenticating against Keystone. +* `implicitTenants`: Indicates whether to use implicit tenants. This can be `true`, `false`, `swift` and `s3`. For more details see the Ceph RadosGW documentation on [multitenancy](https://docs.ceph.com/en/latest/radosgw/multitenancy/). +* `revocationInterval`: The number of seconds between token revocation checks. +* `serviceUserSecretName`: the name of the user secret containing the credentials for the admin user to use by rgw when communicating with Keystone. See [Object Store with Keystone and Swift](../../Storage-Configuration/Object-Storage-RGW/ceph-object-swift.md) for more details on what the secret must contain. +* `tokenCacheSize`: specifies the maximum number of entries in each Keystone token cache. +* `url`: The url of the Keystone API endpoint to use. + +The protocols section is divided into two parts: + +- a section to configure S3 +- a section to configure swift + +#### protocols/S3 settings + +In the `s3` section of the `protocols` section the following options can be configured: + +* `authKeystone`: Whether S3 should also authenticated using Keystone (`true`) or not (`false`). If set to `false` the default S3 auth will be used. +* `enabled`: Whether to enable S3 (`true`) or not (`false`). The default is `true` even if the section is not listed at all! Please note that S3 should not be disabled in a [Ceph Multi Site configuration](https://docs.ceph.com/en/quincy/radosgw/multisite). + +#### protocols/swift settings + +In the `swift` section of the `protocols` section the following options can be configured: + +* `accountInUrl`: Whether or not the Swift account name should be included in the Swift API URL. If set to `false` (the default), the Swift API will listen on a URL formed like `http://host:port//v1`. If set to `true`, the Swift API URL will be `http://host:port//v1/AUTH_`. This option must be set to `true` if radosgw should support publicly-readable containers and temporary URLs. +* `urlPrefix`: The URL prefix for the Swift API, to distinguish it from the S3 API endpoint. The default is `swift`, which makes the Swift API available at the URL `http://host:port/swift/v1` (or `http://host:port/swift/v1/AUTH_%(tenant_id)s` if rgw swift account in url is enabled). "Warning: If you set this option to `/`, the S3 API is automatically disabled. It is not possible to operate radosgw with an urlPrefix of `/` and simultaneously support both the S3 and Swift APIs. [...]" [(see Ceph documentation on swift settings)](https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings). +* `versioningEnabled`: If set to `true`, enables the Object Versioning of OpenStack Object Storage API. This allows clients to put the X-Versions-Location attribute on containers that should be versioned. + ## Gateway Settings The gateway settings correspond to the RGW daemon settings. diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 55e09b32c..fc32110fc 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -1847,6 +1847,34 @@ GatewaySpec +protocols
+ + +ProtocolSpec + + + + +(Optional) +

The protocol specification

+ + + + +auth
+ + +AuthSpec + + + + +(Optional) +

The authentication configuration

+ + + + zone
@@ -2611,6 +2639,38 @@ CIDRList

AnnotationsSpec is the main spec annotation for all daemons

+

AuthSpec +

+

+(Appears on:ObjectStoreSpec) +

+
+

AuthSpec represents the authentication protocol configuration of a Ceph Object Store Gateway

+
+ + + + + + + + + + + + + +
FieldDescription
+keystone
+ + +KeystoneSpec + + +		 +(Optional) +

The spec for Keystone

+

BucketNotificationEvent (string alias)

@@ -7070,6 +7130,32 @@ string +

ImplicitTenantSetting +(string alias)

+

+(Appears on:KeystoneSpec) +

+
+
+ + + + + + + + + + + + + + + + + + +
ValueDescription

""

"false"

"s3"

"swift"

"true"

KafkaEndpointSpec

@@ -7420,6 +7506,95 @@ string +

KeystoneSpec +

+

+(Appears on:AuthSpec) +

+
+

KeystoneSpec represents the Keystone authentication configuration of a Ceph Object Store Gateway

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+url
+ +string + +		 +

The URL for the Keystone server.

+
+serviceUserSecretName
+ +string + +		 +

The name of the secret containing the credentials for the service user account used by RGW. It has to be in the same namespace as the object store resource.

+
+acceptedRoles
+ +[]string + +		 +

The roles requires to serve requests.

+
+implicitTenants
+ + +ImplicitTenantSetting + + +		 +(Optional) +

Create new users in their own tenants of the same name. Possible values are true, false, swift and s3. The latter have the effect of splitting the identity space such that only the indicated protocol will use implicit tenants.

+
+tokenCacheSize
+ +int + +		 +(Optional) +

The maximum number of entries in each Keystone token cache.

+
+revocationInterval
+ +int + +		 +(Optional) +

The number of seconds between token revocation checks.

+

Labels (map[string]string alias)

@@ -9386,6 +9561,34 @@ GatewaySpec +protocols
+ + +ProtocolSpec + + + + +(Optional) +

The protocol specification

+ + + + +auth
+ + +AuthSpec + + + + +(Optional) +

The authentication configuration

+ + + + zone
@@ -10740,6 +10943,52 @@ alive or ready to receive traffic.

+

ProtocolSpec +

+

+(Appears on:ObjectStoreSpec) +

+
+

ProtocolSpec represents a Ceph Object Store protocol specification

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+s3
+ + +S3Spec + + +		 +(Optional) +

The spec for S3

+
+swift
+ + +SwiftSpec + + +		 +(Optional) +

The spec for Swift

+

PullSpec

@@ -11110,6 +11359,48 @@ HybridStorageSpec

ResourceSpec is a collection of ResourceRequirements that describes the compute resource requirements

+

S3Spec +

+

+(Appears on:ProtocolSpec) +

+
+

S3Spec represents Ceph Object Store specification for the S3 API

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+enabled
+ +bool + +		 +(Optional) +

Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.

+
+authUseKeystone
+ +bool + +		 +(Optional) +

Whether to use Keystone for authentication. This option maps directly to the rgw_s3_auth_use_keystone option. Enabling it allows generating S3 credentials via an OpenStack API call, see the docs. If not given, the defaults of the corresponding RGW option apply.

+

SSSDSidecar

@@ -12387,6 +12678,60 @@ string +

SwiftSpec +

+

+(Appears on:ProtocolSpec) +

+
+

SwiftSpec represents Ceph Object Store specification for the Swift API

+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+accountInUrl
+ +bool + +		 +(Optional) +

Whether or not the Swift account name should be included in the Swift API URL. If set to false (the default), then the Swift API will listen on a URL formed like http://host:port//v1. If set to true, the Swift API URL will be http://host:port//v1/AUTH_. You must set this option to true (and update the Keystone service catalog) if you want radosgw to support publicly-readable containers and temporary URLs.

+
+urlPrefix
+ +string + +		 +(Optional) +

The URL prefix for the Swift API, to distinguish it from the S3 API endpoint. The default is swift, which makes the Swift API available at the URL http://host:port/swift/v1 (or http://host:port/swift/v1/AUTH_%(tenant_id)s if rgw swift account in url is enabled).

+
+versioningEnabled
+ +bool + +		 +(Optional) +

Enables the Object Versioning of OpenStack Object Storage API. This allows clients to put the X-Versions-Location attribute on containers that should be versioned.

+

TopicEndpointSpec

diff --git a/Documentation/Contributing/rook-test-framework.md b/Documentation/Contributing/rook-test-framework.md index bd8402f6f..fbb212503 100644 --- a/Documentation/Contributing/rook-test-framework.md +++ b/Documentation/Contributing/rook-test-framework.md @@ -56,7 +56,7 @@ See [environment.go](/tests/framework/installer/environment.go) for the availabl Set the following variables: ```console -export TEST_HELM_PATH=/tmp/rook-tests-scripts-helm/linux-amd64/helm +export TEST_HELM_PATH=/tmp/rook-tests-scripts-helm/helm export TEST_BASE_DIR=WORKING_DIR export TEST_SCRATCH_DEVICE=/dev/vdb ``` diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-swift.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-swift.md new file mode 100644 index 000000000..89dc0544a --- /dev/null +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-swift.md @@ -0,0 +1,143 @@ +--- +title: Object Store with Keystone and Swift +--- + +!!! note + The Object Store with Keystone and Swift is currently in experimental mode. + +Ceph RGW can integrate natively with the Swift API and Keystone via the CephObjectStore CRD. This allows native integration of Rook-operated Ceph RGWs into [OpenStack](https://www.openstack.org/) clouds. + +!!! note + Authentication via the OBC and COSI features is not affected by this configuration. + +## Create a Local Object Store with Keystone and Swift + +This example will create a `CephObjectStore` that starts the RGW service in the cluster providing a Swift API. +Using Swift requires the use of [OpenStack Keystone](https://docs.openstack.org/keystone/latest/) as an authentication provider. + +The OSDs must be located on different nodes, because the [`failureDomain`](../../CRDs/Block-Storage/ceph-block-pool-crd.md#spec) is set to `host` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). + +More details on the settings available for a `CephObjectStore` (including the `Auth` section) can be found in the [Object Store CRD](../../CRDs/Object-Storage/ceph-object-store-crd.md#object-store-settings) document. + +Set the url in the auth section to point to the keystone service url. + +Prior to using keystone as authentication provider an admin user for rook to access and configure the keystone admin api is required. + +The user credentials for this admin user are provided by a secret in the same namespace which is referenced via the `serviceUserSecretName` property. +The secret contains the credentials with names analogue to the environment variables used in an OpenStack `openrc` file. + +!!! note + This example requires *at least 3 bluestore OSDs*, with each OSD located on a *different node*. + This example assumes an existing OpenStack Keystone instance ready to use for authentication. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: usersecret +data: + OS_AUTH_TYPE: cGFzc3dvcmQ= + OS_IDENTITY_API_VERSION: Mw== + OS_PASSWORD: c2VjcmV0 + OS_PROJECT_DOMAIN_NAME: RGVmYXVsdA== + OS_PROJECT_NAME: YWRtaW4= + OS_USER_DOMAIN_NAME: RGVmYXVsdA== + OS_USERNAME: YWRtaW4= +type: Opaque +``` + +```yaml +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: my-store + namespace: rook-ceph +spec: + metadataPool: + failureDomain: host + replicated: + size: 3 + dataPool: + failureDomain: host + erasureCoded: + dataChunks: 2 + codingChunks: 1 + auth: + keystone: + acceptedRoles: + - admin + - member + - service + implicitTenants: "swift" + revocationInterval: 1200 + serviceUserSecretName: usersecret + tokenCacheSize: 1000 + url: https://keystone.rook-ceph.svc/ + protocols: + swift: + accountInUrl: true + urlPrefix: /swift + # note that s3 is enabled by default if protocols.s3.enabled is not explicitly set to false + preservePoolsOnDelete: true + gateway: + sslCertificateRef: + port: 80 + # securePort: 443 + instances: 1 +``` + +After the `CephObjectStore` is created, the Rook operator will create all the pools and other resources necessary to start the service. This may take a minute to complete. + +```console +kubectl create -f object.yaml +``` + +The start of the RGW pod(s) confirms that the object store is configured. + +```console +kubectl -n rook-ceph get pod -l app=rook-ceph-rgw +``` + +The swift service endpoint in OpenStack/Keystone must be created, in order to use the object store in Swift using for example the [OpenStack CLI](https://docs.openstack.org/python-openstackclient/latest/). +The endpoint url should be set to the service endpoint of the created rgw instance. + +```sh +openstack service create --name swift object-store +openstack endpoint create --region default --enable swift admin https://rook-ceph-rgw-default.rook-ceph.svc/swift/v1 +openstack endpoint create --region default --enable swift internal https://rook-ceph-rgw-default.rook-ceph.svc/swift/v1 +``` + +Afterwards any user which has the rights to access the projects resources (as defined in the OpenStack Keystone instance) can access the object store and create container and objects. +Here the username and project are explicitly set to reflect use of the (non-admin) user. + +```sh +export OS_USERNAME=alice +export OS_PROJECT=exampleProject +openstack container create exampleContainer +# put /etc/hosts in the new created container +openstack object create exampleContainer /etc/hosts +# retrieve and save the file +openstack object save --file /tmp/hosts.saved exampleContainer /etc/hosts +openstack object delete exampleContainer /etc/hosts +openstack container delete exampleContainer +``` + +## Basic concepts + +When using Keystone as an authentication provider, Ceph uses the credentials of an admin user (provided in the secret references by `serviceUserSecretName`) to access Keystone. + +For each user accessing the object store using Swift, Ceph implicitly creates a user which must be represented in Keystone with an authorized counterpart. +Keystone checks for a user of the same name. Based on the name and other parameters ((OpenStack Keystone) project, (OpenStack Keystone) role) Keystone allows or disallows access to a swift container or object. Note that the implicitly created users are creaded in addition to any users that are created through other means, so Keystone authentication is not exclusive. + +It is not necessary to create any users in OpenStack Keystone (except for the admin user provided in the `serviceUserSecretName`). + +## Keystone setup + +Keystone must support the v3-API-Version to be used with Rook. Other API versions are not supported. + +The admin user and all users accessing the Object store must exist and their authorizations configured accordingly in Keystone. + +## Openstack setup + +To use the Object Store in OpenStack using Swift the Swift service must be set and the endpoint urls for the Swift service created. +The example configuration "Create a Local Object Store with Keystone and Swift" above contains more details and the corresponding CLI calls. diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md index 241159791..50bfcd176 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md @@ -2,7 +2,7 @@ title: Object Storage Overview --- -Object storage exposes an S3 API to the storage cluster for applications to put and get data. +Object storage exposes an S3 API and or a [Swift API](https://developer.openstack.org/api-ref/object-store/index.html) to the storage cluster for applications to put and get data. ## Prerequisites @@ -12,7 +12,7 @@ This guide assumes a Rook cluster as explained in the [Quickstart](../../Getting Rook can configure the Ceph Object Store for several different scenarios. See each linked section for the configuration details. -1. Create a [local object store](#create-a-local-object-store) with dedicated Ceph pools. This option is recommended if a single object store is required, and is the simplest to get started. +1. Create a [local object store](#create-a-local-object-store-with-s3) with dedicated Ceph pools. This option is recommended if a single object store is required, and is the simplest to get started. 2. Create [one or more object stores with shared Ceph pools](#create-local-object-stores-with-shared-pools). This option is recommended when multiple object stores are required. 3. Connect to an [RGW service in an external Ceph cluster](#connect-to-an-external-object-store), rather than create a local object store. 4. Configure [RGW Multisite](#object-multisite) to synchronize buckets between object stores in different clusters. @@ -20,7 +20,11 @@ Rook can configure the Ceph Object Store for several different scenarios. See ea !!! note Updating the configuration of an object store between these types is not supported. -### Create a Local Object Store +Rook has the ability to either deploy an object store in Kubernetes or to connect to an external RGW service. +Most commonly, the object store will be configured in Kubernetes by Rook. +Alternatively see the [external section](#connect-to-an-external-object-store) to consume an existing Ceph cluster with [Rados Gateways](https://docs.ceph.com/en/quincy/radosgw/index.html) from Rook. + +### Create a Local Object Store with S3 The below sample will create a `CephObjectStore` that starts the RGW service in the cluster with an S3 API. @@ -584,3 +588,10 @@ Multisite is a feature of Ceph that allows object stores to replicate its data o Multisite also allows object stores to be independent and isolated from other object stores in a cluster. For more information on multisite please read the [ceph multisite overview](ceph-object-multisite.md) for how to run it. + +## Using Swift and Keystone + +It is possible to access an object store using the [Swift API](https://developer.openstack.org/api-ref/object-store/index.html). +Using Swift requires the use of [OpenStack Keystone](https://docs.openstack.org/keystone/latest/) as an authentication provider. + +More information on the use of Swift and Keystone can be found in the document on [Object Store with Keystone and Swift](ceph-object-swift.md). diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 6105c3a32..0d2dcc8a7 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -13,3 +13,4 @@ please update the `BucketClass` and `BucketAccessClass` for resolving refer [her - Added support for Ceph Squid (v19) - Allow updating the device class of OSDs, if `allowDeviceClassUpdate: true` is set +- Support for keystone authentication for s3 and swift (see [#9088](https://github.com/rook/rook/issues/9088)). \ No newline at end of file diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 2155c2cd9..263c0f27f 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -10506,6 +10506,41 @@ spec: items: type: string type: array + auth: + description: The authentication configuration + properties: + keystone: + description: The spec for Keystone + nullable: true + properties: + acceptedRoles: + description: The roles requires to serve requests. + items: + type: string + type: array + implicitTenants: + description: Create new users in their own tenants of the same name. Possible values are true, false, swift and s3. The latter have the effect of splitting the identity space such that only the indicated protocol will use implicit tenants. + type: string + revocationInterval: + description: The number of seconds between token revocation checks. + nullable: true + type: integer + serviceUserSecretName: + description: The name of the secret containing the credentials for the service user account used by RGW. It has to be in the same namespace as the object store resource. + type: string + tokenCacheSize: + description: The maximum number of entries in each Keystone token cache. + nullable: true + type: integer + url: + description: The URL for the Keystone server. + type: string + required: + - acceptedRoles + - serviceUserSecretName + - url + type: object + type: object dataPool: description: The data pool settings nullable: true @@ -11882,6 +11917,40 @@ spec: preservePoolsOnDelete: description: Preserve pools on object store deletion type: boolean + protocols: + description: The protocol specification + properties: + s3: + description: The spec for S3 + nullable: true + properties: + authUseKeystone: + description: Whether to use Keystone for authentication. This option maps directly to the rgw_s3_auth_use_keystone option. Enabling it allows generating S3 credentials via an OpenStack API call, see the docs. If not given, the defaults of the corresponding RGW option apply. + nullable: true + type: boolean + enabled: + description: Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. + nullable: true + type: boolean + type: object + swift: + description: The spec for Swift + nullable: true + properties: + accountInUrl: + description: Whether or not the Swift account name should be included in the Swift API URL. If set to false (the default), then the Swift API will listen on a URL formed like http://host:port//v1. If set to true, the Swift API URL will be http://host:port//v1/AUTH_. You must set this option to true (and update the Keystone service catalog) if you want radosgw to support publicly-readable containers and temporary URLs. + nullable: true + type: boolean + urlPrefix: + description: The URL prefix for the Swift API, to distinguish it from the S3 API endpoint. The default is swift, which makes the Swift API available at the URL http://host:port/swift/v1 (or http://host:port/swift/v1/AUTH_%(tenant_id)s if rgw swift account in url is enabled). + nullable: true + type: string + versioningEnabled: + description: Enables the Object Versioning of OpenStack Object Storage API. This allows clients to put the X-Versions-Location attribute on containers that should be versioned. + nullable: true + type: boolean + type: object + type: object security: description: Security represents security settings nullable: true diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 5be751e71..5e87f7a14 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -10497,6 +10497,41 @@ spec: items: type: string type: array + auth: + description: The authentication configuration + properties: + keystone: + description: The spec for Keystone + nullable: true + properties: + acceptedRoles: + description: The roles requires to serve requests. + items: + type: string + type: array + implicitTenants: + description: Create new users in their own tenants of the same name. Possible values are true, false, swift and s3. The latter have the effect of splitting the identity space such that only the indicated protocol will use implicit tenants. + type: string + revocationInterval: + description: The number of seconds between token revocation checks. + nullable: true + type: integer + serviceUserSecretName: + description: The name of the secret containing the credentials for the service user account used by RGW. It has to be in the same namespace as the object store resource. + type: string + tokenCacheSize: + description: The maximum number of entries in each Keystone token cache. + nullable: true + type: integer + url: + description: The URL for the Keystone server. + type: string + required: + - acceptedRoles + - serviceUserSecretName + - url + type: object + type: object dataPool: description: The data pool settings nullable: true @@ -11873,6 +11908,40 @@ spec: preservePoolsOnDelete: description: Preserve pools on object store deletion type: boolean + protocols: + description: The protocol specification + properties: + s3: + description: The spec for S3 + nullable: true + properties: + authUseKeystone: + description: Whether to use Keystone for authentication. This option maps directly to the rgw_s3_auth_use_keystone option. Enabling it allows generating S3 credentials via an OpenStack API call, see the docs. If not given, the defaults of the corresponding RGW option apply. + nullable: true + type: boolean + enabled: + description: Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. + nullable: true + type: boolean + type: object + swift: + description: The spec for Swift + nullable: true + properties: + accountInUrl: + description: Whether or not the Swift account name should be included in the Swift API URL. If set to false (the default), then the Swift API will listen on a URL formed like http://host:port//v1. If set to true, the Swift API URL will be http://host:port//v1/AUTH_. You must set this option to true (and update the Keystone service catalog) if you want radosgw to support publicly-readable containers and temporary URLs. + nullable: true + type: boolean + urlPrefix: + description: The URL prefix for the Swift API, to distinguish it from the S3 API endpoint. The default is swift, which makes the Swift API available at the URL http://host:port/swift/v1 (or http://host:port/swift/v1/AUTH_%(tenant_id)s if rgw swift account in url is enabled). + nullable: true + type: string + versioningEnabled: + description: Enables the Object Versioning of OpenStack Object Storage API. This allows clients to put the X-Versions-Location attribute on containers that should be versioned. + nullable: true + type: boolean + type: object + type: object security: description: Security represents security settings nullable: true diff --git a/design/ceph/object/swift-and-keystone-integration.md b/design/ceph/object/swift-and-keystone-integration.md index 27717ade5..05fac374e 100644 --- a/design/ceph/object/swift-and-keystone-integration.md +++ b/design/ceph/object/swift-and-keystone-integration.md @@ -51,10 +51,13 @@ to further technologies (such as LDAP authentication). create a separate `cephobjectstoreuser`, configure its access rights to the bucket and use those credentials. +* Support for Kubernetes Container Object Storage (COSI) + * Support for authentication technologies other than Keystone (e.g. LDAP) * Exposing options that disable security features (e.g. TLS verification) + ## Proposal details The Object Store CRD will have to be extended to accommodate the new @@ -80,11 +83,11 @@ Annotations: options](https://docs.ceph.com/en/octopus/radosgw/config-ref/#keystone-settings), the corresponding RGW option is formed by prefixing it with `rgw_keystone_` and replacing upper case letters by their lower case - letter followed by an underscore. E.g. `tokenCacheSize` maps to + letter preceded by an underscore. E.g. `tokenCacheSize` maps to `rgw_keystone_token_cache_size`. * `[2]` These settings are required in the `keystone` section if present. -* `[1]` The name of the secret containing the credentials for the +* `[3]` The name of the secret containing the credentials for the service user account used by RGW. It has to be in the same namespace as the object store resource. @@ -173,12 +176,12 @@ Annotations: options](https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings), the corresponding RGW option is formed by prefixing it with `rgw_swift_` and replacing upper case letters by their lower case - letter followed by an underscore. E.g. `urlPrefix` maps to + letter preceded by an underscore. E.g. `urlPrefix` maps to `rgw_swift_url_prefix`. They are optional. If not given, the defaults of the corresponding RGW option apply. -The access to the Swift API is granted by creating a subuser of an RGW -user. While commonly the access is granted via projects +Access to the Swift API is granted by creating a subuser of an RGW +user. While commonly access is granted via projects mapped from Keystone, explicit creation of subusers is supported by extending the `cephobjectstoreuser` resource with a new optional section `spec.subUsers`: diff --git a/go.mod b/go.mod index b6b1c9aa1..a2a0c28e9 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.75.2 github.com/prometheus-operator/prometheus-operator/pkg/client v0.75.2 github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7 + github.com/sethvargo/go-password v0.2.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 0716004f7..e2a6b411c 100644 --- a/go.sum +++ b/go.sum @@ -826,6 +826,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= +github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= diff --git a/pkg/apis/ceph.rook.io/v1/spec_test.go b/pkg/apis/ceph.rook.io/v1/spec_test.go index 3bbbe6142..88e95992c 100644 --- a/pkg/apis/ceph.rook.io/v1/spec_test.go +++ b/pkg/apis/ceph.rook.io/v1/spec_test.go @@ -92,3 +92,78 @@ storage: assert.Equal(t, expectedSpec, clusterSpec) } + +func newTrue() *bool { + t := true + return &t +} + +func newFalse() *bool { + t := false + return &t +} + +func newInt(val int) *int { + return &val +} + +func newString(val string) *string { + return &val +} + +func TestObjectStoreSpecMarshalSwiftAndKeystone(t *testing.T) { + // Assert that the new ObjectStoreSpec fields specified in are correctly parsed + specYaml := []byte(` +auth: + keystone: + url: https://keystone:5000/ + acceptedRoles: ["_member_", "service", "admin"] + implicitTenants: swift + tokenCacheSize: 1000 + revocationInterval: 1200 + serviceUserSecretName: rgw-service-user +protocols: + swift: + accountInUrl: true + urlPrefix: /example + versioningEnabled: false + s3: + enabled: false + authUseKeystone: true +`) + rawJSON, err := yaml.ToJSON(specYaml) + assert.Nil(t, err) + fmt.Printf("rawJSON: %s\n", string(rawJSON)) + + // unmarshal the JSON into a strongly typed storage spec object + var objectStoreSpec ObjectStoreSpec + err = json.Unmarshal(rawJSON, &objectStoreSpec) + assert.Nil(t, err) + + // the unmarshalled storage spec should equal the expected spec below + expectedSpec := ObjectStoreSpec{ + Auth: AuthSpec{ + Keystone: &KeystoneSpec{ + Url: "https://keystone:5000/", + AcceptedRoles: []string{"_member_", "service", "admin"}, + ImplicitTenants: "swift", + TokenCacheSize: newInt(1000), + RevocationInterval: newInt(1200), + ServiceUserSecretName: "rgw-service-user", + }, + }, + Protocols: ProtocolSpec{ + S3: &S3Spec{ + Enabled: newFalse(), + AuthUseKeystone: newTrue(), + }, + Swift: &SwiftSpec{ + AccountInUrl: newTrue(), + UrlPrefix: newString("/example"), + VersioningEnabled: newFalse(), + }, + }, + } + + assert.Equal(t, expectedSpec, objectStoreSpec) +} diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index b0dfeb64a..e305698ab 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -1473,6 +1473,14 @@ type ObjectStoreSpec struct { // +nullable Gateway GatewaySpec `json:"gateway"` + // The protocol specification + // +optional + Protocols ProtocolSpec `json:"protocols,omitempty"` + + // The authentication configuration + // +optional + Auth AuthSpec `json:"auth,omitempty"` + // The multisite info // +optional // +nullable @@ -1644,6 +1652,86 @@ type EndpointAddress struct { Hostname string `json:"hostname,omitempty" protobuf:"bytes,3,opt,name=hostname"` } +// ProtocolSpec represents a Ceph Object Store protocol specification +type ProtocolSpec struct { + // The spec for S3 + // +optional + // +nullable + S3 *S3Spec `json:"s3,omitempty"` + + // The spec for Swift + // +optional + // +nullable + Swift *SwiftSpec `json:"swift"` +} + +// S3Spec represents Ceph Object Store specification for the S3 API +type S3Spec struct { + // Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled. + // +nullable + // +optional + Enabled *bool `json:"enabled,omitempty"` + // Whether to use Keystone for authentication. This option maps directly to the rgw_s3_auth_use_keystone option. Enabling it allows generating S3 credentials via an OpenStack API call, see the docs. If not given, the defaults of the corresponding RGW option apply. + // +nullable + // +optional + AuthUseKeystone *bool `json:"authUseKeystone,omitempty"` +} + +// SwiftSpec represents Ceph Object Store specification for the Swift API +type SwiftSpec struct { + // Whether or not the Swift account name should be included in the Swift API URL. If set to false (the default), then the Swift API will listen on a URL formed like http://host:port//v1. If set to true, the Swift API URL will be http://host:port//v1/AUTH_. You must set this option to true (and update the Keystone service catalog) if you want radosgw to support publicly-readable containers and temporary URLs. + // +nullable + // +optional + AccountInUrl *bool `json:"accountInUrl,omitempty"` + // The URL prefix for the Swift API, to distinguish it from the S3 API endpoint. The default is swift, which makes the Swift API available at the URL http://host:port/swift/v1 (or http://host:port/swift/v1/AUTH_%(tenant_id)s if rgw swift account in url is enabled). + // +nullable + // +optional + UrlPrefix *string `json:"urlPrefix,omitempty"` + // Enables the Object Versioning of OpenStack Object Storage API. This allows clients to put the X-Versions-Location attribute on containers that should be versioned. + // +nullable + // +optional + VersioningEnabled *bool `json:"versioningEnabled,omitempty"` +} + +// AuthSpec represents the authentication protocol configuration of a Ceph Object Store Gateway +type AuthSpec struct { + // The spec for Keystone + // +optional + // +nullable + Keystone *KeystoneSpec `json:"keystone,omitempty"` +} + +// KeystoneSpec represents the Keystone authentication configuration of a Ceph Object Store Gateway +type KeystoneSpec struct { + // The URL for the Keystone server. + Url string `json:"url"` + // The name of the secret containing the credentials for the service user account used by RGW. It has to be in the same namespace as the object store resource. + ServiceUserSecretName string `json:"serviceUserSecretName"` + // The roles requires to serve requests. + AcceptedRoles []string `json:"acceptedRoles"` + // Create new users in their own tenants of the same name. Possible values are true, false, swift and s3. The latter have the effect of splitting the identity space such that only the indicated protocol will use implicit tenants. + // +optional + ImplicitTenants ImplicitTenantSetting `json:"implicitTenants,omitempty"` + // The maximum number of entries in each Keystone token cache. + // +optional + // +nullable + TokenCacheSize *int `json:"tokenCacheSize,omitempty"` + // The number of seconds between token revocation checks. + // +optional + // +nullable + RevocationInterval *int `json:"revocationInterval,omitempty"` +} + +type ImplicitTenantSetting string + +const ( + ImplicitTenantSwift ImplicitTenantSetting = "swift" + ImplicitTenantS3 ImplicitTenantSetting = "s3" + ImplicitTenantTrue ImplicitTenantSetting = "true" + ImplicitTenantFalse ImplicitTenantSetting = "false" + ImplicitTenantDefault ImplicitTenantSetting = "" +) + // ZoneSpec represents a Ceph Object Store Gateway Zone specification type ZoneSpec struct { // RGW Zone the Object Store is in diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go index 8ebe1237a..31082d7bf 100644 --- a/pkg/operator/ceph/object/config.go +++ b/pkg/operator/ceph/object/config.go @@ -110,7 +110,50 @@ func (c *clusterConfig) generateKeyring(rgwConfig *rgwConfig) (string, error) { return keyring, s.CreateOrUpdate(rgwConfig.ResourceName, keyring) } +func mapKeystoneSecretToConfig(cfg map[string]string, secret *v1.Secret) (map[string]string, error) { + + requiredKeys := []string{"OS_PROJECT_DOMAIN_NAME", + "OS_USER_DOMAIN_NAME", + "OS_PROJECT_DOMAIN_NAME", + "OS_USER_DOMAIN_NAME", + "OS_PROJECT_NAME", + "OS_USERNAME", + "OS_PASSWORD"} + + data := make(map[string]string) + for key, value := range secret.Data { + data[key] = string(value[:]) + } + + for _, key := range requiredKeys { + if value, ok := data[key]; !ok || value == "" { + return nil, errors.New(fmt.Sprintf("Missing or empty %s", key)) + } + } + + if authType, ok := data["OS_AUTH_TYPE"]; ok && authType != "password" { + return nil, errors.New(fmt.Sprintf("OS_AUTHTYPE %s is not supported. Only OS_AUTH_TYPE password is supported!", authType)) + } + + if apiVersion, ok := data["OS_IDENTITY_API_VERSION"]; ok && apiVersion != "3" { + return nil, errors.New(fmt.Sprintf("OS_IDENTITY_API_VERSION %s is not supported! Only OS_IDENTITY_API_VERSION 3 is supported!", apiVersion)) + } + + if data["OS_PROJECT_DOMAIN_NAME"] != data["OS_USER_DOMAIN_NAME"] { + return nil, errors.New("The user domain name does not match the project domain name.") + } + + cfg["rgw_keystone_api_version"] = data["OS_IDENTITY_API_VERSION"] + cfg["rgw_keystone_admin_domain"] = data["OS_PROJECT_DOMAIN_NAME"] + cfg["rgw_keystone_admin_project"] = data["OS_PROJECT_NAME"] + cfg["rgw_keystone_admin_user"] = data["OS_USERNAME"] + cfg["rgw_keystone_admin_password"] = data["OS_PASSWORD"] + + return cfg, nil +} + func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { + monStore := cephconfig.GetMonStore(c.context, c.clusterInfo) who := generateCephXUser(rgwConfig.ResourceName) configOptions := make(map[string]string) @@ -126,6 +169,60 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { configOptions["rgw_zone"] = rgwConfig.Zone configOptions["rgw_zonegroup"] = rgwConfig.ZoneGroup + configOptions, err := configureKeystoneAuthentication(rgwConfig, configOptions) + if err != nil { + return err + } + + s3disabled := false + if s3 := rgwConfig.Protocols.S3; s3 != nil { + if s3.Enabled != nil && !*s3.Enabled { + s3disabled = true + } + + if s3.AuthUseKeystone != nil { + configOptions["rgw_s3_auth_use_keystone"] = fmt.Sprintf("%t", *s3.AuthUseKeystone) + } + + } + + if swift := rgwConfig.Protocols.Swift; swift != nil { + + if swift.AccountInUrl != nil { + configOptions["rgw_swift_account_in_url"] = fmt.Sprintf("%t", *swift.AccountInUrl) + } + + if swift.UrlPrefix != nil { + configOptions["rgw_swift_url_prefix"] = *swift.UrlPrefix + + if configOptions["rgw_swift_url_prefix"] == "/" { + logger.Warning("Forcefully disabled S3 as the swift prefix is given as a slash /. Ignoring any S3 options (including Enabled=true)!") + // this will later on disable the s3 api using the rgw_enable_apis setting + s3disabled = true + } + + } + if swift.VersioningEnabled != nil { + configOptions["rgw_swift_versioning_enabled"] = fmt.Sprintf("%t", *swift.VersioningEnabled) + } + + } + + if s3disabled { + // XXX: how to handle enabled APIs? We only configure s3 and + // swift in the resource, `admin` is required for the operator to + // work, `swift_auth` is required to access swift without keystone + // – not sure about the additional APIs + // https://docs.ceph.com/en/quincy/radosgw/config-ref/#confval-rgw_enable_apis + // see also https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings on disabling s3 + // when using '/' as prefix + + // Swift was enabled so far already by default, so perhaps better + // not change that if someone relies on it. + + configOptions["rgw_enable_apis"] = "s3website, swift, swift_auth, admin, sts, iam, notifications" + } + for flag, val := range configOptions { err := monStore.Set(who, flag, val) if err != nil { @@ -136,6 +233,54 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error { return nil } +func configureKeystoneAuthentication(rgwConfig *rgwConfig, configOptions map[string]string) (map[string]string, error) { + + keystone := rgwConfig.Auth.Keystone + if keystone == nil { + logger.Debug("Authentication with keystone is disabled") + return configOptions, nil + } + + logger.Info("Configuring authentication with keystone") + + configOptions["rgw_keystone_url"] = keystone.Url + configOptions["rgw_keystone_accepted_roles"] = strings.Join(keystone.AcceptedRoles, ",") + if keystone.ImplicitTenants != "" { + lc := strings.ToLower(string(keystone.ImplicitTenants)) + + // only four values are valid here (swift, s3, true and false) + // + // https://docs.ceph.com/en/latest/radosgw/keystone/#integrating-with-openstack-keystone + if lc != "true" && + lc != "false" && + lc != "swift" && + lc != "s3" { + + return nil, errors.New(fmt.Sprintf("ImplicitTenantSetting can only be 'swift', 's3', 'true' or 'false', not %q", string(keystone.ImplicitTenants))) + + } + + configOptions["rgw_keystone_implicit_tenants"] = lc + + } + + if keystone.TokenCacheSize != nil { + configOptions["rgw_keystone_token_cache_size"] = fmt.Sprintf("%d", *keystone.TokenCacheSize) + } + + if rgwConfig.KeystoneSecret == nil { + return nil, errors.New("Cannot find keystone secret") + } + + var err error + configOptions, err = mapKeystoneSecretToConfig(configOptions, rgwConfig.KeystoneSecret) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("error mapping keystone secret %s to config", rgwConfig.KeystoneSecret.Name)) + } + + return configOptions, nil +} + func (c *clusterConfig) deleteFlagsMonConfigStore(rgwName string) error { monStore := cephconfig.GetMonStore(c.context, c.clusterInfo) who := generateCephXUser(rgwName) diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index c7feee60e..651978ec9 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -459,8 +459,17 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c return r.setFailedStatus(k8sutil.ObservedGenerationNotAvailable, namespacedName, "failed to configure multisite for object store", err) } - // Create or Update Store - err = cfg.createOrUpdateStore(realmName, zoneGroupName, zoneName) + // Retrieve the keystone secret if specified + var keystoneSecret *corev1.Secret + if ks := cephObjectStore.Spec.Auth.Keystone; ks != nil { + keystoneSecret, err = objContext.Context.Clientset.CoreV1().Secrets(objContext.clusterInfo.Namespace).Get(objContext.clusterInfo.Context, ks.ServiceUserSecretName, metav1.GetOptions{}) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to get the keystone credential secret") + } + } + + // Create or Update store + err = cfg.createOrUpdateStore(realmName, zoneGroupName, zoneName, keystoneSecret) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create object store %q", cephObjectStore.Name) } diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 85acee08b..558100bf4 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -61,6 +61,10 @@ type rgwConfig struct { Realm string ZoneGroup string Zone string + + Auth cephv1.AuthSpec + KeystoneSecret *v1.Secret + Protocols cephv1.ProtocolSpec } var updateDeploymentAndWait = mon.UpdateCephDeploymentAndWait @@ -69,10 +73,10 @@ var ( insecureSkipVerify = "insecureSkipVerify" ) -func (c *clusterConfig) createOrUpdateStore(realmName, zoneGroupName, zoneName string) error { +func (c *clusterConfig) createOrUpdateStore(realmName, zoneGroupName, zoneName string, keystoneSecret *v1.Secret) error { logger.Infof("creating object store %q in namespace %q", c.store.Name, c.store.Namespace) - if err := c.startRGWPods(realmName, zoneGroupName, zoneName); err != nil { + if err := c.startRGWPods(realmName, zoneGroupName, zoneName, keystoneSecret); err != nil { return errors.Wrap(err, "failed to start rgw pods") } @@ -94,7 +98,7 @@ func (c *clusterConfig) createOrUpdateStore(realmName, zoneGroupName, zoneName s return nil } -func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) error { +func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string, keystoneSecret *v1.Secret) error { // backward compatibility, triggered during updates if c.store.Spec.Gateway.Instances < 1 { // Set the minimum of at least one instance @@ -126,11 +130,14 @@ func (c *clusterConfig) startRGWPods(realmName, zoneGroupName, zoneName string) resourceName := fmt.Sprintf("%s-%s-%s", AppName, c.store.Name, daemonLetterID) rgwConfig := &rgwConfig{ - ResourceName: resourceName, - DaemonID: daemonName, - Realm: realmName, - ZoneGroup: zoneGroupName, - Zone: zoneName, + ResourceName: resourceName, + DaemonID: daemonName, + Realm: realmName, + ZoneGroup: zoneGroupName, + Zone: zoneName, + Auth: c.store.Spec.Auth, + Protocols: c.store.Spec.Protocols, + KeystoneSecret: keystoneSecret, } // We set the owner reference of the Secret to the Object controller instead of the replicaset diff --git a/pkg/operator/ceph/object/rgw_test.go b/pkg/operator/ceph/object/rgw_test.go index 65829b354..2d0ddaae1 100644 --- a/pkg/operator/ceph/object/rgw_test.go +++ b/pkg/operator/ceph/object/rgw_test.go @@ -18,6 +18,7 @@ package object import ( "context" + "strings" "testing" "time" @@ -82,7 +83,7 @@ func TestStartRGW(t *testing.T) { t.Run("Deployment is created", func(t *testing.T) { store.Spec.Gateway.Instances = 1 - err := c.startRGWPods(store.Name, store.Name, store.Name) + err := c.startRGWPods(store.Name, store.Name, store.Name, nil) assert.Nil(t, err) validateStart(ctx, t, c, clientset) @@ -95,7 +96,7 @@ func TestStartRGW(t *testing.T) { // Purge store of configurations applied to gateways appliedRgwConfigurations = make(map[string]string) - err := c.startRGWPods(store.Name, store.Name, store.Name) + err := c.startRGWPods(store.Name, store.Name, store.Name, nil) assert.Nil(t, err) assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread") @@ -109,7 +110,7 @@ func TestStartRGW(t *testing.T) { // Purge store of configurations applied to gateways appliedRgwConfigurations = make(map[string]string) - err := c.startRGWPods(store.Name, store.Name, store.Name) + err := c.startRGWPods(store.Name, store.Name, store.Name, nil) assert.Nil(t, err) assert.Contains(t, appliedRgwConfigurations, "rgw_run_sync_thread") @@ -139,9 +140,20 @@ func TestCreateObjectStore(t *testing.T) { } return "", nil } + + timeoutCommand := func(timeout time.Duration, command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + for _, arg := range args { + assert.False(t, strings.Contains(arg, "swift")) + assert.False(t, strings.Contains(arg, "keystone")) + } + return "", nil + } + executor := &exectest.MockExecutor{ MockExecuteCommandWithCombinedOutput: commandWithOutputFunc, MockExecuteCommandWithOutput: commandWithOutputFunc, + MockExecuteCommandWithTimeout: timeoutCommand, } store := simpleStore() @@ -157,7 +169,7 @@ func TestCreateObjectStore(t *testing.T) { r := &ReconcileCephObjectStore{client: cl, scheme: s} ownerInfo := client.NewMinimumOwnerInfoWithOwnerRef() c := &clusterConfig{context, info, store, "1.2.3.4", &cephv1.ClusterSpec{}, ownerInfo, data, r.client} - err := c.createOrUpdateStore(store.Name, store.Name, store.Name) + err := c.createOrUpdateStore(store.Name, store.Name, store.Name, nil) assert.Nil(t, err) } @@ -172,6 +184,57 @@ func simpleStore() *cephv1.CephObjectStore { } } +func TestCreateObjectStoreWithKeystoneAndS3(t *testing.T) { + commandWithOutputFunc := func(command string, args ...string) (string, error) { + logger.Infof("Command: %s %v", command, args) + if command == "ceph" { + if args[1] == "erasure-code-profile" { + return `{"k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}`, nil + } + if args[0] == "auth" && args[1] == "get-or-create-key" { + return `{"key":"mykey"}`, nil + } + } else { + return `{"realms": []}`, nil + } + return "", nil + } + executor := &exectest.MockExecutor{ + MockExecuteCommandWithCombinedOutput: commandWithOutputFunc, + MockExecuteCommandWithOutput: commandWithOutputFunc, + } + + store := simpleStoreWithKeystoneAndS3() + clientset := test.New(t, 3) + context := &clusterd.Context{Executor: executor, Clientset: clientset} + info := clienttest.CreateTestClusterInfo(1) + data := config.NewStatelessDaemonDataPathMap(config.RgwType, "my-fs", "rook-ceph", "/var/lib/rook/") + + // create the pools + s := scheme.Scheme + object := []runtime.Object{&cephv1.CephObjectStore{}} + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + r := &ReconcileCephObjectStore{client: cl, scheme: s} + ownerInfo := client.NewMinimumOwnerInfoWithOwnerRef() + c := &clusterConfig{context, info, store, "1.2.3.4", &cephv1.ClusterSpec{}, ownerInfo, data, r.client} + err := c.createOrUpdateStore(store.Name, store.Name, store.Name, nil) + assert.Nil(t, err) +} + +func simpleStoreWithKeystoneAndS3() *cephv1.CephObjectStore { + authUseKeystone := true + return &cephv1.CephObjectStore{ + ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "mycluster"}, + Spec: cephv1.ObjectStoreSpec{ + MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}, + DataPool: cephv1.PoolSpec{ErasureCoded: cephv1.ErasureCodedSpec{CodingChunks: 1, DataChunks: 2}}, + Gateway: cephv1.GatewaySpec{Port: 123}, + Auth: cephv1.AuthSpec{Keystone: &cephv1.KeystoneSpec{Url: "testurl", ServiceUserSecretName: "testname", AcceptedRoles: []string{"testrole"}}}, + Protocols: cephv1.ProtocolSpec{S3: &cephv1.S3Spec{AuthUseKeystone: &authUseKeystone}}, + }, + } +} + func TestGenerateSecretName(t *testing.T) { cl := fake.NewClientBuilder().Build() diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index f348dfa10..2ff73e3d8 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -365,7 +365,7 @@ func (r *ReconcileObjectStoreUser) createOrUpdateCephUser(u *cephv1.CephObjectSt } // Set access and secret key - if r.userConfig.Keys == nil { + if r.userConfig.Keys == nil || len(r.userConfig.Keys) == 0 { r.userConfig.Keys = make([]admin.UserKeySpec, 1) } r.userConfig.Keys[0].AccessKey = user.Keys[0].AccessKey @@ -446,6 +446,7 @@ func generateUserConfig(user *cephv1.CephObjectStoreUser) admin.User { userConfig := admin.User{ ID: user.Name, DisplayName: displayName, + Keys: make([]admin.UserKeySpec, 0), } defaultMaxBuckets := 1000 diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index a5df65847..a23703fdc 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -18,7 +18,6 @@ package clients import ( "fmt" - "github.com/coreos/pkg/capnslog" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -40,10 +39,11 @@ func CreateObjectOperation(k8sh *utils.K8sHelper, manifests installer.CephManife } // ObjectCreate Function to create a object store in rook -func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32, tlsEnable bool) error { +func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32, tlsEnable bool, swiftAndKeystone bool) error { logger.Info("creating the object store via CRD") - if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(storeName, int(replicaCount), rgwPort, tlsEnable)); err != nil { + + if err := o.k8sh.ResourceOperation("apply", o.manifests.GetObjectStore(storeName, int(replicaCount), rgwPort, tlsEnable, swiftAndKeystone)); err != nil { return err } diff --git a/tests/framework/installer/ceph_helm_installer.go b/tests/framework/installer/ceph_helm_installer.go index 95a5f4359..a8042469c 100644 --- a/tests/framework/installer/ceph_helm_installer.go +++ b/tests/framework/installer/ceph_helm_installer.go @@ -282,7 +282,7 @@ func (h *CephInstaller) CreateFileSystemConfiguration(values map[string]interfac // CreateObjectStoreConfiguration creates an object store configuration func (h *CephInstaller) CreateObjectStoreConfiguration(values map[string]interface{}, name, scName string) error { - testObjectStoreBytes := []byte(h.Manifests.GetObjectStore(name, 2, 8080, false)) + testObjectStoreBytes := []byte(h.Manifests.GetObjectStore(name, 2, 8080, false, false)) var testObjectStoreCRD map[string]interface{} if err := yaml.Unmarshal(testObjectStoreBytes, &testObjectStoreCRD); err != nil { return err diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 32288fa1f..43a72d9bd 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -47,7 +47,7 @@ type CephManifests interface { GetNFS(name string, daemonCount int) string GetNFSPool() string GetRBDMirror(name string, daemonCount int) string - GetObjectStore(name string, replicaCount, port int, tlsEnable bool) string + GetObjectStore(name string, replicaCount, port int, tlsEnable bool, swiftAndKeystone bool) string GetObjectStoreUser(name, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) string GetBucketStorageClass(storeName, storageClassName, reclaimPolicy string) string GetOBC(obcName, storageClassName, bucketName string, maxObject string, createBucket bool) string @@ -464,35 +464,30 @@ spec: requireSafeReplicaSize: false` } -func (m *CephManifestsMaster) GetObjectStore(name string, replicaCount, port int, tlsEnable bool) string { - if tlsEnable { - return `apiVersion: ceph.rook.io/v1 -kind: CephObjectStore -metadata: - name: ` + name + ` - namespace: ` + m.settings.Namespace + ` -spec: - metadataPool: - replicated: - size: 1 - requireSafeReplicaSize: false - compressionMode: passive - dataPool: - replicated: - size: 1 - requireSafeReplicaSize: false - gateway: - resources: null - securePort: ` + strconv.Itoa(port) + ` - instances: ` + strconv.Itoa(replicaCount) + ` - sslCertificateRef: ` + name + ` -` +func (m *CephManifestsMaster) GetObjectStore(name string, replicaCount, port int, tlsEnable bool, swiftAndKeystone bool) string { + type Spec struct { + Name string + TLS bool + Port int + ReplicaCount int + SwiftAndKeystone bool + Manifests *CephManifestsMaster } - return `apiVersion: ceph.rook.io/v1 + + spec := Spec{ + Name: name, + TLS: tlsEnable, + ReplicaCount: replicaCount, + Port: port, + SwiftAndKeystone: swiftAndKeystone, + Manifests: m, + } + + tmpl := `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: - name: ` + name + ` - namespace: ` + m.settings.Namespace + ` + name: {{ .Name }} + namespace: {{ .Manifests.Settings.Namespace }} spec: metadataPool: replicated: @@ -503,11 +498,36 @@ spec: replicated: size: 1 requireSafeReplicaSize: false + {{ if .SwiftAndKeystone }} + auth: + keystone: + acceptedRoles: + - admin + - member + - service + implicitTenants: "true" + revocationInterval: 1200 + serviceUserSecretName: usersecret + tokenCacheSize: 1000 + url: https://keystone.{{ .Manifests.Settings.Namespace }}.svc/ + protocols: + swift: + accountInUrl: false + urlPrefix: foobar + s3: + enabled: true + authUseKeystone: true + {{ end }} gateway: resources: null - port: ` + strconv.Itoa(port) + ` - instances: ` + strconv.Itoa(replicaCount) + ` -` + {{ if .TLS }}securePort: {{ .Port }}{{ else }}port: {{ .Port }}{{ end }} + instances: {{ .ReplicaCount }} + {{ if .SwiftAndKeystone }} + caBundleRef: keystone-bundle + {{ end }} + {{ if .TLS }}sslCertificateRef: {{ .Name }}{{ end }}` + + return renderTemplate(tmpl, spec) } func (m *CephManifestsMaster) GetObjectStoreUser(name, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) string { diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go index 5d9d99b2a..3d48fb66c 100644 --- a/tests/framework/installer/ceph_manifests_previous.go +++ b/tests/framework/installer/ceph_manifests_previous.go @@ -131,8 +131,11 @@ func (m *CephManifestsPreviousVersion) GetNFSPool() string { return m.latest.GetNFSPool() } -func (m *CephManifestsPreviousVersion) GetObjectStore(name string, replicaCount, port int, tlsEnable bool) string { - return m.latest.GetObjectStore(name, replicaCount, port, tlsEnable) +func (m *CephManifestsPreviousVersion) GetObjectStore(name string, replicaCount, port int, tlsEnable bool, swiftAndKeystone bool) string { + if swiftAndKeystone { + panic("Previous version does not support swift or keystone") + } + return m.latest.GetObjectStore(name, replicaCount, port, tlsEnable, false) } func (m *CephManifestsPreviousVersion) GetObjectStoreUser(name, displayName, store, usercaps, maxsize string, maxbuckets, maxobjects int) string { diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index c44602b95..0206f5048 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -18,7 +18,9 @@ package installer import ( "fmt" + "strings" "testing" + "text/template" "github.com/coreos/pkg/capnslog" "github.com/rook/rook/tests/framework/utils" @@ -59,3 +61,16 @@ func checkError(t *testing.T, err error, message string) { } assert.NoError(t, err, "%s. %+v", message, err) } + +func renderTemplate(templateSource string, data any) string { + templateInstance, err := template.New("template").Parse(templateSource) + if err != nil { + panic(fmt.Errorf("syntax error in template: %s", err)) + } + var builder strings.Builder + err = templateInstance.Execute(&builder, data) + if err != nil { + panic(fmt.Errorf("error while rendering the template: %s", err)) + } + return builder.String() +} diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index c69b06dd4..61344c357 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -154,9 +154,9 @@ func (k8sh *K8sHelper) SetDeploymentVersion(namespace, deploymentName, container return err } -// Kubectl is wrapper for executing kubectl commands -func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { - result, err := k8sh.executor.ExecuteCommandWithTimeout(15*time.Second, "kubectl", args...) +// KubectlWithTimeout is wrapper for executing kubectl commands +func (k8sh *K8sHelper) KubectlWithTimeout(timeout time.Duration, args ...string) (string, error) { + result, err := k8sh.executor.ExecuteCommandWithTimeout(timeout*time.Second, "kubectl", args...) if err != nil { k8slogger.Errorf("Failed to execute: %s %+v : %+v. %s", cmd, args, err, result) if args[0] == "delete" { @@ -168,6 +168,13 @@ func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { return result, nil } +// Kubectl is wrapper for executing kubectl commands and a timeout of 15 seconds +func (k8sh *K8sHelper) Kubectl(args ...string) (string, error) { + + return k8sh.KubectlWithTimeout(15, args...) + +} + // KubectlWithStdin is wrapper for executing kubectl commands in stdin func (k8sh *K8sHelper) KubectlWithStdin(stdin string, args ...string) (string, error) { diff --git a/tests/integration/ceph_auth_keystone_test.go b/tests/integration/ceph_auth_keystone_test.go new file mode 100644 index 000000000..cf4625b72 --- /dev/null +++ b/tests/integration/ceph_auth_keystone_test.go @@ -0,0 +1,187 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "testing" + + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/suite" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// *************************************************** +// *** Major scenarios tested by the TestKeystoneAuthSuite *** +// Setup +// - A CephObject store will be created in a test cluster +// Access +// - Create a container +// - Put a file in a container +// - Get a file from the container +// - Try to get the file without valid credentials +// *************************************************** +// *** Keystone/S3/Swift integration test implementation details *** +// ceph_auth_keystone_test.go and ceph_base_keystone_test.go contain +// integration tests for swift/s3 and keystone. +// The tests are done against a minimal but functional setup of +// OpenStack Keystone. +// As it would be very complex to mock all of the involved keystone +// functionality, the decision was made to use a working keystone setup +// instead of implementing mocks. +// The keystone container image used is provided by the Yaook-project +// (see https://yaook.cloud). +// The keystone configuration uses sqlite as the database backend. +// cert-manager and trust-manager are used to provide the certificates +// (cert-manager) and distribute the ca-certificate file(s) using a secret +// (trust-manager). +// To prevent hardcoded credentials, passwords are generated by the tests. +// The integration tests use the aws s3 client (s3-functionality) and the +// OpenStack CLI client (keystone- and swift-functionality). +// (see https://docs.openstack.org/python-openstackclient/latest/). +// This was a conscious design decision to use the standard client tooling +// as close as possible to the used instead of using other go-libraries +// (such as gophercloud) to (re-)implement the client tooling for the test. +// *************************************************** + +func TestCephKeystoneAuthSuite(t *testing.T) { + s := new(KeystoneAuthSuite) + defer func(s *KeystoneAuthSuite) { + HandlePanics(recover(), s.TearDownSuite, s.T) + }(s) + suite.Run(t, s) +} + +type KeystoneAuthSuite struct { + suite.Suite + helper *clients.TestClient + installer *installer.CephInstaller + settings *installer.TestCephSettings + k8shelper *utils.K8sHelper +} + +func (h *KeystoneAuthSuite) SetupSuite() { + namespace := "keystoneauth-ns" + h.settings = &installer.TestCephSettings{ + Namespace: namespace, + OperatorNamespace: namespace, + StorageClassName: "", + UseHelm: true, + UsePVC: false, + Mons: 1, + SkipOSDCreation: false, + // EnableAdmissionController: true, + EnableDiscovery: true, + ChangeHostName: true, + ConnectionsEncrypted: true, + RookVersion: installer.LocalBuildTag, + CephVersion: installer.QuincyVersion, + SkipClusterCleanup: false, + SkipCleanupPolicy: false, + } + h.settings.ApplyEnvVars() + h.installer, h.k8shelper = StartTestCluster(h.T, h.settings) + + // install yaook-keystone here + err := InstallKeystoneInTestCluster(h.k8shelper, namespace) + h.Suite.NoErrorf(err, "Failed to install Keystone in cluster") + + // create usersecret for object store to use + testCtx := context.TODO() + + secrets := map[string][]byte{ + "OS_AUTH_TYPE": []byte("password"), + "OS_IDENTITY_API_VERSION": []byte("3"), + "OS_PROJECT_DOMAIN_NAME": []byte("Default"), + "OS_USER_DOMAIN_NAME": []byte("Default"), + "OS_PROJECT_NAME": []byte(testuserdata["rook-user"]["project"]), + "OS_USERNAME": []byte(testuserdata["rook-user"]["username"]), + "OS_PASSWORD": []byte(testuserdata["rook-user"]["password"]), + } + + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "usersecret", + Namespace: namespace, + }, + Data: secrets, + } + + _, err = h.k8shelper.Clientset.CoreV1().Secrets(namespace).Create(testCtx, secret, metav1.CreateOptions{}) + h.Suite.NoErrorf(err, "Failed to create keystone usersecret") + + h.helper = clients.CreateTestClient(h.k8shelper, h.installer.Manifests) +} + +func (h *KeystoneAuthSuite) TearDownSuite() { + CleanUpKeystoneInTestCluster(h.k8shelper, h.settings.Namespace) + + // remove user secret + if _, err := h.k8shelper.KubectlWithTimeout(30, "delete", "-n", h.settings.Namespace, "secret", "usersecret"); err != nil { + logger.Warningf("Could not remove user secret: %s", err) + } + + h.installer.UninstallRook() +} + +func (h *KeystoneAuthSuite) AfterTest(suiteName, testName string) { + h.installer.CollectOperatorLog(suiteName, testName) +} + +// Test Object StoreCreation on Rook that was installed via helm +func (h *KeystoneAuthSuite) TestObjectStoreOnRookInstalledViaHelmUsingKeystone() { + deleteStore := true + tls := false + swiftAndKeystone := true + + runObjectE2ETestLite(h.T(), h.helper, h.k8shelper, h.installer, h.settings.Namespace, "default", 3, deleteStore, tls, swiftAndKeystone) +} + +func (h *KeystoneAuthSuite) TestWithSwiftAndKeystone() { + deleteStore := true + tls := false + swiftAndKeystone := true + + objectStoreServicePrefix = objectStoreServicePrefixUniq + runSwiftE2ETest(h.T(), h.helper, h.k8shelper, h.installer, h.settings.Namespace, "default", 3, deleteStore, tls, swiftAndKeystone) + cleanUpTLSks(h) + +} + +func (h *KeystoneAuthSuite) TestWithS3AndKeystone() { + deleteStore := true + tls := false + swiftAndKeystone := true + + objectStoreServicePrefix = objectStoreServicePrefixUniq + runS3E2ETest(h.T(), h.helper, h.k8shelper, h.installer, h.settings.Namespace, "default", 3, deleteStore, tls, swiftAndKeystone) + cleanUpTLSks(h) +} + +func cleanUpTLSks(h *KeystoneAuthSuite) { + err := h.k8shelper.Clientset.CoreV1().Secrets(h.settings.Namespace).Delete(context.TODO(), objectTLSSecretName, metav1.DeleteOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + logger.Fatal("failed to deleted store TLS secret") + } + } + logger.Info("successfully deleted store TLS secret") +} diff --git a/tests/integration/ceph_base_keystone_test.go b/tests/integration/ceph_base_keystone_test.go new file mode 100644 index 000000000..7fc019fbc --- /dev/null +++ b/tests/integration/ceph_base_keystone_test.go @@ -0,0 +1,1242 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "os" + "testing" + + "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" + "github.com/rook/rook/tests/framework/utils" + "github.com/sethvargo/go-password/password" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const testProjectName = "testproject" + +var testuserdata = map[string]map[string]string{ + "admin": { + "description": "keystone admin account", + "username": "admin", + "project": "admin", + "role": "admin", + }, + "rook-user": { + "description": "swift admin account", + "project": "admin", + "username": "rook-user", + "role": "admin", + }, + "alice": { + "description": "normal user account", + "username": "alice", + "project": testProjectName, + "role": "member", + }, + "carol": { + "description": "normal user account", + "username": "carol", + "project": testProjectName, + "role": "admin", + }, + "mallory": { + "description": "bad actor user", + "username": "mallory", + "project": testProjectName, + "role": "", + }, +} + +func InstallKeystoneInTestCluster(shelper *utils.K8sHelper, namespace string) error { + + if err := initializePasswords(); err != nil { + + return err + } + + ctx := context.TODO() + + // The namespace keystoneauth-ns is created by SetupSuite + + if err := shelper.CreateNamespace("cert-manager"); err != nil { + + logger.Error("Could not create namespace cert-manager") + return err + + } + + // install cert-manager using helm + // the helm installer uses the rook repository and cannot be used as is + // therefore parts of the installer are adapted here + + // use helm path from environment (the same is used by the helm installer) + helmPath := os.Getenv("TEST_HELM_PATH") + if helmPath == "" { + helmPath = "/tmp/rook-tests-scripts-helm/helm" + } + helmHelper := utils.NewHelmHelper(helmPath) + + // add the cert-manager helm repo + logger.Infof("adding cert-manager helm repo") + cmdArgs := []string{"repo", "add", "jetstack", "https://charts.jetstack.io"} + if _, err := helmHelper.Execute(cmdArgs...); err != nil { + // Continue on error in case the repo already was added + logger.Errorf("failed to add repo cert-manager, err=%v", err) + return err + } + cmdArgs = []string{"repo", "update"} + if _, err := helmHelper.Execute(cmdArgs...); err != nil { + // Continue on error in case the repo already was added + logger.Warningf("failed to update helm repositories, err=%v", err) + return err + } + + if err := installHelmChart(helmHelper, "cert-manager", "cert-manager", "jetstack/cert-manager", "1.13.3", + "installCRDs=true"); err != nil { + return err + } + + // trust-manager does not support k8s<1.25 + // This allows for secrets to be read/written by trust-manager in all namespaces + // This is considered insecure in production environments! This is here only for the quick test setup. + if err := installHelmChart(helmHelper, "cert-manager", "trust-manager", "jetstack/trust-manager", "0.7.0", + "app.trust.namespace="+namespace, "installCRDs=true", "secretTargets.enabled=true", "secretTargets.authorizedSecretsAll=true"); err != nil { + return err + } + + if err := shelper.ResourceOperation("apply", keystoneApiClusterIssuer(namespace)); err != nil { + logger.Errorf("Could not apply ClusterIssuer in namespace %s: %s", namespace, err) + return err + } + + if err := shelper.ResourceOperation("apply", keystoneApiCaCertificate(namespace)); err != nil { + logger.Errorf("Could not apply ClusterIssuer CA Certificate in namespace %s: %s", namespace, err) + return err + } + + if err := shelper.ResourceOperation("apply", keystoneApiCaIssuer(namespace)); err != nil { + logger.Errorf("Could not install CA Issuer in namespace %s: %s", namespace, err) + return err + } + + if err := shelper.ResourceOperation("apply", keystoneApiCertificate(namespace)); err != nil { + logger.Errorf("Could not create Certificate (request) in namespace %s", namespace) + return err + } + + if err := shelper.ResourceOperation("apply", trustManagerBundle(namespace)); err != nil { + logger.Errorf("Could not create CA Certificate Bundle in namespace %s", namespace) + return err + } + + data := getKeystoneApache2CM(namespace) + + keystoneApacheCM := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "keystone-apache2-conf", + Namespace: namespace, + }, + Data: data, + } + + if _, err := shelper.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, keystoneApacheCM, metav1.CreateOptions{}); err != nil { + + logger.Fatalf("failed to create apache2.conf configmap in namespace %s with error %s", namespace, err) + return err + + } + + secretData := getKeystoneConfig() + + keystoneConfSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "keystone-config", + Namespace: namespace, + }, + Data: secretData, + } + + if _, err := shelper.Clientset.CoreV1().Secrets(namespace).Create(ctx, keystoneConfSecret, metav1.CreateOptions{}); err != nil { + logger.Errorf("Could not create keystone config secret in namespace %s", namespace) + return err + } + + if err := shelper.ResourceOperation("apply", keystoneDeployment(namespace, testuserdata["admin"]["password"])); err != nil { + logger.Errorf("Could not create keystone deployment in namespace %s", namespace) + return err + } + + if err := shelper.WaitForPodCount("app=keystone", namespace, 1); err != nil { + logger.Errorf("Wait for keystone pod failed in namespace %s", namespace) + return err + } + + if _, err := shelper.KubectlWithTimeout(315, "wait", "--timeout=300s", "--namespace", namespace, "pod", "--selector=app=keystone", "--for=condition=Ready"); err != nil { + logger.Errorf("Failed to wait for pod keystone in namespace %s", namespace) + return err + } + + if err := shelper.ResourceOperation("apply", keystoneService(namespace)); err != nil { + logger.Errorf("Could not create service for keystone in namespace %s", namespace) + return err + } + + for _, userdata := range testuserdata { + + if err := shelper.ResourceOperation("apply", createOpenStackClient(namespace, userdata["project"], userdata["username"], userdata["password"])); err != nil { + logger.Errorf("Could not create openstack client deployment in namespace %s", namespace) + return err + } + + } + + return nil + +} + +func initializePasswords() error { + + for user := range testuserdata { + + var err error + + if testuserdata[user]["password"], err = password.Generate(20, 2, 0, false, false); err != nil { + + logger.Errorf("Failed to initialize password for user %s: %s", user, err) + return err + + } + + } + + return nil + +} + +func createOpenStackClient(namespace string, project string, username string, password string) string { + return `apiVersion: apps/v1 +kind: Deployment +metadata: + name: osc-` + project + `-` + username + ` + namespace: ` + namespace + ` +spec: + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: osc-` + project + `-` + username + ` + template: + metadata: + creationTimestamp: null + labels: + app: osc-` + project + `-` + username + ` + spec: + containers: + - command: + - sleep + - "7200" + image: nixery.dev/shell/awscli2/openstackclient/jq/busybox + env: + - name: REQUESTS_CA_BUNDLE + value: /etc/ssl/keystone/ca.crt + - name: OS_AUTH_TYPE + value: password + - name: OS_AUTH_URL + value: https://keystone.` + namespace + `.svc/v3 + - name: OS_IDENTITY_API_VERSION + value: "3" + - name: OS_PROJECT_DOMAIN_NAME + value: Default + - name: OS_INTERFACE + value: internal + - name: OS_USER_DOMAIN_NAME + value: Default + - name: OS_PROJECT_NAME + value: ` + project + ` + - name: OS_USERNAME + value: ` + username + ` + - name: OS_PASSWORD + value: "` + password + `" + imagePullPolicy: IfNotPresent + name: openstackclient + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/ssl/keystone + name: keystone-certificate + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - name: keystone-certificate + secret: + defaultMode: 420 + secretName: keystone-api-tls +` +} + +func trustManagerBundle(namespace string) string { + + return `apiVersion: trust.cert-manager.io/v1alpha1 +kind: Bundle +metadata: + name: keystone-bundle + namespace: ` + namespace + ` +spec: + sources: + - useDefaultCAs: true + - secret: + name: "root-secret" + key: "tls.crt" + target: + secret: + key: "cabundle"` + +} + +func installHelmChart(helmHelper *utils.HelmHelper, namespace string, chartName string, chart string, version string, settings ...string) error { + + logger.Infof("installing helm chart %s with version %s", chart, version) + + arguments := []string{"upgrade", "--install", "--debug", "--namespace", namespace, chartName, chart, "--version=" + version, "--wait"} + + for _, setting := range settings { + + arguments = append(arguments, "--set", setting) + + } + + _, err := helmHelper.Execute(arguments...) + if err != nil { + logger.Errorf("failed to install helm chart %s with version %s in namespace: %v, err=%v", chart, version, namespace, err) + return err + } + + return nil + +} + +func keystoneApiCaIssuer(namespace string) string { + + return `apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: my-ca-issuer + namespace: ` + namespace + ` +spec: + ca: + secretName: root-secret +` + +} + +func keystoneApiCaCertificate(namespace string) string { + + return `apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: my-selfsigned-ca + namespace: ` + namespace + ` +spec: + isCA: true + commonName: my-selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: ClusterIssuer + group: cert-manager.io` + +} + +func keystoneApiClusterIssuer(namespace string) string { + + return `apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: selfsigned-issuer + namespace: ` + namespace + ` +spec: + selfSigned: {} +` + +} + +func keystoneService(namespace string) string { + + return `apiVersion: v1 +kind: Service +metadata: + name: keystone + namespace: ` + namespace + ` +spec: + ports: + - name: internal + port: 443 + protocol: TCP + targetPort: 443 + - name: external + port: 5001 + protocol: TCP + targetPort: 5001 + selector: + app: keystone + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {}` + +} + +func keystoneApiCertificate(namespace string) string { + + return ` +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: keystone-api + namespace: ` + namespace + ` +spec: + secretName: keystone-api-tls + duration: 10h + renewBefore: 9h + subject: + organizations: + - rook-integrationtest-keystone-api + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + usages: + - server auth + dnsNames: + - keystone.` + namespace + `.svc + issuerRef: + name: my-ca-issuer + kind: Issuer +` + +} + +func keystoneDeployment(namespace string, adminpassword string) string { + + return `apiVersion: apps/v1 +kind: Deployment +metadata: + name: keystone-api + namespace: ` + namespace + ` + labels: + app: keystone +spec: + replicas: 1 + selector: + matchLabels: + app: keystone + template: + metadata: + labels: + app: keystone + spec: + initContainers: + - name: init-fernet + image: registry.yaook.cloud/yaook/keystone-yoga:3.0.30 + command: ['sh', '-c', 'keystone-manage fernet_setup --keystone-user keystone --keystone-group keystone'] + volumeMounts: + - mountPath: /etc/keystone/keystone.conf + name: keystone-config-vol + subPath: keystone.conf + - mountPath: /var/keystone + name: dbdir + - mountPath: /etc/keystone/fernet-keys + name: keystonefernet + securityContext: + runAsUser: 2500001 + - name: init-db + image: registry.yaook.cloud/yaook/keystone-yoga:3.0.30 + command: ['sh', '-c', 'keystone-manage db_sync'] + volumeMounts: + - mountPath: /etc/keystone/keystone.conf + name: keystone-config-vol + subPath: keystone.conf + - mountPath: /var/keystone + name: dbdir + securityContext: + runAsUser: 2500001 + - name: init-keystone-endpoint + image: registry.yaook.cloud/yaook/keystone-yoga:3.0.30 + command: [ 'sh', '-c', 'keystone-manage bootstrap --bootstrap-password ` + adminpassword + ` --bootstrap-username admin --bootstrap-project-name admin --bootstrap-role-name admin --bootstrap-service-name keystone --bootstrap-region-id RegionOne --bootstrap-admin-url https://keystone.` + namespace + `.svc --bootstrap-internal-url https://keystone.` + namespace + `.svc'] + volumeMounts: + - mountPath: /etc/keystone/keystone.conf + name: keystone-config-vol + subPath: keystone.conf + - mountPath: /etc/keystone/fernet-keys + name: keystonefernet + - mountPath: /var/keystone + name: dbdir + containers: + - env: + - name: REQUESTS_CA_BUNDLE + value: /etc/pki/tls/certs/ca-bundle.crt + - name: WSGI_PROCESSES + value: "3" + image: registry.yaook.cloud/yaook/keystone-yoga:3.0.30 + imagePullPolicy: Always + name: keystone + readinessProbe: + exec: + command: + - curl + - -k + - https://localhost + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + startupProbe: + exec: + command: + - curl + - -k + - https://localhost + failureThreshold: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/keystone + name: dbdir + - mountPath: /etc/keystone/keystone.conf + name: keystone-config-vol + subPath: keystone.conf + - mountPath: /etc/apache2/apache2.conf + name: keystone-apache2-conf + subPath: apache2.conf + - mountPath: /etc/ssl/keystone + name: keystone-certificate + - mountPath: /etc/keystone/fernet-keys + name: keystonefernet + dnsPolicy: ClusterFirst + enableServiceLinks: false + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + shareProcessNamespace: true + terminationGracePeriodSeconds: 30 + volumes: + - name: dbdir + emptyDir: {} + - name: keystone-config-vol + projected: + defaultMode: 420 + sources: + - secret: + items: + - key: keystone.conf + path: keystone.conf + name: keystone-config + - configMap: + defaultMode: 420 + name: keystone-apache2-conf + name: keystone-apache2-conf + - name: keystonefernet + emptyDir: {} + - name: ssl-terminator-config + emptyDir: {} + - name: tls-secret + emptyDir: {} + - name: keystone-certificate + secret: + defaultMode: 420 + secretName: keystone-api-tls` + +} + +func getKeystoneConfig() map[string][]byte { + + returnMap := make(map[string][]byte) + + keystoneConfig := `[DEFAULT] +use_stderr = true +use_json = true +debug = true +insecure_debug = true + +[identity] +driver = sql + +[database] +connection = sqlite:////var/keystone/keystone.db + +[cache] +enabled = false` + + returnMap["keystone.conf"] = []byte(keystoneConfig) + + return returnMap + +} + +func getKeystoneApache2CM(namespace string) map[string]string { + + returnMap := make(map[string]string) + + apache2Config := `LoadModule mpm_event_module modules/mod_mpm_event.so +LoadModule wsgi_module modules/mod_wsgi.so +LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule ssl_module modules/mod_ssl.so + +ServerRoot "/etc/apache2" +Mutex file:/var/lock/apache2 default default +PidFile /run/apache2/apache2.pid +Timeout 60 +KeepAlive On +MaxKeepAliveRequests 100 +KeepAliveTimeout 15 +HostnameLookups Off +LogLevel warn + +User www-data +Group www-data + +Listen 443 + +ErrorLog "/proc/self/fd/2" + + + ServerName keystone-api.` + namespace + `.svc + SSLEngine on + SSLCertificateFile /etc/ssl/keystone/tls.crt + SSLCertificateKeyFile /etc/ssl/keystone/tls.key + SSLCertificateChainFile /etc/ssl/keystone/ca.crt + WSGIDaemonProcess keystone-public processes=${WSGI_PROCESSES} threads=1 user=keystone group=keystone display-name=%{GROUP} home=/usr/local + WSGIProcessGroup keystone-public + WSGIScriptAlias / /usr/local/bin/keystone-wsgi-public + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + + + Require all granted + + + ErrorLogFormat "%M" + LogFormat "{ \"asctime\":\"%{%Y-%m-%d %H:%M:%S}t\", \"remoteIP\":\"%a\", \"name\":\"%V\", \"host\":\"%h\", \"request\":\"%U\", \"query\":\"%q\", \"message\":\"%r\", \"method\":\"%m\", \"status\":\"%>s\", \"userAgent\":\"%{User-agent}i\", \"referer\":\"%{Referer}i\" }" logformat + CustomLog "/dev/stdout" logformat + +TraceEnable Off` + + returnMap["apache2.conf"] = apache2Config + + return returnMap + +} + +func CleanUpKeystoneInTestCluster(shelper *utils.K8sHelper, namespace string) { + + // Un-Install keystone with yaook + err := shelper.DeleteResource("-n", namespace, "configmap", "keystone-apache2-conf") + if err != nil { + logger.Warningf("Could not delete configmap keystone-apache2-conf in namespace %s", namespace) + } + + err = shelper.DeleteResource("-n", namespace, "secret", "keystone-config") + if err != nil { + logger.Warningf("Could not delete secret keystone-config in namespace %s", namespace) + } + + err = shelper.DeleteResource("-n", namespace, "deployment", "keystone-api") + if err != nil { + logger.Warningf("Could not delete deployment keystone-api in namespace %s", namespace) + } + + //cert-manager related resources (including certificates and secrets) are not removed here + //(as they will be removed anyway on uninstalling cert-manager) + +} + +// Test Object StoreCreation on Rook that was installed via helm +func runSwiftE2ETest(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, deleteStore bool, enableTLS bool, swiftAndKeystone bool) { + andDeleting := "" + if deleteStore { + andDeleting = "and deleting" + } + logger.Infof("test creating %s object store %q in namespace %q", andDeleting, storeName, namespace) + + testContainerName := "test-container" + + prepareE2ETest(t, helper, k8sh, installer, namespace, storeName, replicaSize, deleteStore, enableTLS, swiftAndKeystone, testContainerName) + + // test with user with read+write access (member-role) + + t.Run("create container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "container", "create", testContainerName, + ) + + }) + + t.Run("show container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "container", "show", testContainerName, + ) + + }) + + t.Run("create local testfile", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "echo test-content > /tmp/testfile", + ) + + }) + + // openstack object create testContainerName /testfile + t.Run("create object in container (using the local testfile) (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "create", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("list objects in container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "list", testContainerName, + ) + + }) + + t.Run("show testfile object in container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "show", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("save testfile object from container to local disk (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "save", "--file", "/tmp/testfile.saved", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("check testfile (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "diff /tmp/testfile /tmp/testfile.saved", + ) + + }) + + t.Run("delete object in container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "delete", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("delete container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "container", "delete", testContainerName, + ) + + }) + + // unauthorized (?) access + // create container (with alice) + t.Run("prepare container for unauthorized access test (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "container", "create", testContainerName, + ) + + // create object (with alice) + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "echo test-content > /tmp/testfile", + ) + + // openstack object create testContainerName /testfile + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "create", testContainerName, "/tmp/testfile", + ) + + // check whether container got created + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "list", testContainerName, + ) + + }) + + // try access container with id (with mallory, expect: denied) + t.Run("display a container (as unprivileged user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "mallory", false, + "openstack", "container", "show", testContainerName, + ) + + }) + + // try read access object with id (with mallory, expect: denied) + t.Run("show testfile object in container (as unprivileged user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "mallory", false, + "openstack", "object", "show", testContainerName, "/tmp/testfile", + ) + + }) + + // try write access object with id (with mallory, expect: denied) + t.Run("create local testfile", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "mallory", true, + "bash", "-c", "echo bad-content > /tmp/testfile", + ) + + }) + + // openstack object create testContainerName /testfile + t.Run("create object in container (using the local testfile) (as unprivileged user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "mallory", false, + "openstack", "object", "create", testContainerName, "/tmp/testfile", + ) + + }) + + // try deleting object (with mallory, expect: denied) + t.Run("delete object in container (as unprivileged user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "mallory", false, + "openstack", "object", "delete", testContainerName, "/tmp/testfile", + ) + + }) + + // try deleting container (with mallory, expect: denied) + t.Run("delete container (as unprivileged user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "mallory", false, + "openstack", "container", "delete", testContainerName, + ) + + }) + + // try access container with id (with rook-user, expect: success) + t.Run("show container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "container", "show", testContainerName, + ) + + }) + + t.Run("create local testfile (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "bash", "-c", "echo test-content > /tmp/testfile", + ) + + }) + + t.Run("create local testfile (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "bash", "-c", "echo test-content > /tmp/testfile-rook-user", + ) + + }) + + // openstack object create testContainerName /testfile + // try write access object with id (with rook-user, expect: success) + t.Run("create object in container (using the local testfile) (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "object", "create", testContainerName, "/tmp/testfile-rook-user", + ) + + }) + + t.Run("list objects in container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "object", "list", testContainerName, + ) + + }) + + // try read access object with id (with rook-user, expect: success) + t.Run("show testfile object in container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "object", "show", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("save testfile object from container to local disk (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "object", "save", "--file", "/tmp/testfile.saved", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("check testfile (admin-user)", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "bash", "-c", "diff /tmp/testfile /tmp/testfile.saved", + ) + + }) + + // try deleting object (with rook-user, expect: success) + t.Run("delete object in container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "object", "delete", testContainerName, "/tmp/testfile", + ) + + }) + + t.Run("delete object in container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "object", "delete", testContainerName, "/tmp/testfile-rook-user", + ) + + }) + + // try deleting container (with rook-user, expect: success) + t.Run("delete container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "carol", true, + "openstack", "container", "delete", testContainerName, + ) + }) + + cleanupE2ETest(t, k8sh, namespace, storeName, deleteStore, testContainerName) +} + +func testInOpenStackClient(t *testing.T, sh *utils.K8sHelper, namespace string, projectname string, username string, expectNoError bool, command ...string) { + + commandLine := []string{"exec", "-n", namespace, "deployment/osc-" + projectname + "-" + username, "--"} + + commandLine = append(commandLine, command...) + output, err := sh.KubectlWithTimeout(60, commandLine...) + + if err != nil { + logger.Warningf("failed to execute command in openstack cli: %s: %s", commandLine, output) + } + + logger.Infof("%s", output) + + if expectNoError { + + assert.NoError(t, err) + + } else { + + assert.Error(t, err) + + } + +} + +func prepareE2ETest(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, deleteStore bool, enableTLS bool, swiftAndKeystone bool, testContainerName string) { + + t.Run("create test project in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "project", "create", testProjectName, + ) + + }) + + for _, value := range testuserdata { + + if value["username"] == "admin" { + continue + } + + t.Run("create test user "+value["username"]+" in keystone", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "user", "create", "--project", value["project"], "--password", value["password"], value["username"], + ) + }) + + if value["role"] != "" { + + t.Run("assign test user "+value["username"]+" to project "+value["project"]+" in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "role", "add", "--user", value["username"], "--project", value["project"], value["role"], + ) + + }) + + } + + } + + createCephObjectStore(t, helper, k8sh, installer, namespace, storeName, replicaSize, enableTLS, swiftAndKeystone) + + t.Run("create service swift in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "service", "create", "--name", "swift", "object-store", + ) + + }) + + t.Run("create internal swift endpoint in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "endpoint", "create", "--region", "default", "--enable", "swift", "internal", ""+rgwServiceUri(storeName, namespace)+"/foobar/v1", + ) + + }) + + t.Run("create admin swift endpoint in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "endpoint", "create", "--region", "default", "--enable", "swift", "admin", ""+rgwServiceUri(storeName, namespace)+"/foobar/v1", + ) + + }) +} + +func cleanupE2ETest(t *testing.T, k8sh *utils.K8sHelper, namespace, storeName string, deleteStore bool, testContainerName string) { + + t.Run("Delete swift endpoints in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "bash", "-c", "openstack endpoint list -f json | jq '.[] | select(.\"Service Name\" == \"swift\") | .ID' -r | xargs openstack endpoint delete", + ) + + }) + + t.Run("Delete service swift in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "service", "delete", "swift", + ) + + }) + + if deleteStore { + + t.Run("delete object store", func(t *testing.T) { + + deleteObjectStore(t, k8sh, namespace, storeName) + assertObjectStoreDeletion(t, k8sh, namespace, storeName) + + }) + + } + + for _, value := range testuserdata { + + if value["username"] == "admin" { + continue + } + + t.Run("delete test user "+value["username"]+" in keystone", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "user", "delete", value["username"], + ) + }) + + } + + t.Run("delete test project in keystone", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + "admin", "admin", true, + "openstack", "project", "delete", testProjectName, + ) + + }) + +} + +func rgwServiceUri(storeName string, namespace string) string { + return "http://" + RgwServiceName(storeName) + "." + namespace + ".svc" +} + +func runS3E2ETest(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, deleteStore bool, enableTLS bool, swiftAndKeystone bool) { + andDeleting := "" + if deleteStore { + andDeleting = "and deleting" + } + logger.Infof("test creating %s object store %q in namespace %q", andDeleting, storeName, namespace) + + testContainerName := "test-container" + + prepareE2ETest(t, helper, k8sh, installer, namespace, storeName, replicaSize, deleteStore, enableTLS, swiftAndKeystone, testContainerName) + + t.Run("create container (with user being a member)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "container", "create", testContainerName, + ) + + }) + + t.Run("create AWS config file", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "mkdir -p .aws && openstack ec2 credentials create -fjson | jq -r '\"[default]\\naws_access_key_id = \" + .access + \"\\naws_secret_access_key = \" + .secret + \"\\n\"' | tee .aws/credentials && printf '[default]\nregion = idontcare' > .aws/config", + ) + }) + + t.Run("List bucket with S3 with aws debug", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --debug --endpoint-url=http://"+RgwServiceName(storeName)+"."+namespace+".svc s3api list-buckets", + ) + + }) + + t.Run("List bucket with S3", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --endpoint-url="+rgwServiceUri(storeName, namespace)+" s3api list-buckets | jq '.Buckets | .[].Name' -r | grep "+testContainerName, + ) + + }) + + t.Run("List file with S3 created by OS", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "touch testfile2") + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "object", "create", ""+testContainerName+"", "testfile2") + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --endpoint-url="+rgwServiceUri(storeName, namespace)+" s3 ls s3://"+testContainerName+"| grep testfile2", + ) + + }) + + t.Run("Upload test file using S3", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "echo test-content > /tmp/testfile", + ) + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --endpoint-url="+rgwServiceUri(storeName, namespace)+" s3 cp /tmp/testfile s3://"+testContainerName+"/testfile", + ) + + }) + + t.Run("save testfile object from container to local disk", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --endpoint-url="+rgwServiceUri(storeName, namespace)+" s3 cp s3://"+testContainerName+"/testfile /tmp/testfile.saved") + }) + + t.Run("check testfile", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "diff /tmp/testfile /tmp/testfile.saved") + }) + + t.Run("delete object in container", func(t *testing.T) { + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --endpoint-url="+rgwServiceUri(storeName, namespace)+" s3 rm s3://"+testContainerName+"/testfile") + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "bash", "-c", "aws --endpoint-url="+rgwServiceUri(storeName, namespace)+" s3 rm s3://"+testContainerName+"/testfile2") + }) + + t.Run("delete container (admin-user)", func(t *testing.T) { + + testInOpenStackClient(t, k8sh, namespace, + testProjectName, "alice", true, + "openstack", "container", "delete", testContainerName, + ) + }) + + cleanupE2ETest(t, k8sh, namespace, storeName, deleteStore, testContainerName) +} diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index 5f8779a2b..6358935aa 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -42,8 +42,9 @@ import ( ) const ( + rgwPrefix = "rook-ceph-rgw" //nolint:gosec // since this is not leaking any hardcoded credentials, it's just the secret name - objectTLSSecretName = "rook-ceph-rgw-tls-test-store-csr" + objectTLSSecretName = rgwPrefix + "-tls-test-store-csr" ) var ( @@ -66,47 +67,51 @@ var ( ) // Test Object StoreCreation on Rook that was installed via helm -func runObjectE2ETestLite(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, deleteStore bool, enableTLS bool) { +func runObjectE2ETestLite(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, deleteStore bool, enableTLS bool, swiftAndKeystone bool) { andDeleting := "" if deleteStore { andDeleting = "and deleting" } logger.Infof("test creating %s object store %q in namespace %q", andDeleting, storeName, namespace) - createCephObjectStore(t, helper, k8sh, installer, namespace, storeName, replicaSize, enableTLS) + createCephObjectStore(t, helper, k8sh, installer, namespace, storeName, replicaSize, enableTLS, swiftAndKeystone) if deleteStore { t.Run("delete object store", func(t *testing.T) { deleteObjectStore(t, k8sh, namespace, storeName) assertObjectStoreDeletion(t, k8sh, namespace, storeName) }) + // remove user secret } } +func RgwServiceName(storeName string) string { + return rgwPrefix + "-" + storeName +} + // create a CephObjectStore and wait for it to report ready status -func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, tlsEnable bool) { +func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, namespace, storeName string, replicaSize int, tlsEnable bool, swiftAndKeystone bool) { logger.Infof("Create Object Store %q with replica count %d", storeName, replicaSize) - rgwServiceName := "rook-ceph-rgw-" + storeName if tlsEnable { t.Run("generate TLS certs", func(t *testing.T) { - generateRgwTlsCertSecret(t, helper, k8sh, namespace, storeName, rgwServiceName) + generateRgwTlsCertSecret(t, helper, k8sh, namespace, storeName, RgwServiceName(storeName)) }) } t.Run("create CephObjectStore", func(t *testing.T) { - err := helper.ObjectClient.Create(namespace, storeName, int32(replicaSize), tlsEnable) + err := helper.ObjectClient.Create(namespace, storeName, int32(replicaSize), tlsEnable, swiftAndKeystone) assert.Nil(t, err) }) t.Run("wait for RGWs to be running", func(t *testing.T) { // check that ObjectStore is created logger.Infof("Check that RGW pods are Running") - for i := 0; i < 24 && k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, 1, "Running") == false; i++ { + for i := 0; i < 24 && k8sh.CheckPodCountAndState(rgwPrefix, namespace, 1, "Running") == false; i++ { logger.Infof("(%d) RGW pod check sleeping for 5 seconds ...", i) time.Sleep(5 * time.Second) } - assert.True(t, k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running")) + assert.True(t, k8sh.CheckPodCountAndState(rgwPrefix, namespace, replicaSize, "Running")) logger.Info("RGW pods are running") - assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=rook-ceph-rgw", namespace)) + assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app="+rgwPrefix, namespace)) logger.Infof("Object store %q created successfully", storeName) }) @@ -148,7 +153,7 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils t.Run("verify RGW liveness probes show healthy", func(t *testing.T) { err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 90*time.Second, true, func(ctx context.Context) (done bool, err error) { - deployName := "rook-ceph-rgw-" + storeName + "-a" + deployName := RgwServiceName(storeName) + "-a" d, err := k8sh.Clientset.AppsV1().Deployments(namespace).Get(ctx, deployName, metav1.GetOptions{}) if err != nil { logger.Infof("waiting for rgw deployment %q to be ready; failed to get deployment: %v", deployName, err) @@ -164,7 +169,7 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils }) t.Run("verify RGW service is up", func(t *testing.T) { - assert.True(t, k8sh.IsServiceUp("rook-ceph-rgw-"+storeName, namespace)) + assert.True(t, k8sh.IsServiceUp(RgwServiceName(storeName), namespace)) }) t.Run("check if the dashboard-admin user exists in all existing object stores", func(t *testing.T) { diff --git a/tests/integration/ceph_cosi_test.go b/tests/integration/ceph_cosi_test.go index 732771aef..a4cfa8050 100644 --- a/tests/integration/ceph_cosi_test.go +++ b/tests/integration/ceph_cosi_test.go @@ -38,7 +38,7 @@ func testCOSIDriver(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sH assert.NoError(t, err, "failed to create COSI controller") }) - createCephObjectStore(s.T(), helper, k8sh, cephinstaller, namespace, objectStoreCOSI, 1, false) + createCephObjectStore(s.T(), helper, k8sh, cephinstaller, namespace, objectStoreCOSI, 1, false, false) t.Run("Creating CephCOSIDriver CRD", func(t *testing.T) { err := helper.COSIClient.CreateCOSI() diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index 4c64f1713..884573f60 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -105,5 +105,5 @@ func (h *HelmSuite) TestFileStoreOnRookInstalledViaHelm() { func (h *HelmSuite) TestObjectStoreOnRookInstalledViaHelm() { deleteStore := true tls := false - runObjectE2ETestLite(h.T(), h.helper, h.k8shelper, h.installer, h.settings.Namespace, "default", 3, deleteStore, tls) + runObjectE2ETestLite(h.T(), h.helper, h.k8shelper, h.installer, h.settings.Namespace, "default", 3, deleteStore, tls, false) } diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go index ed859ce7d..d51dd59b4 100644 --- a/tests/integration/ceph_object_test.go +++ b/tests/integration/ceph_object_test.go @@ -19,6 +19,7 @@ package integration import ( "context" "encoding/json" + "k8s.io/apimachinery/pkg/api/errors" "testing" "time" @@ -31,7 +32,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -95,8 +95,12 @@ func (s *ObjectSuite) TestWithTLS() { } tls := true + swiftAndKeystone := false objectStoreServicePrefix = objectStoreServicePrefixUniq - runObjectE2ETest(s.helper, s.k8sh, s.installer, &s.Suite, s.settings.Namespace, tls) + runObjectE2ETest(s.helper, s.k8sh, s.installer, &s.Suite, s.settings.Namespace, tls, swiftAndKeystone) + cleanUpTLS(s) +} +func cleanUpTLS(s *ObjectSuite) { err := s.k8sh.Clientset.CoreV1().Secrets(s.settings.Namespace).Delete(context.TODO(), objectTLSSecretName, metav1.DeleteOptions{}) if err != nil { if !errors.IsNotFound(err) { @@ -112,22 +116,23 @@ func (s *ObjectSuite) TestWithoutTLS() { } tls := false + swiftAndKeystone := false objectStoreServicePrefix = objectStoreServicePrefixUniq - runObjectE2ETest(s.helper, s.k8sh, s.installer, &s.Suite, s.settings.Namespace, tls) + runObjectE2ETest(s.helper, s.k8sh, s.installer, &s.Suite, s.settings.Namespace, tls, swiftAndKeystone) } // Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order // Create object store, Create User, Connect to Object Store, Create Bucket, Read/Write/Delete to bucket, // Check issues in MGRs, Delete Bucket and Delete user // Test for ObjectStore with and without TLS enabled -func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, s *suite.Suite, namespace string, tlsEnable bool) { +func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, installer *installer.CephInstaller, s *suite.Suite, namespace string, tlsEnable bool, swiftAndKeystone bool) { storeName := "test-store" if tlsEnable { storeName = objectStoreTLSName } logger.Infof("Running on Rook Cluster %s", namespace) - createCephObjectStore(s.T(), helper, k8sh, installer, namespace, storeName, 3, tlsEnable) + createCephObjectStore(s.T(), helper, k8sh, installer, namespace, storeName, 3, tlsEnable, swiftAndKeystone) // test that a second object store can be created (and deleted) while the first exists s.T().Run("run a second object store", func(t *testing.T) { @@ -135,14 +140,14 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install // The lite e2e test is perfect, as it only creates a cluster, checks that it is healthy, // and then deletes it. deleteStore := true - runObjectE2ETestLite(t, helper, k8sh, installer, namespace, otherStoreName, 1, deleteStore, tlsEnable) + runObjectE2ETestLite(t, helper, k8sh, installer, namespace, otherStoreName, 1, deleteStore, tlsEnable, swiftAndKeystone) }) // now test operation of the first object store - testObjectStoreOperations(s, helper, k8sh, namespace, storeName) + testObjectStoreOperations(s, helper, k8sh, namespace, storeName, swiftAndKeystone) bucketNotificationTestStoreName := "bucket-notification-" + storeName - createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable) + createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable, swiftAndKeystone) testBucketNotifications(s, helper, k8sh, namespace, bucketNotificationTestStoreName) if !tlsEnable { // TODO : need to fix COSI driver to support TLS @@ -153,7 +158,7 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install } } -func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) { +func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string, swiftAndKeystone bool) { ctx := context.TODO() clusterInfo := client.AdminTestClusterInfo(namespace) t := s.T() diff --git a/tests/integration/ceph_smoke_test.go b/tests/integration/ceph_smoke_test.go index c3fa2ec6f..9a9c3c802 100644 --- a/tests/integration/ceph_smoke_test.go +++ b/tests/integration/ceph_smoke_test.go @@ -129,7 +129,7 @@ func (s *SmokeSuite) TestObjectStorage_SmokeTest() { storeName := "lite-store" deleteStore := true tls := false - runObjectE2ETestLite(s.T(), s.helper, s.k8sh, s.installer, s.settings.Namespace, storeName, 2, deleteStore, tls) + runObjectE2ETestLite(s.T(), s.helper, s.k8sh, s.installer, s.settings.Namespace, storeName, 2, deleteStore, tls, false) } // Test to make sure all rook components are installed and Running diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 94b4fcb79..2f77b6edc 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -315,7 +315,7 @@ func (s *UpgradeSuite) deployClusterforUpgrade(baseRookImage, objectUserID, preF logger.Infof("Initializing object before the upgrade") deleteStore := false tls := false - runObjectE2ETestLite(s.T(), s.helper, s.k8sh, s.installer, s.settings.Namespace, installer.ObjectStoreName, 1, deleteStore, tls) + runObjectE2ETestLite(s.T(), s.helper, s.k8sh, s.installer, s.settings.Namespace, installer.ObjectStoreName, 1, deleteStore, tls, false) } logger.Infof("Initializing object user before the upgrade") diff --git a/tests/scripts/create-dev-cluster.sh b/tests/scripts/create-dev-cluster.sh index cead784ce..c47c58215 100755 --- a/tests/scripts/create-dev-cluster.sh +++ b/tests/scripts/create-dev-cluster.sh @@ -112,7 +112,7 @@ setup_minikube_env() { minikube_driver="$(get_minikube_driver)" echo "Setting up minikube env for profile '$ROOK_PROFILE_NAME' (using $minikube_driver driver)" $MINIKUBE delete - $MINIKUBE start --disk-size="$MINIKUBE_DISK_SIZE" --extra-disks="$MINIKUBE_EXTRA_DISKS" --driver "$minikube_driver" -n "$MINIKUBE_NODES" + $MINIKUBE start --disk-size="$MINIKUBE_DISK_SIZE" --extra-disks="$MINIKUBE_EXTRA_DISKS" --driver "$minikube_driver" -n "$MINIKUBE_NODES" $ROOK_MINIKUBE_EXTRA_ARGS eval "$($MINIKUBE docker-env)" }