Joshua Hoblitt
8a72d80127
Configure the CephHelmSuite object store as an RGW multisite zone so the chart's CephObjectRealm, CephObjectZoneGroup, and CephObjectZone templates are exercised on a Helm install. The operator must reconcile the realm, zone group, zone, and store to Ready, which the existing install check already gates on via the RGW pod count. The realm/zone group/zone and shared-pool layout mirror the fixture in tests/integration/object/util/sharedstore: a zone backed by explicit CephBlockPools referenced through sharedPools.poolPlacements. The rgw pools are appended to the block pools already configured for CSI, and are torn down along with the multisite CRs when the default storage CRs are removed. Gate the multisite configuration behind a new UseMultisiteObjectStore setting enabled only for CephHelmSuite. The other Helm-based suites keep the plain object store; the upgrade suite in particular installs an older base chart that lacks the multisite templates and cannot resolve a zone-based store. Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
144 lines
7.2 KiB
Go
144 lines
7.2 KiB
Go
/*
|
|
Copyright 2021 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package installer
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
)
|
|
|
|
// TestCephSettings struct for handling panic and test suite tear down
|
|
type TestCephSettings struct {
|
|
DataDirHostPath string
|
|
ClusterName string
|
|
Namespace string
|
|
OperatorNamespace string
|
|
StorageClassName string
|
|
UseHelm bool
|
|
RetainHelmDefaultStorageCRs bool
|
|
UsePVC bool
|
|
Mons int
|
|
UseCrashPruner bool
|
|
MultipleMgrs bool
|
|
SkipOSDCreation bool
|
|
EnableDiscovery bool
|
|
IsExternal bool
|
|
SkipClusterCleanup bool
|
|
SkipCleanupPolicy bool
|
|
DirectMountToolbox bool
|
|
ConnectionsEncrypted bool
|
|
ConnectionsCompressed bool
|
|
RequireMsgr2 bool
|
|
EnableVolumeReplication bool
|
|
TestNFSCSI bool
|
|
ChangeHostName bool
|
|
RookVersion string
|
|
CephVersion cephv1.CephVersionSpec
|
|
KubernetesVersion string
|
|
ClusterConcurrency int
|
|
|
|
// UseMultisiteObjectStore configures the Helm chart's object store as an RGW
|
|
// multisite zone (realm/zone group/zone/shared pools) instead of a plain
|
|
// store. Only valid for local-chart installs, which carry the multisite
|
|
// templates, so it must stay off for the upgrade suite's older base chart.
|
|
UseMultisiteObjectStore bool
|
|
}
|
|
|
|
func (s *TestCephSettings) ApplyEnvVars() {
|
|
// skip the cleanup by default
|
|
s.SkipClusterCleanup = true
|
|
if os.Getenv("SKIP_TEST_CLEANUP") == "false" {
|
|
s.SkipClusterCleanup = false
|
|
}
|
|
s.SkipCleanupPolicy = true
|
|
if os.Getenv("SKIP_CLEANUP_POLICY") == "false" {
|
|
s.SkipCleanupPolicy = false
|
|
}
|
|
}
|
|
|
|
func (s *TestCephSettings) readManifest(filename string) string {
|
|
manifest := readManifest(filename)
|
|
return replaceNamespaces(manifest, manifest, s.OperatorNamespace, s.Namespace)
|
|
}
|
|
|
|
func (s *TestCephSettings) readManifestFromGitHub(filename string) string {
|
|
return s.readManifestFromGitHubWithClusterNamespace(filename, s.Namespace)
|
|
}
|
|
|
|
func (s *TestCephSettings) readManifestFromGitHubWithClusterNamespace(filename, clusterNamespace string) string {
|
|
manifest := readManifestFromGitHub(s.RookVersion, filename)
|
|
return replaceNamespaces(filename, manifest, s.OperatorNamespace, clusterNamespace)
|
|
}
|
|
|
|
func (s *TestCephSettings) replaceCSIOperatorSettings(csiNamespace, manifest string) string {
|
|
manifest = strings.ReplaceAll(manifest, `namespace: rook-ceph`, `namespace: `+csiNamespace)
|
|
return manifest
|
|
}
|
|
|
|
func (s *TestCephSettings) replaceOperatorSettings(manifest string) string {
|
|
manifest = strings.ReplaceAll(manifest, `ROOK_LOG_LEVEL: "INFO"`, `ROOK_LOG_LEVEL: "DEBUG"`)
|
|
manifest = strings.ReplaceAll(manifest, `# CSI_LOG_LEVEL: "0"`, `CSI_LOG_LEVEL: "5"`)
|
|
manifest = strings.ReplaceAll(manifest, `ROOK_ENABLE_DISCOVERY_DAEMON: "false"`, fmt.Sprintf(`ROOK_ENABLE_DISCOVERY_DAEMON: "%t"`, s.EnableDiscovery))
|
|
manifest = strings.ReplaceAll(manifest, `CSI_ENABLE_VOLUME_REPLICATION: "false"`, fmt.Sprintf(`CSI_ENABLE_VOLUME_REPLICATION: "%t"`, s.EnableVolumeReplication))
|
|
manifest = strings.ReplaceAll(manifest,
|
|
`# ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize" # default allowed configs`,
|
|
`ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS: "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"`)
|
|
if s.ClusterConcurrency > 1 {
|
|
manifest = strings.ReplaceAll(manifest, `ROOK_RECONCILE_CONCURRENT_CLUSTERS: "1"`, fmt.Sprintf(`ROOK_RECONCILE_CONCURRENT_CLUSTERS: "%d"`, s.ClusterConcurrency))
|
|
}
|
|
return manifest
|
|
}
|
|
|
|
func replaceNamespaces(name, manifest, operatorNamespace, clusterNamespace string) string {
|
|
// RBAC and related namespaces
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace)
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:cluster", clusterNamespace)
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph-external # namespace:cluster", clusterNamespace)
|
|
// Double space only needed for v1.5 upgrade test
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph # namespace:operator", operatorNamespace)
|
|
|
|
// SCC namespaces for operator and Ceph daemons
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-system # serviceaccount:namespace:operator", operatorNamespace+":rook-ceph-system")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-default # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-default")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-mgr # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-mgr")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-osd # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-osd")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-rgw # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-rgw")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-ceph-nvmeof # serviceaccount:namespace:cluster", clusterNamespace+":rook-ceph-nvmeof")
|
|
|
|
// SCC namespaces for CSI driver
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-plugin-sa")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-rbd-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-rbd-provisioner-sa")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-plugin-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-plugin-sa")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph:rook-csi-cephfs-provisioner-sa # serviceaccount:namespace:operator", operatorNamespace+":rook-csi-cephfs-provisioner-sa")
|
|
|
|
// CSI Drivers
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph.cephfs.csi.ceph.com # csi-provisioner-name", operatorNamespace+".cephfs.csi.ceph.com")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph.rbd.csi.ceph.com # csi-provisioner-name", operatorNamespace+".rbd.csi.ceph.com")
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph.nfs.csi.ceph.com # csi-provisioner-name", operatorNamespace+".nfs.csi.ceph.com")
|
|
|
|
// Bucket storage class
|
|
manifest = strings.ReplaceAll(manifest, "rook-ceph.ceph.rook.io/bucket # driver:namespace:cluster", clusterNamespace+".ceph.rook.io/bucket")
|
|
if strings.Contains(manifest, "namespace:operator") || strings.Contains(manifest, "namespace:cluster") || strings.Contains(manifest, "driver:namespace:") || strings.Contains(manifest, "serviceaccount:namespace:") {
|
|
logger.Infof("BAD MANIFEST:\n%s", manifest)
|
|
panic(fmt.Sprintf("manifest %s still contains a namespace identifier", name))
|
|
}
|
|
return manifest
|
|
}
|