ceph mds: configure completely from operator
Configure the Ceph mds daemon completely from the operator a la the recent changes to the Ceph mon and mgr operators. Create the mds deployments first and then create the keyring secrets for them with their owner reference as the corresponding deployment. This will mean that the secrets do not need to be micromanaged. When the deployment is deleted, the secret is also deleted. This has not been necessary for the mons or the manager since the mons share a keyring with a lifespan of the cluster, as does the mgr, which currently has single-mgr support only. Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>
This commit is contained in:
35 files changed
+525
-756
No files matched your search
@@ -29,8 +29,8 @@
|
||||
|
||||
- Rook will no longer create a directory-based osd in the `dataDirHostPath` if no directories or
|
||||
devices are specified or if there are no disks on the host.
|
||||
- Containers in `mon` and `mgr` pods have been removed and/or changed names.
|
||||
- Config paths in `mon` and `mgr` containers are now always the Ceph default paths
|
||||
- Containers in `mon`, `mgr`, and `mds` pods have been removed and/or changed names.
|
||||
- Config paths in `mon`, `mgr`, and `mds` containers are now always the Ceph default paths
|
||||
(`/etc/ceph`, `/var/lib/ceph/...`) regardless of the `dataDirHostPath` setting.
|
||||
|
||||
## Known Issues
|
||||
|
||||
@@ -59,7 +59,6 @@ func init() {
|
||||
agentCmd,
|
||||
osdCmd,
|
||||
rgwCmd,
|
||||
mdsCmd,
|
||||
configCmd,
|
||||
nfsCmd)
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package ceph
|
||||
|
||||
import (
|
||||
"github.com/rook/rook/cmd/rook/rook"
|
||||
mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/util/flags"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
mdsName string
|
||||
filesystemID string
|
||||
activeStandby bool
|
||||
keyring string
|
||||
)
|
||||
|
||||
var mdsCmd = &cobra.Command{
|
||||
Use: mdsdaemon.InitCommand,
|
||||
Short: "Generates mds config and runs the mds daemon",
|
||||
}
|
||||
|
||||
func init() {
|
||||
mdsCmd.Flags().StringVar(&mdsName, "mds-name", "", "name of the mds")
|
||||
mdsCmd.Flags().StringVar(&keyring, "mds-keyring", "", "the mds keyring")
|
||||
mdsCmd.Flags().StringVar(&filesystemID, "filesystem-id", "", "ID of the filesystem the mds will serve")
|
||||
mdsCmd.Flags().BoolVar(&activeStandby, "active-standby", true, "whether to start in active standby mode")
|
||||
addCephFlags(mdsCmd)
|
||||
|
||||
flags.SetFlagsFromEnv(mdsCmd.Flags(), rook.RookEnvVarPrefix)
|
||||
|
||||
mdsCmd.RunE = initMds
|
||||
}
|
||||
|
||||
func initMds(cmd *cobra.Command, args []string) error {
|
||||
required := []string{"mon-endpoints", "cluster-name", "admin-secret",
|
||||
"mds-name", "mds-keyring", "filesystem-id"}
|
||||
if err := flags.VerifyRequiredFlags(mdsCmd, required); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := verifyRenamedFlags(mdsCmd); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rook.SetLogLevel()
|
||||
|
||||
rook.LogStartupInfo(mdsCmd.Flags())
|
||||
|
||||
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
|
||||
config := &mdsdaemon.Config{
|
||||
FilesystemID: filesystemID,
|
||||
Name: mdsName,
|
||||
ActiveStandby: activeStandby,
|
||||
Keyring: keyring,
|
||||
ClusterInfo: &clusterInfo,
|
||||
}
|
||||
|
||||
err := mdsdaemon.Initialize(createContext(), config)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -41,6 +41,18 @@ type MonInfo struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
}
|
||||
|
||||
// IsInitialized returns true if the critical information in the ClusterInfo struct has been filled
|
||||
// in. This method exists less out of necessity than the desire to be explicit about the lifecycle
|
||||
// of the ClusterInfo struct during startup, specifically that it is expected to exist after the
|
||||
// Rook operator has started up or connected to the first components of the Ceph cluster.
|
||||
func (c *ClusterInfo) IsInitialized() bool {
|
||||
if c == nil || c.FSID == "" || c.MonitorSecret == "" || c.AdminSecret == "" {
|
||||
logger.Errorf("clusterInfo: %+v", c)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// NewMonInfo returns a new Ceph mon info struct from the given inputs.
|
||||
func NewMonInfo(name, ip string, port int32) *MonInfo {
|
||||
return &MonInfo{Name: name, Endpoint: net.JoinHostPort(ip, fmt.Sprintf("%d", port))}
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package mds
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/model"
|
||||
)
|
||||
|
||||
const (
|
||||
dataPoolSuffix = "data"
|
||||
metadataPoolSuffix = "metadata"
|
||||
appName = "cephfs"
|
||||
)
|
||||
|
||||
// Filesystem represents an instance of a Ceph file system (CephFS)
|
||||
type Filesystem struct {
|
||||
Name string
|
||||
metadataPool *model.Pool
|
||||
dataPools []*model.Pool
|
||||
activeMDSCount int32
|
||||
}
|
||||
|
||||
// NewFS creates a new instance of the file (MDS) service
|
||||
func NewFS(name string, metadataPool *model.Pool, dataPools []*model.Pool, activeMDSCount int32) *Filesystem {
|
||||
|
||||
metadataPool.Name = fmt.Sprintf("%s-%s", name, metadataPoolSuffix)
|
||||
for i, pool := range dataPools {
|
||||
pool.Name = fmt.Sprintf("%s-%s%d", name, dataPoolSuffix, i)
|
||||
}
|
||||
|
||||
return &Filesystem{
|
||||
Name: name,
|
||||
metadataPool: metadataPool,
|
||||
dataPools: dataPools,
|
||||
activeMDSCount: activeMDSCount,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateFilesystem starts the Ceph file daemons and creates the filesystem in Ceph.
|
||||
func (f *Filesystem) CreateFilesystem(context *clusterd.Context, clusterName string) error {
|
||||
_, err := client.GetFilesystem(context, clusterName, f.Name)
|
||||
if err == nil {
|
||||
logger.Infof("file system %s already exists", f.Name)
|
||||
// Even if the fs already exists, the num active mdses may have changed
|
||||
if err := client.SetNumMDSRanks(context, clusterName, f.Name, f.activeMDSCount); err != nil {
|
||||
logger.Errorf(
|
||||
fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", f.activeMDSCount, f.Name) +
|
||||
"this error is not critical, but mdses may not be as failure tolerant as desired. " +
|
||||
fmt.Sprintf("USER should verify that the number of active mdses is %d with 'ceph fs get %s'", f.activeMDSCount, f.Name) +
|
||||
fmt.Sprintf(": %+v", err),
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(f.dataPools) == 0 {
|
||||
return fmt.Errorf("at least one data pool must be specified")
|
||||
}
|
||||
|
||||
fslist, err := client.ListFilesystems(context, clusterName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to list existing file systems: %+v", err)
|
||||
}
|
||||
if len(fslist) > 0 && !client.IsMultiFSEnabled() {
|
||||
return fmt.Errorf("Cannot create multiple filesystems. Enable %s env variable to create more than one", client.MultiFsEnv)
|
||||
}
|
||||
|
||||
logger.Infof("Creating file system %s", f.Name)
|
||||
err = client.CreatePoolWithProfile(context, clusterName, *f.metadataPool, appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create metadata pool '%s': %+v", f.metadataPool.Name, err)
|
||||
}
|
||||
|
||||
var dataPoolNames []string
|
||||
for _, pool := range f.dataPools {
|
||||
dataPoolNames = append(dataPoolNames, pool.Name)
|
||||
err = client.CreatePoolWithProfile(context, clusterName, *pool, appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create data pool %s: %+v", pool.Name, err)
|
||||
}
|
||||
if pool.Type == model.ErasureCoded {
|
||||
// An erasure coded data pool used for a file system must allow overwrites
|
||||
if err := client.SetPoolProperty(context, clusterName, pool.Name, "allow_ec_overwrites", "true"); err != nil {
|
||||
logger.Warningf("failed to set ec pool property: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create the filesystem
|
||||
if err := client.CreateFilesystem(context, clusterName, f.Name, f.metadataPool.Name, dataPoolNames); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("created file system %s on %d data pool(s) and metadata pool %s", f.Name, len(f.dataPools), f.metadataPool.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownFilesystem marks the filesystem as down and the MDS' as failed
|
||||
func DownFilesystem(context *clusterd.Context, clusterName, filesystemName string) error {
|
||||
logger.Infof("Removing file system %s", filesystemName)
|
||||
|
||||
// mark the cephFS instance as cluster_down before removing
|
||||
if err := client.MarkFilesystemAsDown(context, clusterName, filesystemName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// mark each MDS associated with the file system to "failed"
|
||||
fsDetails, err := client.GetFilesystem(context, clusterName, filesystemName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, mdsInfo := range fsDetails.MDSMap.Info {
|
||||
if err := client.FailMDS(context, clusterName, mdsInfo.GID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("Removed file system %s", filesystemName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared
|
||||
// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades.
|
||||
// If the mds is not prepared within the timeout window, an error will be reported.
|
||||
// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/
|
||||
func PrepareForDaemonUpgrade(
|
||||
context *clusterd.Context,
|
||||
clusterName, fsName string,
|
||||
timeout time.Duration,
|
||||
) error {
|
||||
logger.Infof("preparing filesystem %s for daemon upgrade", fsName)
|
||||
// * Beginning of noted section 1
|
||||
// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
|
||||
// See more: https://ceph.com/releases/v13-2-0-mimic-released/
|
||||
// http://docs.ceph.com/docs/mimic/cephfs/upgrading/
|
||||
// As of Oct. 2018, this is only necessary for Luminous and Mimic.
|
||||
if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil {
|
||||
return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err)
|
||||
}
|
||||
if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
// * End of Noted section 1
|
||||
|
||||
logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// FinishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its
|
||||
// ideal state following an upgrade of its daemon(s).
|
||||
func FinishedWithDaemonUpgrade(
|
||||
context *clusterd.Context,
|
||||
clusterName, fsName string,
|
||||
activeMDSCount int32,
|
||||
) error {
|
||||
logger.Debugf("restoring filesystem %s from daemon upgrade", fsName)
|
||||
logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount)
|
||||
// * Beginning of noted section 1
|
||||
// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
|
||||
// See more: https://ceph.com/releases/v13-2-0-mimic-released/
|
||||
// http://docs.ceph.com/docs/mimic/cephfs/upgrading/
|
||||
// TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic.
|
||||
if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil {
|
||||
return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err)
|
||||
} // * End of noted section 1
|
||||
return nil
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package mds provides methods for setting up Ceph configuration for mds daemons.
|
||||
// It also provides methods for creating/deleting/managing Ceph filesystems served by the mds.
|
||||
package mds
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/util"
|
||||
)
|
||||
|
||||
const (
|
||||
// InitCommand is the `rook ceph` subcommand which will perform mds initialization
|
||||
InitCommand = "mds-init"
|
||||
|
||||
keyringTemplate = `
|
||||
[mds.%s]
|
||||
key = %s
|
||||
caps mon = "allow profile mds"
|
||||
caps osd = "allow *"
|
||||
caps mds = "allow"
|
||||
`
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "cephmds")
|
||||
|
||||
// Config contains the necessary parameters Rook needs to know to set up a mds for a Ceph cluster.
|
||||
type Config struct {
|
||||
FilesystemID string
|
||||
Name string
|
||||
Keyring string
|
||||
ActiveStandby bool
|
||||
ClusterInfo *cephconfig.ClusterInfo
|
||||
}
|
||||
|
||||
// Initialize generates configuration files for a Ceph mds
|
||||
func Initialize(context *clusterd.Context, config *Config) error {
|
||||
logger.Infof("Creating config for mds %s [filesystem ID: %s] [active standby?: %+v]",
|
||||
config.Name, config.FilesystemID, config.ActiveStandby)
|
||||
err := generateConfigFiles(context, config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate mds config files: %+v", err)
|
||||
}
|
||||
|
||||
util.WriteFileToLog(logger, cephconfig.DefaultConfigFilePath())
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func generateConfigFiles(context *clusterd.Context, config *Config) error {
|
||||
// write the latest config to the config dir
|
||||
if err := cephconfig.GenerateAdminConnectionConfig(context, config.ClusterInfo); err != nil {
|
||||
return fmt.Errorf("failed to write connection config: %+v", err)
|
||||
}
|
||||
|
||||
settings := map[string]string{
|
||||
"mds_standby_for_fscid": config.FilesystemID,
|
||||
"mds_standby_replay": strconv.FormatBool(config.ActiveStandby),
|
||||
}
|
||||
|
||||
keyringPath := getMdsKeyringPath(context.ConfigDir, config.Name)
|
||||
_, err := cephconfig.GenerateConfigFile(context, config.ClusterInfo,
|
||||
getMdsConfDir(context.ConfigDir, config.Name), fmt.Sprintf("mds.%s", config.Name),
|
||||
keyringPath, nil, settings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mds config file: %+v", err)
|
||||
}
|
||||
|
||||
keyringEval := func(key string) string {
|
||||
r := fmt.Sprintf(keyringTemplate, config.Name, key)
|
||||
return r
|
||||
}
|
||||
|
||||
err = cephconfig.WriteKeyring(keyringPath, config.Keyring, keyringEval)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mds keyring: %+v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getMdsConfDir(dir, name string) string {
|
||||
return path.Join(dir, fmt.Sprintf("mds-%s", name))
|
||||
}
|
||||
|
||||
func getMdsKeyringPath(dir, name string) string {
|
||||
return path.Join(getMdsConfDir(dir, name), "keyring")
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mgr"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
|
||||
@@ -52,6 +53,7 @@ var (
|
||||
)
|
||||
|
||||
type cluster struct {
|
||||
Info *cephconfig.ClusterInfo
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
Spec *cephv1.ClusterSpec
|
||||
@@ -61,9 +63,17 @@ type cluster struct {
|
||||
}
|
||||
|
||||
func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
|
||||
return &cluster{Namespace: c.Namespace, Spec: &c.Spec, context: context,
|
||||
stopCh: make(chan struct{}),
|
||||
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID))}
|
||||
return &cluster{
|
||||
// at this phase of the cluster creation process, the identity components of the cluster are
|
||||
// not yet established. we reserve this struct which is filled in as soon as the cluster's
|
||||
// identity can be established.
|
||||
Info: nil,
|
||||
Namespace: c.Namespace,
|
||||
Spec: &c.Spec,
|
||||
context: context,
|
||||
stopCh: make(chan struct{}),
|
||||
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (string, error) {
|
||||
@@ -143,20 +153,29 @@ func (c *cluster) createInstance(rookImage string) error {
|
||||
}
|
||||
|
||||
// Start the mon pods
|
||||
c.mons = mon.New(c.context, c.Namespace, c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon, cephv1.GetMonPlacement(c.Spec.Placement),
|
||||
c.Spec.Network.HostNetwork, cephv1.GetMonResources(c.Spec.Resources), c.ownerRef)
|
||||
c.mons = mon.New(c.Info, c.context, c.Namespace,
|
||||
c.Spec.DataDirHostPath, rookImage, c.Spec.CephVersion, c.Spec.Mon,
|
||||
cephv1.GetMonPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork,
|
||||
cephv1.GetMonResources(c.Spec.Resources), c.ownerRef)
|
||||
clusterInfo, err := c.mons.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the mons. %+v", err)
|
||||
}
|
||||
c.Info = clusterInfo // mons return the cluster's info
|
||||
|
||||
// The cluster Identity must be established at this point
|
||||
if !c.Info.IsInitialized() {
|
||||
return fmt.Errorf("the cluster identity was not established: %+v", c.Info)
|
||||
}
|
||||
|
||||
err = c.createInitialCrushMap()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create initial crushmap: %+v", err)
|
||||
}
|
||||
|
||||
mgrs := mgr.New(c.context, c.Namespace, rookImage, c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement),
|
||||
c.Spec.Network.HostNetwork, c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef, clusterInfo)
|
||||
mgrs := mgr.New(c.Info, c.context, c.Namespace, rookImage,
|
||||
c.Spec.CephVersion, cephv1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork,
|
||||
c.Spec.Dashboard, cephv1.GetMgrResources(c.Spec.Resources), c.ownerRef)
|
||||
err = mgrs.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the ceph mgr. %+v", err)
|
||||
|
||||
@@ -238,7 +238,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
|
||||
objectStoreUserController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start file system CRD watcher
|
||||
fileController := file.NewFilesystemController(c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
|
||||
fileController := file.NewFilesystemController(cluster.Info, c.context, c.rookImage, cluster.Spec.CephVersion, cluster.Spec.Network.HostNetwork, cluster.ownerRef)
|
||||
fileController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start nfs ganesha CRD watcher
|
||||
|
||||
@@ -44,6 +44,7 @@ const (
|
||||
|
||||
// Cluster represents the Rook and environment configuration settings needed to set up Ceph mgrs.
|
||||
type Cluster struct {
|
||||
clusterInfo *cephconfig.ClusterInfo
|
||||
Namespace string
|
||||
Replicas int
|
||||
placement rookalpha.Placement
|
||||
@@ -56,13 +57,22 @@ type Cluster struct {
|
||||
cephVersion cephv1.CephVersionSpec
|
||||
rookVersion string
|
||||
exitCode func(err error) (int, bool)
|
||||
clusterInfo *cephconfig.ClusterInfo
|
||||
}
|
||||
|
||||
// New creates an instance of the mgr
|
||||
func New(context *clusterd.Context, namespace, rookVersion string, cephVersion cephv1.CephVersionSpec, placement rookalpha.Placement, hostNetwork bool, dashboard cephv1.DashboardSpec,
|
||||
resources v1.ResourceRequirements, ownerRef metav1.OwnerReference, cInfo *cephconfig.ClusterInfo) *Cluster {
|
||||
func New(
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
context *clusterd.Context,
|
||||
namespace, rookVersion string,
|
||||
cephVersion cephv1.CephVersionSpec,
|
||||
placement rookalpha.Placement,
|
||||
hostNetwork bool,
|
||||
dashboard cephv1.DashboardSpec,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *Cluster {
|
||||
return &Cluster{
|
||||
clusterInfo: clusterInfo,
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
placement: placement,
|
||||
@@ -75,7 +85,6 @@ func New(context *clusterd.Context, namespace, rookVersion string, cephVersion c
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
exitCode: getExitCode,
|
||||
clusterInfo: cInfo,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +123,7 @@ func (c *Cluster) Start() error {
|
||||
return fmt.Errorf("failed to create mgr deployment %s. %+v", resourceName, err)
|
||||
}
|
||||
logger.Infof("deployment for mgr %s already exists. updating if needed", resourceName)
|
||||
if err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil {
|
||||
if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil {
|
||||
return fmt.Errorf("failed to update mgr deployment %s. %+v", resourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
|
||||
optest "github.com/rook/rook/pkg/operator/test"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -52,7 +52,9 @@ func TestStartMGR(t *testing.T) {
|
||||
Executor: executor,
|
||||
ConfigDir: configDir,
|
||||
Clientset: testop.New(3)}
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
c := New(
|
||||
clusterInfo,
|
||||
context,
|
||||
"ns",
|
||||
"myversion",
|
||||
@@ -62,7 +64,6 @@ func TestStartMGR(t *testing.T) {
|
||||
cephv1.DashboardSpec{Enabled: true},
|
||||
v1.ResourceRequirements{},
|
||||
metav1.OwnerReference{},
|
||||
optest.CreateConfigDir(1),
|
||||
)
|
||||
defer os.RemoveAll(c.dataDir)
|
||||
|
||||
|
||||
@@ -118,8 +118,7 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule
|
||||
// L: config-key set mgr/<mod>/server_addr $(ROOK_CEPH_<MOD>_SERVER_ADDR)
|
||||
// M: config set mgr.a mgr/<mod>/server_addr $(ROOK_CEPH_<MOD>_SERVER_ADDR)
|
||||
// N: config set mgr.a mgr/<mod>/server_addr $(ROOK_CEPH_<MOD>_SERVER_ADDR) --force
|
||||
envVarName := fmt.Sprintf("ROOK_CEPH_%s_SERVER_ADDR", strings.ToUpper(mgrModule))
|
||||
envVarReference := fmt.Sprintf("$(%s)", envVarName)
|
||||
podIPEnvVar := "ROOK_POD_IP"
|
||||
cfgSetArgs := []string{"config", "set"}
|
||||
if c.cephVersion.Name == cephv1.Luminous || c.cephVersion.Name == "" {
|
||||
cfgSetArgs[0] = "config-key"
|
||||
@@ -127,7 +126,7 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule
|
||||
cfgSetArgs = append(cfgSetArgs, fmt.Sprintf("mgr.%s", mgrConfig.DaemonID))
|
||||
}
|
||||
cfgPath := fmt.Sprintf("mgr/%s/server_addr", mgrModule)
|
||||
cfgSetArgs = append(cfgSetArgs, cfgPath, envVarReference)
|
||||
cfgSetArgs = append(cfgSetArgs, cfgPath, opspec.ContainerEnvVarReference(podIPEnvVar))
|
||||
if cephv1.VersionAtLeast(c.cephVersion.Name, cephv1.Nautilus) {
|
||||
cfgSetArgs = append(cfgSetArgs, "--force")
|
||||
}
|
||||
@@ -149,7 +148,7 @@ func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule
|
||||
),
|
||||
Env: append(
|
||||
opspec.DaemonEnvVars(),
|
||||
k8sutil.PodIPEnvVar(envVarName),
|
||||
k8sutil.PodIPEnvVar(podIPEnvVar),
|
||||
),
|
||||
Resources: c.resources,
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ package mgr
|
||||
import (
|
||||
"testing"
|
||||
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
@@ -33,7 +34,9 @@ import (
|
||||
)
|
||||
|
||||
func TestPodSpec(t *testing.T) {
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
c := New(
|
||||
clusterInfo,
|
||||
&clusterd.Context{Clientset: optest.New(1)},
|
||||
"ns",
|
||||
"rook/rook:myversion",
|
||||
@@ -50,7 +53,6 @@ func TestPodSpec(t *testing.T) {
|
||||
},
|
||||
},
|
||||
metav1.OwnerReference{},
|
||||
optest.CreateConfigDir(1),
|
||||
)
|
||||
|
||||
mgrTestConfig := mgrConfig{
|
||||
@@ -73,7 +75,9 @@ func TestPodSpec(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestServiceSpec(t *testing.T) {
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
c := New(
|
||||
clusterInfo,
|
||||
&clusterd.Context{Clientset: optest.New(1)},
|
||||
"ns",
|
||||
"myversion",
|
||||
@@ -83,7 +87,6 @@ func TestServiceSpec(t *testing.T) {
|
||||
cephv1.DashboardSpec{},
|
||||
v1.ResourceRequirements{},
|
||||
metav1.OwnerReference{},
|
||||
optest.CreateConfigDir(1),
|
||||
)
|
||||
|
||||
s := c.makeMetricsService("rook-mgr")
|
||||
@@ -93,7 +96,9 @@ func TestServiceSpec(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestHostNetwork(t *testing.T) {
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
c := New(
|
||||
clusterInfo,
|
||||
&clusterd.Context{Clientset: optest.New(1)},
|
||||
"ns",
|
||||
"myversion",
|
||||
@@ -103,7 +108,6 @@ func TestHostNetwork(t *testing.T) {
|
||||
cephv1.DashboardSpec{},
|
||||
v1.ResourceRequirements{},
|
||||
metav1.OwnerReference{},
|
||||
optest.CreateConfigDir(1),
|
||||
)
|
||||
|
||||
mgrTestConfig := mgrConfig{
|
||||
|
||||
@@ -184,19 +184,19 @@ func createClusterAccessSecret(clientset kubernetes.Interface, namespace string,
|
||||
var err error
|
||||
|
||||
// store the secrets for internal usage of the rook pods
|
||||
secrets := map[string]string{
|
||||
clusterSecretName: clusterInfo.Name,
|
||||
fsidSecretName: clusterInfo.FSID,
|
||||
monSecretName: clusterInfo.MonitorSecret,
|
||||
adminSecretName: clusterInfo.AdminSecret,
|
||||
secrets := map[string][]byte{
|
||||
clusterSecretName: []byte(clusterInfo.Name),
|
||||
fsidSecretName: []byte(clusterInfo.FSID),
|
||||
monSecretName: []byte(clusterInfo.MonitorSecret),
|
||||
adminSecretName: []byte(clusterInfo.AdminSecret),
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: appName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
StringData: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
Data: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
k8sutil.SetOwnerRef(clientset, namespace, &secret.ObjectMeta, ownerRef)
|
||||
|
||||
|
||||
@@ -55,9 +55,9 @@ func TestCheckHealth(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
c := New(test.CreateConfigDir(1), context, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
logger.Infof("initial mons: %v", c.clusterInfo.Monitors)
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
@@ -108,9 +108,9 @@ func TestCheckHealthNotFound(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(2)
|
||||
c := New(test.CreateConfigDir(2), context, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
|
||||
@@ -180,9 +180,9 @@ func TestCheckHealthTwoMonsOneNode(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(2)
|
||||
c := New(test.CreateConfigDir(2), context, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
|
||||
@@ -290,9 +290,9 @@ func TestCheckMonsValid(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
c := New(test.CreateConfigDir(1), context, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
|
||||
@@ -378,10 +378,10 @@ func TestAddRemoveMons(t *testing.T) {
|
||||
ConfigDir: configDir,
|
||||
Executor: executor,
|
||||
}
|
||||
c := New(context, "ns", "", "myversion", cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 5, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c := New(test.CreateConfigDir(0), context, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 5, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.maxMonID = 0
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
c.waitForStart = false
|
||||
defer os.RemoveAll(c.context.ConfigDir)
|
||||
|
||||
|
||||
@@ -77,6 +77,7 @@ const (
|
||||
|
||||
// Cluster represents the Rook and environment configuration settings needed to set up Ceph mons.
|
||||
type Cluster struct {
|
||||
clusterInfo *cephconfig.ClusterInfo
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
Keyring string
|
||||
@@ -86,7 +87,6 @@ type Cluster struct {
|
||||
AllowMultiplePerNode bool
|
||||
MonCountMutex sync.Mutex
|
||||
Port int32
|
||||
clusterInfo *cephconfig.ClusterInfo
|
||||
placement rookalpha.Placement
|
||||
maxMonID int
|
||||
waitForStart bool
|
||||
@@ -130,6 +130,7 @@ type NodeInfo struct {
|
||||
|
||||
// New creates an instance of a mon cluster
|
||||
func New(
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
context *clusterd.Context,
|
||||
namespace, dataDirHostPath, rookVersion string,
|
||||
cephVersion cephv1.CephVersionSpec,
|
||||
@@ -140,6 +141,7 @@ func New(
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *Cluster {
|
||||
return &Cluster{
|
||||
clusterInfo: clusterInfo,
|
||||
context: context,
|
||||
placement: placement,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
@@ -172,6 +174,7 @@ func (c *Cluster) Start() (*cephconfig.ClusterInfo, error) {
|
||||
|
||||
logger.Infof("start running mons")
|
||||
|
||||
logger.Debugf("establishing ceph cluster info")
|
||||
if err := c.initClusterInfo(); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize ceph cluster info. %+v", err)
|
||||
}
|
||||
@@ -489,7 +492,7 @@ func (c *Cluster) startMon(m *monConfig, hostname string) error {
|
||||
// image or Ceph image has changed.
|
||||
if p.Spec.Template.Spec.Containers[0].Image != d.Spec.Template.Spec.Containers[0].Image ||
|
||||
p.Spec.Template.Spec.InitContainers[0].Image != d.Spec.Template.Spec.InitContainers[0].Image {
|
||||
if err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil {
|
||||
if _, err := updateDeploymentAndWait(c.context, d, c.Namespace); err != nil {
|
||||
return fmt.Errorf("failed to update mon deployment %s. %+v", m.ResourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,7 @@ func newTestStartClusterWithQuorumResponse(namespace string, monResponse func()
|
||||
|
||||
func newCluster(context *clusterd.Context, namespace string, hostNetwork bool, allowMultiplePerNode bool, resources v1.ResourceRequirements) *Cluster {
|
||||
return &Cluster{
|
||||
clusterInfo: nil,
|
||||
HostNetwork: hostNetwork,
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
@@ -134,16 +135,14 @@ func TestStartMonPods(t *testing.T) {
|
||||
c := newCluster(context, namespace, false, true, v1.ResourceRequirements{})
|
||||
|
||||
// start a basic cluster
|
||||
i, err := c.Start()
|
||||
_, err := c.Start()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, i, c.clusterInfo)
|
||||
|
||||
validateStart(t, c)
|
||||
|
||||
// starting again should be a no-op, but still results in an error
|
||||
i, err = c.Start()
|
||||
_, err = c.Start()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, i, c.clusterInfo)
|
||||
|
||||
validateStart(t, c)
|
||||
}
|
||||
@@ -155,18 +154,18 @@ func TestOperatorRestart(t *testing.T) {
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
|
||||
// start a basic cluster
|
||||
i, err := c.Start()
|
||||
info, err := c.Start()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, i, c.clusterInfo)
|
||||
assert.True(t, info.IsInitialized())
|
||||
|
||||
validateStart(t, c)
|
||||
|
||||
c = newCluster(context, namespace, false, true, v1.ResourceRequirements{})
|
||||
|
||||
// starting again should be a no-op, but will not result in an error
|
||||
i, err = c.Start()
|
||||
info, err = c.Start()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, i, c.clusterInfo)
|
||||
assert.True(t, info.IsInitialized())
|
||||
|
||||
validateStart(t, c)
|
||||
}
|
||||
@@ -181,9 +180,9 @@ func TestOperatorRestartHostNetwork(t *testing.T) {
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
|
||||
// start a basic cluster
|
||||
i, err := c.Start()
|
||||
info, err := c.Start()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, i, c.clusterInfo)
|
||||
assert.True(t, info.IsInitialized())
|
||||
|
||||
validateStart(t, c)
|
||||
|
||||
@@ -191,9 +190,9 @@ func TestOperatorRestartHostNetwork(t *testing.T) {
|
||||
c = newCluster(context, namespace, true, false, v1.ResourceRequirements{})
|
||||
|
||||
// starting again should be a no-op, but still results in an error
|
||||
i, err = c.Start()
|
||||
info, err = c.Start()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, i, c.clusterInfo)
|
||||
assert.True(t, info.IsInitialized(), info)
|
||||
|
||||
validateStart(t, c)
|
||||
}
|
||||
@@ -201,7 +200,7 @@ func TestOperatorRestartHostNetwork(t *testing.T) {
|
||||
func validateStart(t *testing.T, c *Cluster) {
|
||||
s, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{})
|
||||
assert.Nil(t, err) // there shouldn't be an error due the secret existing
|
||||
assert.Equal(t, 4, len(s.StringData))
|
||||
assert.Equal(t, 4, len(s.Data))
|
||||
|
||||
// there is only one pod created. the other two won't be created since the first one doesn't start
|
||||
_, err = c.context.Clientset.Apps().Deployments(c.Namespace).Get("rook-ceph-mon-a", metav1.GetOptions{})
|
||||
@@ -212,10 +211,10 @@ func TestSaveMonEndpoints(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: configDir}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
c := New(test.CreateConfigDir(1), &clusterd.Context{Clientset: clientset, ConfigDir: configDir},
|
||||
"ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{}, false,
|
||||
v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(1)
|
||||
|
||||
// create the initial config map
|
||||
err := c.saveMonConfig()
|
||||
|
||||
@@ -30,10 +30,10 @@ import (
|
||||
|
||||
func TestAvailableMonNodes(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(nodes))
|
||||
@@ -57,10 +57,9 @@ func TestAvailableMonNodes(t *testing.T) {
|
||||
|
||||
func TestAvailableNodesInUse(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// all three nodes are available by default
|
||||
nodes, err := c.getMonNodes()
|
||||
@@ -99,10 +98,9 @@ func TestAvailableNodesInUse(t *testing.T) {
|
||||
|
||||
func TestTaintedNodes(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
@@ -134,10 +132,9 @@ func TestTaintedNodes(t *testing.T) {
|
||||
|
||||
func TestNodeAffinity(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
nodes, err := c.getMonNodes()
|
||||
assert.Nil(t, err)
|
||||
@@ -190,10 +187,9 @@ func TestHostNetworkSameNode(t *testing.T) {
|
||||
|
||||
func TestHostNetwork(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: false}, rookalpha.Placement{},
|
||||
false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
c := New(test.CreateConfigDir(0), &clusterd.Context{Clientset: clientset}, "ns", "", "myversion",
|
||||
cephv1.CephVersionSpec{}, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: false},
|
||||
rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
c.HostNetwork = true
|
||||
|
||||
@@ -222,11 +218,6 @@ func TestGetNodeInfoFromNode(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset}, "ns", "", "myversion", cephv1.CephVersionSpec{},
|
||||
cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, rookalpha.Placement{},
|
||||
true, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
c.clusterInfo = test.CreateConfigDir(0)
|
||||
|
||||
var info *NodeInfo
|
||||
info, err = getNodeInfoFromNode(*node)
|
||||
assert.Nil(t, err)
|
||||
|
||||
@@ -152,6 +152,7 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container {
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
|
||||
podIPEnvVar := "ROOK_POD_IP"
|
||||
container := v1.Container{
|
||||
Name: "mon",
|
||||
Command: []string{
|
||||
@@ -161,7 +162,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
|
||||
opspec.DaemonFlags(c.clusterInfo, config.MonType, monConfig.DaemonName),
|
||||
"--foreground",
|
||||
config.NewFlag("public-addr", monConfig.PublicIP),
|
||||
config.NewFlag("public-bind-addr", "$(ROOK_PRIVATE_IP)"),
|
||||
config.NewFlag("public-bind-addr", opspec.ContainerEnvVarReference(podIPEnvVar)),
|
||||
),
|
||||
Image: c.cephVersion.Image,
|
||||
VolumeMounts: opspec.DaemonVolumeMounts(monConfig.DataPathMap, keyringStoreName),
|
||||
@@ -175,7 +176,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
|
||||
},
|
||||
Env: append(
|
||||
opspec.DaemonEnvVars(),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(podIPEnvVar),
|
||||
),
|
||||
Resources: c.resources,
|
||||
}
|
||||
|
||||
@@ -22,13 +22,11 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
test_opceph "github.com/rook/rook/pkg/operator/ceph/test"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -45,6 +43,7 @@ func TestPodSpecs(t *testing.T) {
|
||||
func testPodSpec(t *testing.T, monID string) {
|
||||
clientset := testop.New(1)
|
||||
c := New(
|
||||
testop.CreateConfigDir(0),
|
||||
&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"},
|
||||
"ns",
|
||||
"/var/lib/rook",
|
||||
@@ -63,7 +62,6 @@ func testPodSpec(t *testing.T, monID string) {
|
||||
},
|
||||
metav1.OwnerReference{},
|
||||
)
|
||||
c.clusterInfo = testop.CreateConfigDir(0)
|
||||
monConfig := testGenMonConfig(monID)
|
||||
|
||||
pod := c.makeMonPod(monConfig, "foo")
|
||||
@@ -129,7 +127,7 @@ func testPodSpec(t *testing.T, monID string) {
|
||||
Args: append(commonFlags,
|
||||
[]string{"--foreground"},
|
||||
[]string{"--public-addr=2.4.6.1"},
|
||||
[]string{"--public-bind-addr=$(ROOK_PRIVATE_IP)"}),
|
||||
[]string{"--public-bind-addr=$(ROOK_POD_IP)"}),
|
||||
VolumeMountNames: cephVolumeMountNames,
|
||||
EnvCount: &monDaemonEnvs,
|
||||
Ports: []v1.ContainerPort{
|
||||
|
||||
@@ -320,7 +320,7 @@ func (c *Cluster) startOSDDaemonsOnNode(nodeName string, config *provisionConfig
|
||||
continue
|
||||
}
|
||||
logger.Infof("deployment for osd %d already exists. updating if needed", osd.ID)
|
||||
if err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil {
|
||||
if _, err = k8sutil.UpdateDeploymentAndWait(c.context, dp, c.Namespace); err != nil {
|
||||
config.addError(fmt.Sprintf("failed to update osd deployment %d. %+v", osd.ID, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,10 +17,35 @@ limitations under the License.
|
||||
// Package config provides default configurations which Rook will set in Ceph clusters.
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// DefaultFlags returns the default configuration flags Rook will set on the command line for all
|
||||
// calls to Ceph daemons and tools. Values specified here will not be able to be overridden using
|
||||
// the mon's central KV store, and that is (and should be) by intent.
|
||||
func DefaultFlags(fsid, mountedKeyringPath string) []string {
|
||||
return defaultFlagConfigs(fsid, mountedKeyringPath).GlobalFlags()
|
||||
}
|
||||
|
||||
// DefaultFlagsAsConfigFile is the same as DefaultFlags except that it returns the ceph.conf file
|
||||
// text which corresponds to the flags. In some versions of Ceph and for some Daemons, the default
|
||||
// flags are not accepted in flag form.
|
||||
func DefaultFlagsAsConfigFile(fsid, mountedKeyringPath string) (string, error) {
|
||||
c := defaultFlagConfigs(fsid, mountedKeyringPath)
|
||||
f, err := c.IniFile()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate text for default flags compatible with ceph.conf. %v", err)
|
||||
}
|
||||
b := new(bytes.Buffer)
|
||||
if _, err := f.WriteTo(b); err != nil {
|
||||
return "", fmt.Errorf("failed to generate text for default flags compatible with ceph.conf. %v", err)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
func defaultFlagConfigs(fsid, mountedKeyringPath string) *Config {
|
||||
c := NewConfig()
|
||||
c.Section("global").
|
||||
// fsid unnecessary but is a safety to make sure daemons can only connect to their cluster
|
||||
@@ -32,10 +57,9 @@ func DefaultFlags(fsid, mountedKeyringPath string) []string {
|
||||
Set("mon-cluster-log-to-stderr", "true").
|
||||
Set("log-stderr-prefix", "debug ")
|
||||
// ^ differentiate debug text from audit text, and the space after 'debug' is critical
|
||||
return append(
|
||||
c.GlobalFlags(),
|
||||
StoredMonHostEnvVarFlags()...,
|
||||
)
|
||||
m := StoredMonHostEnvVarReferences()
|
||||
c.Merge(m)
|
||||
return c
|
||||
}
|
||||
|
||||
// DefaultCentralizedConfigs returns the default configuration options Rook will set in Ceph's
|
||||
@@ -68,7 +92,10 @@ func DefaultLegacyConfigs() *Config {
|
||||
Set("osd pool default pgp num", "100").
|
||||
//
|
||||
Set("rbd_default_features", "3"). // TODO: still needed?
|
||||
//
|
||||
Set("fatal signal handlers", "false") // TODO: still needed?
|
||||
// Setting fatal signal handlers to true (the default) will print a lot of extra information
|
||||
// from daemons when they encounter a failures, but it is VERY verbose. When the mon kv
|
||||
// store is available, it will probably be best to set this to false by default if it is
|
||||
// unset, but leave it set to true if the user has specified it as true during runtime.
|
||||
Set("fatal signal handlers", "false")
|
||||
return c
|
||||
}
|
||||
@@ -256,10 +256,13 @@ func StoredMonHostEnvVars() []v1.EnvVar {
|
||||
}
|
||||
}
|
||||
|
||||
// StoredMonHostEnvVarFlags returns a flag compatible with Ceph daemons and tools to read "mon_host"
|
||||
// and "mon_initial_members" information from the StoredMonHostEnvVar.
|
||||
func StoredMonHostEnvVarFlags() []string {
|
||||
return []string{
|
||||
NewFlag(monHostKey, "$(ROOK_CEPH_MON_HOST)"),
|
||||
NewFlag(monInitialMembersKey, "$(ROOK_CEPH_MON_INITIAL_MEMBERS)")}
|
||||
// StoredMonHostEnvVarReferences returns a small Ceph Config which references "mon_host" and
|
||||
// "mon_initial_members" information from the StoredMonHostEnvVars. This config can be used to
|
||||
// generate flags referencing the env vars or to generate the string representation of a Config.
|
||||
func StoredMonHostEnvVarReferences() *Config {
|
||||
c := NewConfig()
|
||||
c.Section("global").
|
||||
Set(monHostKey, "$(ROOK_CEPH_MON_HOST)").
|
||||
Set(monInitialMembersKey, "$(ROOK_CEPH_MON_INITIAL_MEMBERS)")
|
||||
return c
|
||||
}
|
||||
@@ -204,7 +204,7 @@ func TestEnvVarsAndFlags(t *testing.T) {
|
||||
s.CreateOrUpdate(testop.CreateConfigDir(3))
|
||||
|
||||
v := StoredMonHostEnvVars()
|
||||
f := StoredMonHostEnvVarFlags()
|
||||
f := StoredMonHostEnvVarReferences().GlobalFlags()
|
||||
|
||||
// make sure the env var names and flags are matching pairs
|
||||
mh := v[0].Name
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
Copyright 2019 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
)
|
||||
|
||||
const (
|
||||
keyringTemplate = `
|
||||
[mds.%s]
|
||||
key = %s
|
||||
caps mon = "allow profile mds"
|
||||
caps osd = "allow *"
|
||||
caps mds = "allow"
|
||||
`
|
||||
)
|
||||
|
||||
func (c *cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
|
||||
user := fmt.Sprintf("mds.%s", m.DaemonID)
|
||||
access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"}
|
||||
ownerRef := &metav1.OwnerReference{
|
||||
UID: deploymentUID,
|
||||
APIVersion: "v1",
|
||||
Kind: "deployment",
|
||||
Name: m.ResourceName,
|
||||
}
|
||||
s := keyring.GetSecretStore(c.context, c.fs.Namespace, ownerRef)
|
||||
|
||||
key, err := s.GenerateKey(m.ResourceName, user, access)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete legacy key store for upgrade from Rook v0.9.x to v1.0.x
|
||||
err = c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Delete(m.ResourceName, &metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Debugf("legacy mgr key %s is already removed", m.ResourceName)
|
||||
} else {
|
||||
logger.Warningf("legacy mgr key %s could not be removed: %+v", m.ResourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
keyring := fmt.Sprintf(keyringTemplate, m.DaemonID, key)
|
||||
return s.CreateOrUpdate(m.ResourceName, keyring)
|
||||
}
|
||||
@@ -21,13 +21,13 @@ import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
cephbeta "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-file")
|
||||
|
||||
// FilesystemResource represents the file system custom resource
|
||||
// FilesystemResource represents the filesystem custom resource
|
||||
var FilesystemResource = opkit.CustomResource{
|
||||
Name: "cephfilesystem",
|
||||
Plural: "cephfilesystems",
|
||||
@@ -55,8 +55,9 @@ var filesystemResourceRookLegacy = opkit.CustomResource{
|
||||
Kind: reflect.TypeOf(cephbeta.Filesystem{}).Name(),
|
||||
}
|
||||
|
||||
// FilesystemController represents a controller for file system custom resources
|
||||
// FilesystemController represents a controller for filesystem custom resources
|
||||
type FilesystemController struct {
|
||||
clusterInfo *cephconfig.ClusterInfo
|
||||
context *clusterd.Context
|
||||
rookVersion string
|
||||
cephVersion cephv1.CephVersionSpec
|
||||
@@ -64,8 +65,9 @@ type FilesystemController struct {
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// NewFilesystemController create controller for watching file system custom resources created
|
||||
// NewFilesystemController create controller for watching filesystem custom resources created
|
||||
func NewFilesystemController(
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
context *clusterd.Context,
|
||||
rookVersion string,
|
||||
cephVersion cephv1.CephVersionSpec,
|
||||
@@ -73,6 +75,7 @@ func NewFilesystemController(
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *FilesystemController {
|
||||
return &FilesystemController{
|
||||
clusterInfo: clusterInfo,
|
||||
context: context,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
@@ -114,9 +117,9 @@ func (c *FilesystemController) onAdd(obj interface{}) {
|
||||
return
|
||||
}
|
||||
|
||||
err = createFilesystem(c.context, *filesystem, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(filesystem))
|
||||
err = createFilesystem(c.clusterInfo, c.context, *filesystem, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(filesystem))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create file system %s: %+v", filesystem.Name, err)
|
||||
logger.Errorf("failed to create filesystem %s: %+v", filesystem.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,11 +147,11 @@ func (c *FilesystemController) onUpdate(oldObj, newObj interface{}) {
|
||||
return
|
||||
}
|
||||
|
||||
// if the file system is modified, allow the file system to be created if it wasn't already
|
||||
// if the filesystem is modified, allow the filesystem to be created if it wasn't already
|
||||
logger.Infof("updating filesystem %s", newFS.Name)
|
||||
err = createFilesystem(c.context, *newFS, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(newFS))
|
||||
err = createFilesystem(c.clusterInfo, c.context, *newFS, c.rookVersion, c.cephVersion, c.hostNetwork, c.filesystemOwners(newFS))
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create (modify) file system %s: %+v", newFS.Name, err)
|
||||
logger.Errorf("failed to create (modify) filesystem %s: %+v", newFS.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,7 +169,7 @@ func (c *FilesystemController) onDelete(obj interface{}) {
|
||||
|
||||
err = deleteFilesystem(c.context, *filesystem)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete file system %s: %+v", filesystem.Name, err)
|
||||
logger.Errorf("failed to delete filesystem %s: %+v", filesystem.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package mds to manage a rook file system.
|
||||
// Package mds to manage a rook filesystem.
|
||||
package file
|
||||
|
||||
import (
|
||||
@@ -25,6 +25,8 @@ import (
|
||||
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
@@ -83,7 +85,9 @@ func TestMigrateFilesystemObject(t *testing.T) {
|
||||
Clientset: clientset,
|
||||
RookClientset: rookfake.NewSimpleClientset(legacyFilesystem),
|
||||
}
|
||||
controller := NewFilesystemController(context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{})
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
|
||||
controller := NewFilesystemController(clusterInfo, context, "", cephv1.CephVersionSpec{}, false, metav1.OwnerReference{})
|
||||
|
||||
// convert the legacy filesystem object in memory and assert that a migration is needed
|
||||
convertedFilesystem, migrationNeeded, err := getFilesystemObject(legacyFilesystem)
|
||||
|
||||
@@ -18,18 +18,34 @@ package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/model"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
dataPoolSuffix = "data"
|
||||
metadataPoolSuffix = "metadata"
|
||||
appName = "cephfs"
|
||||
)
|
||||
|
||||
// Filesystem represents an instance of a Ceph filesystem (CephFS)
|
||||
type Filesystem struct {
|
||||
Name string
|
||||
metadataPool *model.Pool
|
||||
dataPools []*model.Pool
|
||||
activeMDSCount int32
|
||||
}
|
||||
|
||||
// createFilesystem creates a Ceph filesystem with metadata servers
|
||||
func createFilesystem(
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
context *clusterd.Context,
|
||||
fs cephv1.CephFilesystem,
|
||||
rookVersion string,
|
||||
@@ -46,15 +62,15 @@ func createFilesystem(
|
||||
for _, p := range fs.Spec.DataPools {
|
||||
dataPools = append(dataPools, p.ToModel(""))
|
||||
}
|
||||
f := mdsdaemon.NewFS(fs.Name, fs.Spec.MetadataPool.ToModel(""), dataPools, fs.Spec.MetadataServer.ActiveCount)
|
||||
if err := f.CreateFilesystem(context, fs.Namespace); err != nil {
|
||||
return fmt.Errorf("failed to create file system %s: %+v", fs.Name, err)
|
||||
f := newFS(fs.Name, fs.Spec.MetadataPool.ToModel(""), dataPools, fs.Spec.MetadataServer.ActiveCount)
|
||||
if err := f.doFilesystemCreate(context, fs.Namespace); err != nil {
|
||||
return fmt.Errorf("failed to create filesystem %s: %+v", fs.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
filesystem, err := client.GetFilesystem(context, fs.Namespace, fs.Name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get file system %s: %+v", fs.Name, err)
|
||||
return fmt.Errorf("failed to get filesystem %s: %+v", fs.Name, err)
|
||||
}
|
||||
|
||||
// set the number of active mds instances
|
||||
@@ -64,8 +80,8 @@ func createFilesystem(
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("start running mdses for file system %s", fs.Name)
|
||||
c := newCluster(context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs)
|
||||
logger.Infof("start running mdses for filesystem %s", fs.Name)
|
||||
c := newCluster(clusterInfo, context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs)
|
||||
if err := c.start(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -73,15 +89,15 @@ func createFilesystem(
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteFileSystem deletes the file system and the metadata servers
|
||||
// deleteFileSystem deletes the filesystem and the metadata servers
|
||||
func deleteFilesystem(context *clusterd.Context, fs cephv1.CephFilesystem) error {
|
||||
// The most important part of deletion is that the filesystem gets removed from Ceph
|
||||
if err := mdsdaemon.DownFilesystem(context, fs.Namespace, fs.Name); err != nil {
|
||||
if err := downFilesystem(context, fs.Namespace, fs.Name); err != nil {
|
||||
// If the fs isn't deleted from Ceph, leave the daemons so it can still be used.
|
||||
return fmt.Errorf("failed to down filesystem %s: %+v", fs.Name, err)
|
||||
}
|
||||
|
||||
// Permanently remove the file system if it was created by rook
|
||||
// Permanently remove the filesystem if it was created by rook
|
||||
if len(fs.Spec.DataPools) != 0 {
|
||||
if err := client.RemoveFilesystem(context, fs.Namespace, fs.Name); err != nil {
|
||||
return fmt.Errorf("failed to remove filesystem %s: %+v", fs.Name, err)
|
||||
@@ -116,3 +132,148 @@ func validateFilesystem(context *clusterd.Context, f cephv1.CephFilesystem) erro
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// newFS creates a new instance of the file (MDS) service
|
||||
func newFS(name string, metadataPool *model.Pool, dataPools []*model.Pool, activeMDSCount int32) *Filesystem {
|
||||
|
||||
metadataPool.Name = fmt.Sprintf("%s-%s", name, metadataPoolSuffix)
|
||||
for i, pool := range dataPools {
|
||||
pool.Name = fmt.Sprintf("%s-%s%d", name, dataPoolSuffix, i)
|
||||
}
|
||||
|
||||
return &Filesystem{
|
||||
Name: name,
|
||||
metadataPool: metadataPool,
|
||||
dataPools: dataPools,
|
||||
activeMDSCount: activeMDSCount,
|
||||
}
|
||||
}
|
||||
|
||||
// doFilesystemCreate starts the Ceph file daemons and creates the filesystem in Ceph.
|
||||
func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterName string) error {
|
||||
_, err := client.GetFilesystem(context, clusterName, f.Name)
|
||||
if err == nil {
|
||||
logger.Infof("filesystem %s already exists", f.Name)
|
||||
// Even if the fs already exists, the num active mdses may have changed
|
||||
if err := client.SetNumMDSRanks(context, clusterName, f.Name, f.activeMDSCount); err != nil {
|
||||
logger.Errorf(
|
||||
fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", f.activeMDSCount, f.Name) +
|
||||
"this error is not critical, but mdses may not be as failure tolerant as desired. " +
|
||||
fmt.Sprintf("USER should verify that the number of active mdses is %d with 'ceph fs get %s'", f.activeMDSCount, f.Name) +
|
||||
fmt.Sprintf(": %+v", err),
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(f.dataPools) == 0 {
|
||||
return fmt.Errorf("at least one data pool must be specified")
|
||||
}
|
||||
|
||||
fslist, err := client.ListFilesystems(context, clusterName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to list existing filesystem: %+v", err)
|
||||
}
|
||||
if len(fslist) > 0 && !client.IsMultiFSEnabled() {
|
||||
return fmt.Errorf("Cannot create multiple filesystems. Enable %s env variable to create more than one", client.MultiFsEnv)
|
||||
}
|
||||
|
||||
logger.Infof("Creating filesystem %s", f.Name)
|
||||
err = client.CreatePoolWithProfile(context, clusterName, *f.metadataPool, appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create metadata pool '%s': %+v", f.metadataPool.Name, err)
|
||||
}
|
||||
|
||||
var dataPoolNames []string
|
||||
for _, pool := range f.dataPools {
|
||||
dataPoolNames = append(dataPoolNames, pool.Name)
|
||||
err = client.CreatePoolWithProfile(context, clusterName, *pool, appName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create data pool %s: %+v", pool.Name, err)
|
||||
}
|
||||
if pool.Type == model.ErasureCoded {
|
||||
// An erasure coded data pool used for a filesystem must allow overwrites
|
||||
if err := client.SetPoolProperty(context, clusterName, pool.Name, "allow_ec_overwrites", "true"); err != nil {
|
||||
logger.Warningf("failed to set ec pool property: %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create the filesystem
|
||||
if err := client.CreateFilesystem(context, clusterName, f.Name, f.metadataPool.Name, dataPoolNames); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("created filesystem%s on %d data pool(s) and metadata pool %s", f.Name, len(f.dataPools), f.metadataPool.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// downFilesystem marks the filesystem as down and the MDS' as failed
|
||||
func downFilesystem(context *clusterd.Context, clusterName, filesystemName string) error {
|
||||
logger.Infof("Removing filesystem %s", filesystemName)
|
||||
|
||||
// mark the cephFS instance as cluster_down before removing
|
||||
if err := client.MarkFilesystemAsDown(context, clusterName, filesystemName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// mark each MDS associated with the filesystem to "failed"
|
||||
fsDetails, err := client.GetFilesystem(context, clusterName, filesystemName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, mdsInfo := range fsDetails.MDSMap.Info {
|
||||
if err := client.FailMDS(context, clusterName, mdsInfo.GID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("Removed filesystem %s", filesystemName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared
|
||||
// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades.
|
||||
// If the mds is not prepared within the timeout window, an error will be reported.
|
||||
// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/
|
||||
func prepareForDaemonUpgrade(
|
||||
context *clusterd.Context,
|
||||
clusterName, fsName string,
|
||||
timeout time.Duration,
|
||||
) error {
|
||||
logger.Infof("preparing filesystem %s for daemon upgrade", fsName)
|
||||
// * Beginning of noted section 1
|
||||
// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
|
||||
// See more: https://ceph.com/releases/v13-2-0-mimic-released/
|
||||
// http://docs.ceph.com/docs/mimic/cephfs/upgrading/
|
||||
// As of Oct. 2018, this is only necessary for Luminous and Mimic.
|
||||
if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil {
|
||||
return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err)
|
||||
}
|
||||
if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
// * End of Noted section 1
|
||||
|
||||
logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its
|
||||
// ideal state following an upgrade of its daemon(s).
|
||||
func finishedWithDaemonUpgrade(
|
||||
context *clusterd.Context,
|
||||
clusterName, fsName string,
|
||||
activeMDSCount int32,
|
||||
) error {
|
||||
logger.Debugf("restoring filesystem %s from daemon upgrade", fsName)
|
||||
logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount)
|
||||
// * Beginning of noted section 1
|
||||
// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
|
||||
// See more: https://ceph.com/releases/v13-2-0-mimic-released/
|
||||
// http://docs.ceph.com/docs/mimic/cephfs/upgrading/
|
||||
// TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic.
|
||||
if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil {
|
||||
return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err)
|
||||
} // * End of noted section 1
|
||||
return nil
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
cephtest "github.com/rook/rook/pkg/daemon/ceph/test"
|
||||
testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
@@ -71,7 +72,7 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
|
||||
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
// Output to check multiple file system creation
|
||||
// Output to check multiple filesystem creation
|
||||
fses := `[{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":1,"data_pool_ids":[2],"data_pools":["myfs-data0"]}]`
|
||||
|
||||
executor := &exectest.MockExecutor{
|
||||
@@ -109,16 +110,17 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
|
||||
// start a basic cluster
|
||||
err := createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
err := createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
assert.ElementsMatch(t, []string{}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
testopk8s.ClearDeploymentsUpdated(deploymentsUpdated)
|
||||
|
||||
// starting again should be a no-op
|
||||
err = createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
assert.ElementsMatch(t, []string{"rook-ceph-mds-myfs-a", "rook-ceph-mds-myfs-b"}, testopk8s.DeploymentNamesUpdated(deploymentsUpdated))
|
||||
@@ -139,13 +141,13 @@ func TestCreateFilesystem(t *testing.T) {
|
||||
Clientset: testop.New(3)}
|
||||
|
||||
//Create another filesystem which should fail
|
||||
err = createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Equal(t, "failed to create file system myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error())
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Equal(t, "failed to create filesystem myfs: Cannot create multiple filesystems. Enable ROOK_ALLOW_MULTIPLE_FILESYSTEMS env variable to create more than one", err.Error())
|
||||
}
|
||||
|
||||
func TestCreateNopoolFilesystem(t *testing.T) {
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
// Output to check multiple file system creation
|
||||
// Output to check multiple filesystem creation
|
||||
fses := `[{"name":"myfs"}]`
|
||||
|
||||
executor := &exectest.MockExecutor{
|
||||
@@ -173,14 +175,15 @@ func TestCreateNopoolFilesystem(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
|
||||
// start a basic cluster
|
||||
err := createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
err := createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
|
||||
// starting again should be a no-op
|
||||
err = createFilesystem(context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
err = createFilesystem(clusterInfo, context, fs, "v0.1", cephv1.CephVersionSpec{}, false, []metav1.OwnerReference{})
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, context, fs)
|
||||
|
||||
|
||||
@@ -24,10 +24,10 @@ import (
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@@ -43,6 +43,7 @@ const (
|
||||
)
|
||||
|
||||
type cluster struct {
|
||||
clusterInfo *cephconfig.ClusterInfo
|
||||
context *clusterd.Context
|
||||
rookVersion string
|
||||
cephVersion cephv1.CephVersionSpec
|
||||
@@ -52,7 +53,14 @@ type cluster struct {
|
||||
ownerRefs []metav1.OwnerReference
|
||||
}
|
||||
|
||||
type mdsConfig struct {
|
||||
ResourceName string
|
||||
DaemonID string
|
||||
DataPathMap *config.DataPathMap // location to store data in container
|
||||
}
|
||||
|
||||
func newCluster(
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
context *clusterd.Context,
|
||||
rookVersion string,
|
||||
cephVersion cephv1.CephVersionSpec,
|
||||
@@ -62,6 +70,7 @@ func newCluster(
|
||||
ownerRefs []metav1.OwnerReference,
|
||||
) *cluster {
|
||||
return &cluster{
|
||||
clusterInfo: clusterInfo,
|
||||
context: context,
|
||||
rookVersion: rookVersion,
|
||||
cephVersion: cephVersion,
|
||||
@@ -72,31 +81,6 @@ func newCluster(
|
||||
}
|
||||
}
|
||||
|
||||
type mdsConfig struct {
|
||||
ResourceName string
|
||||
DaemonName string
|
||||
}
|
||||
|
||||
// return true if an attempt was made to prepare the filesystem associated for upgrade
|
||||
func (c *cluster) deleteLegacyMdsDeployment() bool {
|
||||
legacyName := fmt.Sprintf("%s-%s", AppName, c.fs.Name) // rook-ceph-mds-<fsname>
|
||||
logger.Debugf("getting legacy mds deployment %s for filesystem %s if it exists", legacyName, c.fs.Name)
|
||||
_, err := c.context.Clientset.Apps().Deployments(c.fs.Namespace).Get(legacyName, metav1.GetOptions{})
|
||||
if err != nil && errors.IsNotFound(err) {
|
||||
logger.Infof("legacy mds deployment %s not found, no update needed", legacyName)
|
||||
return false
|
||||
// if any other error, failed to get legacy dep., but it exists, so still try deleting it.
|
||||
}
|
||||
logger.Infof("deleting legacy mds deployment %s", legacyName)
|
||||
if err := mdsdaemon.PrepareForDaemonUpgrade(c.context, c.fs.Namespace, c.fs.Name, fsWaitForActiveTimeout); err != nil {
|
||||
logger.Errorf("failed to prepare filesystem %s for mds upgrade while deleting legacy deployment %s, continuing anyway: %+v", c.fs.Name, legacyName, err)
|
||||
}
|
||||
if err := k8sutil.DeleteDeployment(c.context.Clientset, c.fs.Namespace, legacyName); err != nil {
|
||||
logger.Errorf("failed to delete legacy deployment %s, USER should delete the legacy mds deployment manually if it still exists: %+v", legacyName, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait
|
||||
|
||||
func (c *cluster) start() error {
|
||||
@@ -105,15 +89,12 @@ func (c *cluster) start() error {
|
||||
var fsPreparedForUpgrade = false
|
||||
defer func() {
|
||||
if fsPreparedForUpgrade {
|
||||
if err := mdsdaemon.FinishedWithDaemonUpgrade(c.context, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil {
|
||||
if err := finishedWithDaemonUpgrade(c.context, c.fs.Namespace, c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount); err != nil {
|
||||
logger.Errorf("for filesystem %s, USER should make sure the Ceph fs max_mds property is set to %d: %+v",
|
||||
c.fs.Name, c.fs.Spec.MetadataServer.ActiveCount, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
// Delete legacy daemonset, and remove it if it exists.
|
||||
// This is relevant to the upgrade from Rook 0.8 to 0.9.
|
||||
fsPreparedForUpgrade = c.deleteLegacyMdsDeployment()
|
||||
|
||||
// Always create double the number of metadata servers to have standby mdses available
|
||||
replicas := c.fs.Spec.MetadataServer.ActiveCount * 2
|
||||
@@ -129,18 +110,14 @@ func (c *cluster) start() error {
|
||||
|
||||
mdsConfig := &mdsConfig{
|
||||
ResourceName: resourceName,
|
||||
DaemonName: daemonName,
|
||||
}
|
||||
|
||||
// create unique key for each mds
|
||||
if err := c.getOrCreateKeyring(mdsConfig); err != nil {
|
||||
return fmt.Errorf("failed to create mds keyring for filesystem %s: %+v", c.fs.Name, err)
|
||||
DaemonID: daemonName,
|
||||
DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, daemonName),
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
d := c.makeDeployment(mdsConfig)
|
||||
logger.Debugf("starting mds: %+v", d)
|
||||
_, err := c.context.Clientset.Apps().Deployments(c.fs.Namespace).Create(d)
|
||||
createdDeployment, err := c.context.Clientset.Apps().Deployments(c.fs.Namespace).Create(d)
|
||||
if err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create mds deployment %s: %+v", mdsConfig.ResourceName, err)
|
||||
@@ -149,11 +126,16 @@ func (c *cluster) start() error {
|
||||
// TODO: need to prepare for upgrade here each time. Also, before a given deployment is
|
||||
// terminated, I think we should somehow make sure that it isn't running the single
|
||||
// active daemon. If it is, then we should have another daemon take over as active. @Jan?
|
||||
if err := updateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil {
|
||||
if createdDeployment, err = updateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil {
|
||||
return fmt.Errorf("failed to update mds deployment %s. %+v", mdsConfig.ResourceName, err)
|
||||
}
|
||||
}
|
||||
desiredDeployments[d.GetName()] = true // add deployment name to improvised set
|
||||
|
||||
// create unique key for each mds saved to k8s secret
|
||||
if err := c.generateKeyring(mdsConfig, createdDeployment.UID); err != nil {
|
||||
return fmt.Errorf("failed to generate keyring for %s. %+v", resourceName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove extraneous mds deployments if they exist
|
||||
@@ -204,62 +186,6 @@ func (c *cluster) start() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *cluster) getOrCreateKeyring(mdsConfig *mdsConfig) error {
|
||||
_, err := c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Get(
|
||||
mdsConfig.ResourceName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
logger.Infof("the mds keyring was already generated")
|
||||
return nil
|
||||
}
|
||||
if !errors.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to get mds secrets: %+v", err)
|
||||
}
|
||||
|
||||
// get or create key for the mds user
|
||||
key, err := getOrCreateKey(c.context, c.fs.Namespace, mdsConfig.DaemonName)
|
||||
if err != nil {
|
||||
return err // there isn't any additional useful information to add here
|
||||
}
|
||||
|
||||
// Store the keyring in a secret
|
||||
secrets := map[string]string{
|
||||
keyringSecretKeyName: key,
|
||||
}
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mdsConfig.ResourceName,
|
||||
Namespace: c.fs.Namespace,
|
||||
},
|
||||
StringData: secrets,
|
||||
Type: k8sutil.RookType,
|
||||
}
|
||||
k8sutil.SetOwnerRefs(c.context.Clientset, c.fs.Namespace, &secret.ObjectMeta, c.ownerRefs)
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().Secrets(c.fs.Namespace).Create(secret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save mds secret: %+v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// create a keyring for the mds cluster with a limited set of privileges
|
||||
func getOrCreateKey(context *clusterd.Context, clusterName, daemonName string) (string, error) {
|
||||
access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"}
|
||||
username := fmt.Sprintf("mds.%s", daemonName)
|
||||
// "mds."" without a letter ID creates a key that can work for all mdses. This also means that
|
||||
// it *could* work for mdses serving a different filesystem, though we want to have a unique key
|
||||
// for each filesystem so that access can be quickly revoked by deleting the secret.
|
||||
|
||||
// get-or-create-key for the user account
|
||||
key, err := client.AuthGetOrCreateKey(context, clusterName, username, access)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get or create auth key for user %s: %+v", username, err)
|
||||
}
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func deleteMdsCluster(context *clusterd.Context, namespace, fsName string) error {
|
||||
// Try to delete all mds deployments and secret keys serving the filesystem, and aggregate
|
||||
// failures together to report all at once at the end.
|
||||
|
||||
@@ -19,8 +19,7 @@ package file
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds"
|
||||
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
opspec "github.com/rook/rook/pkg/operator/ceph/spec"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
@@ -40,14 +39,12 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
|
||||
Annotations: map[string]string{},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
InitContainers: []v1.Container{
|
||||
c.makeConfigInitContainer(mdsConfig),
|
||||
},
|
||||
InitContainers: []v1.Container{},
|
||||
Containers: []v1.Container{
|
||||
c.makeMdsDaemonContainer(mdsConfig),
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: opspec.PodVolumes(""),
|
||||
Volumes: opspec.DaemonVolumes(mdsConfig.DataPathMap, mdsConfig.ResourceName),
|
||||
HostNetwork: c.HostNetwork,
|
||||
},
|
||||
}
|
||||
@@ -61,6 +58,7 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: mdsConfig.ResourceName,
|
||||
Namespace: c.fs.Namespace,
|
||||
Labels: c.podLabels(mdsConfig),
|
||||
},
|
||||
Spec: apps.DeploymentSpec{
|
||||
Selector: &metav1.LabelSelector{
|
||||
@@ -77,60 +75,29 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *cluster) makeConfigInitContainer(mdsConfig *mdsConfig) v1.Container {
|
||||
return v1.Container{
|
||||
Name: opspec.ConfigInitContainerName,
|
||||
Args: []string{
|
||||
"ceph",
|
||||
mdsdaemon.InitCommand,
|
||||
"--config-dir", k8sutil.DataDir,
|
||||
"--mds-name", mdsConfig.DaemonName,
|
||||
"--filesystem-id", c.fsID,
|
||||
"--active-standby", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby),
|
||||
},
|
||||
Image: k8sutil.MakeRookImage(c.rookVersion),
|
||||
Env: []v1.EnvVar{
|
||||
// Set '--mds-keyring' flag with an env var sourced from the secret
|
||||
{Name: "ROOK_MDS_KEYRING",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
SecretKeyRef: &v1.SecretKeySelector{
|
||||
LocalObjectReference: v1.LocalObjectReference{Name: mdsConfig.ResourceName},
|
||||
Key: keyringSecretKeyName,
|
||||
}}},
|
||||
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
|
||||
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
|
||||
opmon.ClusterNameEnvVar(c.fs.Namespace),
|
||||
opmon.EndpointEnvVar(),
|
||||
opmon.SecretEnvVar(),
|
||||
opmon.AdminSecretEnvVar(),
|
||||
k8sutil.ConfigOverrideEnvVar(),
|
||||
},
|
||||
VolumeMounts: opspec.RookVolumeMounts(),
|
||||
Resources: c.fs.Spec.MetadataServer.Resources,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
|
||||
return v1.Container{
|
||||
Name: "mgr",
|
||||
Name: "mds",
|
||||
Command: []string{
|
||||
mdsDaemonCommand,
|
||||
"ceph-mds",
|
||||
},
|
||||
Args: []string{
|
||||
Args: append(
|
||||
opspec.DaemonFlags(c.clusterInfo, config.MdsType, mdsConfig.DaemonID),
|
||||
"--foreground",
|
||||
"--id", mdsConfig.DaemonName,
|
||||
// do not add the '--cluster/--conf/--keyring' flags; rook wants their default values
|
||||
},
|
||||
config.NewFlag("mds-standby-for-fscid", c.fsID),
|
||||
config.NewFlag("mds-standby-replay", strconv.FormatBool(c.fs.Spec.MetadataServer.ActiveStandby)),
|
||||
),
|
||||
Image: c.cephVersion.Image,
|
||||
Env: k8sutil.ClusterDaemonEnvVars(),
|
||||
VolumeMounts: opspec.CephVolumeMounts(),
|
||||
// TODO: mds doesn't need ports?
|
||||
VolumeMounts: opspec.DaemonVolumeMounts(mdsConfig.DataPathMap, mdsConfig.ResourceName),
|
||||
Env: append(
|
||||
opspec.DaemonEnvVars(),
|
||||
),
|
||||
Resources: c.fs.Spec.MetadataServer.Resources,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cluster) podLabels(mdsConfig *mdsConfig) map[string]string {
|
||||
labels := opspec.PodLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonName)
|
||||
labels := opspec.PodLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID)
|
||||
labels["rook_file_system"] = c.fs.Name
|
||||
return labels
|
||||
}
|
||||
@@ -19,13 +19,14 @@ package file
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
|
||||
mdsdaemon "github.com/rook/rook/pkg/daemon/ceph/mds"
|
||||
|
||||
cephtest "github.com/rook/rook/pkg/operator/ceph/test"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
@@ -49,7 +50,10 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment {
|
||||
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
|
||||
},
|
||||
}}}}
|
||||
clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
|
||||
|
||||
c := newCluster(
|
||||
clusterInfo,
|
||||
&clusterd.Context{Clientset: testop.New(1)},
|
||||
"rook/rook:myversion",
|
||||
cephv1.CephVersionSpec{Image: "ceph/ceph:testversion", Name: "mimic"},
|
||||
@@ -59,8 +63,9 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment {
|
||||
[]metav1.OwnerReference{{}},
|
||||
)
|
||||
mdsTestConfig := &mdsConfig{
|
||||
DaemonName: "myfs-a",
|
||||
DaemonID: "myfs-a",
|
||||
ResourceName: "rook-ceph-mds-myfs-a",
|
||||
DataPathMap: config.NewStatelessDaemonDataPathMap(config.MdsType, "myfs-a"),
|
||||
}
|
||||
return c.makeDeployment(mdsTestConfig)
|
||||
}
|
||||
@@ -69,84 +74,15 @@ func TestPodSpecs(t *testing.T) {
|
||||
d := testDeploymentObject(false) // no host network
|
||||
|
||||
assert.NotNil(t, d)
|
||||
assert.Equal(t, "rook-ceph-mds-myfs-a", d.Name)
|
||||
assert.Equal(t, "rook-ceph-mds-myfs-a", d.ObjectMeta.Name)
|
||||
assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy)
|
||||
assert.Equal(t, 0, len(d.ObjectMeta.Annotations))
|
||||
|
||||
pod := d.Spec.Template
|
||||
// Deployment should have Ceph labels
|
||||
cephtest.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels,
|
||||
config.MdsType, "myfs-a", "rook-ceph-mds", "ns")
|
||||
|
||||
assert.Nil(t, cephtest.VerifyPodLabels("rook-ceph-mds", "ns", "mds", "myfs-a", pod.ObjectMeta.Labels))
|
||||
assert.Equal(t, "myfs", pod.ObjectMeta.Labels["rook_file_system"])
|
||||
assert.Equal(t, 0, len(d.ObjectMeta.Annotations))
|
||||
assert.Nil(t, testop.VolumeExists("rook-data", pod.Spec.Volumes))
|
||||
assert.Nil(t, testop.VolumeExists(cephconfig.DefaultConfigMountName, pod.Spec.Volumes))
|
||||
assert.Nil(t, testop.VolumeExists(k8sutil.ConfigOverrideName, pod.Spec.Volumes))
|
||||
|
||||
assert.Equal(t, 1, len(pod.Spec.InitContainers))
|
||||
assert.Equal(t, 1, len(pod.Spec.Containers))
|
||||
|
||||
configImage := "rook/rook:myversion"
|
||||
configEnvs := 8
|
||||
configContainerDefinition := cephtest.ContainerTestDefinition{
|
||||
Image: &configImage,
|
||||
Command: []string{}, // no command
|
||||
Args: [][]string{
|
||||
{"ceph"},
|
||||
{mdsdaemon.InitCommand},
|
||||
{"--config-dir", "/var/lib/rook"},
|
||||
{"--mds-name", "myfs-a"},
|
||||
{"--filesystem-id", "15"},
|
||||
{"--active-standby", "false"}},
|
||||
InOrderArgs: map[int]string{
|
||||
0: "ceph", // ceph must be first arg
|
||||
1: mdsdaemon.InitCommand}, // mds init command must be second arg
|
||||
VolumeMountNames: []string{
|
||||
"rook-data",
|
||||
cephconfig.DefaultConfigMountName,
|
||||
k8sutil.ConfigOverrideName},
|
||||
EnvCount: &configEnvs,
|
||||
Ports: []v1.ContainerPort{},
|
||||
IsPrivileged: nil, // not set in spec
|
||||
}
|
||||
cont := &pod.Spec.InitContainers[0]
|
||||
configContainerDefinition.TestContainer(t, "config init", cont, logger)
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
daemonImage := "ceph/ceph:testversion"
|
||||
daemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
|
||||
daemonContainerDefinition := cephtest.ContainerTestDefinition{
|
||||
Image: &daemonImage,
|
||||
Command: []string{
|
||||
"ceph-mds"},
|
||||
Args: [][]string{
|
||||
{"--foreground"},
|
||||
{"--id", "myfs-a"}},
|
||||
VolumeMountNames: []string{
|
||||
"rook-data",
|
||||
cephconfig.DefaultConfigMountName},
|
||||
EnvCount: &daemonEnvs,
|
||||
Ports: []v1.ContainerPort{},
|
||||
IsPrivileged: nil, // not set in spec
|
||||
}
|
||||
keyringMount, err := testop.GetEnv("ROOK_MDS_KEYRING", cont.Env)
|
||||
assert.Nil(t, err) // keyring should get the secret named by the filesystem without daemon id
|
||||
assert.Equal(t, "rook-ceph-mds-myfs-a", keyringMount.ValueFrom.SecretKeyRef.LocalObjectReference.Name)
|
||||
|
||||
cont = &pod.Spec.Containers[0]
|
||||
daemonContainerDefinition.TestContainer(t, "main mon daemon", cont, logger)
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
|
||||
// Verify that all the mounts have volumes and that there are no extraneous volumes
|
||||
volsMountsTestDef := testop.VolumesAndMountsTestDefinition{
|
||||
VolumesSpec: &testop.VolumesSpec{Moniker: "mon pod volumes", Volumes: pod.Spec.Volumes},
|
||||
MountsSpecItems: []*testop.MountsSpec{
|
||||
{Moniker: "mds config init mounts", Mounts: pod.Spec.InitContainers[0].VolumeMounts},
|
||||
{Moniker: "mds daemon mounts", Mounts: pod.Spec.Containers[0].VolumeMounts}},
|
||||
}
|
||||
volsMountsTestDef.TestMountsMatchVolumes(t)
|
||||
podTemplate := cephtest.NewPodTemplateSpecTester(t, &d.Spec.Template)
|
||||
podTemplate.RunFullSuite(config.MdsType, "myfs-a", "rook-ceph-mds", "ns", "ceph/ceph:testversion",
|
||||
"100", "1337" /* resources */)
|
||||
}
|
||||
|
||||
func TestHostNetwork(t *testing.T) {
|
||||
|
||||
@@ -92,20 +92,23 @@ func DaemonVolumeMounts(dataPaths *config.DataPathMap, keyringResourceName strin
|
||||
}
|
||||
|
||||
// DaemonFlags returns the command line flags used by all Ceph daemons.
|
||||
func DaemonFlags(
|
||||
clusterInfo *cephconfig.ClusterInfo,
|
||||
daemonType config.DaemonType, daemonID string,
|
||||
) []string {
|
||||
func DaemonFlags(cluster *cephconfig.ClusterInfo, daemonType config.DaemonType, daemonID string) []string {
|
||||
return append(
|
||||
config.DefaultFlags(clusterInfo.FSID, keyring.VolumeMount().KeyringFilePath()),
|
||||
config.DefaultFlags(cluster.FSID, keyring.VolumeMount().KeyringFilePath()),
|
||||
// all daemons are named in the format <type>.<id>
|
||||
config.NewFlag("name", fmt.Sprintf("%s.%s", string(daemonType), daemonID)),
|
||||
)
|
||||
}
|
||||
|
||||
// AdminFlags returns the command line flags used for Ceph commands requiring admin authentication.
|
||||
func AdminFlags(clusterInfo *cephconfig.ClusterInfo) []string {
|
||||
return config.DefaultFlags(clusterInfo.FSID, keyring.VolumeMount().AdminKeyringFilePath())
|
||||
func AdminFlags(cluster *cephconfig.ClusterInfo) []string {
|
||||
return config.DefaultFlags(cluster.FSID, keyring.VolumeMount().AdminKeyringFilePath())
|
||||
}
|
||||
|
||||
// ContainerEnvVarReference returns a reference to a Kubernetes container env var of the given name
|
||||
// which can be used in command or argument fields.
|
||||
func ContainerEnvVarReference(envVarName string) string {
|
||||
return fmt.Sprintf("$(%s)", envVarName)
|
||||
}
|
||||
|
||||
// DaemonEnvVars returns the container environment variables used by all Ceph daemons.
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -121,7 +122,10 @@ func (ct *ContainersTester) AssertArgReferencesMatchEnvVars() {
|
||||
}
|
||||
// also make sure there are no extraneous env vars
|
||||
// the only allowed extraneous vars are the required vars
|
||||
assert.ElementsMatch(ct.t, ct.allNonrequiredVarNames(), ct.allNonrequiredArgEnvReferences())
|
||||
assert.ElementsMatch(ct.t, ct.allNonrequiredVarNames(), ct.allNonrequiredArgEnvReferences(),
|
||||
"there are extra arguments or references which do not have a corresponding ref/arg",
|
||||
fmt.Sprintf("%+v", ct.allNonrequiredVarNames()),
|
||||
fmt.Sprintf("%+v", ct.allNonrequiredArgEnvReferences()))
|
||||
}
|
||||
|
||||
// AssertCephImagesMatch asserts that for all Ceph containers under test, the Ceph image used is the
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"k8s.io/api/apps/v1"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -80,15 +81,17 @@ func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, co
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) error {
|
||||
// UpdateDeploymentAndWait updates a deployment and waits until it is running to return. It will
|
||||
// error if the deployment does not exist to be updated or if it takes too long.
|
||||
func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) {
|
||||
original, err := context.Clientset.Apps().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err)
|
||||
return nil, fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err)
|
||||
}
|
||||
|
||||
logger.Infof("updating deployment %s", deployment.Name)
|
||||
if _, err := context.Clientset.Apps().Deployments(namespace).Update(deployment); err != nil {
|
||||
return fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err)
|
||||
return nil, fmt.Errorf("failed to update deployment %s. %+v", deployment.Name, err)
|
||||
}
|
||||
|
||||
// wait for the deployment to be restarted
|
||||
@@ -102,18 +105,18 @@ func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deploym
|
||||
// check for the status of the deployment
|
||||
d, err := context.Clientset.Apps().Deployments(namespace).Get(deployment.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err)
|
||||
return nil, fmt.Errorf("failed to get deployment %s. %+v", deployment.Name, err)
|
||||
}
|
||||
if d.Status.ObservedGeneration != original.Status.ObservedGeneration && d.Status.UpdatedReplicas > 0 && d.Status.ReadyReplicas > 0 {
|
||||
logger.Infof("finished waiting for updated deployment %s", d.Name)
|
||||
return nil
|
||||
return d, nil
|
||||
}
|
||||
|
||||
logger.Debugf("deployment %s status=%v", d.Name, d.Status)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
|
||||
return fmt.Errorf("gave up waiting for deployment %s to update", deployment.Name)
|
||||
return nil, fmt.Errorf("gave up waiting for deployment %s to update", deployment.Name)
|
||||
}
|
||||
|
||||
// GetDeployments returns a list of deployment names labels matching a given selector
|
||||
|
||||
@@ -2,7 +2,9 @@ package test
|
||||
|
||||
import (
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"k8s.io/api/apps/v1"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// UpdateDeploymentAndWaitStub returns a stub replacement for the UpdateDeploymentAndWait function
|
||||
@@ -14,13 +16,13 @@ import (
|
||||
// returns a pointer to this slice which the calling func may use to verify the expected contents of
|
||||
// deploymentsUpdated based on expected behavior.
|
||||
func UpdateDeploymentAndWaitStub() (
|
||||
stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace string) error,
|
||||
stubFunc func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error),
|
||||
deploymentsUpdated *[]*apps.Deployment,
|
||||
) {
|
||||
deploymentsUpdated = &[]*apps.Deployment{}
|
||||
stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace string) error {
|
||||
stubFunc = func(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) {
|
||||
*deploymentsUpdated = append(*deploymentsUpdated, deployment)
|
||||
return nil
|
||||
return &v1.Deployment{ObjectMeta: metav1.ObjectMeta{UID: "stub-deployment-uid"}}, nil
|
||||
}
|
||||
return stubFunc, deploymentsUpdated
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user