diff --git a/cmd/rook/ceph/mgr.go b/cmd/rook/ceph/mgr.go index 78f2e6a38..322e313fc 100644 --- a/cmd/rook/ceph/mgr.go +++ b/cmd/rook/ceph/mgr.go @@ -25,8 +25,10 @@ import ( ) var ( - mgrName string - mgrKeyring string + mgrName string + mgrKeyring string + mgrModuleServerAddr string + cephVersionName string ) var mgrCmd = &cobra.Command{ @@ -38,6 +40,8 @@ var mgrCmd = &cobra.Command{ func init() { mgrCmd.Flags().StringVar(&mgrName, "mgr-name", "", "name of the mgr") mgrCmd.Flags().StringVar(&mgrKeyring, "mgr-keyring", "", "the mgr keyring") + mgrCmd.Flags().StringVar(&mgrModuleServerAddr, "mgr-module-server-addr", "", "the server_addr to set for the mgr module bindings") + mgrCmd.Flags().StringVar(&cephVersionName, "ceph-version-name", "", "the major version of ceph running") addCephFlags(mgrCmd) flags.SetFlagsFromEnv(mgrCmd.Flags(), rook.RookEnvVarPrefix) @@ -47,7 +51,7 @@ func init() { func initMgr(cmd *cobra.Command, args []string) error { required := []string{ - "mgr-name", "mgr-keyring", + "mgr-name", "mgr-keyring", "ceph-version-name", "mgr-module-server-addr", "mon-endpoints", "cluster-name", "mon-secret", "admin-secret"} if err := flags.VerifyRequiredFlags(mgrCmd, required); err != nil { return err @@ -63,9 +67,11 @@ func initMgr(cmd *cobra.Command, args []string) error { clusterInfo.Monitors = mondaemon.ParseMonEndpoints(cfg.monEndpoints) config := &mgrdaemon.Config{ - Name: mgrName, - Keyring: mgrKeyring, - ClusterInfo: &clusterInfo, + Name: mgrName, + Keyring: mgrKeyring, + ClusterInfo: &clusterInfo, + ModuleServerAddr: mgrModuleServerAddr, + CephVersionName: cephVersionName, } err := mgrdaemon.Initialize(createContext(), config) diff --git a/pkg/daemon/ceph/client/mgr.go b/pkg/daemon/ceph/client/mgr.go index 1b9da688b..82a070f41 100755 --- a/pkg/daemon/ceph/client/mgr.go +++ b/pkg/daemon/ceph/client/mgr.go @@ -18,9 +18,15 @@ func MgrDisableModule(context *clusterd.Context, clusterName, name string) error return enableModule(context, clusterName, name, false, "disable") } -func MgrSetConfig(context *clusterd.Context, clusterName, cephVersionName, key, val string) (bool, error) { - var getArgs, setArgs []string +// MgrSetAllConfig applies a setting for all mgr daemons +func MgrSetAllConfig(context *clusterd.Context, clusterName, cephVersionName, key, val string) (bool, error) { + return MgrSetConfig(context, clusterName, "", cephVersionName, key, val) +} +// MgrSetConfig applies a setting for a single mgr daemon +func MgrSetConfig(context *clusterd.Context, clusterName, mgrName, cephVersionName, key, val string) (bool, error) { + var getArgs, setArgs []string + mgrID := fmt.Sprintf("mgr.%s", mgrName) if cephVersionName == cephv1.Luminous || cephVersionName == "" { getArgs = append(getArgs, "config-key", "get", key) if val == "" { @@ -29,11 +35,11 @@ func MgrSetConfig(context *clusterd.Context, clusterName, cephVersionName, key, setArgs = append(setArgs, "config-key", "set", key, val) } } else { - getArgs = append(getArgs, "config", "get", "mgr.", key) + getArgs = append(getArgs, "config", "get", mgrID, key) if val == "" { - setArgs = append(setArgs, "config", "rm", "mgr", key) + setArgs = append(setArgs, "config", "rm", mgrID, key) } else { - setArgs = append(setArgs, "config", "set", "mgr", key, val) + setArgs = append(setArgs, "config", "set", mgrID, key, val) } } diff --git a/pkg/daemon/ceph/mgr/init.go b/pkg/daemon/ceph/mgr/init.go index 1e61ecb64..98b289040 100644 --- a/pkg/daemon/ceph/mgr/init.go +++ b/pkg/daemon/ceph/mgr/init.go @@ -18,10 +18,12 @@ package mgr import ( "fmt" + "os" "path" "github.com/coreos/pkg/capnslog" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/util" ) @@ -46,9 +48,11 @@ const ( // Config contains the necessary parameters Rook needs to know to set up a mgr for a Ceph cluster. type Config struct { - ClusterInfo *cephconfig.ClusterInfo - Name string - Keyring string + ClusterInfo *cephconfig.ClusterInfo + Name string + Keyring string + ModuleServerAddr string + CephVersionName string } // Initialize generates configuration files for a Ceph mgr @@ -61,6 +65,36 @@ func Initialize(context *clusterd.Context, config *Config) error { util.WriteFileToLog(logger, cephconfig.DefaultConfigFilePath()) + return setServerAddr(context, config) +} + +func setServerAddr(context *clusterd.Context, config *Config) error { + logger.Infof("setting server_addr for the prometheus and dashboard modules") + + // use the admin keyring for these operations + adminKeyringEval := func(key string) string { + return fmt.Sprintf(cephconfig.AdminKeyringTemplate, key) + } + + keyringPath := path.Join(cephconfig.DefaultConfigDir, "keyring") + if err := cephconfig.WriteKeyring(keyringPath, config.ClusterInfo.AdminSecret, adminKeyringEval); err != nil { + return fmt.Errorf("failed to write admin keyring. %+v", err) + } + + clusterName := "ceph" + context.ConfigDir = "/etc" + modules := []string{"prometheus", "dashboard"} + for _, module := range modules { + settingPath := fmt.Sprintf("mgr/%s/server_addr", module) + if _, err := client.MgrSetConfig(context, clusterName, config.Name, config.CephVersionName, settingPath, config.ModuleServerAddr); err != nil { + return fmt.Errorf("setting %s server_addr failed. %+v", module, err) + } + } + + // remove the admin keyring + if err := os.Remove(keyringPath); err != nil { + logger.Warningf("failed to remove admin keyring. %+v", err) + } return nil } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 04f6582b9..71255d1b5 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -116,6 +116,8 @@ func (c *cluster) detectCephMajorVersion(image string, timeout time.Duration) (s // delete the job since we're done with it k8sutil.DeleteBatchJob(c.context.Clientset, c.Namespace, job.Name, false) + + logger.Infof("Detected ceph image version: %s", version) return version, nil } diff --git a/pkg/operator/ceph/cluster/mgr/dashboard.go b/pkg/operator/ceph/cluster/mgr/dashboard.go index 97a1e06c5..639db7264 100755 --- a/pkg/operator/ceph/cluster/mgr/dashboard.go +++ b/pkg/operator/ceph/cluster/mgr/dashboard.go @@ -102,7 +102,7 @@ func (c *Cluster) toggleDashboardModule() error { } func (c *Cluster) configureDashboardModule() error { - hasChanged, err := client.MgrSetConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/url_prefix", c.dashboard.UrlPrefix) + hasChanged, err := client.MgrSetAllConfig(c.context, c.Namespace, c.cephVersion.Name, "mgr/dashboard/url_prefix", c.dashboard.UrlPrefix) if err != nil { return err } diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 544d69371..704ac7033 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -98,6 +98,8 @@ func (c *Cluster) makeConfigInitContainer(mgrConfig *mgrConfig) v1.Container { }}}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), + k8sutil.PodIPEnvVar("ROOK_MGR_MODULE_SERVER_ADDR"), + {Name: "ROOK_CEPH_VERSION_NAME", Value: c.cephVersion.Name}, opmon.EndpointEnvVar(), opmon.SecretEnvVar(), opmon.AdminSecretEnvVar(), diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go index c6f970511..966bb2d66 100644 --- a/pkg/operator/ceph/cluster/mgr/spec_test.go +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -84,7 +84,7 @@ func TestPodSpec(t *testing.T) { assert.Equal(t, 1, len(pod.Spec.Containers)) configImage := "rook/rook:myversion" - configEnvs := 7 + configEnvs := 9 configContainerDefinition := cephtest.ContainerTestDefinition{ Image: &configImage, Command: []string{}, // no command