diff --git a/cmd/rookd/main.go b/cmd/rookd/main.go index 272dbefbc..209a7ebca 100644 --- a/cmd/rookd/main.go +++ b/cmd/rookd/main.go @@ -41,6 +41,7 @@ type config struct { dataDir string forceFormat bool location string + debug bool } func newConfig() *config { @@ -71,6 +72,8 @@ func init() { "true to force the format of any specified devices, even if they already have a filesystem. BE CAREFUL!") rootCmd.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement") + rootCmd.PersistentFlags().BoolVar(&cfg.debug, "debug", false, "true to enable debug logging/tracing") + rootCmd.MarkFlagRequired("private-ipv4") rootCmd.MarkFlagRequired("public-ipv4") @@ -136,7 +139,8 @@ func joinCluster() error { } // start the cluster orchestration services - context, err := clusterd.StartJoinCluster(services, procMan, cfg.dataDir, cfg.nodeID, cfg.discoveryURL, cfg.etcdMembers, cfg.publicIPv4, cfg.privateIPv4) + context, err := clusterd.StartJoinCluster(services, procMan, cfg.dataDir, cfg.nodeID, cfg.discoveryURL, + cfg.etcdMembers, cfg.publicIPv4, cfg.privateIPv4, cfg.debug) if err != nil { return err } diff --git a/pkg/cephmgr/config.go b/pkg/cephmgr/config.go index 97ccee32a..54f1cbc60 100644 --- a/pkg/cephmgr/config.go +++ b/pkg/cephmgr/config.go @@ -56,7 +56,8 @@ func ConnectToClusterAsAdmin(context *clusterd.Context, factory client.Connectio return nil, err } - return connectToCluster(context, factory, cluster, getMonRunDirPath(context.ConfigDir, monName), "admin", getMonKeyringPath(context.ConfigDir, monName)) + return connectToCluster(context, factory, cluster, getMonRunDirPath(context.ConfigDir, monName), + "admin", getMonKeyringPath(context.ConfigDir, monName), context.Debug) } // get the path of a given monitor's config file @@ -65,12 +66,14 @@ func getConfFilePath(root, clusterName string) string { } // generates and writes the monitor config file to disk -func generateConnectionConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string) (string, error) { - return generateConfigFile(context, cluster, pathRoot, user, keyringPath, false, nil, nil) +func generateConnectionConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string, debug bool) (string, error) { + return generateConfigFile(context, cluster, pathRoot, user, keyringPath, debug, false, nil, nil) } // generates and writes the monitor config file to disk -func generateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string, bluestore bool, userConfig *cephConfig, clientSettings map[string]string) (string, error) { +func generateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string, + debug, bluestore bool, userConfig *cephConfig, clientSettings map[string]string) (string, error) { + if pathRoot == "" { pathRoot = getMonRunDirPath(context.ConfigDir, getFirstMonitor(cluster)) } @@ -80,7 +83,7 @@ func generateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoo fmt.Printf("failed to create config directory at %s: %+v", pathRoot, err) } - configFile, err := createGlobalConfigFileSection(cluster, pathRoot, bluestore, userConfig) + configFile, err := createGlobalConfigFileSection(cluster, pathRoot, debug, bluestore, userConfig) if err != nil { return "", fmt.Errorf("failed to create global config section, %+v", err) } @@ -121,10 +124,12 @@ func getFirstMonitor(cluster *ClusterInfo) string { } // opens a connection to the cluster that can be used for management operations -func connectToCluster(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo, basePath, user, keyringPath string) (client.Connection, error) { +func connectToCluster(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo, + basePath, user, keyringPath string, debug bool) (client.Connection, error) { + log.Printf("connecting to ceph cluster %s with user %s", cluster.Name, user) - confFilePath, err := generateConnectionConfigFile(context, cluster, basePath, user, keyringPath) + confFilePath, err := generateConnectionConfigFile(context, cluster, basePath, user, keyringPath, debug) if err != nil { return nil, fmt.Errorf("failed to generate config file: %v", err) } @@ -145,7 +150,7 @@ func connectToCluster(context *clusterd.Context, factory client.ConnectionFactor return conn, nil } -func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool) *cephConfig { +func createDefaultCephConfig(cluster *ClusterInfo, runDir string, debug, bluestore bool) *cephConfig { // extract a list of just the monitor names, which will populate the "mon initial members" // global config field monMembers := make([]string, len(cluster.Monitors)) @@ -162,6 +167,11 @@ func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool store = "bluestore" } + loggingLevel := 0 + if debug { + loggingLevel = 20 + } + return &cephConfig{ &cephGlobalConfig{ EnableExperimental: experimental, @@ -170,10 +180,10 @@ func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool MonMembers: strings.Join(monMembers, " "), LogFile: "/dev/stdout", MonClusterLogFile: "/dev/stdout", - DebugLogDefaultLevel: 0, - DebugLogRadosLevel: 0, - DebugLogMonLevel: 0, - DebugLogBluestoreLevel: 0, + DebugLogDefaultLevel: loggingLevel, + DebugLogRadosLevel: loggingLevel, + DebugLogMonLevel: loggingLevel, + DebugLogBluestoreLevel: loggingLevel, OsdPgBits: 11, OsdPgpBits: 11, OsdPoolDefaultSize: 1, @@ -187,14 +197,14 @@ func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool } } -func createGlobalConfigFileSection(cluster *ClusterInfo, runDir string, bluestore bool, userConfig *cephConfig) (*ini.File, error) { +func createGlobalConfigFileSection(cluster *ClusterInfo, runDir string, debug, bluestore bool, userConfig *cephConfig) (*ini.File, error) { var ceph *cephConfig if userConfig != nil { // use the user config since it was provided ceph = userConfig } else { - ceph = createDefaultCephConfig(cluster, runDir, bluestore) + ceph = createDefaultCephConfig(cluster, runDir, debug, bluestore) } configFile := ini.Empty() diff --git a/pkg/cephmgr/config_test.go b/pkg/cephmgr/config_test.go new file mode 100644 index 000000000..800ef6251 --- /dev/null +++ b/pkg/cephmgr/config_test.go @@ -0,0 +1,57 @@ +package cephmgr + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreateDefaultCephConfig(t *testing.T) { + clusterInfo := &ClusterInfo{ + FSID: "id", + MonitorSecret: "monsecret", + AdminSecret: "adminsecret", + Name: "foo-cluster", + Monitors: map[string]*CephMonitorConfig{ + "node0": &CephMonitorConfig{Name: "mon0", Endpoint: "10.0.0.1:6790"}, + "node1": &CephMonitorConfig{Name: "mon1", Endpoint: "10.0.0.2:6790"}, + }, + } + + monMembers := "mon0 mon1" + + cephConfig := createDefaultCephConfig(clusterInfo, "/var/lib/rook1", false, false) + verifyConfig(t, cephConfig, monMembers, "", "filestore", 0) + + cephConfig = createDefaultCephConfig(clusterInfo, "/var/lib/rook1", false, true) + verifyConfig(t, cephConfig, monMembers, "bluestore rocksdb", "bluestore", 0) + + cephConfig = createDefaultCephConfig(clusterInfo, "/var/lib/rook1", true, false) + verifyConfig(t, cephConfig, monMembers, "", "filestore", 20) + + cephConfig = createDefaultCephConfig(clusterInfo, "/var/lib/rook1", true, true) + verifyConfig(t, cephConfig, monMembers, "bluestore rocksdb", "bluestore", 20) +} + +func verifyConfig(t *testing.T, cephConfig *cephConfig, expectedMonMembers, experimental, objectStore string, loggingLevel int) { + + for _, expectedMon := range strings.Split(expectedMonMembers, " ") { + contained := false + for _, actualMon := range strings.Split(cephConfig.MonMembers, " ") { + if expectedMon == actualMon { + contained = true + break + } + } + + assert.True(t, contained, "expectedMons: %+v, actualMons: %+v", expectedMonMembers, cephConfig.MonMembers) + } + + assert.Equal(t, experimental, cephConfig.EnableExperimental) + assert.Equal(t, objectStore, cephConfig.OsdObjectStore) + assert.Equal(t, loggingLevel, cephConfig.DebugLogDefaultLevel) + assert.Equal(t, loggingLevel, cephConfig.DebugLogMonLevel) + assert.Equal(t, loggingLevel, cephConfig.DebugLogRadosLevel) + assert.Equal(t, loggingLevel, cephConfig.DebugLogBluestoreLevel) +} diff --git a/pkg/cephmgr/monagent.go b/pkg/cephmgr/monagent.go index d6c241656..58d0edc24 100644 --- a/pkg/cephmgr/monagent.go +++ b/pkg/cephmgr/monagent.go @@ -98,7 +98,8 @@ func (a *monAgent) makeMonitorFileSystem(context *clusterd.Context, cluster *Clu } // write the config file to disk - confFilePath, err := generateConnectionConfigFile(context, cluster, getMonRunDirPath(context.ConfigDir, monName), "admin", getMonKeyringPath(context.ConfigDir, monName)) + confFilePath, err := generateConnectionConfigFile(context, cluster, getMonRunDirPath(context.ConfigDir, monName), + "admin", getMonKeyringPath(context.ConfigDir, monName), context.Debug) if err != nil { return err } diff --git a/pkg/cephmgr/osd.go b/pkg/cephmgr/osd.go index 0dc12f134..309bf94f3 100644 --- a/pkg/cephmgr/osd.go +++ b/pkg/cephmgr/osd.go @@ -359,9 +359,9 @@ func loadOSDInfo(config *osdConfig) error { } func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context, - bootstrapConn client.Connection, cluster *ClusterInfo, location string, executor exec.Executor) error { + bootstrapConn client.Connection, cluster *ClusterInfo, location string, debug bool, executor exec.Executor) error { - cephConfig := createDefaultCephConfig(cluster, config.rootPath, config.bluestore) + cephConfig := createDefaultCephConfig(cluster, config.rootPath, debug, config.bluestore) if !config.bluestore { // using the local file system requires some config overrides // http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended @@ -377,7 +377,8 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context // write the OSD config file to disk keyringPath := getOSDKeyringPath(config.rootPath) - _, err = generateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id), keyringPath, config.bluestore, cephConfig, settings) + _, err = generateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id), + keyringPath, debug, config.bluestore, cephConfig, settings) if err != nil { return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err) } @@ -399,7 +400,8 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context } // open a connection to the cluster using the OSDs creds - osdConn, err := connectToCluster(context, factory, cluster, path.Join(config.rootPath, "tmp"), fmt.Sprintf("osd.%d", config.id), keyringPath) + osdConn, err := connectToCluster(context, factory, cluster, path.Join(config.rootPath, "tmp"), + fmt.Sprintf("osd.%d", config.id), keyringPath, debug) if err != nil { return err } diff --git a/pkg/cephmgr/osdagent.go b/pkg/cephmgr/osdagent.go index 294f8b76f..8e30c72d2 100644 --- a/pkg/cephmgr/osdagent.go +++ b/pkg/cephmgr/osdagent.go @@ -241,7 +241,8 @@ func (a *osdAgent) createDesiredOSDs(adminConn client.Connection, context *clust } // connect to the cluster using the bootstrap-osd creds, this connection will be used for config operations - bootstrapConn, err := connectToCluster(context, a.factory, a.cluster, getBootstrapOSDDir(context.ConfigDir), "bootstrap-osd", getBootstrapOSDKeyringPath(context.ConfigDir, a.cluster.Name)) + bootstrapConn, err := connectToCluster(context, a.factory, a.cluster, getBootstrapOSDDir(context.ConfigDir), + "bootstrap-osd", getBootstrapOSDKeyringPath(context.ConfigDir, a.cluster.Name), context.Debug) if err != nil { return err } @@ -320,7 +321,7 @@ func (a *osdAgent) createOrStartOSD(context *clusterd.Context, connection client } // osd_data_dir/whoami does not exist yet, create/initialize the OSD - err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Executor) + err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Debug, context.Executor) if err != nil { return fmt.Errorf("failed to initialize OSD at %s: %+v", config.rootPath, err) } diff --git a/pkg/clusterd/clusterservice.go b/pkg/clusterd/clusterservice.go index da887d91a..8c5f2d5c5 100644 --- a/pkg/clusterd/clusterservice.go +++ b/pkg/clusterd/clusterservice.go @@ -69,6 +69,9 @@ type Context struct { // The root configuration directory used by services ConfigDir string + + // A value indicating if debug logging/tracing should be enabled + Debug bool } func copyContext(c *Context) *Context { @@ -80,6 +83,7 @@ func copyContext(c *Context) *Context { ProcMan: c.ProcMan, Inventory: c.Inventory, ConfigDir: c.ConfigDir, + Debug: c.Debug, } } diff --git a/pkg/clusterd/join.go b/pkg/clusterd/join.go index 69d84d294..413690f1f 100644 --- a/pkg/clusterd/join.go +++ b/pkg/clusterd/join.go @@ -13,9 +13,11 @@ import ( ) // StartJoinCluster initializes the cluster services to enable joining the cluster and listening for orchestration. -func StartJoinCluster(services []*ClusterService, procMan *proc.ProcManager, configDir, nodeID, discoveryURL, etcdMembers, publicIPv4, privateIPv4 string) (*Context, error) { - log.Printf("Starting cluster. config=%s, nodeID=%s, url=%s, members=%s, publicIPv4=%s, privateIPv4=%s", - configDir, nodeID, discoveryURL, etcdMembers, publicIPv4, privateIPv4) +func StartJoinCluster(services []*ClusterService, procMan *proc.ProcManager, configDir, nodeID, discoveryURL, + etcdMembers, publicIPv4, privateIPv4 string, debug bool) (*Context, error) { + + log.Printf("Starting cluster. configDir=%s, nodeID=%s, url=%s, members=%s, publicIPv4=%s, privateIPv4=%s, debug=%t", + configDir, nodeID, discoveryURL, etcdMembers, publicIPv4, privateIPv4, debug) etcdClients := []string{} if etcdMembers != "" { @@ -58,6 +60,7 @@ func StartJoinCluster(services []*ClusterService, procMan *proc.ProcManager, con Services: services, ProcMan: procMan, ConfigDir: configDir, + Debug: debug, } clusterLeader := newServicesLeader(context) clusterMember := newClusterMember(context, leaseManager, clusterLeader)