Merge pull request #114 from jbw976/debug-logs

Enable debug logging from the cmd line with --debug

cmd/rookd/main.go

@@ -41,6 +41,7 @@ type config struct {
 	dataDir      string
 	forceFormat  bool
 	location     string
+	debug        bool
 }
 
 func newConfig() *config {
@@ -71,6 +72,8 @@ func init() {
 		"true to force the format of any specified devices, even if they already have a filesystem.  BE CAREFUL!")
 	rootCmd.Flags().StringVar(&cfg.location, "location", "", "location of this node for CRUSH placement")
 
+	rootCmd.PersistentFlags().BoolVar(&cfg.debug, "debug", false, "true to enable debug logging/tracing")
+
 	rootCmd.MarkFlagRequired("private-ipv4")
 	rootCmd.MarkFlagRequired("public-ipv4")
 
@@ -136,7 +139,8 @@ func joinCluster() error {
 	}
 
 	// start the cluster orchestration services
-	context, err := clusterd.StartJoinCluster(services, procMan, cfg.dataDir, cfg.nodeID, cfg.discoveryURL, cfg.etcdMembers, cfg.publicIPv4, cfg.privateIPv4)
+	context, err := clusterd.StartJoinCluster(services, procMan, cfg.dataDir, cfg.nodeID, cfg.discoveryURL,
+		cfg.etcdMembers, cfg.publicIPv4, cfg.privateIPv4, cfg.debug)
 	if err != nil {
 		return err
 	}

pkg/cephmgr/config.go

@@ -56,7 +56,8 @@ func ConnectToClusterAsAdmin(context *clusterd.Context, factory client.Connectio
 		return nil, err
 	}
 
-	return connectToCluster(context, factory, cluster, getMonRunDirPath(context.ConfigDir, monName), "admin", getMonKeyringPath(context.ConfigDir, monName))
+	return connectToCluster(context, factory, cluster, getMonRunDirPath(context.ConfigDir, monName),
+		"admin", getMonKeyringPath(context.ConfigDir, monName), context.Debug)
 }
 
 // get the path of a given monitor's config file
@@ -65,12 +66,14 @@ func getConfFilePath(root, clusterName string) string {
 }
 
 // generates and writes the monitor config file to disk
-func generateConnectionConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string) (string, error) {
-	return generateConfigFile(context, cluster, pathRoot, user, keyringPath, false, nil, nil)
+func generateConnectionConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string, debug bool) (string, error) {
+	return generateConfigFile(context, cluster, pathRoot, user, keyringPath, debug, false, nil, nil)
 }
 
 // generates and writes the monitor config file to disk
-func generateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string, bluestore bool, userConfig *cephConfig, clientSettings map[string]string) (string, error) {
+func generateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string,
+	debug, bluestore bool, userConfig *cephConfig, clientSettings map[string]string) (string, error) {
+
 	if pathRoot == "" {
 		pathRoot = getMonRunDirPath(context.ConfigDir, getFirstMonitor(cluster))
 	}
@@ -80,7 +83,7 @@ func generateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoo
 		fmt.Printf("failed to create config directory at %s: %+v", pathRoot, err)
 	}
 
-	configFile, err := createGlobalConfigFileSection(cluster, pathRoot, bluestore, userConfig)
+	configFile, err := createGlobalConfigFileSection(cluster, pathRoot, debug, bluestore, userConfig)
 	if err != nil {
 		return "", fmt.Errorf("failed to create global config section, %+v", err)
 	}
@@ -121,10 +124,12 @@ func getFirstMonitor(cluster *ClusterInfo) string {
 }
 
 // opens a connection to the cluster that can be used for management operations
-func connectToCluster(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo, basePath, user, keyringPath string) (client.Connection, error) {
+func connectToCluster(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo,
+	basePath, user, keyringPath string, debug bool) (client.Connection, error) {
+
 	log.Printf("connecting to ceph cluster %s with user %s", cluster.Name, user)
 
-	confFilePath, err := generateConnectionConfigFile(context, cluster, basePath, user, keyringPath)
+	confFilePath, err := generateConnectionConfigFile(context, cluster, basePath, user, keyringPath, debug)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate config file: %v", err)
 	}
@@ -145,7 +150,7 @@ func connectToCluster(context *clusterd.Context, factory client.ConnectionFactor
 	return conn, nil
 }
 
-func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool) *cephConfig {
+func createDefaultCephConfig(cluster *ClusterInfo, runDir string, debug, bluestore bool) *cephConfig {
 	// extract a list of just the monitor names, which will populate the "mon initial members"
 	// global config field
 	monMembers := make([]string, len(cluster.Monitors))
@@ -162,6 +167,11 @@ func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool
 		store = "bluestore"
 	}
 
+	loggingLevel := 0
+	if debug {
+		loggingLevel = 20
+	}
+
 	return &cephConfig{
 		&cephGlobalConfig{
 			EnableExperimental:     experimental,
@@ -170,10 +180,10 @@ func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool
 			MonMembers:             strings.Join(monMembers, " "),
 			LogFile:                "/dev/stdout",
 			MonClusterLogFile:      "/dev/stdout",
-			DebugLogDefaultLevel:   0,
-			DebugLogRadosLevel:     0,
-			DebugLogMonLevel:       0,
-			DebugLogBluestoreLevel: 0,
+			DebugLogDefaultLevel:   loggingLevel,
+			DebugLogRadosLevel:     loggingLevel,
+			DebugLogMonLevel:       loggingLevel,
+			DebugLogBluestoreLevel: loggingLevel,
 			OsdPgBits:              11,
 			OsdPgpBits:             11,
 			OsdPoolDefaultSize:     1,
@@ -187,14 +197,14 @@ func createDefaultCephConfig(cluster *ClusterInfo, runDir string, bluestore bool
 	}
 }
 
-func createGlobalConfigFileSection(cluster *ClusterInfo, runDir string, bluestore bool, userConfig *cephConfig) (*ini.File, error) {
+func createGlobalConfigFileSection(cluster *ClusterInfo, runDir string, debug, bluestore bool, userConfig *cephConfig) (*ini.File, error) {
 	var ceph *cephConfig
 
 	if userConfig != nil {
 		// use the user config since it was provided
 		ceph = userConfig
 	} else {
-		ceph = createDefaultCephConfig(cluster, runDir, bluestore)
+		ceph = createDefaultCephConfig(cluster, runDir, debug, bluestore)
 	}
 
 	configFile := ini.Empty()

pkg/cephmgr/config_test.go

@@ -0,0 +1,57 @@
+package cephmgr
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateDefaultCephConfig(t *testing.T) {
+	clusterInfo := &ClusterInfo{
+		FSID:          "id",
+		MonitorSecret: "monsecret",
+		AdminSecret:   "adminsecret",
+		Name:          "foo-cluster",
+		Monitors: map[string]*CephMonitorConfig{
+			"node0": &CephMonitorConfig{Name: "mon0", Endpoint: "10.0.0.1:6790"},
+			"node1": &CephMonitorConfig{Name: "mon1", Endpoint: "10.0.0.2:6790"},
+		},
+	}
+
+	monMembers := "mon0 mon1"
+
+	cephConfig := createDefaultCephConfig(clusterInfo, "/var/lib/rook1", false, false)
+	verifyConfig(t, cephConfig, monMembers, "", "filestore", 0)
+
+	cephConfig = createDefaultCephConfig(clusterInfo, "/var/lib/rook1", false, true)
+	verifyConfig(t, cephConfig, monMembers, "bluestore rocksdb", "bluestore", 0)
+
+	cephConfig = createDefaultCephConfig(clusterInfo, "/var/lib/rook1", true, false)
+	verifyConfig(t, cephConfig, monMembers, "", "filestore", 20)
+
+	cephConfig = createDefaultCephConfig(clusterInfo, "/var/lib/rook1", true, true)
+	verifyConfig(t, cephConfig, monMembers, "bluestore rocksdb", "bluestore", 20)
+}
+
+func verifyConfig(t *testing.T, cephConfig *cephConfig, expectedMonMembers, experimental, objectStore string, loggingLevel int) {
+
+	for _, expectedMon := range strings.Split(expectedMonMembers, " ") {
+		contained := false
+		for _, actualMon := range strings.Split(cephConfig.MonMembers, " ") {
+			if expectedMon == actualMon {
+				contained = true
+				break
+			}
+		}
+
+		assert.True(t, contained, "expectedMons: %+v, actualMons: %+v", expectedMonMembers, cephConfig.MonMembers)
+	}
+
+	assert.Equal(t, experimental, cephConfig.EnableExperimental)
+	assert.Equal(t, objectStore, cephConfig.OsdObjectStore)
+	assert.Equal(t, loggingLevel, cephConfig.DebugLogDefaultLevel)
+	assert.Equal(t, loggingLevel, cephConfig.DebugLogMonLevel)
+	assert.Equal(t, loggingLevel, cephConfig.DebugLogRadosLevel)
+	assert.Equal(t, loggingLevel, cephConfig.DebugLogBluestoreLevel)
+}

pkg/cephmgr/monagent.go

@@ -98,7 +98,8 @@ func (a *monAgent) makeMonitorFileSystem(context *clusterd.Context, cluster *Clu
 	}
 
 	// write the config file to disk
-	confFilePath, err := generateConnectionConfigFile(context, cluster, getMonRunDirPath(context.ConfigDir, monName), "admin", getMonKeyringPath(context.ConfigDir, monName))
+	confFilePath, err := generateConnectionConfigFile(context, cluster, getMonRunDirPath(context.ConfigDir, monName),
+		"admin", getMonKeyringPath(context.ConfigDir, monName), context.Debug)
 	if err != nil {
 		return err
 	}

pkg/cephmgr/osd.go

@@ -359,9 +359,9 @@ func loadOSDInfo(config *osdConfig) error {
 }
 
 func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context *clusterd.Context,
-	bootstrapConn client.Connection, cluster *ClusterInfo, location string, executor exec.Executor) error {
+	bootstrapConn client.Connection, cluster *ClusterInfo, location string, debug bool, executor exec.Executor) error {
 
-	cephConfig := createDefaultCephConfig(cluster, config.rootPath, config.bluestore)
+	cephConfig := createDefaultCephConfig(cluster, config.rootPath, debug, config.bluestore)
 	if !config.bluestore {
 		// using the local file system requires some config overrides
 		// http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/#not-recommended
@@ -377,7 +377,8 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context
 
 	// write the OSD config file to disk
 	keyringPath := getOSDKeyringPath(config.rootPath)
-	_, err = generateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id), keyringPath, config.bluestore, cephConfig, settings)
+	_, err = generateConfigFile(context, cluster, config.rootPath, fmt.Sprintf("osd.%d", config.id),
+		keyringPath, debug, config.bluestore, cephConfig, settings)
 	if err != nil {
 		return fmt.Errorf("failed to write OSD %d config file: %+v", config.id, err)
 	}
@@ -399,7 +400,8 @@ func initializeOSD(config *osdConfig, factory client.ConnectionFactory, context
 	}
 
 	// open a connection to the cluster using the OSDs creds
-	osdConn, err := connectToCluster(context, factory, cluster, path.Join(config.rootPath, "tmp"), fmt.Sprintf("osd.%d", config.id), keyringPath)
+	osdConn, err := connectToCluster(context, factory, cluster, path.Join(config.rootPath, "tmp"),
+		fmt.Sprintf("osd.%d", config.id), keyringPath, debug)
 	if err != nil {
 		return err
 	}

pkg/cephmgr/osdagent.go

@@ -241,7 +241,8 @@ func (a *osdAgent) createDesiredOSDs(adminConn client.Connection, context *clust
 	}
 
 	// connect to the cluster using the bootstrap-osd creds, this connection will be used for config operations
-	bootstrapConn, err := connectToCluster(context, a.factory, a.cluster, getBootstrapOSDDir(context.ConfigDir), "bootstrap-osd", getBootstrapOSDKeyringPath(context.ConfigDir, a.cluster.Name))
+	bootstrapConn, err := connectToCluster(context, a.factory, a.cluster, getBootstrapOSDDir(context.ConfigDir),
+		"bootstrap-osd", getBootstrapOSDKeyringPath(context.ConfigDir, a.cluster.Name), context.Debug)
 	if err != nil {
 		return err
 	}
@@ -320,7 +321,7 @@ func (a *osdAgent) createOrStartOSD(context *clusterd.Context, connection client
 		}
 
 		// osd_data_dir/whoami does not exist yet, create/initialize the OSD
-		err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Executor)
+		err := initializeOSD(config, a.factory, context, connection, a.cluster, a.location, context.Debug, context.Executor)
 		if err != nil {
 			return fmt.Errorf("failed to initialize OSD at %s: %+v", config.rootPath, err)
 		}

pkg/clusterd/clusterservice.go

@@ -69,6 +69,9 @@ type Context struct {
 
 	// The root configuration directory used by services
 	ConfigDir string
+
+	// A value indicating if debug logging/tracing should be enabled
+	Debug bool
 }
 
 func copyContext(c *Context) *Context {
@@ -80,6 +83,7 @@ func copyContext(c *Context) *Context {
 		ProcMan:    c.ProcMan,
 		Inventory:  c.Inventory,
 		ConfigDir:  c.ConfigDir,
+		Debug:      c.Debug,
 	}
 }
 

pkg/clusterd/join.go

@@ -13,9 +13,11 @@ import (
 )
 
 // StartJoinCluster initializes the cluster services to enable joining the cluster and listening for orchestration.
-func StartJoinCluster(services []*ClusterService, procMan *proc.ProcManager, configDir, nodeID, discoveryURL, etcdMembers, publicIPv4, privateIPv4 string) (*Context, error) {
-	log.Printf("Starting cluster. config=%s, nodeID=%s, url=%s, members=%s, publicIPv4=%s, privateIPv4=%s",
-		configDir, nodeID, discoveryURL, etcdMembers, publicIPv4, privateIPv4)
+func StartJoinCluster(services []*ClusterService, procMan *proc.ProcManager, configDir, nodeID, discoveryURL,
+	etcdMembers, publicIPv4, privateIPv4 string, debug bool) (*Context, error) {
+
+	log.Printf("Starting cluster. configDir=%s, nodeID=%s, url=%s, members=%s, publicIPv4=%s, privateIPv4=%s, debug=%t",
+		configDir, nodeID, discoveryURL, etcdMembers, publicIPv4, privateIPv4, debug)
 
 	etcdClients := []string{}
 	if etcdMembers != "" {
@@ -58,6 +60,7 @@ func StartJoinCluster(services []*ClusterService, procMan *proc.ProcManager, con
 		Services:   services,
 		ProcMan:    procMan,
 		ConfigDir:  configDir,
+		Debug:      debug,
 	}
 	clusterLeader := newServicesLeader(context)
 	clusterMember := newClusterMember(context, leaseManager, clusterLeader)