Files
my-rook-config/pkg/cephmgr/mon/config.go
T

387 lines
13 KiB
Go

/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mon
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/coreos/pkg/capnslog"
"github.com/go-ini/ini"
"github.com/rook/rook/pkg/cephmgr/client"
"github.com/rook/rook/pkg/clusterd"
)
type CephMonitorConfig struct {
Name string `json:"name"`
Endpoint string `json:"endpoint"`
}
type cephConfig struct {
*GlobalConfig `ini:"global,omitempty"`
}
type GlobalConfig struct {
EnableExperimental string `ini:"enable experimental unrecoverable data corrupting features"`
FSID string `ini:"fsid,omitempty"`
RunDir string `ini:"run dir,omitempty"`
MonMembers string `ini:"mon initial members,omitempty"`
LogFile string `ini:"log file,omitempty"`
MonClusterLogFile string `ini:"mon cluster log file,omitempty"`
PublicAddr string `ini:"public addr,omitempty"`
PublicNetwork string `ini:"public network,omitempty"`
ClusterAddr string `ini:"cluster addr,omitempty"`
ClusterNetwork string `ini:"cluster network,omitempty"`
MonKeyValueDb string `ini:"mon keyvaluedb"`
DebugLogDefaultLevel int `ini:"debug default"`
DebugLogRadosLevel int `ini:"debug rados"`
DebugLogMonLevel int `ini:"debug mon"`
DebugLogOSDLevel int `ini:"debug osd"`
DebugLogBluestoreLevel int `ini:"debug bluestore"`
DebugLogFilestoreLevel int `ini:"debug filestore"`
DebugLogJournalLevel int `ini:"debug journal"`
DebugLogLevelDBLevel int `ini:"debug leveldb"`
FileStoreOmapBackend string `ini:"filestore_omap_backend"`
OsdPgBits int `ini:"osd pg bits,omitempty"`
OsdPgpBits int `ini:"osd pgp bits,omitempty"`
OsdPoolDefaultSize int `ini:"osd pool default size,omitempty"`
OsdPoolDefaultMinSize int `ini:"osd pool default min size,omitempty"`
OsdPoolDefaultPgNum int `ini:"osd pool default pg num,omitempty"`
OsdPoolDefaultPgpNum int `ini:"osd pool default pgp num,omitempty"`
OsdMaxObjectNameLen int `ini:"osd max object name len,omitempty"`
OsdMaxObjectNamespaceLen int `ini:"osd max object namespace len,omitempty"`
OsdObjectStore string `ini:"osd objectstore"`
RbdDefaultFeatures int `ini:"rbd_default_features,omitempty"`
CrushtoolPath string `ini:"crushtool"`
FatalSignalHandlers string `ini:"fatal signal handlers"`
}
// get the path of a given monitor's run dir
func getMonRunDirPath(configDir, monName string) string {
return path.Join(configDir, monName)
}
// get the path of a given monitor's keyring
func getMonKeyringPath(configDir, monName string) string {
return filepath.Join(getMonRunDirPath(configDir, monName), "keyring")
}
// get the path of a given monitor's data dir
func getMonDataDirPath(configDir, monName string) string {
return filepath.Join(getMonRunDirPath(configDir, monName), fmt.Sprintf("mon.%s", monName))
}
func ConnectToClusterAsAdmin(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo) (client.Connection, error) {
if len(cluster.Monitors) == 0 {
return nil, errors.New("no monitors")
}
// write the monitor keyring to disk
monName := getFirstMonitor(cluster)
keyringPath := getMonKeyringPath(context.ConfigDir, monName)
if err := writeMonitorKeyring(monName, cluster, keyringPath); err != nil {
return nil, err
}
return ConnectToCluster(context, factory, cluster, getMonRunDirPath(context.ConfigDir, monName),
"admin", getMonKeyringPath(context.ConfigDir, monName))
}
// get the path of a given monitor's config file
func GetConfFilePath(root, clusterName string) string {
return fmt.Sprintf("%s/%s.config", root, clusterName)
}
func GenerateTempConfigFiles(context *clusterd.Context, cluster *ClusterInfo) (string, string, string, error) {
root := path.Join(context.ConfigDir, "tmp")
keyring := path.Join(root, "keyring")
user := getFirstMonitor(cluster)
err := writeMonitorKeyring(user, cluster, keyring)
if err != nil {
return "", "", "", fmt.Errorf("failed to write keyring to %s", root)
}
configFile, err := GenerateConfigFile(context, cluster, root, user, keyring, false, nil, nil)
return configFile, keyring, user, err
}
// generates and writes the monitor config file to disk
func GenerateConnectionConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string) (string, error) {
return GenerateConfigFile(context, cluster, pathRoot, user, keyringPath, false, nil, nil)
}
// generates and writes the monitor config file to disk
func GenerateConfigFile(context *clusterd.Context, cluster *ClusterInfo, pathRoot, user, keyringPath string,
bluestore bool, userConfig *cephConfig, clientSettings map[string]string) (string, error) {
if pathRoot == "" {
pathRoot = getMonRunDirPath(context.ConfigDir, getFirstMonitor(cluster))
}
// create the config directory
if err := os.MkdirAll(pathRoot, 0744); err != nil {
logger.Warningf("failed to create config directory at %s: %+v", pathRoot, err)
}
configFile, err := createGlobalConfigFileSection(context, cluster, pathRoot, bluestore, userConfig)
if err != nil {
return "", fmt.Errorf("failed to create global config section, %+v", err)
}
if err := addClientConfigFileSection(configFile, getQualifiedUser(user), keyringPath, clientSettings); err != nil {
return "", fmt.Errorf("failed to add admin client config section, %+v", err)
}
if err := addInitialMonitorsConfigFileSections(configFile, cluster); err != nil {
return "", fmt.Errorf("failed to add initial monitor config sections, %+v", err)
}
// if there's a config file override path given, process the given config file
if context.ConfigFileOverride != "" {
err := configFile.Append(context.ConfigFileOverride)
if err != nil {
// log the config file override failure as a warning, but proceed without it
logger.Warningf("failed to add config file override from '%s': %+v", context.ConfigFileOverride, err)
}
}
// write the entire config to disk
filePath := GetConfFilePath(pathRoot, cluster.Name)
logger.Debugf("writing config file %s", filePath)
if err := configFile.SaveTo(filePath); err != nil {
return "", fmt.Errorf("failed to save config file %s. %+v", filePath, err)
}
return filePath, nil
}
// create a keyring for access to the cluster, with the desired set of privileges
func CreateKeyring(conn client.Connection, username, keyringPath string, access []string, generateContents func(string) string) error {
_, err := os.Stat(keyringPath)
if err == nil {
// no error, the file exists, bail out with no error
logger.Debugf("keyring already exists at %s", keyringPath)
return nil
} else if !os.IsNotExist(err) {
// some other error besides "does not exist", bail out with error
return fmt.Errorf("failed to stat %s: %+v", keyringPath, err)
}
// get-or-create-key for the user account
key, err := client.AuthGetOrCreateKey(conn, username, access)
if err != nil {
return fmt.Errorf("failed to get or create auth key for %s. %+v", username, err)
}
return WriteKeyring(keyringPath, key, generateContents)
}
func WriteKeyring(keyringPath, keyring string, generateContents func(string) string) error {
// write the keyring to disk
keyringDir := filepath.Dir(keyringPath)
if err := os.MkdirAll(keyringDir, 0744); err != nil {
return fmt.Errorf("failed to create keyring dir at %s: %+v", keyringDir, err)
}
contents := generateContents(keyring)
logger.Debugf("Writing keyring to: %s", keyringPath)
if err := ioutil.WriteFile(keyringPath, []byte(contents), 0644); err != nil {
return fmt.Errorf("failed to write keyring to %s: %+v", keyringPath, err)
}
return nil
}
// prepends "client." if a user namespace is not already specified
func getQualifiedUser(user string) string {
if strings.Index(user, ".") == -1 {
return fmt.Sprintf("client.%s", user)
}
return user
}
func getFirstMonitor(cluster *ClusterInfo) string {
// Get the first monitor
for _, m := range cluster.Monitors {
return m.Name
}
return ""
}
// opens a connection to the cluster that can be used for management operations
func ConnectToCluster(context *clusterd.Context, factory client.ConnectionFactory, cluster *ClusterInfo,
basePath, user, keyringPath string) (client.Connection, error) {
logger.Infof("connecting to ceph cluster %s with user %s", cluster.Name, user)
confFilePath, err := GenerateConnectionConfigFile(context, cluster, basePath, user, keyringPath)
if err != nil {
return nil, fmt.Errorf("failed to generate config file: %v", err)
}
conn, err := factory.NewConnWithClusterAndUser(cluster.Name, getQualifiedUser(user))
if err != nil {
return nil, fmt.Errorf("failed to create rados connection for cluster %s and user %s: %+v", cluster.Name, user, err)
}
if err = conn.ReadConfigFile(confFilePath); err != nil {
return nil, fmt.Errorf("failed to read config file for cluster %s: %+v", cluster.Name, err)
}
if err = conn.Connect(); err != nil {
return nil, fmt.Errorf("failed to connect to cluster %s: %+v", cluster.Name, err)
}
logger.Infof("successfully connected to cluster %s with user %s", cluster.Name, user)
return conn, nil
}
func CreateDefaultCephConfig(context *clusterd.Context, cluster *ClusterInfo, runDir string, bluestore bool) *cephConfig {
// extract a list of just the monitor names, which will populate the "mon initial members"
// global config field
monMembers := make([]string, len(cluster.Monitors))
i := 0
for _, monitor := range cluster.Monitors {
monMembers[i] = monitor.Name
i++
}
experimental := ""
store := "filestore"
if bluestore {
experimental = "bluestore rocksdb"
store = "bluestore"
}
cephLogLevel := logLevelToCephLogLevel(context.LogLevel)
return &cephConfig{
&GlobalConfig{
EnableExperimental: experimental,
FSID: cluster.FSID,
RunDir: runDir,
MonMembers: strings.Join(monMembers, " "),
LogFile: "/dev/stdout",
MonClusterLogFile: "/dev/stdout",
PublicAddr: context.NetworkInfo.PublicAddrIPv4,
PublicNetwork: context.NetworkInfo.PublicNetwork,
ClusterAddr: context.NetworkInfo.ClusterAddrIPv4,
ClusterNetwork: context.NetworkInfo.ClusterNetwork,
MonKeyValueDb: "rocksdb",
DebugLogDefaultLevel: cephLogLevel,
DebugLogRadosLevel: cephLogLevel,
DebugLogMonLevel: cephLogLevel,
DebugLogOSDLevel: cephLogLevel,
DebugLogBluestoreLevel: cephLogLevel,
DebugLogFilestoreLevel: cephLogLevel,
DebugLogJournalLevel: cephLogLevel,
DebugLogLevelDBLevel: cephLogLevel,
FileStoreOmapBackend: "rocksdb",
OsdPgBits: 11,
OsdPgpBits: 11,
OsdPoolDefaultSize: 1,
OsdPoolDefaultMinSize: 1,
OsdPoolDefaultPgNum: 100,
OsdPoolDefaultPgpNum: 100,
OsdObjectStore: store,
RbdDefaultFeatures: 3,
CrushtoolPath: "",
FatalSignalHandlers: "false",
},
}
}
func createGlobalConfigFileSection(context *clusterd.Context, cluster *ClusterInfo, runDir string,
bluestore bool, userConfig *cephConfig) (*ini.File, error) {
var ceph *cephConfig
if userConfig != nil {
// use the user config since it was provided
ceph = userConfig
} else {
ceph = CreateDefaultCephConfig(context, cluster, runDir, bluestore)
}
configFile := ini.Empty()
err := ini.ReflectFrom(configFile, ceph)
return configFile, err
}
func addClientConfigFileSection(configFile *ini.File, clientName, keyringPath string, settings map[string]string) error {
s, err := configFile.NewSection(clientName)
if err != nil {
return err
}
if _, err := s.NewKey("keyring", keyringPath); err != nil {
return err
}
for key, val := range settings {
if _, err := s.NewKey(key, val); err != nil {
return fmt.Errorf("failed to add key %s. %v", key, err)
}
}
return nil
}
func addInitialMonitorsConfigFileSections(configFile *ini.File, cluster *ClusterInfo) error {
// write the config for each individual monitor member of the cluster to the content buffer
for _, mon := range cluster.Monitors {
s, err := configFile.NewSection(fmt.Sprintf("mon.%s", mon.Name))
if err != nil {
return err
}
if _, err := s.NewKey("name", mon.Name); err != nil {
return err
}
if _, err := s.NewKey("mon addr", mon.Endpoint); err != nil {
return err
}
}
return nil
}
func logLevelToCephLogLevel(logLevel capnslog.LogLevel) int {
switch logLevel {
case capnslog.CRITICAL:
case capnslog.ERROR:
case capnslog.WARNING:
return -1
case capnslog.NOTICE:
case capnslog.INFO:
return 0
case capnslog.DEBUG:
return 10
case capnslog.TRACE:
return 100
}
return 0
}