my-rook-config/pkg/daemon/agent/flexvolume/manager/ceph/manager.go

/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ceph

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/coreos/pkg/capnslog"
	"github.com/rook/rook/pkg/clusterd"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	cephmon "github.com/rook/rook/pkg/daemon/ceph/mon"
	"github.com/rook/rook/pkg/daemon/ceph/util"
	"github.com/rook/rook/pkg/model"
	"github.com/rook/rook/pkg/operator/cluster/ceph/mon"
	"github.com/rook/rook/pkg/util/sys"
)

const (
	findDevicePathMaxRetries = 10
	rbdKernelModuleName      = "rbd"
)

var logger = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-volumeattacher")

// VolumeManager represents an object for perform volume attachment requests for Ceph volumes
type VolumeManager struct {
	context          *clusterd.Context
	devicePathFinder model.DevicePathFinder
}

type devicePathFinder struct{}

// NewVolumeManager create attacher for ceph volumes
func NewVolumeManager(context *clusterd.Context) *VolumeManager {
	vm := &VolumeManager{
		context:          context,
		devicePathFinder: &devicePathFinder{},
	}
	vm.Init()
	return vm
}

// Init the ceph volume manager
func (vm *VolumeManager) Init() error {

	// check to see if the rbd kernel module has single_major support
	hasSingleMajor, err := sys.CheckKernelModuleParam(rbdKernelModuleName, "single_major", vm.context.Executor)
	if err != nil {
		logger.Noticef("failed %s single_major check, assuming it's unsupported: %+v", rbdKernelModuleName, err)
		hasSingleMajor = false
	}

	opts := []string{}
	if hasSingleMajor {
		opts = append(opts, "single_major=Y")
	}

	// load the rbd kernel module with options
	// TODO: should this fail if modprobe fails?
	if err := sys.LoadKernelModule(rbdKernelModuleName, opts, vm.context.Executor); err != nil {
		logger.Noticef("failed to load kernel module %s: %+v", rbdKernelModuleName, err)
	}

	return nil
}

// Attach a ceph image to the node
func (vm *VolumeManager) Attach(image, pool, clusterName string) (string, error) {

	// check if the volume is already attached
	devicePath, err := vm.isAttached(image, pool, clusterName)
	if err != nil {
		return "", fmt.Errorf("failed to check if volume %s/%s is already attached: %+v", pool, image, err)
	}
	if devicePath != "" {
		logger.Infof("volume %s/%s is already attached. The device path is %s", pool, image, devicePath)
		return devicePath, nil
	}

	// attach and poll until volume is mapped
	logger.Infof("attaching volume %s/%s cluster %s", pool, image, clusterName)
	monitors, keyring, err := getClusterInfo(vm.context, clusterName)
	defer os.Remove(keyring)
	if err != nil {
		return "", fmt.Errorf("failed to load cluster information from cluster %s: %+v", clusterName, err)
	}

	err = cephclient.MapImage(vm.context, image, pool, clusterName, keyring, monitors)
	if err != nil {
		return "", fmt.Errorf("failed to map image %s/%s cluster %s. %+v", pool, image, clusterName, err)
	}

	// poll for device path
	retryCount := 0
	for {
		devicePath, err := vm.devicePathFinder.FindDevicePath(image, pool, clusterName)
		if err != nil {
			return "", fmt.Errorf("failed to poll for mapped image %s/%s cluster %s. %+v", pool, image, clusterName, err)
		}

		if devicePath != "" {
			return devicePath, nil
		}

		retryCount++
		if retryCount >= findDevicePathMaxRetries {
			return "", fmt.Errorf("exceeded retry count while finding device path: %+v", err)
		}

		logger.Infof("failed to find device path, sleeping 1 second: %+v", err)
		<-time.After(time.Second)
	}
}

// Detach the volume
func (vm *VolumeManager) Detach(image, pool, clusterName string, force bool) error {
	// check if the volume is attached
	devicePath, err := vm.isAttached(image, pool, clusterName)
	if err != nil {
		return fmt.Errorf("failed to check if volume %s/%s is attached cluster %s", pool, image, clusterName)
	}
	if devicePath == "" {
		logger.Infof("volume %s/%s is already detached cluster %s", pool, image, clusterName)
		return nil
	}

	logger.Infof("detaching volume %s/%s cluster %s", pool, image, clusterName)
	monitors, keyring, err := getClusterInfo(vm.context, clusterName)
	defer os.Remove(keyring)
	if err != nil {
		return fmt.Errorf("failed to load cluster information from cluster %s: %+v", clusterName, err)
	}

	err = cephclient.UnMapImage(vm.context, image, pool, clusterName, keyring, monitors, force)
	if err != nil {
		return fmt.Errorf("failed to detach volume %s/%s cluster %s. %+v", pool, image, clusterName, err)
	}
	logger.Infof("detached volume %s/%s", pool, image)
	return nil
}

// Check if the volume is attached
func (vm *VolumeManager) isAttached(image, pool, clusterName string) (string, error) {
	devicePath, err := vm.devicePathFinder.FindDevicePath(image, pool, clusterName)
	if err != nil {
		return "", err
	}
	return devicePath, nil
}

func getClusterInfo(context *clusterd.Context, clusterName string) (string, string, error) {
	clusterInfo, _, _, err := mon.LoadClusterInfo(context, clusterName)
	if err != nil {
		return "", "", fmt.Errorf("failed to load cluster information from cluster %s: %+v", clusterName, err)
	}

	// create temp keyring file
	keyringFile, err := ioutil.TempFile("", clusterName+".keyring")
	if err != nil {
		return "", "", err
	}

	keyring := fmt.Sprintf(cephmon.AdminKeyringTemplate, clusterInfo.AdminSecret)
	if err := ioutil.WriteFile(keyringFile.Name(), []byte(keyring), 0644); err != nil {
		return "", "", fmt.Errorf("failed to write monitor keyring to %s: %+v", keyringFile.Name(), err)
	}

	monEndpoints := make([]string, 0, len(clusterInfo.Monitors))
	for _, monitor := range clusterInfo.Monitors {
		monEndpoints = append(monEndpoints, monitor.Endpoint)
	}
	return strings.Join(monEndpoints, ","), keyringFile.Name(), nil
}

// FindDevicePath polls and wait for the mapped ceph image device to show up
func (f *devicePathFinder) FindDevicePath(image, pool, clusterName string) (string, error) {
	mappedFile, err := util.FindRBDMappedFile(image, pool, util.RBDSysBusPathDefault)
	if err != nil {
		return "", fmt.Errorf("failed to find mapped image: %+v", err)
	}

	if mappedFile != "" {
		devicePath := util.RBDDevicePathPrefix + mappedFile
		if _, err := os.Lstat(devicePath); err != nil {
			return "", fmt.Errorf("sysfs information for image '%s' in pool '%s' found but the rbd device path %s does not exist", image, pool, devicePath)
		}
		return devicePath, nil
	}
	return "", nil
}