2017-09-14 09:40:34 -07:00
|
|
|
|
/*
|
|
|
|
|
|
Copyright 2017 The Rook Authors. All rights reserved.
|
|
|
|
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
|
|
limitations under the License.
|
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
// Package flexvolume to manage Kubernetes storage attach events.
|
|
|
|
|
|
package flexvolume
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
|
"fmt"
|
|
|
|
|
|
"io"
|
|
|
|
|
|
"net"
|
|
|
|
|
|
"net/rpc"
|
|
|
|
|
|
"os"
|
|
|
|
|
|
"path"
|
2018-05-18 14:32:20 -07:00
|
|
|
|
"regexp"
|
|
|
|
|
|
"strings"
|
2017-09-14 09:40:34 -07:00
|
|
|
|
|
2017-10-31 08:12:21 -07:00
|
|
|
|
"k8s.io/kubernetes/pkg/util/version"
|
2017-09-14 09:40:34 -07:00
|
|
|
|
|
|
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
|
|
|
|
"github.com/rook/rook/pkg/clusterd"
|
2017-10-31 08:12:21 -07:00
|
|
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
2017-09-14 09:40:34 -07:00
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
|
|
UnixSocketName = ".rook.sock"
|
2018-05-18 14:32:20 -07:00
|
|
|
|
FlexvolumeVendor = "ceph.rook.io"
|
|
|
|
|
|
FlexvolumeVendorLegacy = "rook.io"
|
2018-05-11 14:02:16 -07:00
|
|
|
|
FlexDriverName = "rook"
|
2017-10-09 12:13:25 -07:00
|
|
|
|
flexvolumeDriverFileName = "rookflex"
|
2018-05-11 14:02:16 -07:00
|
|
|
|
flexMountPath = "/flexmnt/%s~%s"
|
2017-10-09 12:13:25 -07:00
|
|
|
|
usrBinDir = "/usr/local/bin/"
|
2017-10-31 08:12:21 -07:00
|
|
|
|
serverVersionV180 = "v1.8.0"
|
2017-09-14 09:40:34 -07:00
|
|
|
|
)
|
|
|
|
|
|
|
2017-12-04 08:33:19 -08:00
|
|
|
|
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "flexvolume")
|
2017-09-14 09:40:34 -07:00
|
|
|
|
|
|
|
|
|
|
// FlexvolumeServer start a unix domain socket server to interact with the flexvolume driver
|
|
|
|
|
|
type FlexvolumeServer struct {
|
2017-10-31 08:12:21 -07:00
|
|
|
|
context *clusterd.Context
|
2017-12-04 08:33:19 -08:00
|
|
|
|
controller *Controller
|
2018-05-11 14:02:16 -07:00
|
|
|
|
listeners map[string]net.Listener
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// NewFlexvolumeServer creates an Flexvolume server
|
2018-08-21 14:48:09 +08:00
|
|
|
|
func NewFlexvolumeServer(context *clusterd.Context, controller *Controller) *FlexvolumeServer {
|
2017-09-14 09:40:34 -07:00
|
|
|
|
return &FlexvolumeServer{
|
2017-10-31 08:12:21 -07:00
|
|
|
|
context: context,
|
2017-09-14 09:40:34 -07:00
|
|
|
|
controller: controller,
|
2018-05-11 14:02:16 -07:00
|
|
|
|
listeners: make(map[string]net.Listener),
|
2017-10-25 12:23:15 -07:00
|
|
|
|
}
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Start configures the flexvolume driver on the host and starts the unix domain socket server to communicate with the driver
|
2018-05-18 14:32:20 -07:00
|
|
|
|
func (s *FlexvolumeServer) Start(driverVendor, driverName string) error {
|
2017-10-09 12:13:25 -07:00
|
|
|
|
|
2017-10-31 08:12:21 -07:00
|
|
|
|
// first install the flexvolume driver
|
2018-05-11 14:02:16 -07:00
|
|
|
|
// /usr/local/bin/rookflex
|
|
|
|
|
|
driverFile := path.Join(usrBinDir, flexvolumeDriverFileName)
|
|
|
|
|
|
// /flexmnt/rook.io~rook-system
|
2018-05-18 14:32:20 -07:00
|
|
|
|
flexVolumeDriverDir := fmt.Sprintf(flexMountPath, driverVendor, driverName)
|
2018-05-11 14:02:16 -07:00
|
|
|
|
|
|
|
|
|
|
err := configureFlexVolume(driverFile, flexVolumeDriverDir, driverName)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
if err != nil {
|
2018-05-18 14:32:20 -07:00
|
|
|
|
return fmt.Errorf("unable to configure flexvolume %s: %v", flexVolumeDriverDir, err)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
unixSocketFile := path.Join(flexVolumeDriverDir, UnixSocketName) // /flextmnt/rook.io~rook-system/.rook.sock
|
|
|
|
|
|
if _, ok := s.listeners[unixSocketFile]; ok {
|
|
|
|
|
|
logger.Infof("flex server already running at %s", unixSocketFile)
|
|
|
|
|
|
return nil
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// remove unix socket if it existed previously
|
|
|
|
|
|
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
|
|
|
|
|
|
logger.Info("Deleting unix domain socket file.")
|
|
|
|
|
|
os.Remove(unixSocketFile)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
listener, err := net.Listen("unix", unixSocketFile)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
if err != nil {
|
|
|
|
|
|
return fmt.Errorf("unable to listen at %s: %v", unixSocketFile, err)
|
|
|
|
|
|
}
|
2018-05-11 14:02:16 -07:00
|
|
|
|
s.listeners[unixSocketFile] = listener
|
2017-09-14 09:40:34 -07:00
|
|
|
|
|
|
|
|
|
|
if err := os.Chmod(unixSocketFile, 0770); err != nil {
|
|
|
|
|
|
return fmt.Errorf("unable to set file permission to unix socket %s: %v", unixSocketFile, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
go rpc.Accept(listener)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
logger.Infof("Listening on unix socket for Kubernetes volume attach commands: %s", unixSocketFile)
|
2017-10-31 08:12:21 -07:00
|
|
|
|
|
|
|
|
|
|
// flexvolume driver was installed OK. If running on pre 1.8 Kubernetes, then remind the user
|
|
|
|
|
|
// to restart the Kubelet. We do this last so that it's the last message in the log, making it
|
|
|
|
|
|
// harder for the user to miss.
|
|
|
|
|
|
checkIfKubeletRestartRequired(s.context)
|
|
|
|
|
|
|
2017-09-14 09:40:34 -07:00
|
|
|
|
return nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Stop the unix domain socket server and deletes the socket file
|
2018-05-11 14:02:16 -07:00
|
|
|
|
func (s *FlexvolumeServer) StopAll() {
|
|
|
|
|
|
logger.Infof("Stopping %d unix socket rpc server(s).", len(s.listeners))
|
|
|
|
|
|
for unixSocketFile, listener := range s.listeners {
|
|
|
|
|
|
if err := listener.Close(); err != nil {
|
2017-09-14 09:40:34 -07:00
|
|
|
|
logger.Errorf("Failed to stop unix socket rpc server: %+v", err)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
// closing the listener should remove the unix socket file. But lets try it remove it just in case.
|
|
|
|
|
|
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
|
|
|
|
|
|
logger.Infof("Deleting unix domain socket file %s.", unixSocketFile)
|
|
|
|
|
|
os.Remove(unixSocketFile)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
s.listeners = make(map[string]net.Listener)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
func RookDriverName(context *clusterd.Context) (string, error) {
|
|
|
|
|
|
kubeVersion, err := k8sutil.GetK8SVersion(context.Clientset)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return "", fmt.Errorf("Error getting server version: %v", err)
|
|
|
|
|
|
}
|
|
|
|
|
|
// K8s 1.7 returns an error when trying to run multiple drivers under the same rook.io provider,
|
|
|
|
|
|
// so we will fall back to the rook driver name in that case.
|
|
|
|
|
|
if kubeVersion.AtLeast(version.MustParseSemantic(serverVersionV180)) {
|
|
|
|
|
|
// the driver name needs to be the same as the namespace so that we can support multiple namespaces
|
|
|
|
|
|
// without the drivers conflicting with each other
|
|
|
|
|
|
return os.Getenv(k8sutil.PodNamespaceEnvVar), nil
|
|
|
|
|
|
}
|
|
|
|
|
|
// fall back to the rook driver name where multiple system namespaces are not supported
|
|
|
|
|
|
return FlexDriverName, nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func configureFlexVolume(driverFile, driverDir, driverName string) error {
|
2017-09-14 09:40:34 -07:00
|
|
|
|
// copying flex volume
|
|
|
|
|
|
if _, err := os.Stat(driverDir); os.IsNotExist(err) {
|
2018-05-11 14:02:16 -07:00
|
|
|
|
err := os.Mkdir(driverDir, 0755)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
logger.Errorf("failed to create dir %s. %+v", driverDir, err)
|
|
|
|
|
|
}
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
2017-10-09 12:13:25 -07:00
|
|
|
|
|
2018-05-11 14:02:16 -07:00
|
|
|
|
destFile := path.Join(driverDir, "."+driverName) // /flextmnt/rook.io~rook-system/.rook-system
|
|
|
|
|
|
finalDestFile := path.Join(driverDir, driverName) // /flextmnt/rook.io~rook-system/rook-system
|
2017-10-09 12:13:25 -07:00
|
|
|
|
err := copyFile(driverFile, destFile)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
if err != nil {
|
2017-10-09 12:13:25 -07:00
|
|
|
|
return fmt.Errorf("unable to copy flexvolume from %s to %s: %+v", driverFile, destFile, err)
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// renaming flex volume. Rename is an atomic execution while copying is not.
|
|
|
|
|
|
if _, err := os.Stat(finalDestFile); !os.IsNotExist(err) {
|
|
|
|
|
|
// Delete old plugin if it exists
|
|
|
|
|
|
err = os.Remove(finalDestFile)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
logger.Warningf("Could not delete old Rook Flexvolume driver at %s: %v", finalDestFile, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
}
|
2017-10-31 08:12:21 -07:00
|
|
|
|
|
|
|
|
|
|
if err := os.Rename(destFile, finalDestFile); err != nil {
|
|
|
|
|
|
return fmt.Errorf("failed to rename %s to %s: %+v", destFile, finalDestFile, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return nil
|
2017-09-14 09:40:34 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func copyFile(src, dest string) error {
|
|
|
|
|
|
srcFile, err := os.Open(src)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return fmt.Errorf("error opening source file %s: %v", src, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
defer srcFile.Close()
|
|
|
|
|
|
|
|
|
|
|
|
destFile, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) // creates if file doesn't exist
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return fmt.Errorf("error creating destination file %s: %v", dest, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
defer destFile.Close()
|
|
|
|
|
|
|
|
|
|
|
|
_, err = io.Copy(destFile, srcFile)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
return fmt.Errorf("error copying file from %s to %s: %v", src, dest, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
return destFile.Sync()
|
|
|
|
|
|
}
|
2017-10-31 08:12:21 -07:00
|
|
|
|
|
|
|
|
|
|
func checkIfKubeletRestartRequired(context *clusterd.Context) {
|
|
|
|
|
|
kubeVersion, err := k8sutil.GetK8SVersion(context.Clientset)
|
|
|
|
|
|
if err != nil || kubeVersion.LessThan(version.MustParseSemantic(serverVersionV180)) {
|
|
|
|
|
|
logger.Warning("NOTE: The Kubelet must be restarted on this node since this pod appears to " +
|
|
|
|
|
|
"be running on a Kubernetes version prior to 1.8. More details can be found in the Rook docs at " +
|
2018-03-23 14:23:30 -07:00
|
|
|
|
"https://rook.io/docs/rook/master/common-issues.html#kubelet-restart")
|
2017-10-31 08:12:21 -07:00
|
|
|
|
}
|
|
|
|
|
|
}
|
2018-05-18 14:32:20 -07:00
|
|
|
|
|
|
|
|
|
|
// Gets the flex driver info (vendor, driver name) from a given path where the flex driver exists.
|
|
|
|
|
|
// The given path may look something like this:
|
|
|
|
|
|
// /usr/libexec/kubernetes/kubelet-plugins/volume/exec/rook.io~rook/rook/
|
|
|
|
|
|
// In which case, the vendor is rook.io and the driver name is rook.
|
|
|
|
|
|
func getFlexDriverInfo(flexDriverPath string) (vendor, driver string, err error) {
|
|
|
|
|
|
parts := strings.Split(flexDriverPath, string(os.PathSeparator))
|
|
|
|
|
|
for i := len(parts) - 1; i >= 0; i-- {
|
|
|
|
|
|
p := parts[i]
|
|
|
|
|
|
if matched, _ := regexp.Match(".+~.+", []byte(p)); matched {
|
|
|
|
|
|
// found a match for the flex driver directory name pattern
|
|
|
|
|
|
flexInfo := strings.Split(p, "~")
|
|
|
|
|
|
if len(flexInfo) > 2 {
|
|
|
|
|
|
return "", "", fmt.Errorf("unexpected number of items in flex driver info %+v from path %s", flexInfo, flexDriverPath)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return flexInfo[0], flexInfo[1], nil
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return "", "", fmt.Errorf("failed to find flex driver info from path %s", flexDriverPath)
|
|
|
|
|
|
}
|