flex: base driver name on the namespace

Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
travisn committed 2018-05-11 14:02:16 -07:00
1 parent 656d2233d7
commit 7bd850e7fc
9 files changed
+112 -45

No files matched your search

+1 -1
View File
@@ -51,7 +51,7 @@ func getRPCClient() (*rpc.Client, error) {
if err != nil {
return nil, fmt.Errorf("error getting path of the Rook flexvolume driver: %v", err)
}
unixSocketFile := path.Join(path.Dir(ex), path.Join(flexvolume.UnixSocketName)) // /usr/libexec/kubernetes/plugin/volume/rook.io~rook/.rook.sock
unixSocketFile := path.Join(path.Dir(ex), flexvolume.UnixSocketName) // /usr/libexec/kubernetes/kubelet-plugins/volume/exec/rook.io~rook/rook/.rook.sock
conn, err := net.Dial("unix", unixSocketFile)
if err != nil {
return nil, fmt.Errorf("error connecting to socket %s: %+v", unixSocketFile, err)
+23 -2
View File
@@ -19,6 +19,7 @@ package agent
import (
"fmt"
"net/rpc"
"os"
"os/signal"
"syscall"
@@ -62,7 +63,27 @@ func (a *Agent) Run() error {
volumeManager,
)
flexvolumeServer.Start()
err = rpc.Register(flexvolumeController)
if err != nil {
return fmt.Errorf("unable to register rpc: %v", err)
}
driverName, err := flexvolume.RookDriverName(a.context)
if err != nil {
return fmt.Errorf("failed to get driver name. %+v", err)
}
err = flexvolumeServer.Start(driverName)
if err != nil {
return fmt.Errorf("failed to start flex volume server %s, %+v", driverName, err)
}
// Register drivers both with the name of the namespace and the name "rook"
// for the volume plugins not based on the namespace.
err = flexvolumeServer.Start(flexvolume.FlexDriverName)
if err != nil {
return fmt.Errorf("failed to start flex volume server %s. %+v", flexvolume.FlexDriverName, err)
}
// create a cluster controller and tell it to start watching for changes to clusters
clusterController := cluster.NewClusterController(
@@ -79,7 +100,7 @@ func (a *Agent) Run() error {
select {
case <-sigc:
logger.Infof("shutdown signal received, exiting...")
flexvolumeServer.Stop()
flexvolumeServer.StopAll()
close(stopChan)
return nil
}
+5 -1
View File
@@ -325,7 +325,11 @@ func (c *Controller) GetAttachInfoFromMountDir(mountDir string, attachOptions *A
// GetGlobalMountPath generate the global mount path where the device path is mounted.
// It is based on the kubelet root dir, which defaults to /var/lib/kubelet
func (c *Controller) GetGlobalMountPath(volumeName string, globalMountPath *string) error {
*globalMountPath = path.Join(c.getKubeletRootDir(), "plugins", FlexvolumeVendor, FlexvolumeDriver, "mounts", volumeName)
driverName, err := RookDriverName(c.context)
if err != nil {
return err
}
*globalMountPath = path.Join(c.getKubeletRootDir(), "plugins", FlexvolumeVendor, driverName, "mounts", volumeName)
return nil
}
+53 -31
View File
@@ -35,21 +35,20 @@ import (
const (
UnixSocketName = ".rook.sock"
FlexvolumeVendor = "rook.io"
FlexvolumeDriver = "rook"
FlexDriverName = "rook"
flexvolumeDriverFileName = "rookflex"
flexMountPath = "/flexmnt/%s~%s"
usrBinDir = "/usr/local/bin/"
serverVersionV170 = "v1.7.0"
serverVersionV180 = "v1.8.0"
)
var flexVolumeDriverDir = fmt.Sprintf("/flexmnt/%s~%s", FlexvolumeVendor, FlexvolumeDriver)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "flexvolume")
// FlexvolumeServer start a unix domain socket server to interact with the flexvolume driver
type FlexvolumeServer struct {
context *clusterd.Context
controller *Controller
listener net.Listener
listeners map[string]net.Listener
}
// NewFlexvolumeServer creates an Flexvolume server
@@ -57,26 +56,29 @@ func NewFlexvolumeServer(context *clusterd.Context, controller *Controller, mana
return &FlexvolumeServer{
context: context,
controller: controller,
listeners: make(map[string]net.Listener),
}
}
// Start configures the flexvolume driver on the host and starts the unix domain socket server to communicate with the driver
func (s *FlexvolumeServer) Start() error {
func (s *FlexvolumeServer) Start(driverName string) error {
// first install the flexvolume driver
driverFile := path.Join(usrBinDir, flexvolumeDriverFileName) // /usr/local/bin/rookflex
err := configureFlexVolume(driverFile, flexVolumeDriverDir)
// /usr/local/bin/rookflex
driverFile := path.Join(usrBinDir, flexvolumeDriverFileName)
// /flexmnt/rook.io~rook-system
flexVolumeDriverDir := fmt.Sprintf(flexMountPath, FlexvolumeVendor, driverName)
err := configureFlexVolume(driverFile, flexVolumeDriverDir, driverName)
if err != nil {
return fmt.Errorf("unable to configure flexvolume: %v", err)
}
err = rpc.Register(s.controller)
if err != nil {
return fmt.Errorf("unable to register rpc: %v", err)
unixSocketFile := path.Join(flexVolumeDriverDir, UnixSocketName) // /flextmnt/rook.io~rook-system/.rook.sock
if _, ok := s.listeners[unixSocketFile]; ok {
logger.Infof("flex server already running at %s", unixSocketFile)
return nil
}
logger.Info("Rook Flexvolume configured")
unixSocketFile := path.Join(flexVolumeDriverDir, path.Join(UnixSocketName)) // /flextmnt/rook.io~rook/.rook.sock
// remove unix socket if it existed previously
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
@@ -84,18 +86,19 @@ func (s *FlexvolumeServer) Start() error {
os.Remove(unixSocketFile)
}
s.listener, err = net.Listen("unix", unixSocketFile)
listener, err := net.Listen("unix", unixSocketFile)
if err != nil {
return fmt.Errorf("unable to listen at %s: %v", unixSocketFile, err)
}
s.listeners[unixSocketFile] = listener
if err := os.Chmod(unixSocketFile, 0770); err != nil {
return fmt.Errorf("unable to set file permission to unix socket %s: %v", unixSocketFile, err)
}
go rpc.Accept(s.listener)
go rpc.Accept(listener)
logger.Info("Listening on unix socket for Kubernetes volume attach commands.")
logger.Infof("Listening on unix socket for Kubernetes volume attach commands: %s", unixSocketFile)
// flexvolume driver was installed OK. If running on pre 1.8 Kubernetes, then remind the user
// to restart the Kubelet. We do this last so that it's the last message in the log, making it
@@ -106,30 +109,49 @@ func (s *FlexvolumeServer) Start() error {
}
// Stop the unix domain socket server and deletes the socket file
func (s *FlexvolumeServer) Stop() {
if s.listener != nil {
logger.Info("Stopping unix socket rpc server.")
if err := s.listener.Close(); err != nil {
func (s *FlexvolumeServer) StopAll() {
logger.Infof("Stopping %d unix socket rpc server(s).", len(s.listeners))
for unixSocketFile, listener := range s.listeners {
if err := listener.Close(); err != nil {
logger.Errorf("Failed to stop unix socket rpc server: %+v", err)
}
}
// closing the listener should remove the unix socket file. But lets try it remove it just in case.
unixSocketFile := path.Join(flexVolumeDriverDir, path.Join(UnixSocketName)) // /flextmnt/rook.io~rook/.rook.sock
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
logger.Info("Deleting unix domain socket file.")
os.Remove(unixSocketFile)
}
// closing the listener should remove the unix socket file. But lets try it remove it just in case.
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
logger.Infof("Deleting unix domain socket file %s.", unixSocketFile)
os.Remove(unixSocketFile)
}
}
s.listeners = make(map[string]net.Listener)
}
func configureFlexVolume(driverFile, driverDir string) error {
func RookDriverName(context *clusterd.Context) (string, error) {
kubeVersion, err := k8sutil.GetK8SVersion(context.Clientset)
if err != nil {
return "", fmt.Errorf("Error getting server version: %v", err)
}
// K8s 1.7 returns an error when trying to run multiple drivers under the same rook.io provider,
// so we will fall back to the rook driver name in that case.
if kubeVersion.AtLeast(version.MustParseSemantic(serverVersionV180)) {
// the driver name needs to be the same as the namespace so that we can support multiple namespaces
// without the drivers conflicting with each other
return os.Getenv(k8sutil.PodNamespaceEnvVar), nil
}
// fall back to the rook driver name where multiple system namespaces are not supported
return FlexDriverName, nil
}
func configureFlexVolume(driverFile, driverDir, driverName string) error {
// copying flex volume
if _, err := os.Stat(driverDir); os.IsNotExist(err) {
os.Mkdir(driverDir, 0755)
err := os.Mkdir(driverDir, 0755)
if err != nil {
logger.Errorf("failed to create dir %s. %+v", driverDir, err)
}
}
destFile := path.Join(driverDir, "."+FlexvolumeDriver) // /flextmnt/rook.io~rook/.rook
finalDestFile := path.Join(driverDir, path.Join(FlexvolumeDriver)) // /flextmnt/rook.io~rook/rook
destFile := path.Join(driverDir, "."+driverName) // /flextmnt/rook.io~rook-system/.rook-system
finalDestFile := path.Join(driverDir, driverName) // /flextmnt/rook.io~rook-system/rook-system
err := copyFile(driverFile, destFile)
if err != nil {
return fmt.Errorf("unable to copy flexvolume from %s to %s: %+v", driverFile, destFile, err)
+4 -2
View File
@@ -32,9 +32,11 @@ func TestConfigureFlexVolume(t *testing.T) {
driverFile := path.Join(driverDir, flexvolumeDriverFileName)
os.OpenFile(driverFile, os.O_RDONLY|os.O_CREATE, 0755)
err := configureFlexVolume(driverFile, driverDir)
driverName := "rook"
os.Setenv("POD_NAMESPACE", driverName)
defer os.Setenv("POD_NAMESPACE", "")
err := configureFlexVolume(driverFile, driverDir, driverName)
assert.Nil(t, err)
_, err = os.Stat(path.Join(driverDir, "rook"))
assert.False(t, os.IsNotExist(err))
}
+6 -1
View File
@@ -37,7 +37,6 @@ const (
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-provisioner")
var flexdriver = fmt.Sprintf("%s/%s", flexvolume.FlexvolumeVendor, flexvolume.FlexvolumeDriver)
// RookVolumeProvisioner is used to provision Rook volumes on Kubernetes
type RookVolumeProvisioner struct {
@@ -95,6 +94,12 @@ func (p *RookVolumeProvisioner) Provision(options controller.VolumeOptions) (*v1
return nil, err
}
driverName, err := flexvolume.RookDriverName(p.context)
if err != nil {
return nil, fmt.Errorf("failed to get driver name. %+v", err)
}
flexdriver := fmt.Sprintf("%s/%s", flexvolume.FlexvolumeVendor, driverName)
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: imageName,
+3 -1
View File
@@ -40,6 +40,8 @@ func TestProvisionImage(t *testing.T) {
clientset := test.New(3)
namespace := "ns"
configDir, _ := ioutil.TempDir("", "")
os.Setenv("POD_NAMESPACE", "rook-system")
defer os.Setenv("POD_NAMESPACE", "")
defer os.RemoveAll(configDir)
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) {
@@ -72,7 +74,7 @@ func TestProvisionImage(t *testing.T) {
assert.Equal(t, "pvc-uid-1-1", pv.Name)
assert.NotNil(t, pv.Spec.PersistentVolumeSource.FlexVolume)
assert.Equal(t, "rook.io/rook", pv.Spec.PersistentVolumeSource.FlexVolume.Driver)
assert.Equal(t, "rook.io/rook-system", pv.Spec.PersistentVolumeSource.FlexVolume.Driver)
assert.Equal(t, "ext3", pv.Spec.PersistentVolumeSource.FlexVolume.FSType)
assert.Equal(t, "class-1", pv.Spec.PersistentVolumeSource.FlexVolume.Options["storageClass"])
assert.Equal(t, "testpool", pv.Spec.PersistentVolumeSource.FlexVolume.Options["pool"])
+2 -2
View File
@@ -415,12 +415,12 @@ func (k8sh *K8sHelper) IsPodTerminated(name string, namespace string) bool {
k8slogger.Infof("Pod %s in namespace %s terminated ", name, namespace)
return true
}
k8slogger.Infof("waiting for Pod %s in namespace %s to terminated, status : %v", name, namespace, pod.Status.Phase)
k8slogger.Infof("waiting for Pod %s in namespace %s to terminate, status : %v", name, namespace, pod.Status.Phase)
time.Sleep(RetryInterval * time.Second)
inc++
}
k8slogger.Infof("Pod %s in namespace %s did not terminated", name, namespace)
k8slogger.Infof("Pod %s in namespace %s did not terminate", name, namespace)
return false
}
+15 -4
View File
@@ -19,10 +19,13 @@ package integration
import (
"fmt"
"github.com/rook/rook/pkg/daemon/agent/flexvolume"
"github.com/rook/rook/tests/framework/clients"
"github.com/rook/rook/tests/framework/installer"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"k8s.io/kubernetes/pkg/util/version"
)
var (
@@ -96,14 +99,22 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
}
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error {
_, err := k8sh.ResourceOperation(action, getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath))
driverName := installer.SystemNamespace(namespace)
v := version.MustParseSemantic(k8sh.GetK8sServerVersion())
if v.LessThan(version.MustParseSemantic("1.10.0")) {
// k8s 1.10 and newer requires the new driver name to avoid conflicts in the test
driverName = flexvolume.FlexDriverName
}
testPod := getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName)
_, err := k8sh.ResourceOperation(action, testPod)
if err != nil {
return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath), err)
return fmt.Errorf("failed to %s pod -- %s. %+v", action, testPod, err)
}
return nil
}
func getFilesystemTestPod(podname string, namespace string, filesystemName string, fileMountPath string) string {
func getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName string) string {
return `apiVersion: v1
kind: Pod
metadata:
@@ -124,7 +135,7 @@ spec:
volumes:
- name: ` + filesystemName + `
flexVolume:
driver: rook.io/rook
driver: rook.io/` + driverName + `
fsType: ceph
options:
fsName: ` + filesystemName + `