diff --git a/cmd/rookflex/cmd/root.go b/cmd/rookflex/cmd/root.go index 206978760..0c3026be7 100644 --- a/cmd/rookflex/cmd/root.go +++ b/cmd/rookflex/cmd/root.go @@ -51,7 +51,7 @@ func getRPCClient() (*rpc.Client, error) { if err != nil { return nil, fmt.Errorf("error getting path of the Rook flexvolume driver: %v", err) } - unixSocketFile := path.Join(path.Dir(ex), path.Join(flexvolume.UnixSocketName)) // /usr/libexec/kubernetes/plugin/volume/rook.io~rook/.rook.sock + unixSocketFile := path.Join(path.Dir(ex), flexvolume.UnixSocketName) // /usr/libexec/kubernetes/kubelet-plugins/volume/exec/rook.io~rook/rook/.rook.sock conn, err := net.Dial("unix", unixSocketFile) if err != nil { return nil, fmt.Errorf("error connecting to socket %s: %+v", unixSocketFile, err) diff --git a/pkg/daemon/agent/agent.go b/pkg/daemon/agent/agent.go index 0bcbc7a99..0174706b4 100644 --- a/pkg/daemon/agent/agent.go +++ b/pkg/daemon/agent/agent.go @@ -19,6 +19,7 @@ package agent import ( "fmt" + "net/rpc" "os" "os/signal" "syscall" @@ -62,7 +63,27 @@ func (a *Agent) Run() error { volumeManager, ) - flexvolumeServer.Start() + err = rpc.Register(flexvolumeController) + if err != nil { + return fmt.Errorf("unable to register rpc: %v", err) + } + + driverName, err := flexvolume.RookDriverName(a.context) + if err != nil { + return fmt.Errorf("failed to get driver name. %+v", err) + } + + err = flexvolumeServer.Start(driverName) + if err != nil { + return fmt.Errorf("failed to start flex volume server %s, %+v", driverName, err) + } + + // Register drivers both with the name of the namespace and the name "rook" + // for the volume plugins not based on the namespace. + err = flexvolumeServer.Start(flexvolume.FlexDriverName) + if err != nil { + return fmt.Errorf("failed to start flex volume server %s. %+v", flexvolume.FlexDriverName, err) + } // create a cluster controller and tell it to start watching for changes to clusters clusterController := cluster.NewClusterController( @@ -79,7 +100,7 @@ func (a *Agent) Run() error { select { case <-sigc: logger.Infof("shutdown signal received, exiting...") - flexvolumeServer.Stop() + flexvolumeServer.StopAll() close(stopChan) return nil } diff --git a/pkg/daemon/agent/flexvolume/controller.go b/pkg/daemon/agent/flexvolume/controller.go index b61576c55..e33fcfed3 100644 --- a/pkg/daemon/agent/flexvolume/controller.go +++ b/pkg/daemon/agent/flexvolume/controller.go @@ -325,7 +325,11 @@ func (c *Controller) GetAttachInfoFromMountDir(mountDir string, attachOptions *A // GetGlobalMountPath generate the global mount path where the device path is mounted. // It is based on the kubelet root dir, which defaults to /var/lib/kubelet func (c *Controller) GetGlobalMountPath(volumeName string, globalMountPath *string) error { - *globalMountPath = path.Join(c.getKubeletRootDir(), "plugins", FlexvolumeVendor, FlexvolumeDriver, "mounts", volumeName) + driverName, err := RookDriverName(c.context) + if err != nil { + return err + } + *globalMountPath = path.Join(c.getKubeletRootDir(), "plugins", FlexvolumeVendor, driverName, "mounts", volumeName) return nil } diff --git a/pkg/daemon/agent/flexvolume/server.go b/pkg/daemon/agent/flexvolume/server.go index c92f4a378..fb52d3229 100644 --- a/pkg/daemon/agent/flexvolume/server.go +++ b/pkg/daemon/agent/flexvolume/server.go @@ -35,21 +35,20 @@ import ( const ( UnixSocketName = ".rook.sock" FlexvolumeVendor = "rook.io" - FlexvolumeDriver = "rook" + FlexDriverName = "rook" flexvolumeDriverFileName = "rookflex" + flexMountPath = "/flexmnt/%s~%s" usrBinDir = "/usr/local/bin/" - serverVersionV170 = "v1.7.0" serverVersionV180 = "v1.8.0" ) -var flexVolumeDriverDir = fmt.Sprintf("/flexmnt/%s~%s", FlexvolumeVendor, FlexvolumeDriver) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "flexvolume") // FlexvolumeServer start a unix domain socket server to interact with the flexvolume driver type FlexvolumeServer struct { context *clusterd.Context controller *Controller - listener net.Listener + listeners map[string]net.Listener } // NewFlexvolumeServer creates an Flexvolume server @@ -57,26 +56,29 @@ func NewFlexvolumeServer(context *clusterd.Context, controller *Controller, mana return &FlexvolumeServer{ context: context, controller: controller, + listeners: make(map[string]net.Listener), } } // Start configures the flexvolume driver on the host and starts the unix domain socket server to communicate with the driver -func (s *FlexvolumeServer) Start() error { +func (s *FlexvolumeServer) Start(driverName string) error { // first install the flexvolume driver - driverFile := path.Join(usrBinDir, flexvolumeDriverFileName) // /usr/local/bin/rookflex - err := configureFlexVolume(driverFile, flexVolumeDriverDir) + // /usr/local/bin/rookflex + driverFile := path.Join(usrBinDir, flexvolumeDriverFileName) + // /flexmnt/rook.io~rook-system + flexVolumeDriverDir := fmt.Sprintf(flexMountPath, FlexvolumeVendor, driverName) + + err := configureFlexVolume(driverFile, flexVolumeDriverDir, driverName) if err != nil { return fmt.Errorf("unable to configure flexvolume: %v", err) } - err = rpc.Register(s.controller) - if err != nil { - return fmt.Errorf("unable to register rpc: %v", err) + unixSocketFile := path.Join(flexVolumeDriverDir, UnixSocketName) // /flextmnt/rook.io~rook-system/.rook.sock + if _, ok := s.listeners[unixSocketFile]; ok { + logger.Infof("flex server already running at %s", unixSocketFile) + return nil } - logger.Info("Rook Flexvolume configured") - - unixSocketFile := path.Join(flexVolumeDriverDir, path.Join(UnixSocketName)) // /flextmnt/rook.io~rook/.rook.sock // remove unix socket if it existed previously if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) { @@ -84,18 +86,19 @@ func (s *FlexvolumeServer) Start() error { os.Remove(unixSocketFile) } - s.listener, err = net.Listen("unix", unixSocketFile) + listener, err := net.Listen("unix", unixSocketFile) if err != nil { return fmt.Errorf("unable to listen at %s: %v", unixSocketFile, err) } + s.listeners[unixSocketFile] = listener if err := os.Chmod(unixSocketFile, 0770); err != nil { return fmt.Errorf("unable to set file permission to unix socket %s: %v", unixSocketFile, err) } - go rpc.Accept(s.listener) + go rpc.Accept(listener) - logger.Info("Listening on unix socket for Kubernetes volume attach commands.") + logger.Infof("Listening on unix socket for Kubernetes volume attach commands: %s", unixSocketFile) // flexvolume driver was installed OK. If running on pre 1.8 Kubernetes, then remind the user // to restart the Kubelet. We do this last so that it's the last message in the log, making it @@ -106,30 +109,49 @@ func (s *FlexvolumeServer) Start() error { } // Stop the unix domain socket server and deletes the socket file -func (s *FlexvolumeServer) Stop() { - if s.listener != nil { - logger.Info("Stopping unix socket rpc server.") - if err := s.listener.Close(); err != nil { +func (s *FlexvolumeServer) StopAll() { + logger.Infof("Stopping %d unix socket rpc server(s).", len(s.listeners)) + for unixSocketFile, listener := range s.listeners { + if err := listener.Close(); err != nil { logger.Errorf("Failed to stop unix socket rpc server: %+v", err) } - } - // closing the listener should remove the unix socket file. But lets try it remove it just in case. - unixSocketFile := path.Join(flexVolumeDriverDir, path.Join(UnixSocketName)) // /flextmnt/rook.io~rook/.rook.sock - if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) { - logger.Info("Deleting unix domain socket file.") - os.Remove(unixSocketFile) - } + // closing the listener should remove the unix socket file. But lets try it remove it just in case. + if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) { + logger.Infof("Deleting unix domain socket file %s.", unixSocketFile) + os.Remove(unixSocketFile) + } + } + s.listeners = make(map[string]net.Listener) } -func configureFlexVolume(driverFile, driverDir string) error { +func RookDriverName(context *clusterd.Context) (string, error) { + kubeVersion, err := k8sutil.GetK8SVersion(context.Clientset) + if err != nil { + return "", fmt.Errorf("Error getting server version: %v", err) + } + // K8s 1.7 returns an error when trying to run multiple drivers under the same rook.io provider, + // so we will fall back to the rook driver name in that case. + if kubeVersion.AtLeast(version.MustParseSemantic(serverVersionV180)) { + // the driver name needs to be the same as the namespace so that we can support multiple namespaces + // without the drivers conflicting with each other + return os.Getenv(k8sutil.PodNamespaceEnvVar), nil + } + // fall back to the rook driver name where multiple system namespaces are not supported + return FlexDriverName, nil +} + +func configureFlexVolume(driverFile, driverDir, driverName string) error { // copying flex volume if _, err := os.Stat(driverDir); os.IsNotExist(err) { - os.Mkdir(driverDir, 0755) + err := os.Mkdir(driverDir, 0755) + if err != nil { + logger.Errorf("failed to create dir %s. %+v", driverDir, err) + } } - destFile := path.Join(driverDir, "."+FlexvolumeDriver) // /flextmnt/rook.io~rook/.rook - finalDestFile := path.Join(driverDir, path.Join(FlexvolumeDriver)) // /flextmnt/rook.io~rook/rook + destFile := path.Join(driverDir, "."+driverName) // /flextmnt/rook.io~rook-system/.rook-system + finalDestFile := path.Join(driverDir, driverName) // /flextmnt/rook.io~rook-system/rook-system err := copyFile(driverFile, destFile) if err != nil { return fmt.Errorf("unable to copy flexvolume from %s to %s: %+v", driverFile, destFile, err) diff --git a/pkg/daemon/agent/flexvolume/server_test.go b/pkg/daemon/agent/flexvolume/server_test.go index 0813b6379..fa3bf3adf 100644 --- a/pkg/daemon/agent/flexvolume/server_test.go +++ b/pkg/daemon/agent/flexvolume/server_test.go @@ -32,9 +32,11 @@ func TestConfigureFlexVolume(t *testing.T) { driverFile := path.Join(driverDir, flexvolumeDriverFileName) os.OpenFile(driverFile, os.O_RDONLY|os.O_CREATE, 0755) - err := configureFlexVolume(driverFile, driverDir) + driverName := "rook" + os.Setenv("POD_NAMESPACE", driverName) + defer os.Setenv("POD_NAMESPACE", "") + err := configureFlexVolume(driverFile, driverDir, driverName) assert.Nil(t, err) _, err = os.Stat(path.Join(driverDir, "rook")) assert.False(t, os.IsNotExist(err)) - } diff --git a/pkg/operator/provisioner/provisioner.go b/pkg/operator/provisioner/provisioner.go index e5009b0ce..afb542cf9 100644 --- a/pkg/operator/provisioner/provisioner.go +++ b/pkg/operator/provisioner/provisioner.go @@ -37,7 +37,6 @@ const ( ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-provisioner") -var flexdriver = fmt.Sprintf("%s/%s", flexvolume.FlexvolumeVendor, flexvolume.FlexvolumeDriver) // RookVolumeProvisioner is used to provision Rook volumes on Kubernetes type RookVolumeProvisioner struct { @@ -95,6 +94,12 @@ func (p *RookVolumeProvisioner) Provision(options controller.VolumeOptions) (*v1 return nil, err } + driverName, err := flexvolume.RookDriverName(p.context) + if err != nil { + return nil, fmt.Errorf("failed to get driver name. %+v", err) + } + + flexdriver := fmt.Sprintf("%s/%s", flexvolume.FlexvolumeVendor, driverName) pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: imageName, diff --git a/pkg/operator/provisioner/provisioner_test.go b/pkg/operator/provisioner/provisioner_test.go index 277a12079..c96f0ba85 100644 --- a/pkg/operator/provisioner/provisioner_test.go +++ b/pkg/operator/provisioner/provisioner_test.go @@ -40,6 +40,8 @@ func TestProvisionImage(t *testing.T) { clientset := test.New(3) namespace := "ns" configDir, _ := ioutil.TempDir("", "") + os.Setenv("POD_NAMESPACE", "rook-system") + defer os.Setenv("POD_NAMESPACE", "") defer os.RemoveAll(configDir) executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) { @@ -72,7 +74,7 @@ func TestProvisionImage(t *testing.T) { assert.Equal(t, "pvc-uid-1-1", pv.Name) assert.NotNil(t, pv.Spec.PersistentVolumeSource.FlexVolume) - assert.Equal(t, "rook.io/rook", pv.Spec.PersistentVolumeSource.FlexVolume.Driver) + assert.Equal(t, "rook.io/rook-system", pv.Spec.PersistentVolumeSource.FlexVolume.Driver) assert.Equal(t, "ext3", pv.Spec.PersistentVolumeSource.FlexVolume.FSType) assert.Equal(t, "class-1", pv.Spec.PersistentVolumeSource.FlexVolume.Options["storageClass"]) assert.Equal(t, "testpool", pv.Spec.PersistentVolumeSource.FlexVolume.Options["pool"]) diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index 41ab9dff5..1d246879c 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -415,12 +415,12 @@ func (k8sh *K8sHelper) IsPodTerminated(name string, namespace string) bool { k8slogger.Infof("Pod %s in namespace %s terminated ", name, namespace) return true } - k8slogger.Infof("waiting for Pod %s in namespace %s to terminated, status : %v", name, namespace, pod.Status.Phase) + k8slogger.Infof("waiting for Pod %s in namespace %s to terminate, status : %v", name, namespace, pod.Status.Phase) time.Sleep(RetryInterval * time.Second) inc++ } - k8slogger.Infof("Pod %s in namespace %s did not terminated", name, namespace) + k8slogger.Infof("Pod %s in namespace %s did not terminate", name, namespace) return false } diff --git a/tests/integration/base_file_test.go b/tests/integration/base_file_test.go index ddafc94c1..46578140e 100644 --- a/tests/integration/base_file_test.go +++ b/tests/integration/base_file_test.go @@ -19,10 +19,13 @@ package integration import ( "fmt" + "github.com/rook/rook/pkg/daemon/agent/flexvolume" "github.com/rook/rook/tests/framework/clients" + "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "k8s.io/kubernetes/pkg/util/version" ) var ( @@ -96,14 +99,22 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su } func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error { - _, err := k8sh.ResourceOperation(action, getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath)) + driverName := installer.SystemNamespace(namespace) + v := version.MustParseSemantic(k8sh.GetK8sServerVersion()) + if v.LessThan(version.MustParseSemantic("1.10.0")) { + // k8s 1.10 and newer requires the new driver name to avoid conflicts in the test + driverName = flexvolume.FlexDriverName + } + + testPod := getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName) + _, err := k8sh.ResourceOperation(action, testPod) if err != nil { - return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath), err) + return fmt.Errorf("failed to %s pod -- %s. %+v", action, testPod, err) } return nil } -func getFilesystemTestPod(podname string, namespace string, filesystemName string, fileMountPath string) string { +func getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName string) string { return `apiVersion: v1 kind: Pod metadata: @@ -124,7 +135,7 @@ spec: volumes: - name: ` + filesystemName + ` flexVolume: - driver: rook.io/rook + driver: rook.io/` + driverName + ` fsType: ceph options: fsName: ` + filesystemName + `