flex: base driver name on the namespace
Signed-off-by: travisn <tnielsen@redhat.com>
This commit is contained in:
9 files changed
+112
-45
No files matched your search
@@ -51,7 +51,7 @@ func getRPCClient() (*rpc.Client, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting path of the Rook flexvolume driver: %v", err)
|
||||
}
|
||||
unixSocketFile := path.Join(path.Dir(ex), path.Join(flexvolume.UnixSocketName)) // /usr/libexec/kubernetes/plugin/volume/rook.io~rook/.rook.sock
|
||||
unixSocketFile := path.Join(path.Dir(ex), flexvolume.UnixSocketName) // /usr/libexec/kubernetes/kubelet-plugins/volume/exec/rook.io~rook/rook/.rook.sock
|
||||
conn, err := net.Dial("unix", unixSocketFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error connecting to socket %s: %+v", unixSocketFile, err)
|
||||
|
||||
@@ -19,6 +19,7 @@ package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/rpc"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
@@ -62,7 +63,27 @@ func (a *Agent) Run() error {
|
||||
volumeManager,
|
||||
)
|
||||
|
||||
flexvolumeServer.Start()
|
||||
err = rpc.Register(flexvolumeController)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to register rpc: %v", err)
|
||||
}
|
||||
|
||||
driverName, err := flexvolume.RookDriverName(a.context)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get driver name. %+v", err)
|
||||
}
|
||||
|
||||
err = flexvolumeServer.Start(driverName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start flex volume server %s, %+v", driverName, err)
|
||||
}
|
||||
|
||||
// Register drivers both with the name of the namespace and the name "rook"
|
||||
// for the volume plugins not based on the namespace.
|
||||
err = flexvolumeServer.Start(flexvolume.FlexDriverName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start flex volume server %s. %+v", flexvolume.FlexDriverName, err)
|
||||
}
|
||||
|
||||
// create a cluster controller and tell it to start watching for changes to clusters
|
||||
clusterController := cluster.NewClusterController(
|
||||
@@ -79,7 +100,7 @@ func (a *Agent) Run() error {
|
||||
select {
|
||||
case <-sigc:
|
||||
logger.Infof("shutdown signal received, exiting...")
|
||||
flexvolumeServer.Stop()
|
||||
flexvolumeServer.StopAll()
|
||||
close(stopChan)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -325,7 +325,11 @@ func (c *Controller) GetAttachInfoFromMountDir(mountDir string, attachOptions *A
|
||||
// GetGlobalMountPath generate the global mount path where the device path is mounted.
|
||||
// It is based on the kubelet root dir, which defaults to /var/lib/kubelet
|
||||
func (c *Controller) GetGlobalMountPath(volumeName string, globalMountPath *string) error {
|
||||
*globalMountPath = path.Join(c.getKubeletRootDir(), "plugins", FlexvolumeVendor, FlexvolumeDriver, "mounts", volumeName)
|
||||
driverName, err := RookDriverName(c.context)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*globalMountPath = path.Join(c.getKubeletRootDir(), "plugins", FlexvolumeVendor, driverName, "mounts", volumeName)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -35,21 +35,20 @@ import (
|
||||
const (
|
||||
UnixSocketName = ".rook.sock"
|
||||
FlexvolumeVendor = "rook.io"
|
||||
FlexvolumeDriver = "rook"
|
||||
FlexDriverName = "rook"
|
||||
flexvolumeDriverFileName = "rookflex"
|
||||
flexMountPath = "/flexmnt/%s~%s"
|
||||
usrBinDir = "/usr/local/bin/"
|
||||
serverVersionV170 = "v1.7.0"
|
||||
serverVersionV180 = "v1.8.0"
|
||||
)
|
||||
|
||||
var flexVolumeDriverDir = fmt.Sprintf("/flexmnt/%s~%s", FlexvolumeVendor, FlexvolumeDriver)
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "flexvolume")
|
||||
|
||||
// FlexvolumeServer start a unix domain socket server to interact with the flexvolume driver
|
||||
type FlexvolumeServer struct {
|
||||
context *clusterd.Context
|
||||
controller *Controller
|
||||
listener net.Listener
|
||||
listeners map[string]net.Listener
|
||||
}
|
||||
|
||||
// NewFlexvolumeServer creates an Flexvolume server
|
||||
@@ -57,26 +56,29 @@ func NewFlexvolumeServer(context *clusterd.Context, controller *Controller, mana
|
||||
return &FlexvolumeServer{
|
||||
context: context,
|
||||
controller: controller,
|
||||
listeners: make(map[string]net.Listener),
|
||||
}
|
||||
}
|
||||
|
||||
// Start configures the flexvolume driver on the host and starts the unix domain socket server to communicate with the driver
|
||||
func (s *FlexvolumeServer) Start() error {
|
||||
func (s *FlexvolumeServer) Start(driverName string) error {
|
||||
|
||||
// first install the flexvolume driver
|
||||
driverFile := path.Join(usrBinDir, flexvolumeDriverFileName) // /usr/local/bin/rookflex
|
||||
err := configureFlexVolume(driverFile, flexVolumeDriverDir)
|
||||
// /usr/local/bin/rookflex
|
||||
driverFile := path.Join(usrBinDir, flexvolumeDriverFileName)
|
||||
// /flexmnt/rook.io~rook-system
|
||||
flexVolumeDriverDir := fmt.Sprintf(flexMountPath, FlexvolumeVendor, driverName)
|
||||
|
||||
err := configureFlexVolume(driverFile, flexVolumeDriverDir, driverName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to configure flexvolume: %v", err)
|
||||
}
|
||||
|
||||
err = rpc.Register(s.controller)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to register rpc: %v", err)
|
||||
unixSocketFile := path.Join(flexVolumeDriverDir, UnixSocketName) // /flextmnt/rook.io~rook-system/.rook.sock
|
||||
if _, ok := s.listeners[unixSocketFile]; ok {
|
||||
logger.Infof("flex server already running at %s", unixSocketFile)
|
||||
return nil
|
||||
}
|
||||
logger.Info("Rook Flexvolume configured")
|
||||
|
||||
unixSocketFile := path.Join(flexVolumeDriverDir, path.Join(UnixSocketName)) // /flextmnt/rook.io~rook/.rook.sock
|
||||
|
||||
// remove unix socket if it existed previously
|
||||
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
|
||||
@@ -84,18 +86,19 @@ func (s *FlexvolumeServer) Start() error {
|
||||
os.Remove(unixSocketFile)
|
||||
}
|
||||
|
||||
s.listener, err = net.Listen("unix", unixSocketFile)
|
||||
listener, err := net.Listen("unix", unixSocketFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to listen at %s: %v", unixSocketFile, err)
|
||||
}
|
||||
s.listeners[unixSocketFile] = listener
|
||||
|
||||
if err := os.Chmod(unixSocketFile, 0770); err != nil {
|
||||
return fmt.Errorf("unable to set file permission to unix socket %s: %v", unixSocketFile, err)
|
||||
}
|
||||
|
||||
go rpc.Accept(s.listener)
|
||||
go rpc.Accept(listener)
|
||||
|
||||
logger.Info("Listening on unix socket for Kubernetes volume attach commands.")
|
||||
logger.Infof("Listening on unix socket for Kubernetes volume attach commands: %s", unixSocketFile)
|
||||
|
||||
// flexvolume driver was installed OK. If running on pre 1.8 Kubernetes, then remind the user
|
||||
// to restart the Kubelet. We do this last so that it's the last message in the log, making it
|
||||
@@ -106,30 +109,49 @@ func (s *FlexvolumeServer) Start() error {
|
||||
}
|
||||
|
||||
// Stop the unix domain socket server and deletes the socket file
|
||||
func (s *FlexvolumeServer) Stop() {
|
||||
if s.listener != nil {
|
||||
logger.Info("Stopping unix socket rpc server.")
|
||||
if err := s.listener.Close(); err != nil {
|
||||
func (s *FlexvolumeServer) StopAll() {
|
||||
logger.Infof("Stopping %d unix socket rpc server(s).", len(s.listeners))
|
||||
for unixSocketFile, listener := range s.listeners {
|
||||
if err := listener.Close(); err != nil {
|
||||
logger.Errorf("Failed to stop unix socket rpc server: %+v", err)
|
||||
}
|
||||
}
|
||||
// closing the listener should remove the unix socket file. But lets try it remove it just in case.
|
||||
unixSocketFile := path.Join(flexVolumeDriverDir, path.Join(UnixSocketName)) // /flextmnt/rook.io~rook/.rook.sock
|
||||
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
|
||||
logger.Info("Deleting unix domain socket file.")
|
||||
os.Remove(unixSocketFile)
|
||||
}
|
||||
|
||||
// closing the listener should remove the unix socket file. But lets try it remove it just in case.
|
||||
if _, err := os.Stat(unixSocketFile); !os.IsNotExist(err) {
|
||||
logger.Infof("Deleting unix domain socket file %s.", unixSocketFile)
|
||||
os.Remove(unixSocketFile)
|
||||
}
|
||||
}
|
||||
s.listeners = make(map[string]net.Listener)
|
||||
}
|
||||
|
||||
func configureFlexVolume(driverFile, driverDir string) error {
|
||||
func RookDriverName(context *clusterd.Context) (string, error) {
|
||||
kubeVersion, err := k8sutil.GetK8SVersion(context.Clientset)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Error getting server version: %v", err)
|
||||
}
|
||||
// K8s 1.7 returns an error when trying to run multiple drivers under the same rook.io provider,
|
||||
// so we will fall back to the rook driver name in that case.
|
||||
if kubeVersion.AtLeast(version.MustParseSemantic(serverVersionV180)) {
|
||||
// the driver name needs to be the same as the namespace so that we can support multiple namespaces
|
||||
// without the drivers conflicting with each other
|
||||
return os.Getenv(k8sutil.PodNamespaceEnvVar), nil
|
||||
}
|
||||
// fall back to the rook driver name where multiple system namespaces are not supported
|
||||
return FlexDriverName, nil
|
||||
}
|
||||
|
||||
func configureFlexVolume(driverFile, driverDir, driverName string) error {
|
||||
// copying flex volume
|
||||
if _, err := os.Stat(driverDir); os.IsNotExist(err) {
|
||||
os.Mkdir(driverDir, 0755)
|
||||
err := os.Mkdir(driverDir, 0755)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create dir %s. %+v", driverDir, err)
|
||||
}
|
||||
}
|
||||
|
||||
destFile := path.Join(driverDir, "."+FlexvolumeDriver) // /flextmnt/rook.io~rook/.rook
|
||||
finalDestFile := path.Join(driverDir, path.Join(FlexvolumeDriver)) // /flextmnt/rook.io~rook/rook
|
||||
destFile := path.Join(driverDir, "."+driverName) // /flextmnt/rook.io~rook-system/.rook-system
|
||||
finalDestFile := path.Join(driverDir, driverName) // /flextmnt/rook.io~rook-system/rook-system
|
||||
err := copyFile(driverFile, destFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to copy flexvolume from %s to %s: %+v", driverFile, destFile, err)
|
||||
|
||||
@@ -32,9 +32,11 @@ func TestConfigureFlexVolume(t *testing.T) {
|
||||
driverFile := path.Join(driverDir, flexvolumeDriverFileName)
|
||||
os.OpenFile(driverFile, os.O_RDONLY|os.O_CREATE, 0755)
|
||||
|
||||
err := configureFlexVolume(driverFile, driverDir)
|
||||
driverName := "rook"
|
||||
os.Setenv("POD_NAMESPACE", driverName)
|
||||
defer os.Setenv("POD_NAMESPACE", "")
|
||||
err := configureFlexVolume(driverFile, driverDir, driverName)
|
||||
assert.Nil(t, err)
|
||||
_, err = os.Stat(path.Join(driverDir, "rook"))
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
|
||||
}
|
||||
@@ -37,7 +37,6 @@ const (
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-provisioner")
|
||||
var flexdriver = fmt.Sprintf("%s/%s", flexvolume.FlexvolumeVendor, flexvolume.FlexvolumeDriver)
|
||||
|
||||
// RookVolumeProvisioner is used to provision Rook volumes on Kubernetes
|
||||
type RookVolumeProvisioner struct {
|
||||
@@ -95,6 +94,12 @@ func (p *RookVolumeProvisioner) Provision(options controller.VolumeOptions) (*v1
|
||||
return nil, err
|
||||
}
|
||||
|
||||
driverName, err := flexvolume.RookDriverName(p.context)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get driver name. %+v", err)
|
||||
}
|
||||
|
||||
flexdriver := fmt.Sprintf("%s/%s", flexvolume.FlexvolumeVendor, driverName)
|
||||
pv := &v1.PersistentVolume{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: imageName,
|
||||
|
||||
@@ -40,6 +40,8 @@ func TestProvisionImage(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
namespace := "ns"
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
os.Setenv("POD_NAMESPACE", "rook-system")
|
||||
defer os.Setenv("POD_NAMESPACE", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, args ...string) (string, error) {
|
||||
@@ -72,7 +74,7 @@ func TestProvisionImage(t *testing.T) {
|
||||
|
||||
assert.Equal(t, "pvc-uid-1-1", pv.Name)
|
||||
assert.NotNil(t, pv.Spec.PersistentVolumeSource.FlexVolume)
|
||||
assert.Equal(t, "rook.io/rook", pv.Spec.PersistentVolumeSource.FlexVolume.Driver)
|
||||
assert.Equal(t, "rook.io/rook-system", pv.Spec.PersistentVolumeSource.FlexVolume.Driver)
|
||||
assert.Equal(t, "ext3", pv.Spec.PersistentVolumeSource.FlexVolume.FSType)
|
||||
assert.Equal(t, "class-1", pv.Spec.PersistentVolumeSource.FlexVolume.Options["storageClass"])
|
||||
assert.Equal(t, "testpool", pv.Spec.PersistentVolumeSource.FlexVolume.Options["pool"])
|
||||
|
||||
@@ -415,12 +415,12 @@ func (k8sh *K8sHelper) IsPodTerminated(name string, namespace string) bool {
|
||||
k8slogger.Infof("Pod %s in namespace %s terminated ", name, namespace)
|
||||
return true
|
||||
}
|
||||
k8slogger.Infof("waiting for Pod %s in namespace %s to terminated, status : %v", name, namespace, pod.Status.Phase)
|
||||
k8slogger.Infof("waiting for Pod %s in namespace %s to terminate, status : %v", name, namespace, pod.Status.Phase)
|
||||
time.Sleep(RetryInterval * time.Second)
|
||||
inc++
|
||||
|
||||
}
|
||||
k8slogger.Infof("Pod %s in namespace %s did not terminated", name, namespace)
|
||||
k8slogger.Infof("Pod %s in namespace %s did not terminate", name, namespace)
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
@@ -19,10 +19,13 @@ package integration
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/agent/flexvolume"
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
"github.com/rook/rook/tests/framework/installer"
|
||||
"github.com/rook/rook/tests/framework/utils"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"k8s.io/kubernetes/pkg/util/version"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -96,14 +99,22 @@ func fileTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, s su
|
||||
}
|
||||
|
||||
func podWithFilesystem(k8sh *utils.K8sHelper, s suite.Suite, podname string, namespace string, filesystemName string, fileMountPath string, action string) error {
|
||||
_, err := k8sh.ResourceOperation(action, getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath))
|
||||
driverName := installer.SystemNamespace(namespace)
|
||||
v := version.MustParseSemantic(k8sh.GetK8sServerVersion())
|
||||
if v.LessThan(version.MustParseSemantic("1.10.0")) {
|
||||
// k8s 1.10 and newer requires the new driver name to avoid conflicts in the test
|
||||
driverName = flexvolume.FlexDriverName
|
||||
}
|
||||
|
||||
testPod := getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName)
|
||||
_, err := k8sh.ResourceOperation(action, testPod)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to %s pod -- %s. %+v", action, getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath), err)
|
||||
return fmt.Errorf("failed to %s pod -- %s. %+v", action, testPod, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFilesystemTestPod(podname string, namespace string, filesystemName string, fileMountPath string) string {
|
||||
func getFilesystemTestPod(podname, namespace, filesystemName, fileMountPath, driverName string) string {
|
||||
return `apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
@@ -124,7 +135,7 @@ spec:
|
||||
volumes:
|
||||
- name: ` + filesystemName + `
|
||||
flexVolume:
|
||||
driver: rook.io/rook
|
||||
driver: rook.io/` + driverName + `
|
||||
fsType: ceph
|
||||
options:
|
||||
fsName: ` + filesystemName + `
|
||||
|
||||
Reference in new issue
Block a user