Merge pull request #1255 from jbw976/volume-plugin-dir-1.8

Get flexvolume plugin dir for all k8s versions
This commit is contained in:
Jared Watts authored and GitHub committed 2017-11-27 13:14:57 -08:00
commit 40d129d2e0
5 files changed
+105 -29

No files matched your search

+49 -27
View File
@@ -97,6 +97,10 @@ func (a *Agent) createAgentDaemonSet(namespace string) error {
if err != nil {
return err
}
flexvolumeDirPath, source := a.discoverFlexvolumeDir()
logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath)
privileged := true
ds := &extensions.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
@@ -148,7 +152,7 @@ func (a *Agent) createAgentDaemonSet(namespace string) error {
Name: "flexvolume",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: a.discoverFlexvolumeDir(),
Path: flexvolumeDirPath,
},
},
},
@@ -196,7 +200,7 @@ func (a *Agent) createAgentDaemonSet(namespace string) error {
}
func (a *Agent) discoverFlexvolumeDir() string {
func (a *Agent) discoverFlexvolumeDir() (flexvolumeDirPath, source string) {
//copy flexvolume to flexvolume dir
nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
if nodeName == "" {
@@ -205,7 +209,6 @@ func (a *Agent) discoverFlexvolumeDir() string {
}
// determining where the path of the flexvolume dir on the node
var flexvolumeDirPath string
nodeConfigURI, err := k8sutil.NodeConfigURI()
if err != nil {
logger.Warningf(err.Error())
@@ -217,38 +220,57 @@ func (a *Agent) discoverFlexvolumeDir() string {
return getDefaultFlexvolumeDir()
}
// unmarshall to a NodeConfigKubelet
// unmarshal to a NodeConfigKubelet
configKubelet := NodeConfigKubelet{}
if err := json.Unmarshal(nodeConfig, &configKubelet); err != nil {
logger.Warningf("unable to parse node config from Kubelet: %+v", err)
return getDefaultFlexvolumeDir()
}
flexvolumeDirPath = configKubelet.ComponentConfig.VolumePluginDir
if flexvolumeDirPath == "" {
// unmarshall to a NodeConfigControllerManager
configControllerManager := NodeConfigControllerManager{}
if err := json.Unmarshal(nodeConfig, &configControllerManager); err != nil {
logger.Warningf("unable to parse node config from controller manager: %+v", err)
return getDefaultFlexvolumeDir()
}
flexvolumeDirPath = configControllerManager.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir
if flexvolumeDirPath == "" {
flexvolumeDirPath = getDefaultFlexvolumeDir()
}
} else {
flexvolumeDirPath = configKubelet.ComponentConfig.VolumePluginDir
}
logger.Infof("using flexvolume dir path: %s", flexvolumeDirPath)
return flexvolumeDirPath
if flexvolumeDirPath != "" {
return flexvolumeDirPath, "NodeConfigKubelet"
}
// unmarshal to a NodeConfigControllerManager
configControllerManager := NodeConfigControllerManager{}
if err := json.Unmarshal(nodeConfig, &configControllerManager); err != nil {
logger.Warningf("unable to parse node config from controller manager: %+v", err)
} else {
flexvolumeDirPath = configControllerManager.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir
}
if flexvolumeDirPath != "" {
return flexvolumeDirPath, "NodeConfigControllerManager"
}
// unmarshal to a KubeletConfiguration
kubeletConfiguration := KubeletConfiguration{}
if err := json.Unmarshal(nodeConfig, &kubeletConfiguration); err != nil {
logger.Warningf("unable to parse node config as kubelet configuration: %+v", err)
} else {
flexvolumeDirPath = kubeletConfiguration.KubeletConfig.VolumePluginDir
}
if flexvolumeDirPath != "" {
return flexvolumeDirPath, "KubeletConfiguration"
}
return getDefaultFlexvolumeDir()
}
func getDefaultFlexvolumeDir() string {
logger.Info("getting flexvolume dir path from provided env var")
flexvolumeDirPath := os.Getenv(flexvolumePathDirEnv)
if flexvolumeDirPath == "" {
logger.Infof("flexvolume dir path is not provided. Defaulting to: %s", flexvolumeDefaultDirPath)
flexvolumeDirPath = flexvolumeDefaultDirPath
func getDefaultFlexvolumeDir() (flexvolumeDirPath, source string) {
logger.Infof("getting flexvolume dir path from %s env var", flexvolumePathDirEnv)
flexvolumeDirPath = os.Getenv(flexvolumePathDirEnv)
if flexvolumeDirPath != "" {
return flexvolumeDirPath, "env var"
}
return flexvolumeDirPath
logger.Infof("flexvolume dir path env var %s is not provided. Defaulting to: %s",
flexvolumePathDirEnv, flexvolumeDefaultDirPath)
flexvolumeDirPath = flexvolumeDefaultDirPath
return flexvolumeDirPath, "default"
}
func getContainerImage(clientset kubernetes.Interface) (string, error) {
+7
View File
@@ -40,3 +40,10 @@ type NodeConfigControllerManager struct {
type NodeConfigKubelet struct {
ComponentConfig kubeletcomponentconfig.KubeletConfiguration `json:"componentconfig"`
}
// KubeletConfiguration represents the response from the node config URI (configz) in Kubernetes 1.8+
type KubeletConfiguration struct {
KubeletConfig struct {
VolumePluginDir string `json:"volumePluginDir"`
} `json:"kubeletconfig"`
}
+2
View File
@@ -73,6 +73,7 @@ func CreateRestAPIClient(k8sHelper *utils.K8sHelper, namespace string) *RestAPIC
client := rclient.NewRookNetworkRestClient(endpoint, httpclient)
//make sure rest client is up and available
logger.Infof("ensuring API service is accessible by client on external endpoint %s", endpoint)
inc := 0
for inc < utils.RetryLoop {
_, err := client.GetStatusDetails()
@@ -83,6 +84,7 @@ func CreateRestAPIClient(k8sHelper *utils.K8sHelper, namespace string) *RestAPIC
}
inc++
logger.Infof("failed to access API service, will retry: %+v", err)
time.Sleep(time.Second * utils.RetryInterval)
}
+29
View File
@@ -17,6 +17,8 @@ limitations under the License.
package integration
import (
"regexp"
"strings"
"testing"
"github.com/rook/rook/tests/framework/clients"
@@ -26,6 +28,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// ************************************************
@@ -115,3 +119,28 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() {
checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3)
}
func (suite *SmokeSuite) TestOperatorGetFlexvolumePath() {
// get the operator pod
sysNamespace := installer.SystemNamespace(suite.namespace)
listOpts := metav1.ListOptions{LabelSelector: "app=rook-operator"}
podList, err := suite.k8sh.Clientset.Pods(sysNamespace).List(listOpts)
require.Nil(suite.T(), err)
require.Equal(suite.T(), 1, len(podList.Items))
// get the raw log for the operator pod
opPodName := podList.Items[0].Name
rawLog, err := suite.k8sh.Clientset.Pods(sysNamespace).GetLogs(opPodName, &v1.PodLogOptions{}).Do().Raw()
require.Nil(suite.T(), err)
r := regexp.MustCompile(`discovered flexvolume dir path from source.*\n`)
logStmt := string(r.Find(rawLog))
logger.Infof("flexvolume discovery log statement: %s", logStmt)
// verify that the volume plugin dir was discovered by the operator pod and that it did not come from
// an env var or the default
require.NotEmpty(suite.T(), logStmt)
assert.True(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source"))
assert.False(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source env var"))
assert.False(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source default"))
}
+18 -2
View File
@@ -21,11 +21,17 @@ usage(){
install_master(){
# This is needed on K8S 1.6 to fix a regression. https://github.com/kubernetes/kubernetes/issues/47109
if [[ $KUBE_VERSION == v1.6* ]] ;
then
if [[ $KUBE_VERSION == v1.6* ]] ; then
cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-disable_attachdetach_controller.conf
[Service]
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --enable-controller-attach-detach=false"
EOF
sudo systemctl daemon-reload
elif [[ $KUBE_VERSION == v1.8* ]] ; then
# for k8s 1.8, use a non default value for volume plugins
cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-volume_plugin_dir.conf
[Service]
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
EOF
sudo systemctl daemon-reload
fi
@@ -60,6 +66,16 @@ EOF
#install k8s node
install_node(){
echo "inside install node function"
# for k8s 1.8, use a non default value for volume plugins
if [[ $KUBE_VERSION == v1.8* ]] ; then
cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-volume_plugin_dir.conf
[Service]
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
EOF
sudo systemctl daemon-reload
fi
echo "kubeadm join ${1} ${2} ${3} ${4} ${5} --skip-preflight-checks"
sudo kubeadm join ${1} ${2} ${3} ${4} ${5} --skip-preflight-checks || true
}