diff --git a/pkg/operator/agent/agent.go b/pkg/operator/agent/agent.go index 4cece8d7e..41d5b09e5 100644 --- a/pkg/operator/agent/agent.go +++ b/pkg/operator/agent/agent.go @@ -97,6 +97,10 @@ func (a *Agent) createAgentDaemonSet(namespace string) error { if err != nil { return err } + + flexvolumeDirPath, source := a.discoverFlexvolumeDir() + logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath) + privileged := true ds := &extensions.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ @@ -148,7 +152,7 @@ func (a *Agent) createAgentDaemonSet(namespace string) error { Name: "flexvolume", VolumeSource: v1.VolumeSource{ HostPath: &v1.HostPathVolumeSource{ - Path: a.discoverFlexvolumeDir(), + Path: flexvolumeDirPath, }, }, }, @@ -196,7 +200,7 @@ func (a *Agent) createAgentDaemonSet(namespace string) error { } -func (a *Agent) discoverFlexvolumeDir() string { +func (a *Agent) discoverFlexvolumeDir() (flexvolumeDirPath, source string) { //copy flexvolume to flexvolume dir nodeName := os.Getenv(k8sutil.NodeNameEnvVar) if nodeName == "" { @@ -205,7 +209,6 @@ func (a *Agent) discoverFlexvolumeDir() string { } // determining where the path of the flexvolume dir on the node - var flexvolumeDirPath string nodeConfigURI, err := k8sutil.NodeConfigURI() if err != nil { logger.Warningf(err.Error()) @@ -217,38 +220,57 @@ func (a *Agent) discoverFlexvolumeDir() string { return getDefaultFlexvolumeDir() } - // unmarshall to a NodeConfigKubelet + // unmarshal to a NodeConfigKubelet configKubelet := NodeConfigKubelet{} if err := json.Unmarshal(nodeConfig, &configKubelet); err != nil { logger.Warningf("unable to parse node config from Kubelet: %+v", err) - return getDefaultFlexvolumeDir() - } - flexvolumeDirPath = configKubelet.ComponentConfig.VolumePluginDir - if flexvolumeDirPath == "" { - // unmarshall to a NodeConfigControllerManager - configControllerManager := NodeConfigControllerManager{} - if err := json.Unmarshal(nodeConfig, &configControllerManager); err != nil { - logger.Warningf("unable to parse node config from controller manager: %+v", err) - return getDefaultFlexvolumeDir() - } - flexvolumeDirPath = configControllerManager.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir - if flexvolumeDirPath == "" { - flexvolumeDirPath = getDefaultFlexvolumeDir() - } + } else { + flexvolumeDirPath = configKubelet.ComponentConfig.VolumePluginDir } - logger.Infof("using flexvolume dir path: %s", flexvolumeDirPath) - return flexvolumeDirPath + if flexvolumeDirPath != "" { + return flexvolumeDirPath, "NodeConfigKubelet" + } + + // unmarshal to a NodeConfigControllerManager + configControllerManager := NodeConfigControllerManager{} + if err := json.Unmarshal(nodeConfig, &configControllerManager); err != nil { + logger.Warningf("unable to parse node config from controller manager: %+v", err) + } else { + flexvolumeDirPath = configControllerManager.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir + } + + if flexvolumeDirPath != "" { + return flexvolumeDirPath, "NodeConfigControllerManager" + } + + // unmarshal to a KubeletConfiguration + kubeletConfiguration := KubeletConfiguration{} + if err := json.Unmarshal(nodeConfig, &kubeletConfiguration); err != nil { + logger.Warningf("unable to parse node config as kubelet configuration: %+v", err) + } else { + flexvolumeDirPath = kubeletConfiguration.KubeletConfig.VolumePluginDir + } + + if flexvolumeDirPath != "" { + return flexvolumeDirPath, "KubeletConfiguration" + } + + return getDefaultFlexvolumeDir() } -func getDefaultFlexvolumeDir() string { - logger.Info("getting flexvolume dir path from provided env var") - flexvolumeDirPath := os.Getenv(flexvolumePathDirEnv) - if flexvolumeDirPath == "" { - logger.Infof("flexvolume dir path is not provided. Defaulting to: %s", flexvolumeDefaultDirPath) - flexvolumeDirPath = flexvolumeDefaultDirPath +func getDefaultFlexvolumeDir() (flexvolumeDirPath, source string) { + logger.Infof("getting flexvolume dir path from %s env var", flexvolumePathDirEnv) + flexvolumeDirPath = os.Getenv(flexvolumePathDirEnv) + if flexvolumeDirPath != "" { + return flexvolumeDirPath, "env var" } - return flexvolumeDirPath + + logger.Infof("flexvolume dir path env var %s is not provided. Defaulting to: %s", + flexvolumePathDirEnv, flexvolumeDefaultDirPath) + flexvolumeDirPath = flexvolumeDefaultDirPath + + return flexvolumeDirPath, "default" } func getContainerImage(clientset kubernetes.Interface) (string, error) { diff --git a/pkg/operator/agent/types.go b/pkg/operator/agent/types.go index a86e9c992..416c3d7a9 100644 --- a/pkg/operator/agent/types.go +++ b/pkg/operator/agent/types.go @@ -40,3 +40,10 @@ type NodeConfigControllerManager struct { type NodeConfigKubelet struct { ComponentConfig kubeletcomponentconfig.KubeletConfiguration `json:"componentconfig"` } + +// KubeletConfiguration represents the response from the node config URI (configz) in Kubernetes 1.8+ +type KubeletConfiguration struct { + KubeletConfig struct { + VolumePluginDir string `json:"volumePluginDir"` + } `json:"kubeletconfig"` +} diff --git a/tests/framework/clients/rest_client.go b/tests/framework/clients/rest_client.go index 73956fa97..d02aa700c 100644 --- a/tests/framework/clients/rest_client.go +++ b/tests/framework/clients/rest_client.go @@ -73,6 +73,7 @@ func CreateRestAPIClient(k8sHelper *utils.K8sHelper, namespace string) *RestAPIC client := rclient.NewRookNetworkRestClient(endpoint, httpclient) //make sure rest client is up and available + logger.Infof("ensuring API service is accessible by client on external endpoint %s", endpoint) inc := 0 for inc < utils.RetryLoop { _, err := client.GetStatusDetails() @@ -83,6 +84,7 @@ func CreateRestAPIClient(k8sHelper *utils.K8sHelper, namespace string) *RestAPIC } inc++ + logger.Infof("failed to access API service, will retry: %+v", err) time.Sleep(time.Second * utils.RetryInterval) } diff --git a/tests/integration/smoke_test.go b/tests/integration/smoke_test.go index 680f8218d..a07e03769 100644 --- a/tests/integration/smoke_test.go +++ b/tests/integration/smoke_test.go @@ -17,6 +17,8 @@ limitations under the License. package integration import ( + "regexp" + "strings" "testing" "github.com/rook/rook/tests/framework/clients" @@ -26,6 +28,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // ************************************************ @@ -115,3 +119,28 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() { func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() { checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3) } + +func (suite *SmokeSuite) TestOperatorGetFlexvolumePath() { + // get the operator pod + sysNamespace := installer.SystemNamespace(suite.namespace) + listOpts := metav1.ListOptions{LabelSelector: "app=rook-operator"} + podList, err := suite.k8sh.Clientset.Pods(sysNamespace).List(listOpts) + require.Nil(suite.T(), err) + require.Equal(suite.T(), 1, len(podList.Items)) + + // get the raw log for the operator pod + opPodName := podList.Items[0].Name + rawLog, err := suite.k8sh.Clientset.Pods(sysNamespace).GetLogs(opPodName, &v1.PodLogOptions{}).Do().Raw() + require.Nil(suite.T(), err) + + r := regexp.MustCompile(`discovered flexvolume dir path from source.*\n`) + logStmt := string(r.Find(rawLog)) + logger.Infof("flexvolume discovery log statement: %s", logStmt) + + // verify that the volume plugin dir was discovered by the operator pod and that it did not come from + // an env var or the default + require.NotEmpty(suite.T(), logStmt) + assert.True(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source")) + assert.False(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source env var")) + assert.False(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source default")) +} diff --git a/tests/scripts/kubeadm.sh b/tests/scripts/kubeadm.sh index 116f49331..353147405 100755 --- a/tests/scripts/kubeadm.sh +++ b/tests/scripts/kubeadm.sh @@ -21,11 +21,17 @@ usage(){ install_master(){ # This is needed on K8S 1.6 to fix a regression. https://github.com/kubernetes/kubernetes/issues/47109 - if [[ $KUBE_VERSION == v1.6* ]] ; - then + if [[ $KUBE_VERSION == v1.6* ]] ; then cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-disable_attachdetach_controller.conf [Service] Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --enable-controller-attach-detach=false" +EOF + sudo systemctl daemon-reload + elif [[ $KUBE_VERSION == v1.8* ]] ; then + # for k8s 1.8, use a non default value for volume plugins + cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-volume_plugin_dir.conf +[Service] +Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins" EOF sudo systemctl daemon-reload fi @@ -60,6 +66,16 @@ EOF #install k8s node install_node(){ echo "inside install node function" + + # for k8s 1.8, use a non default value for volume plugins + if [[ $KUBE_VERSION == v1.8* ]] ; then + cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-volume_plugin_dir.conf +[Service] +Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins" +EOF + sudo systemctl daemon-reload + fi + echo "kubeadm join ${1} ${2} ${3} ${4} ${5} --skip-preflight-checks" sudo kubeadm join ${1} ${2} ${3} ${4} ${5} --skip-preflight-checks || true }