Merge pull request #1255 from jbw976/volume-plugin-dir-1.8
Get flexvolume plugin dir for all k8s versions
This commit is contained in:
5 files changed
+105
-29
No files matched your search
+49
-27
@@ -97,6 +97,10 @@ func (a *Agent) createAgentDaemonSet(namespace string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
flexvolumeDirPath, source := a.discoverFlexvolumeDir()
|
||||
logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath)
|
||||
|
||||
privileged := true
|
||||
ds := &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -148,7 +152,7 @@ func (a *Agent) createAgentDaemonSet(namespace string) error {
|
||||
Name: "flexvolume",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: a.discoverFlexvolumeDir(),
|
||||
Path: flexvolumeDirPath,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -196,7 +200,7 @@ func (a *Agent) createAgentDaemonSet(namespace string) error {
|
||||
|
||||
}
|
||||
|
||||
func (a *Agent) discoverFlexvolumeDir() string {
|
||||
func (a *Agent) discoverFlexvolumeDir() (flexvolumeDirPath, source string) {
|
||||
//copy flexvolume to flexvolume dir
|
||||
nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
|
||||
if nodeName == "" {
|
||||
@@ -205,7 +209,6 @@ func (a *Agent) discoverFlexvolumeDir() string {
|
||||
}
|
||||
|
||||
// determining where the path of the flexvolume dir on the node
|
||||
var flexvolumeDirPath string
|
||||
nodeConfigURI, err := k8sutil.NodeConfigURI()
|
||||
if err != nil {
|
||||
logger.Warningf(err.Error())
|
||||
@@ -217,38 +220,57 @@ func (a *Agent) discoverFlexvolumeDir() string {
|
||||
return getDefaultFlexvolumeDir()
|
||||
}
|
||||
|
||||
// unmarshall to a NodeConfigKubelet
|
||||
// unmarshal to a NodeConfigKubelet
|
||||
configKubelet := NodeConfigKubelet{}
|
||||
if err := json.Unmarshal(nodeConfig, &configKubelet); err != nil {
|
||||
logger.Warningf("unable to parse node config from Kubelet: %+v", err)
|
||||
return getDefaultFlexvolumeDir()
|
||||
}
|
||||
flexvolumeDirPath = configKubelet.ComponentConfig.VolumePluginDir
|
||||
if flexvolumeDirPath == "" {
|
||||
// unmarshall to a NodeConfigControllerManager
|
||||
configControllerManager := NodeConfigControllerManager{}
|
||||
if err := json.Unmarshal(nodeConfig, &configControllerManager); err != nil {
|
||||
logger.Warningf("unable to parse node config from controller manager: %+v", err)
|
||||
return getDefaultFlexvolumeDir()
|
||||
}
|
||||
flexvolumeDirPath = configControllerManager.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir
|
||||
if flexvolumeDirPath == "" {
|
||||
flexvolumeDirPath = getDefaultFlexvolumeDir()
|
||||
}
|
||||
} else {
|
||||
flexvolumeDirPath = configKubelet.ComponentConfig.VolumePluginDir
|
||||
}
|
||||
|
||||
logger.Infof("using flexvolume dir path: %s", flexvolumeDirPath)
|
||||
return flexvolumeDirPath
|
||||
if flexvolumeDirPath != "" {
|
||||
return flexvolumeDirPath, "NodeConfigKubelet"
|
||||
}
|
||||
|
||||
// unmarshal to a NodeConfigControllerManager
|
||||
configControllerManager := NodeConfigControllerManager{}
|
||||
if err := json.Unmarshal(nodeConfig, &configControllerManager); err != nil {
|
||||
logger.Warningf("unable to parse node config from controller manager: %+v", err)
|
||||
} else {
|
||||
flexvolumeDirPath = configControllerManager.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir
|
||||
}
|
||||
|
||||
if flexvolumeDirPath != "" {
|
||||
return flexvolumeDirPath, "NodeConfigControllerManager"
|
||||
}
|
||||
|
||||
// unmarshal to a KubeletConfiguration
|
||||
kubeletConfiguration := KubeletConfiguration{}
|
||||
if err := json.Unmarshal(nodeConfig, &kubeletConfiguration); err != nil {
|
||||
logger.Warningf("unable to parse node config as kubelet configuration: %+v", err)
|
||||
} else {
|
||||
flexvolumeDirPath = kubeletConfiguration.KubeletConfig.VolumePluginDir
|
||||
}
|
||||
|
||||
if flexvolumeDirPath != "" {
|
||||
return flexvolumeDirPath, "KubeletConfiguration"
|
||||
}
|
||||
|
||||
return getDefaultFlexvolumeDir()
|
||||
}
|
||||
|
||||
func getDefaultFlexvolumeDir() string {
|
||||
logger.Info("getting flexvolume dir path from provided env var")
|
||||
flexvolumeDirPath := os.Getenv(flexvolumePathDirEnv)
|
||||
if flexvolumeDirPath == "" {
|
||||
logger.Infof("flexvolume dir path is not provided. Defaulting to: %s", flexvolumeDefaultDirPath)
|
||||
flexvolumeDirPath = flexvolumeDefaultDirPath
|
||||
func getDefaultFlexvolumeDir() (flexvolumeDirPath, source string) {
|
||||
logger.Infof("getting flexvolume dir path from %s env var", flexvolumePathDirEnv)
|
||||
flexvolumeDirPath = os.Getenv(flexvolumePathDirEnv)
|
||||
if flexvolumeDirPath != "" {
|
||||
return flexvolumeDirPath, "env var"
|
||||
}
|
||||
return flexvolumeDirPath
|
||||
|
||||
logger.Infof("flexvolume dir path env var %s is not provided. Defaulting to: %s",
|
||||
flexvolumePathDirEnv, flexvolumeDefaultDirPath)
|
||||
flexvolumeDirPath = flexvolumeDefaultDirPath
|
||||
|
||||
return flexvolumeDirPath, "default"
|
||||
}
|
||||
|
||||
func getContainerImage(clientset kubernetes.Interface) (string, error) {
|
||||
|
||||
@@ -40,3 +40,10 @@ type NodeConfigControllerManager struct {
|
||||
type NodeConfigKubelet struct {
|
||||
ComponentConfig kubeletcomponentconfig.KubeletConfiguration `json:"componentconfig"`
|
||||
}
|
||||
|
||||
// KubeletConfiguration represents the response from the node config URI (configz) in Kubernetes 1.8+
|
||||
type KubeletConfiguration struct {
|
||||
KubeletConfig struct {
|
||||
VolumePluginDir string `json:"volumePluginDir"`
|
||||
} `json:"kubeletconfig"`
|
||||
}
|
||||
@@ -73,6 +73,7 @@ func CreateRestAPIClient(k8sHelper *utils.K8sHelper, namespace string) *RestAPIC
|
||||
client := rclient.NewRookNetworkRestClient(endpoint, httpclient)
|
||||
|
||||
//make sure rest client is up and available
|
||||
logger.Infof("ensuring API service is accessible by client on external endpoint %s", endpoint)
|
||||
inc := 0
|
||||
for inc < utils.RetryLoop {
|
||||
_, err := client.GetStatusDetails()
|
||||
@@ -83,6 +84,7 @@ func CreateRestAPIClient(k8sHelper *utils.K8sHelper, namespace string) *RestAPIC
|
||||
|
||||
}
|
||||
inc++
|
||||
logger.Infof("failed to access API service, will retry: %+v", err)
|
||||
time.Sleep(time.Second * utils.RetryInterval)
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,8 @@ limitations under the License.
|
||||
package integration
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/tests/framework/clients"
|
||||
@@ -26,6 +28,8 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// ************************************************
|
||||
@@ -115,3 +119,28 @@ func (suite *SmokeSuite) TestObjectStorage_SmokeTest() {
|
||||
func (suite *SmokeSuite) TestRookClusterInstallation_smokeTest() {
|
||||
checkIfRookClusterIsInstalled(suite.Suite, suite.k8sh, installer.SystemNamespace(suite.namespace), suite.namespace, 3)
|
||||
}
|
||||
|
||||
func (suite *SmokeSuite) TestOperatorGetFlexvolumePath() {
|
||||
// get the operator pod
|
||||
sysNamespace := installer.SystemNamespace(suite.namespace)
|
||||
listOpts := metav1.ListOptions{LabelSelector: "app=rook-operator"}
|
||||
podList, err := suite.k8sh.Clientset.Pods(sysNamespace).List(listOpts)
|
||||
require.Nil(suite.T(), err)
|
||||
require.Equal(suite.T(), 1, len(podList.Items))
|
||||
|
||||
// get the raw log for the operator pod
|
||||
opPodName := podList.Items[0].Name
|
||||
rawLog, err := suite.k8sh.Clientset.Pods(sysNamespace).GetLogs(opPodName, &v1.PodLogOptions{}).Do().Raw()
|
||||
require.Nil(suite.T(), err)
|
||||
|
||||
r := regexp.MustCompile(`discovered flexvolume dir path from source.*\n`)
|
||||
logStmt := string(r.Find(rawLog))
|
||||
logger.Infof("flexvolume discovery log statement: %s", logStmt)
|
||||
|
||||
// verify that the volume plugin dir was discovered by the operator pod and that it did not come from
|
||||
// an env var or the default
|
||||
require.NotEmpty(suite.T(), logStmt)
|
||||
assert.True(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source"))
|
||||
assert.False(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source env var"))
|
||||
assert.False(suite.T(), strings.Contains(logStmt, "discovered flexvolume dir path from source default"))
|
||||
}
|
||||
@@ -21,11 +21,17 @@ usage(){
|
||||
install_master(){
|
||||
|
||||
# This is needed on K8S 1.6 to fix a regression. https://github.com/kubernetes/kubernetes/issues/47109
|
||||
if [[ $KUBE_VERSION == v1.6* ]] ;
|
||||
then
|
||||
if [[ $KUBE_VERSION == v1.6* ]] ; then
|
||||
cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-disable_attachdetach_controller.conf
|
||||
[Service]
|
||||
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --enable-controller-attach-detach=false"
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
elif [[ $KUBE_VERSION == v1.8* ]] ; then
|
||||
# for k8s 1.8, use a non default value for volume plugins
|
||||
cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-volume_plugin_dir.conf
|
||||
[Service]
|
||||
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
fi
|
||||
@@ -60,6 +66,16 @@ EOF
|
||||
#install k8s node
|
||||
install_node(){
|
||||
echo "inside install node function"
|
||||
|
||||
# for k8s 1.8, use a non default value for volume plugins
|
||||
if [[ $KUBE_VERSION == v1.8* ]] ; then
|
||||
cat << EOF | sudo tee -a /etc/systemd/system/kubelet.service.d/11-volume_plugin_dir.conf
|
||||
[Service]
|
||||
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
|
||||
EOF
|
||||
sudo systemctl daemon-reload
|
||||
fi
|
||||
|
||||
echo "kubeadm join ${1} ${2} ${3} ${4} ${5} --skip-preflight-checks"
|
||||
sudo kubeadm join ${1} ${2} ${3} ${4} ${5} --skip-preflight-checks || true
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user