From 35e2fbe4e075574995c209be7db7952a298471e9 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Fri, 5 Apr 2019 10:16:20 -0600 Subject: [PATCH] optest: wait longer for deployment image The Ceph upgrade test is the only one which uses the `WaitForDeploymentImage` method, and it has to be configured to wait longer after upgrade at this point. Since the wait time is still hard-coded, this method is moved to the operator's test dir to make it clear that the method is suitable only for tests currently. Signed-off-by: Blaine Gardner --- pkg/operator/k8sutil/deployment.go | 37 ----------------- pkg/operator/test/deployment.go | 64 ++++++++++++++++++++++++++++++ tests/integration/upgrade_test.go | 3 +- 3 files changed, 66 insertions(+), 38 deletions(-) create mode 100644 pkg/operator/test/deployment.go diff --git a/pkg/operator/k8sutil/deployment.go b/pkg/operator/k8sutil/deployment.go index 8acef5a25..e48a4cce7 100644 --- a/pkg/operator/k8sutil/deployment.go +++ b/pkg/operator/k8sutil/deployment.go @@ -45,43 +45,6 @@ func GetDeploymentSpecImage(clientset kubernetes.Interface, d apps.Deployment, c return image, nil } -func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, container string, initContainer bool, desiredImage string) error { - - sleepTime := 3 - attempts := 30 - for i := 0; i < attempts; i++ { - deployments, err := clientset.Apps().Deployments(namespace).List(metav1.ListOptions{LabelSelector: label}) - if err != nil { - return fmt.Errorf("failed to list deployments with label %s. %v", label, err) - } - - matches := 0 - for _, d := range deployments.Items { - image, err := GetDeploymentSpecImage(clientset, d, container, initContainer) - if err != nil { - logger.Infof("failed to get image for deployment %s. %+v", d.Name, err) - continue - } - if image == desiredImage { - matches++ - } - } - - if matches == len(deployments.Items) && matches > 0 { - logger.Infof("all %d %s deployments are on image %s", matches, label, desiredImage) - return nil - } - - if len(deployments.Items) == 0 { - logger.Infof("waiting for at least one deployment to start to see the version") - } else { - logger.Infof("%d/%d %s deployments match image %s", matches, len(deployments.Items), label, desiredImage) - } - time.Sleep(time.Duration(sleepTime) * time.Second) - } - return fmt.Errorf("failed to wait for image %s in label %s", desiredImage, label) -} - // UpdateDeploymentAndWait updates a deployment and waits until it is running to return. It will // error if the deployment does not exist to be updated or if it takes too long. func UpdateDeploymentAndWait(context *clusterd.Context, deployment *apps.Deployment, namespace string) (*v1.Deployment, error) { diff --git a/pkg/operator/test/deployment.go b/pkg/operator/test/deployment.go new file mode 100644 index 000000000..2cf41b864 --- /dev/null +++ b/pkg/operator/test/deployment.go @@ -0,0 +1,64 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + "time" + + "github.com/rook/rook/pkg/operator/k8sutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// WaitForDeploymentImage waits for all deployments with the given labels are running. +func WaitForDeploymentImage(clientset kubernetes.Interface, namespace, label, container string, initContainer bool, desiredImage string) error { + + sleepTime := 3 + attempts := 120 + for i := 0; i < attempts; i++ { + deployments, err := clientset.Apps().Deployments(namespace).List(metav1.ListOptions{LabelSelector: label}) + if err != nil { + return fmt.Errorf("failed to list deployments with label %s. %v", label, err) + } + + matches := 0 + for _, d := range deployments.Items { + image, err := k8sutil.GetDeploymentSpecImage(clientset, d, container, initContainer) + if err != nil { + logger.Infof("failed to get image for deployment %s. %+v", d.Name, err) + continue + } + if image == desiredImage { + matches++ + } + } + + if matches == len(deployments.Items) && matches > 0 { + logger.Infof("all %d %s deployments are on image %s", matches, label, desiredImage) + return nil + } + + if len(deployments.Items) == 0 { + logger.Infof("waiting for at least one deployment to start to see the version") + } else { + logger.Infof("%d/%d %s deployments match image %s", matches, len(deployments.Items), label, desiredImage) + } + time.Sleep(time.Duration(sleepTime) * time.Second) + } + return fmt.Errorf("failed to wait for image %s in label %s", desiredImage, label) +} diff --git a/tests/integration/upgrade_test.go b/tests/integration/upgrade_test.go index a8d8a95b5..3dc163647 100644 --- a/tests/integration/upgrade_test.go +++ b/tests/integration/upgrade_test.go @@ -22,6 +22,7 @@ import ( opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" + optest "github.com/rook/rook/pkg/operator/test" "github.com/rook/rook/tests/framework/clients" "github.com/rook/rook/tests/framework/installer" "github.com/rook/rook/tests/framework/utils" @@ -126,7 +127,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { require.Nil(s.T(), err) // wait for the osd pods to be updated - err = k8sutil.WaitForDeploymentImage(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd", opspec.ConfigInitContainerName, true, "rook/ceph:master") + err = optest.WaitForDeploymentImage(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd", opspec.ConfigInitContainerName, true, "rook/ceph:master") require.Nil(s.T(), err) err = s.k8sh.WaitForLabeledPodsToRun("app=rook-ceph-osd", s.namespace)