From 173b4fa20d3e6011a093b6e064b031e3fb9838b3 Mon Sep 17 00:00:00 2001
From: Divesh Angula <divesh.angula@quantum.com>
Date: Wed, 17 May 2017 11:59:23 -0700
Subject: [PATCH] support multiple parallel builds.

---
 .gitignore                                    |   3 +-
 .../manager/scripts/rook-dind-cluster-v1.6.sh | 128 +-----------------
 e2e/images/Dockerfile                         |  10 +-
 e2e/scripts/smoke_test.sh                     |  73 +++++++---
 4 files changed, 70 insertions(+), 144 deletions(-)

diff --git a/.gitignore b/.gitignore
index d43bd4be6..9994b7a7a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,4 +20,5 @@ demo/standalone/cloud-config.yml
 e2e/results
 e2e/vendor
 e2e/.glide
-tests/smoke/temp_image.tar
+e2e/tests/smoke/temp_image.tar
+forRookImage.tar
diff --git a/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh b/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh
index 6121833d5..197924751 100755
--- a/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh
+++ b/e2e/framework/manager/scripts/rook-dind-cluster-v1.6.sh
@@ -140,15 +140,6 @@ function dind::prepare-sys-mounts {
     if [[ -d /lib/modules ]]; then
       sys_volume_args+=(-v /lib/modules:/lib/modules)
     fi
-    if [[ -d /dev ]]; then
-      sys_volume_args+=(-v /dev:/dev)
-    fi
-    if [[ -d /sys ]]; then
-      sys_volume_args+=(-v /sys/bus:/sys/bus)
-    fi
-    if [[ -d /var/run/docker.sock ]]; then
-      sys_volume_args+=(-v /var/run/docker.sock:/var/run/docker.sock)
-    fi
     return 0
   fi
   if ! dind::volume-exists kubeadm-dind-sys; then
@@ -320,7 +311,7 @@ function dind::ensure-binaries {
 
 function dind::ensure-network {
   if ! docker network inspect kubeadm-dind-net >&/dev/null; then
-    docker network create --subnet=10.192.0.0/16 kubeadm-dind-net >/dev/null
+    docker network create --subnet="${DIND_SUBNET}/16" kubeadm-dind-net >/dev/null
   fi
 }
 
@@ -399,6 +390,9 @@ function dind::run {
          --hostname "${container_name}" \
          -l mirantis.kubeadm_dind_cluster \
          -v ${volume_name}:/dind \
+         -v /lib/modules:/lib/modules \
+         -v /sbin/modprobe:/sbin/modprobe \
+         -v /sys/bus:/sys/bus \
          -v /dev:/dev \
          -v /var/run/docker.sock:/opt/outer-docker.sock \
          ${opts[@]+"${opts[@]}"} \
@@ -652,119 +646,9 @@ function dind::up {
     flannel)
       if dind::use-rbac; then
         curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml?raw=true" | "${kubectl}" create -f -
-        # FIXME: current kube-flannel yaml causes problems after cluster restart
-        # kube-dns and kubernetes-dashboard stay in "ContainerCreating" state.
-        # Flannel pods become 'Running' for a brief amount of time and then
-        # die while trying to contact apiserver.
-        # Following the advice from https://github.com/kubernetes/kubernetes/issues/39701 (comments)
-        # we hardcode apiserver host into flannel pod definition
-        # (KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT)
-        # We could use 'kubectl convert' + 'jq' but we don't want
-        # to require jq on the host, and doing this via docker is cumbersome,
-        # so for now we're embedding patched flannel yaml here.
-        # For some reason the problem happens only on 1.6.
-        "${kubectl}" create -f - <<EOF
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: flannel
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kube-flannel-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-data:
-  cni-conf.json: |
-    {
-      "name": "cbr0",
-      "type": "flannel",
-      "delegate": {
-        "isDefaultGateway": true
-      }
-    }
-  net-conf.json: |
-    {
-      "Network": "10.244.0.0/16",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }
----
-apiVersion: extensions/v1beta1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
-      serviceAccountName: flannel
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.7.0-amd64
-        command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: KUBERNETES_SERVICE_HOST
-          value: "kube-master"
-        - name: KUBERNETES_SERVICE_PORT
-          value: "6443"
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.7.0-amd64
-        command: [ "/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done" ]
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
-EOF
-      else
-        # without --validate=false this will fail on older k8s versions
-        curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" | "${kubectl}" create --validate=false -f -
       fi
+      # without --validate=false this will fail on older k8s versions
+      curl -sSL "https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml?raw=true" | "${kubectl}" create --validate=false -f -
       ;;
     calico)
       if dind::use-rbac; then
diff --git a/e2e/images/Dockerfile b/e2e/images/Dockerfile
index 46355ad7a..2d0028bd7 100644
--- a/e2e/images/Dockerfile
+++ b/e2e/images/Dockerfile
@@ -56,14 +56,14 @@ ENV PATH $PATH:$GOBIN:$GOPATH:$GOROOT
 #Install glide
 RUN curl https://glide.sh/get | sh
 
-
-VOLUME /sys/fs/cgroup
-VOLUME /lib/modules
-VOLUME /sys
+VOLUME /var/lib/docker
+#VOLUME /sys/fs/cgroup
+#VOLUME /lib/modules
+#VOLUME /sys
 
 RUN apt-get clean -y
 
 STOPSIGNAL SIGRTMIN+3
 
 CMD ["/bin/bash", "-c", "exec /sbin/init --log-target=journal 3>&1"]
-#CMD /sbin/init
+#CMD /sbin/init
\ No newline at end of file
diff --git a/e2e/scripts/smoke_test.sh b/e2e/scripts/smoke_test.sh
index b3b753213..07ae9be84 100755
--- a/e2e/scripts/smoke_test.sh
+++ b/e2e/scripts/smoke_test.sh
@@ -4,27 +4,49 @@ test_source_repo=$(pwd)
 docker_test_repo=/workspace/go/src/github.com/rook/rook
 git_smoke_test_directory=github.com/rook/rook/e2e/tests/smoke
 git_test_directory=github.com/rook/rook/e2e/tests
-container_image=quay.io/quantum/rook-test
-tmp_docker_sock_path=/tmp/docker.sock
+container_image=quay.io/rook/rook-test
+tmp_docker_sock_path=/var/run/docker.sock
 results_dir=results
 results_filename=test.log
+temp_archive=forRookImage.tar
 
 #create the rook infrastructure container
 rook_infra::create() {
     export id=$(docker run \
-        --net=host \
         -d \
         -e GOPATH=/workspace/go \
         -e GOROOT=/usr/lib/go \
         --privileged \
         --security-opt=seccomp:unconfined \
+        -v /lib/modules:/lib/modules \
+        -v /sbin/modprobe:/sbin/modprobe \
+        -v /sys/bus:/sys/bus \
+        -v /dev:/dev \
         -v /var/run/docker.sock:${tmp_docker_sock_path} \
         -v ${test_source_repo}:${docker_test_repo} \
         -w ${docker_test_repo}/e2e \
         ${container_image} \
         /sbin/init)
 
-        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+
+    local attempt_num=1
+    local max_attempts=10
+
+    until [[ $(docker inspect -f {{.State.Running}} $id) == "true" ]]
+    do
+        if (( attempt_num == max_attempts ))
+        then
+            echo "rook-test infra container failed to start"
+            return 1
+        else
+            sleep 1
+            attempt_num++
+
+        fi
+    done
+
+    echo "rook-test container is running..."
 }
 
 #prepare the rook_infra container to run tests
@@ -33,15 +55,6 @@ rook_infra::init() {
     docker exec ${id} mkdir -p ${results_dir}
     rc=$?; if [[ $rc != 0 ]]; then set -e; fi
 
-    echo Removing infra docker.sock
-    docker exec ${id} rm -rfv /var/run/docker.sock
-    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
-
-    echo Creating sysmlink to host docker.sock
-    docker exec ${id} ln -s ${tmp_docker_sock_path} /var/run/docker.sock
-    rc=$?; if [[ $rc != 0 ]]; then set -e; fi
-    echo Success...
-
     echo Installing rook-test-framework dependencies...
     docker exec ${id} /bin/bash -c \
     "glide install"
@@ -61,6 +74,34 @@ rook_infra::run_test() {
     rc=$?; if [[ $rc != 0 ]]; then set -e; fi
 }
 
+rook_infra::try_copy_docker_image_to_rook_infra() {
+    local tag_name=$1
+
+    echo Searching local docker registry for ${tag_name}
+    export imageId=$(docker images -q ${tag_name})
+
+    if [ -z "$imageId" ]; then
+        echo Image not found
+    else
+        echo Image found...
+
+        echo Archiving the docker image
+        docker save -o ${temp_archive} ${tag_name}
+        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+        echo success...
+
+        echo Copying archived image to rook-infra
+        docker cp forRookImage.tar ${id}:/${temp_archive}
+        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+        echo success...
+
+        echo Importing image into rook-infra image registry
+        docker exec ${id} /bin/bash -c "docker load -i /${temp_archive}"
+        rc=$?; if [[ $rc != 0 ]]; then set -e; fi
+        echo success...
+    fi
+}
+
 rook_infra::gather_results() {
     echo Gathering results...
 
@@ -84,6 +125,7 @@ rook_infra::cleanup() {
     docker kill ${id} || true
     docker rm ${id} || true
     docker images|grep 'rook-test\|kubeadm-dind-cluster\|ceph/base'|xargs docker rmi > /dev/null 2>&1 || true
+    docker volume ls -qf dangling=true | xargs -r docker volume rm > /dev/null 2>&1|| true
 
 }
 
@@ -108,15 +150,14 @@ rook_infra::cleanup() {
 { #try
 
     rook_infra::create
-    sleep 5
+    rook_infra::try_copy_docker_image_to_rook_infra quay.io/rook/rookd:${tag_name}
+    rook_infra::try_copy_docker_image_to_rook_infra quay.io/rook/rook:${tag_name}
     rook_infra::init
 
     rook_infra::run_test SmokeSuite ${tag_name} ${rook_platform} ${k8s_version}
 
     rook_infra::gather_results
 
-  # Delete the rook infrastructure container
-
 } || { #catch
     rook_infra::gather_results
 }