From c27bfc3bb41c8c0204e028241918a0625c56f1be Mon Sep 17 00:00:00 2001 From: Illya Chekrygin Date: Sun, 4 Feb 2018 12:27:52 -0800 Subject: [PATCH] Define CRD's as a part of the operator provisionning and remove it from operator init phase Remove empty try/catch (erroneous commit) Update Teardown documentation Update PendingReleaseNotes with Operator resource changes Signed-off-by: Illya Chekrygin --- Documentation/teardown.md | 2 - PendingReleaseNotes.md | 1 + .../charts/rook/templates/clusterrole.yaml | 23 ++- cluster/charts/rook/templates/resources.yaml | 115 ++++++++++++ .../kubernetes/1.6/rook-operator.yaml | 173 ++++++++++++++++++ .../examples/kubernetes/rook-operator.yaml | 92 ++++++++-- pkg/operator/operator.go | 30 --- tests/framework/installer/install_data.go | 126 +++++++++++-- tests/framework/installer/install_helper.go | 27 ++- tests/scripts/minikube.sh | 2 +- 10 files changed, 524 insertions(+), 67 deletions(-) create mode 100644 cluster/charts/rook/templates/resources.yaml create mode 100644 cluster/examples/kubernetes/1.6/rook-operator.yaml diff --git a/Documentation/teardown.md b/Documentation/teardown.md index 67b405d69..8d7d50687 100644 --- a/Documentation/teardown.md +++ b/Documentation/teardown.md @@ -43,8 +43,6 @@ kubectl -n rook get cluster ## Delete the Operator and Agent This will begin the process of all cluster resources being cleaned up, after which you can delete the rest of the deployment with the following: ```console -kubectl delete thirdpartyresources cluster.rook.io pool.rook.io objectstore.rook.io filesystem.rook.io volumeattachment.rook.io # ignore errors if on K8s 1.7+ -kubectl delete crd clusters.rook.io pools.rook.io objectstores.rook.io filesystems.rook.io volumeattachments.rook.io # ignore errors if on K8s 1.5 and 1.6 kubectl delete -n rook-system daemonset rook-agent kubectl delete -f rook-operator.yaml kubectl delete clusterroles rook-agent diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 933c45312..961d001e5 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -5,6 +5,7 @@ ## Notable Features - Ceph tools can be run [from any rook pod](Documentation/common-issues.md#ceph-tools). - Output from stderr will be included in error messages returned from the `exec` of external tools +- Rook-Operator no longer creates the resources CRD's or TPR's at the runtime. Instead, those resources are provisioned during deployment via `helm` or `kubectl`. ## Breaking Changes diff --git a/cluster/charts/rook/templates/clusterrole.yaml b/cluster/charts/rook/templates/clusterrole.yaml index 850f94f53..faacaea4e 100644 --- a/cluster/charts/rook/templates/clusterrole.yaml +++ b/cluster/charts/rook/templates/clusterrole.yaml @@ -32,7 +32,6 @@ rules: - apiGroups: - extensions resources: - - thirdpartyresources - deployments - daemonsets - replicasets @@ -43,16 +42,30 @@ rules: - create - update - delete +- apiGroups: + - extensions + resources: + - thirdpartyresources + resourceNames: + - cluster.rook.io + - pool.rook.io + - objectstore.rook.io + - filesystem.rook.io + - volumeattachment.rook.io + verbs: + - get - apiGroups: - apiextensions.k8s.io resources: - customresourcedefinitions + resourceNames: + - clusters.rook.io + - pools.rook.io + - objectstores.rook.io + - filesystems.rook.io + - volumeattachments.rook.io verbs: - get - - list - - watch - - create - - delete - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/cluster/charts/rook/templates/resources.yaml b/cluster/charts/rook/templates/resources.yaml new file mode 100644 index 000000000..f7ca92f4b --- /dev/null +++ b/cluster/charts/rook/templates/resources.yaml @@ -0,0 +1,115 @@ +{{- if .Capabilities.APIVersions.Has "apiextensions.k8s.io/v1beta1" }} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusters.rook.io +spec: + group: rook.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: filesystems.rook.io +spec: + group: rook.io + names: + kind: Filesystem + listKind: FilesystemList + plural: filesystems + singular: filesystem + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectstores.rook.io +spec: + group: rook.io + names: + kind: ObjectStore + listKind: ObjectStoreList + plural: objectstores + singular: objectstore + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pools.rook.io +spec: + group: rook.io + names: + kind: Pool + listKind: PoolList + plural: pools + singular: pool + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumeattachments.rook.io +spec: + group: rook.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + singular: volumeattachment + scope: Namespaced + version: v1alpha1 +--- +{{- else }} +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for cluster +kind: ThirdPartyResource +metadata: + name: cluster.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for filesystem +kind: ThirdPartyResource +metadata: + name: filesystem.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for objectstore +kind: ThirdPartyResource +metadata: + name: objectstore.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for pool +kind: ThirdPartyResource +metadata: + name: pool.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for volumeattachment +kind: ThirdPartyResource +metadata: + name: volumeattachment.rook.io +versions: +- name: v1alpha1 +--- +{{- end }} \ No newline at end of file diff --git a/cluster/examples/kubernetes/1.6/rook-operator.yaml b/cluster/examples/kubernetes/1.6/rook-operator.yaml new file mode 100644 index 000000000..b6fe01752 --- /dev/null +++ b/cluster/examples/kubernetes/1.6/rook-operator.yaml @@ -0,0 +1,173 @@ +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for cluster +kind: ThirdPartyResource +metadata: + name: cluster.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for filesystem +kind: ThirdPartyResource +metadata: + name: filesystem.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for objectstore +kind: ThirdPartyResource +metadata: + name: objectstore.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for pool +kind: ThirdPartyResource +metadata: + name: pool.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for volumeattachment +kind: ThirdPartyResource +metadata: + name: volumeattachment.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: v1 +kind: Namespace +metadata: + name: rook-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-operator +rules: +- apiGroups: + - "" + resources: + - namespaces + - serviceaccounts + - secrets + - pods + - services + - nodes + - nodes/proxy + - configmaps + - events + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - delete +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-operator + namespace: rook-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-operator + namespace: rook-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-operator +subjects: +- kind: ServiceAccount + name: rook-operator + namespace: rook-system +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rook-operator + namespace: rook-system +spec: + replicas: 1 + template: + metadata: + labels: + app: rook-operator + spec: + serviceAccountName: rook-operator + containers: + - name: rook-operator + image: rook/rook:master + args: ["operator"] + env: + # To disable RBAC, uncomment the following: + # - name: RBAC_ENABLED + # value: "false" + # Rook Agent toleration. Will tolerate all taints with all keys. + # Choose between NoSchedule, PreferNoSchedule and NoExecute: + # - name: AGENT_TOLERATION + # value: "NoSchedule" + # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate + # - name: AGENT_TOLERATION_KEY + # value: "" + # Set the path where the Rook agent can find the flex volumes + # - name: FLEXVOLUME_DIR_PATH + # value: "" + # The interval to check if every mon is in the quorum. + - name: ROOK_MON_HEALTHCHECK_INTERVAL + value: "45s" + # The duration to wait before trying to failover or remove/replace the + # current mon with a new mon (useful for compensating flapping network). + - name: ROOK_MON_OUT_TIMEOUT + value: "300s" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/cluster/examples/kubernetes/rook-operator.yaml b/cluster/examples/kubernetes/rook-operator.yaml index 8f0324c3c..2b4bf1d8d 100644 --- a/cluster/examples/kubernetes/rook-operator.yaml +++ b/cluster/examples/kubernetes/rook-operator.yaml @@ -1,10 +1,76 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: rook-system --- -kind: ClusterRole +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusters.rook.io +spec: + group: rook.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: filesystems.rook.io +spec: + group: rook.io + names: + kind: Filesystem + listKind: FilesystemList + plural: filesystems + singular: filesystem + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectstores.rook.io +spec: + group: rook.io + names: + kind: ObjectStore + listKind: ObjectStoreList + plural: objectstores + singular: objectstore + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pools.rook.io +spec: + group: rook.io + names: + kind: Pool + listKind: PoolList + plural: pools + singular: pool + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumeattachments.rook.io +spec: + group: rook.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + singular: volumeattachment + scope: Namespaced + version: v1alpha1 +--- apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole metadata: name: rook-operator rules: @@ -33,7 +99,6 @@ rules: - apiGroups: - extensions resources: - - thirdpartyresources - deployments - daemonsets - replicasets @@ -44,16 +109,6 @@ rules: - create - update - delete -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - apiGroups: - rbac.authorization.k8s.io resources: @@ -85,6 +140,11 @@ rules: - "*" --- apiVersion: v1 +kind: Namespace +metadata: + name: rook-system +--- +apiVersion: v1 kind: ServiceAccount metadata: name: rook-operator diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 010faafb5..6718f740f 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "syscall" - "time" "github.com/coreos/pkg/capnslog" opkit "github.com/rook/operator-kit" @@ -39,10 +38,6 @@ import ( "k8s.io/api/core/v1" ) -const ( - initRetryDelay = 10 * time.Second -) - // volume provisioner constant const ( provisionerName = "rook.io/block" @@ -85,15 +80,6 @@ func (o *Operator) Run() error { return fmt.Errorf("Rook operator namespace is not provided. Expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar) } - for { - err := o.initResources() - if err == nil { - break - } - logger.Errorf("failed to init resources. %+v. retrying...", err) - <-time.After(initRetryDelay) - } - rookAgent := agent.New(o.context.Clientset) if err := rookAgent.Start(namespace, o.rookImage); err != nil { @@ -132,19 +118,3 @@ func (o *Operator) Run() error { } } } - -func (o *Operator) initResources() error { - kitCtx := opkit.Context{ - Clientset: o.context.Clientset, - APIExtensionClientset: o.context.APIExtensionClientset, - Interval: 500 * time.Millisecond, - Timeout: 60 * time.Second, - } - - // Create and wait for CRD resources - err := opkit.CreateCustomResources(kitCtx, o.resources) - if err != nil { - return fmt.Errorf("failed to create custom resource. %+v", err) - } - return nil -} diff --git a/tests/framework/installer/install_data.go b/tests/framework/installer/install_data.go index 3210b4d59..c44de0fb8 100644 --- a/tests/framework/installer/install_data.go +++ b/tests/framework/installer/install_data.go @@ -29,6 +29,121 @@ func NewK8sInstallData() *InstallData { return &InstallData{} } +func (i *InstallData) GetRookTPRs(namespace string) string { + return `apiVersion: extensions/v1beta1 +description: ThirdPartyResource for cluster +kind: ThirdPartyResource +metadata: + name: cluster.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for filesystem +kind: ThirdPartyResource +metadata: + name: filesystem.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for objectstore +kind: ThirdPartyResource +metadata: + name: objectstore.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for pool +kind: ThirdPartyResource +metadata: + name: pool.rook.io +versions: +- name: v1alpha1 +--- +apiVersion: extensions/v1beta1 +description: ThirdPartyResource for volumeattachment +kind: ThirdPartyResource +metadata: + name: volumeattachment.rook.io +versions: +- name: v1alpha1` +} + +func (i *InstallData) GetRookCRDs(namespace string) string { + return `apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusters.rook.io +spec: + group: rook.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: filesystems.rook.io +spec: + group: rook.io + names: + kind: Filesystem + listKind: FilesystemList + plural: filesystems + singular: filesystem + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectstores.rook.io +spec: + group: rook.io + names: + kind: ObjectStore + listKind: ObjectStoreList + plural: objectstores + singular: objectstore + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: pools.rook.io +spec: + group: rook.io + names: + kind: Pool + listKind: PoolList + plural: pools + singular: pool + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: volumeattachments.rook.io +spec: + group: rook.io + names: + kind: VolumeAttachment + listKind: VolumeAttachmentList + plural: volumeattachments + singular: volumeattachment + scope: Namespaced + version: v1alpha1 +` +} + //GetRookOperator returns rook Operator manifest func (i *InstallData) GetRookOperator(namespace string) string { @@ -67,7 +182,6 @@ rules: - apiGroups: - extensions resources: - - thirdpartyresources - deployments - daemonsets - replicasets @@ -77,16 +191,6 @@ rules: - watch - create - delete -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index 3dc1299b0..7c46a7d92 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -63,12 +63,34 @@ type InstallHelper struct { T func() *testing.T } +func (h *InstallHelper) CreateK8sRookResources(namespace string) (err error) { + var resources string + if h.k8shelper.VersionAtLeast("v1.7.0") { + logger.Info("Creating Rook CRD's") + + resources = h.installData.GetRookCRDs(namespace) + } else { + logger.Info("Creating Rook TPR's") + + resources = h.installData.GetRookTPRs(namespace) + } + + _, err = h.k8shelper.KubectlWithStdin(resources, createArgs...) + + return +} + //CreateK8sRookOperator creates rook-operator via kubectl func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) { logger.Infof("Starting Rook Operator") //creating clusterrolebinding for kubeadm env. h.k8shelper.CreateAnonSystemClusterBinding() + //creating rook resources + if err = h.CreateK8sRookResources(namespace); err != nil { + return err + } + rookOperator := h.installData.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createArgs...) @@ -253,6 +275,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT } } else { onamespace = SystemNamespace(namespace) + err := h.CreateK8sRookOperator(onamespace) if err != nil { logger.Errorf("Rook Operator not installed ,error -> %v", err) @@ -327,10 +350,10 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa logger.Infof("removing the operator from namespace %s", systemNamespace) if h.k8shelper.VersionAtLeast("v1.7.0") { - _, err = h.k8shelper.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io", "filesystems.rook.io"}) + _, err = h.k8shelper.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io", "filesystems.rook.io", "volumeattachments.rook.io"}) h.checkError(err, "cannot delete CRDs") } else { - _, err = h.k8shelper.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io", "objectstore.rook.io", "filesystem.rook.io"}) + _, err = h.k8shelper.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io", "objectstore.rook.io", "filesystem.rook.io", "volumeattachment.rook.io"}) h.checkError(err, "cannot delete TPRs") } diff --git a/tests/scripts/minikube.sh b/tests/scripts/minikube.sh index 0a99d57e9..f9140af66 100755 --- a/tests/scripts/minikube.sh +++ b/tests/scripts/minikube.sh @@ -70,7 +70,7 @@ function wait_for_ssh() { function copy_image_to_cluster() { local build_image=$1 local final_image=$2 - docker save ${build_image} | (eval $(minikube docker-env) && docker load && docker tag ${build_image} ${final_image}) + docker save ${build_image} | (eval $(minikube docker-env --shell bash) && docker load && docker tag ${build_image} ${final_image}) } # Deletes pods with 'rook-' prefix. Namespace is expected as the first argument