feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #1471 from ichekrygin/illya/crds

Browse Source 
Refactor Rook Operator CRD Initialization
This commit is contained in:
Jared Watts authored and GitHub committed 2018-04-06 16:12:06 -07:00
2 parents 457189288c c27bfc3bb4
commit 2716b7caf8
10 files changed
+524 -67
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/teardown.md
-2
View File
@@ -43,8 +43,6 @@ kubectl -n rook get cluster
## Delete the Operator and Agent
This will begin the process of all cluster resources being cleaned up, after which you can delete the rest of the deployment with the following:
```console
kubectl delete thirdpartyresources cluster.rook.io pool.rook.io objectstore.rook.io filesystem.rook.io volumeattachment.rook.io # ignore errors if on K8s 1.7+
kubectl delete crd clusters.rook.io pools.rook.io objectstores.rook.io filesystems.rook.io volumeattachments.rook.io # ignore errors if on K8s 1.5 and 1.6
kubectl delete -n rook-system daemonset rook-agent
kubectl delete -f rook-operator.yaml
kubectl delete clusterroles rook-agent
PendingReleaseNotes.md
+1
View File
@@ -5,6 +5,7 @@
## Notable Features
- Ceph tools can be run [from any rook pod](Documentation/common-issues.md#ceph-tools).
- Output from stderr will be included in error messages returned from the `exec` of external tools
- Rook-Operator no longer creates the resources CRD's or TPR's at the runtime. Instead, those resources are provisioned during deployment via `helm` or `kubectl`.
## Breaking Changes
cluster/charts/rook/templates/clusterrole.yaml
+18 -5
View File
@@ -32,7 +32,6 @@ rules:
- apiGroups:
- extensions
resources:
- thirdpartyresources
- deployments
- daemonsets
- replicasets
@@ -43,16 +42,30 @@ rules:
- create
- update
- delete
- apiGroups:
- extensions
resources:
- thirdpartyresources
resourceNames:
- cluster.rook.io
- pool.rook.io
- objectstore.rook.io
- filesystem.rook.io
- volumeattachment.rook.io
verbs:
- get
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
resourceNames:
- clusters.rook.io
- pools.rook.io
- objectstores.rook.io
- filesystems.rook.io
- volumeattachments.rook.io
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
cluster/charts/rook/templates/resources.yaml
+115
View File
@@ -0,0 +1,115 @@
{{- if .Capabilities.APIVersions.Has "apiextensions.k8s.io/v1beta1" }}
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusters.rook.io
spec:
group: rook.io
names:
kind: Cluster
listKind: ClusterList
plural: clusters
singular: cluster
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: filesystems.rook.io
spec:
group: rook.io
names:
kind: Filesystem
listKind: FilesystemList
plural: filesystems
singular: filesystem
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectstores.rook.io
spec:
group: rook.io
names:
kind: ObjectStore
listKind: ObjectStoreList
plural: objectstores
singular: objectstore
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: pools.rook.io
spec:
group: rook.io
names:
kind: Pool
listKind: PoolList
plural: pools
singular: pool
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: volumeattachments.rook.io
spec:
group: rook.io
names:
kind: VolumeAttachment
listKind: VolumeAttachmentList
plural: volumeattachments
singular: volumeattachment
scope: Namespaced
version: v1alpha1
---
{{- else }}
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for cluster
kind: ThirdPartyResource
metadata:
name: cluster.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for filesystem
kind: ThirdPartyResource
metadata:
name: filesystem.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for objectstore
kind: ThirdPartyResource
metadata:
name: objectstore.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for pool
kind: ThirdPartyResource
metadata:
name: pool.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for volumeattachment
kind: ThirdPartyResource
metadata:
name: volumeattachment.rook.io
versions:
- name: v1alpha1
---
{{- end }}
cluster/examples/kubernetes/1.6/rook-operator.yaml
+173
View File
@@ -0,0 +1,173 @@
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for cluster
kind: ThirdPartyResource
metadata:
name: cluster.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for filesystem
kind: ThirdPartyResource
metadata:
name: filesystem.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for objectstore
kind: ThirdPartyResource
metadata:
name: objectstore.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for pool
kind: ThirdPartyResource
metadata:
name: pool.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for volumeattachment
kind: ThirdPartyResource
metadata:
name: volumeattachment.rook.io
versions:
- name: v1alpha1
---
apiVersion: v1
kind: Namespace
metadata:
name: rook-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-operator
rules:
- apiGroups:
- ""
resources:
- namespaces
- serviceaccounts
- secrets
- pods
- services
- nodes
- nodes/proxy
- configmaps
- events
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
- clusterrolebindings
- roles
- rolebindings
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- delete
- apiGroups:
- rook.io
resources:
- "*"
verbs:
- "*"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-operator
namespace: rook-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-operator
namespace: rook-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-operator
subjects:
- kind: ServiceAccount
name: rook-operator
namespace: rook-system
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: rook-operator
namespace: rook-system
spec:
replicas: 1
template:
metadata:
labels:
app: rook-operator
spec:
serviceAccountName: rook-operator
containers:
- name: rook-operator
image: rook/rook:master
args: ["operator"]
env:
# To disable RBAC, uncomment the following:
# - name: RBAC_ENABLED
# value: "false"
# Rook Agent toleration. Will tolerate all taints with all keys.
# Choose between NoSchedule, PreferNoSchedule and NoExecute:
# - name: AGENT_TOLERATION
# value: "NoSchedule"
# (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
# - name: AGENT_TOLERATION_KEY
# value: "<KeyOfTheTaintToTolerate>"
# Set the path where the Rook agent can find the flex volumes
# - name: FLEXVOLUME_DIR_PATH
# value: "<PathToFlexVolumes>"
# The interval to check if every mon is in the quorum.
- name: ROOK_MON_HEALTHCHECK_INTERVAL
value: "45s"
# The duration to wait before trying to failover or remove/replace the
# current mon with a new mon (useful for compensating flapping network).
- name: ROOK_MON_OUT_TIMEOUT
value: "300s"
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
cluster/examples/kubernetes/rook-operator.yaml
+76 -16
View File
@@ -1,10 +1,76 @@
apiVersion: v1
kind: Namespace
metadata:
name: rook-system
---
kind: ClusterRole
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusters.rook.io
spec:
group: rook.io
names:
kind: Cluster
listKind: ClusterList
plural: clusters
singular: cluster
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: filesystems.rook.io
spec:
group: rook.io
names:
kind: Filesystem
listKind: FilesystemList
plural: filesystems
singular: filesystem
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectstores.rook.io
spec:
group: rook.io
names:
kind: ObjectStore
listKind: ObjectStoreList
plural: objectstores
singular: objectstore
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: pools.rook.io
spec:
group: rook.io
names:
kind: Pool
listKind: PoolList
plural: pools
singular: pool
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: volumeattachments.rook.io
spec:
group: rook.io
names:
kind: VolumeAttachment
listKind: VolumeAttachmentList
plural: volumeattachments
singular: volumeattachment
scope: Namespaced
version: v1alpha1
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-operator
rules:
@@ -33,7 +99,6 @@ rules:
- apiGroups:
- extensions
resources:
- thirdpartyresources
- deployments
- daemonsets
- replicasets
@@ -44,16 +109,6 @@ rules:
- create
- update
- delete
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
@@ -85,6 +140,11 @@ rules:
- "*"
---
apiVersion: v1
kind: Namespace
metadata:
name: rook-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-operator
pkg/operator/operator.go
-30
View File
@@ -22,7 +22,6 @@ import (
"os"
"os/signal"
"syscall"
"time"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
@@ -39,10 +38,6 @@ import (
"k8s.io/api/core/v1"
)
const (
initRetryDelay = 10 * time.Second
)
// volume provisioner constant
const (
provisionerName = "rook.io/block"
@@ -85,15 +80,6 @@ func (o *Operator) Run() error {
return fmt.Errorf("Rook operator namespace is not provided. Expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
}
for {
err := o.initResources()
if err == nil {
break
}
logger.Errorf("failed to init resources. %+v. retrying...", err)
<-time.After(initRetryDelay)
}
rookAgent := agent.New(o.context.Clientset)
if err := rookAgent.Start(namespace, o.rookImage); err != nil {
@@ -132,19 +118,3 @@ func (o *Operator) Run() error {
}
}
}
func (o *Operator) initResources() error {
kitCtx := opkit.Context{
Clientset: o.context.Clientset,
APIExtensionClientset: o.context.APIExtensionClientset,
Interval: 500 * time.Millisecond,
Timeout: 60 * time.Second,
}
// Create and wait for CRD resources
err := opkit.CreateCustomResources(kitCtx, o.resources)
if err != nil {
return fmt.Errorf("failed to create custom resource. %+v", err)
}
return nil
}
tests/framework/installer/install_data.go
+115 -11
View File
@@ -29,6 +29,121 @@ func NewK8sInstallData() *InstallData {
return &InstallData{}
}
func (i *InstallData) GetRookTPRs(namespace string) string {
return `apiVersion: extensions/v1beta1
description: ThirdPartyResource for cluster
kind: ThirdPartyResource
metadata:
name: cluster.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for filesystem
kind: ThirdPartyResource
metadata:
name: filesystem.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for objectstore
kind: ThirdPartyResource
metadata:
name: objectstore.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for pool
kind: ThirdPartyResource
metadata:
name: pool.rook.io
versions:
- name: v1alpha1
---
apiVersion: extensions/v1beta1
description: ThirdPartyResource for volumeattachment
kind: ThirdPartyResource
metadata:
name: volumeattachment.rook.io
versions:
- name: v1alpha1`
}
func (i *InstallData) GetRookCRDs(namespace string) string {
return `apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: clusters.rook.io
spec:
group: rook.io
names:
kind: Cluster
listKind: ClusterList
plural: clusters
singular: cluster
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: filesystems.rook.io
spec:
group: rook.io
names:
kind: Filesystem
listKind: FilesystemList
plural: filesystems
singular: filesystem
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectstores.rook.io
spec:
group: rook.io
names:
kind: ObjectStore
listKind: ObjectStoreList
plural: objectstores
singular: objectstore
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: pools.rook.io
spec:
group: rook.io
names:
kind: Pool
listKind: PoolList
plural: pools
singular: pool
scope: Namespaced
version: v1alpha1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: volumeattachments.rook.io
spec:
group: rook.io
names:
kind: VolumeAttachment
listKind: VolumeAttachmentList
plural: volumeattachments
singular: volumeattachment
scope: Namespaced
version: v1alpha1
`
}
//GetRookOperator returns rook Operator manifest
func (i *InstallData) GetRookOperator(namespace string) string {
@@ -67,7 +182,6 @@ rules:
- apiGroups:
- extensions
resources:
- thirdpartyresources
- deployments
- daemonsets
- replicasets
@@ -77,16 +191,6 @@ rules:
- watch
- create
- delete
- apiGroups:
- apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
- get
- list
- watch
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
tests/framework/installer/install_helper.go
+25 -2
View File
@@ -63,12 +63,34 @@ type InstallHelper struct {
T func() *testing.T
}
func (h *InstallHelper) CreateK8sRookResources(namespace string) (err error) {
var resources string
if h.k8shelper.VersionAtLeast("v1.7.0") {
logger.Info("Creating Rook CRD's")
resources = h.installData.GetRookCRDs(namespace)
} else {
logger.Info("Creating Rook TPR's")
resources = h.installData.GetRookTPRs(namespace)
}
_, err = h.k8shelper.KubectlWithStdin(resources, createArgs...)
return
}
//CreateK8sRookOperator creates rook-operator via kubectl
func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
logger.Infof("Starting Rook Operator")
//creating clusterrolebinding for kubeadm env.
h.k8shelper.CreateAnonSystemClusterBinding()
//creating rook resources
if err = h.CreateK8sRookResources(namespace); err != nil {
return err
}
rookOperator := h.installData.GetRookOperator(namespace)
_, err = h.k8shelper.KubectlWithStdin(rookOperator, createArgs...)
@@ -253,6 +275,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
}
} else {
onamespace = SystemNamespace(namespace)
err := h.CreateK8sRookOperator(onamespace)
if err != nil {
logger.Errorf("Rook Operator not installed ,error -> %v", err)
@@ -327,10 +350,10 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
logger.Infof("removing the operator from namespace %s", systemNamespace)
if h.k8shelper.VersionAtLeast("v1.7.0") {
_, err = h.k8shelper.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io", "filesystems.rook.io"})
_, err = h.k8shelper.DeleteResource([]string{"crd", "clusters.rook.io", "pools.rook.io", "objectstores.rook.io", "filesystems.rook.io", "volumeattachments.rook.io"})
h.checkError(err, "cannot delete CRDs")
} else {
_, err = h.k8shelper.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io", "objectstore.rook.io", "filesystem.rook.io"})
_, err = h.k8shelper.DeleteResource([]string{"thirdpartyresources", "cluster.rook.io", "pool.rook.io", "objectstore.rook.io", "filesystem.rook.io", "volumeattachment.rook.io"})
h.checkError(err, "cannot delete TPRs")
}
tests/scripts/minikube.sh
+1 -1
View File
@@ -70,7 +70,7 @@ function wait_for_ssh() {
function copy_image_to_cluster() {
local build_image=$1
local final_image=$2
docker save ${build_image} | (eval $(minikube docker-env) && docker load && docker tag ${build_image} ${final_image})
docker save ${build_image} | (eval $(minikube docker-env --shell bash) && docker load && docker tag ${build_image} ${final_image})
}
# Deletes pods with 'rook-' prefix. Namespace is expected as the first argument