From 6502cda3b4ec3b80800166d3bc66a81768178f63 Mon Sep 17 00:00:00 2001 From: travisn Date: Wed, 30 May 2018 21:29:07 -0700 Subject: [PATCH 1/2] refactor cluster roles for minimal rbac access Signed-off-by: travisn --- Documentation/rbac.md | 8 +- Documentation/upgrade.md | 58 +++++ PendingReleaseNotes.md | 2 + .../rook-ceph/templates/clusterrole.yaml | 45 ++-- .../templates/clusterrolebinding.yaml | 18 +- .../rook-ceph/templates/deployment.yaml | 2 +- cluster/charts/rook-ceph/templates/role.yaml | 35 +++ .../rook-ceph/templates/rolebinding.yaml | 19 ++ .../rook-ceph/templates/serviceaccount.yaml | 2 +- cluster/examples/kubernetes/ceph/cluster.yaml | 48 ++++ .../examples/kubernetes/ceph/operator.yaml | 113 +++++++-- cmd/rook/ceph/operator.go | 11 +- cmd/rook/cockroachdb/operator.go | 9 +- cmd/rook/minio/operator.go | 9 +- design/security-model.md | 26 +- pkg/apis/ceph.rook.io/v1alpha1/types.go | 3 + pkg/operator/ceph/agent/agent.go | 41 +--- pkg/operator/ceph/agent/agent_test.go | 34 +-- pkg/operator/ceph/cluster/controller.go | 2 +- pkg/operator/ceph/cluster/osd/osd.go | 25 +- pkg/operator/ceph/cluster/osd/osd_test.go | 10 +- pkg/operator/ceph/cluster/osd/pod.go | 12 +- pkg/operator/ceph/cluster/osd/pod_test.go | 19 +- pkg/operator/ceph/operator.go | 14 +- pkg/operator/ceph/operator_test.go | 2 +- pkg/operator/discover/discover.go | 13 +- pkg/operator/discover/discover_test.go | 10 +- pkg/operator/k8sutil/pod.go | 33 +-- pkg/operator/k8sutil/rbac.go | 177 -------------- pkg/operator/k8sutil/rbac_test.go | 228 ------------------ tests/framework/installer/install_data.go | 166 +++++++++++-- tests/framework/installer/install_helper.go | 55 +++-- tests/framework/utils/k8s_helper.go | 5 + .../integration/multi_cluster_deploy_test.go | 15 +- tests/longhaul/base_block_test.go | 1 - 35 files changed, 606 insertions(+), 664 deletions(-) create mode 100644 cluster/charts/rook-ceph/templates/role.yaml create mode 100644 cluster/charts/rook-ceph/templates/rolebinding.yaml delete mode 100644 pkg/operator/k8sutil/rbac.go delete mode 100644 pkg/operator/k8sutil/rbac_test.go diff --git a/Documentation/rbac.md b/Documentation/rbac.md index d89188393..aabfd5f94 100644 --- a/Documentation/rbac.md +++ b/Documentation/rbac.md @@ -95,18 +95,18 @@ kind: Namespace metadata: name: rook-system --- -# Allow the rook-ceph-agent serviceAccount to use the privileged PSP +# Allow the rook-ceph-system serviceAccount to use the privileged PSP apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: rook-ceph-agent-psp + name: rook-ceph-system-psp roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: privileged-psp-user subjects: - kind: ServiceAccount - name: rook-ceph-agent + name: rook-ceph-system namespace: rook-system ``` @@ -148,6 +148,6 @@ roleRef: name: privileged-psp-user subjects: - kind: ServiceAccount - name: rook-ceph-osd + name: rook-ceph-cluster namespace: rook-ceph ``` diff --git a/Documentation/upgrade.md b/Documentation/upgrade.md index 2a0ba2a29..e210300f7 100644 --- a/Documentation/upgrade.md +++ b/Documentation/upgrade.md @@ -145,6 +145,64 @@ kubectl delete clusterrolebindings rook-agent Now when the operator is recreated, the agent daemonset will automatically be created again with the new version. +### Operator Access to Clusters +No longer is the operator given privileges across every namespace. The operator will need privileges to manage each Ceph cluster that is configured. +The following service account, role, and role bindings are included in `cluster.yaml` when creating a new cluster. Save the following yaml as `cluster-privs.yaml`, +modifying the `namespace:` attribute of each resource if you are not using the default `rook` or `rook-system` namespaces. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cluster + namespace: rook +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster + namespace: rook +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: [ "get", "list", "watch", "create", "update", "delete" ] +--- +# Allow the operator to create resources in this cluster's namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster-mgmt + namespace: rook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-cluster-mgmt +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: rook-system +--- +# Allow the pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster + namespace: rook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cluster +subjects: +- kind: ServiceAccount + name: rook-ceph-cluster + namespace: rook +``` + +Now create the objects in the cluster: +```bash +kubectl create -f cluster-privs.yaml +``` + ### Operator The Rook operator is the management brains of the cluster, so it should be upgraded first before other components. In the event that the new version requires a migration of metadata or config, the operator is the one that would understand how to perform that migration. diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index d28f42145..c5c3d9491 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -30,6 +30,8 @@ - Namespaces: The example namespaces are now backend-specific. Instead of `rook-system` and `rook`, you will see `rook-ceph-system` and `rook-ceph`. - Volume plugins: The dynamic provisioner and flex driver are now based on `ceph.rook.io` instead of `rook.io` - Ceph container images now use CentOS 7 as a base +- Minimal privileges are configured with a new cluster role for the operator and Ceph daemons, following the new [security design](design/security-model.md). + - A role binding must be defined for each cluster to be managed by the operator. ### Removal of the API service and rookctl tool diff --git a/cluster/charts/rook-ceph/templates/clusterrole.yaml b/cluster/charts/rook-ceph/templates/clusterrole.yaml index d56b49d7c..b13e1214a 100644 --- a/cluster/charts/rook-ceph/templates/clusterrole.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrole.yaml @@ -1,27 +1,20 @@ {{- if .Values.rbacEnable }} +# The cluster role for managing all the cluster-specific resources in a namespace apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: - name: rook-ceph-operator + name: rook-ceph-cluster-mgmt labels: operator: rook storage-backend: ceph - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" rules: - apiGroups: - "" resources: - - namespaces - - serviceaccounts - secrets - pods - services - - nodes - - nodes/proxy - configmaps - - events - - persistentvolumes - - persistentvolumeclaims verbs: - get - list @@ -43,17 +36,40 @@ rules: - create - update - delete +--- +# The cluster role for managing the Rook CRDs +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph +rules: - apiGroups: - - rbac.authorization.k8s.io + - "" resources: - - clusterroles - - clusterrolebindings - - roles - - rolebindings + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy verbs: - get - list - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch - create - update - delete @@ -65,7 +81,6 @@ rules: - get - list - watch - - delete - apiGroups: - ceph.rook.io resources: diff --git a/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml b/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml index 797d06d0a..845eb6d7b 100644 --- a/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml @@ -1,8 +1,9 @@ {{- if .Values.rbacEnable }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: rook-ceph-operator + name: rook-ceph-global labels: operator: rook storage-backend: ceph @@ -10,17 +11,17 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-operator + name: rook-ceph-global subjects: - kind: ServiceAccount - name: rook-ceph-operator + name: rook-ceph-system namespace: {{ .Release.Namespace }} {{- if .Values.pspEnable }} --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: rook-ceph-operator-psp-users + name: rook-ceph-system-psp-users labels: operator: rook storage-backend: ceph @@ -28,13 +29,10 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-operator-psp-user + name: rook-ceph-system-psp-user subjects: - kind: ServiceAccount - name: rook-ceph-operator - namespace: {{ .Release.Namespace }} -- kind: ServiceAccount - name: rook-ceph-agent + name: rook-ceph-system namespace: {{ .Release.Namespace }} {{- end }} {{- end }} diff --git a/cluster/charts/rook-ceph/templates/deployment.yaml b/cluster/charts/rook-ceph/templates/deployment.yaml index 6cf3b2810..e2b90eb10 100644 --- a/cluster/charts/rook-ceph/templates/deployment.yaml +++ b/cluster/charts/rook-ceph/templates/deployment.yaml @@ -78,5 +78,5 @@ spec: resources: {{ toYaml .Values.resources | indent 10 }} {{- if .Values.rbacEnable }} - serviceAccountName: rook-ceph-operator + serviceAccountName: rook-ceph-system {{- end }} diff --git a/cluster/charts/rook-ceph/templates/role.yaml b/cluster/charts/rook-ceph/templates/role.yaml new file mode 100644 index 000000000..ae4c6f63a --- /dev/null +++ b/cluster/charts/rook-ceph/templates/role.yaml @@ -0,0 +1,35 @@ +{{- if .Values.rbacEnable }} +# The role for the operator to manage resources in the system namespace +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get + - list + - watch + - create + - update + - delete +{{- end }} \ No newline at end of file diff --git a/cluster/charts/rook-ceph/templates/rolebinding.yaml b/cluster/charts/rook-ceph/templates/rolebinding.yaml new file mode 100644 index 000000000..5ae28cba4 --- /dev/null +++ b/cluster/charts/rook-ceph/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbacEnable }} +# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-system + namespace: {{ .Release.Namespace }} + labels: + operator: rook + storage-backend: ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/cluster/charts/rook-ceph/templates/serviceaccount.yaml b/cluster/charts/rook-ceph/templates/serviceaccount.yaml index bb6ec388c..7b42de173 100644 --- a/cluster/charts/rook-ceph/templates/serviceaccount.yaml +++ b/cluster/charts/rook-ceph/templates/serviceaccount.yaml @@ -1,8 +1,8 @@ apiVersion: v1 kind: ServiceAccount metadata: + name: rook-ceph-system labels: operator: rook storage-backend: ceph chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - name: rook-ceph-operator diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 4eb546910..2d12fe820 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -3,12 +3,60 @@ kind: Namespace metadata: name: rook-ceph --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cluster + namespace: rook-ceph +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster + namespace: rook-ceph +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: [ "get", "list", "watch", "create", "update", "delete" ] +--- +# Allow the operator to create resources in this cluster's namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster-mgmt + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-cluster-mgmt +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph-system +--- +# Allow the pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cluster +subjects: +- kind: ServiceAccount + name: rook-ceph-cluster + namespace: rook-ceph +--- apiVersion: ceph.rook.io/v1alpha1 kind: Cluster metadata: name: rook-ceph namespace: rook-ceph spec: + # The service account under which to run the daemon pods in this cluster if the default account is not sufficient (OSDs) + serviceAccount: rook-ceph-cluster # The path on the host where configuration files will be persisted. If not specified, a kubernetes emptyDir will be created (not recommended). # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster. # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment. diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml index ab697b592..c6939fd9d 100644 --- a/cluster/examples/kubernetes/ceph/operator.yaml +++ b/cluster/examples/kubernetes/ceph/operator.yaml @@ -83,10 +83,11 @@ spec: scope: Namespaced version: v1alpha2 --- +# The cluster role for managing all the cluster-specific resources in a namespace apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: - name: rook-ceph-operator + name: rook-ceph-cluster-mgmt labels: operator: rook storage-backend: ceph @@ -94,17 +95,10 @@ rules: - apiGroups: - "" resources: - - namespaces - - serviceaccounts - secrets - pods - services - - nodes - - nodes/proxy - configmaps - - events - - persistentvolumes - - persistentvolumeclaims verbs: - get - list @@ -126,17 +120,75 @@ rules: - create - update - delete +--- +# The role for the operator to manage resources in the system namespace +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +rules: - apiGroups: - - rbac.authorization.k8s.io + - "" resources: - - clusterroles - - clusterrolebindings - - roles - - rolebindings + - pods + - configmaps verbs: - get - list - watch + - patch + - create + - update + - delete +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# The cluster role for managing the Rook CRDs +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch - create - update - delete @@ -148,7 +200,6 @@ rules: - get - list - watch - - delete - apiGroups: - ceph.rook.io resources: @@ -162,19 +213,39 @@ rules: verbs: - "*" --- +# The rook system service account used by the operator, agent, and discovery pods apiVersion: v1 kind: ServiceAccount metadata: - name: rook-ceph-operator + name: rook-ceph-system namespace: rook-ceph-system labels: operator: rook storage-backend: ceph --- +# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-system + namespace: rook-ceph-system + labels: + operator: rook + storage-backend: ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph-system +--- +# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: rook-ceph-operator + name: rook-ceph-global namespace: rook-ceph-system labels: operator: rook @@ -182,12 +253,13 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-operator + name: rook-ceph-global subjects: - kind: ServiceAccount - name: rook-ceph-operator + name: rook-ceph-system namespace: rook-ceph-system --- +# The deployment for the rook operator apiVersion: apps/v1beta1 kind: Deployment metadata: @@ -203,7 +275,7 @@ spec: labels: app: rook-ceph-operator spec: - serviceAccountName: rook-ceph-operator + serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator image: rook/ceph:master @@ -235,6 +307,9 @@ spec: # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027 - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS value: "false" + # The logging level for the operator: INFO | DEBUG + - name: ROOK_LOG_LEVEL + value: "INFO" # The interval to check if every mon is in the quorum. - name: ROOK_MON_HEALTHCHECK_INTERVAL value: "45s" diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go index e6c4011ce..a6569b4a5 100644 --- a/cmd/rook/ceph/operator.go +++ b/cmd/rook/ceph/operator.go @@ -69,13 +69,18 @@ func startOperator(cmd *cobra.Command, args []string) error { rook.TerminateFatal(err) } - // Using the rook-ceph-operator image to deploy other rook pods - rookImage, err := k8sutil.GetContainerImage(clientset, containerName) + // Using the current image version to deploy other rook pods + pod, err := k8sutil.GetRunningPod(clientset) + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err)) + } + + rookImage, err := k8sutil.GetContainerImage(pod, containerName) if err != nil { rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err)) } - op := operator.New(context, volumeAttachment, rookImage) + op := operator.New(context, volumeAttachment, rookImage, pod.Spec.ServiceAccountName) err = op.Run() if err != nil { rook.TerminateFatal(fmt.Errorf("failed to run operator. %+v\n", err)) diff --git a/cmd/rook/cockroachdb/operator.go b/cmd/rook/cockroachdb/operator.go index 2b893f4a0..0af1f56cd 100644 --- a/cmd/rook/cockroachdb/operator.go +++ b/cmd/rook/cockroachdb/operator.go @@ -58,8 +58,13 @@ func startOperator(cmd *cobra.Command, args []string) error { context.APIExtensionClientset = apiExtClientset context.RookClientset = rookClientset - // Using the cockroachdb-operator image to deploy other pods - rookImage, err := k8sutil.GetContainerImage(clientset, containerName) + // Using the current image version to deploy other rook pods + pod, err := k8sutil.GetRunningPod(clientset) + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err)) + } + + rookImage, err := k8sutil.GetContainerImage(pod, containerName) if err != nil { rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err)) } diff --git a/cmd/rook/minio/operator.go b/cmd/rook/minio/operator.go index 2ccb9c520..90b75849d 100644 --- a/cmd/rook/minio/operator.go +++ b/cmd/rook/minio/operator.go @@ -61,8 +61,13 @@ func startOperator(cmd *cobra.Command, args []string) error { rook.TerminateFatal(err) } - // Using the rook-operator image to deploy other rook pods - rookImage, err := k8sutil.GetContainerImage(clientset, containerName) + // Using the current image version to deploy other rook pods + pod, err := k8sutil.GetRunningPod(clientset) + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err)) + } + + rookImage, err := k8sutil.GetContainerImage(pod, containerName) if err != nil { rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err)) } diff --git a/design/security-model.md b/design/security-model.md index 761085ecb..c5279579c 100644 --- a/design/security-model.md +++ b/design/security-model.md @@ -12,7 +12,7 @@ The Rook operator currently uses a highly privileged service account with permis Today the cluster admin creates the rook system namespace, rook-operator service account and RBAC rules as follows: -``` +```yaml apiVersion: v1 kind: Namespace metadata: @@ -67,7 +67,7 @@ subjects: Once the rook operator is up and running it will automatically create the service account for the rook agent and the following RBAC rules: -``` +```yaml apiVersion: v1 kind: ServiceAccount metadata: @@ -106,7 +106,7 @@ subjects: When the cluster admin create a new Rook cluster they do so by adding a namespace and the rook cluster spec: -``` +```yaml apiVersion: v1 kind: Namespace metadata: @@ -124,7 +124,7 @@ At this point the rook operator will notice that a new rook cluster CRD showed u The `rook-api` service account and RBAC rules are as follows: -``` +```yaml apiVersion: v1 kind: ServiceAccount metadata: @@ -167,7 +167,7 @@ subjects: The `rook-ceph-osd` service account and RBAC rules are as follows: -``` +```yaml apiVersion: v1 kind: ServiceAccount metadata: @@ -203,7 +203,7 @@ subjects: Just as we do today the cluster admin is responsible for creating the `rook-system` namespace. I propose we have a single service account in this namespace and call it `rook-system` by default. The names used are inconsequential and can be set to something different by the cluster admin. -``` +```yaml apiVersion: v1 kind: Namespace metadata: @@ -218,7 +218,7 @@ metadata: The `rook-system` service account is responsible for launching all pods, services, daemonsets, etc. for Rook and should have enough privilege to do and nothing more. I've not audited all the RBAC rules but a good tool to do is [here](https://github.com/liggitt/audit2rbac). For example: -``` +```yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -252,13 +252,13 @@ subjects: namespace: rook-system ``` -Notably absent here are privileges to set other RBAC rules read cluster-wide secrets, etc. Because the admin created the `rook-system` namespace and service account they are free to set policies on them using PSP or namespace quotas. Similar to the ones defined [here](https://github.com/rook/rook/blob/master/Documentation/kubernetes.md#rbac-for-podsecuritypolicies)). +Notably absent here are privileges to set other RBAC rules and create read cluster-wide secrets and other resources. Because the admin created the `rook-system` namespace and service account they are free to set policies on them using PSP or namespace quotas. Similar to the ones defined [here](https://github.com/rook/rook/blob/master/Documentation/kubernetes.md#rbac-for-podsecuritypolicies)). Also note that while we use a `ClusterRole` for rook-system we only use a `RoleBinding` to grant it access to the `rook-system` namespace. It does not have cluster-wide privileges. When creating a Rook cluster the cluster admin will continue to define the namespace and cluster CRD as follows: -``` +```yaml apiVersion: v1 kind: Namespace metadata: @@ -274,7 +274,7 @@ metadata: In addition we will require that the cluster-admin define a service account and role binding as follows: -``` +```yaml apiVersion: v1 kind: ServiceAccount metadata: @@ -284,7 +284,7 @@ metadata: kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: rook-ceph-osd + name: rook-cluster namespace: mycluster rules: - apiGroups: [""] @@ -331,11 +331,11 @@ Also all rook pods except the rook operator pod should run using `rook-cluster` Finally, we should support running multiple rook clusters in the same namespaces. While namespaces are a great organizational unit for pods etc. they are also a unit of policy and quotas. While we can force the cluster admin to go to an approach where they need to manage multiple namespaces, we would be better off if we give the option to cluster admin decide how they use namespace. -For example, it should be possible to run rook-operator, rook-agent, and multiple independent rook clusters in a single namespace. This is going to require setting a prefix for pod names and other resources that could collide. It also means that removing a rook cluster would require enumerating each of the resources and removing them. +For example, it should be possible to run rook-operator, rook-agent, and multiple independent rook clusters in a single namespace. This is going to require setting a prefix for pod names and other resources that could collide. The following should be possible: -``` +```yaml apiVersion: v1 kind: Namespace metadata: diff --git a/pkg/apis/ceph.rook.io/v1alpha1/types.go b/pkg/apis/ceph.rook.io/v1alpha1/types.go index ee7282003..a18cf31f2 100644 --- a/pkg/apis/ceph.rook.io/v1alpha1/types.go +++ b/pkg/apis/ceph.rook.io/v1alpha1/types.go @@ -63,6 +63,9 @@ type ClusterSpec struct { // Resources set resource requests and limits Resources rook.ResourceSpec `json:"resources,omitempty"` + // The service account in which to start the cluster resources + ServiceAccount string `json:"serviceAccount,omitempty"` + // The path on the host where config and data can be persisted. DataDirHostPath string `json:"dataDirHostPath,omitempty"` diff --git a/pkg/operator/ceph/agent/agent.go b/pkg/operator/ceph/agent/agent.go index e31f94981..d8ed9d7d2 100644 --- a/pkg/operator/ceph/agent/agent.go +++ b/pkg/operator/ceph/agent/agent.go @@ -23,14 +23,9 @@ import ( "os" "github.com/coreos/pkg/capnslog" - cephv1alpha1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1alpha1" - rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" - "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" - opcluster "github.com/rook/rook/pkg/operator/ceph/cluster" "github.com/rook/rook/pkg/operator/k8sutil" "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" - "k8s.io/api/rbac/v1beta1" kserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -46,29 +41,6 @@ const ( var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-agent") -var clusterAccessRules = []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes", "nodes/proxy"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{rookv1alpha2.CustomResourceGroup}, - Resources: []string{attachment.CustomResourceNamePlural}, - Verbs: []string{"get", "list", "create", "delete", "update"}, - }, - { - APIGroups: []string{cephv1alpha1.CustomResourceGroup}, - Resources: []string{opcluster.CustomResourceNamePlural}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"get"}, - }, -} - // New creates an instance of Agent func New(clientset kubernetes.Interface) *Agent { return &Agent{ @@ -77,21 +49,16 @@ func New(clientset kubernetes.Interface) *Agent { } // Start the agent -func (a *Agent) Start(namespace, agentImage string) error { +func (a *Agent) Start(namespace, agentImage, serviceAccount string) error { - err := k8sutil.MakeClusterRole(a.clientset, namespace, agentDaemonsetName, clusterAccessRules, nil) - if err != nil { - return fmt.Errorf("failed to init RBAC for rook-ceph-agents. %+v", err) - } - - err = a.createAgentDaemonSet(namespace, agentImage) + err := a.createAgentDaemonSet(namespace, agentImage, serviceAccount) if err != nil { return fmt.Errorf("Error starting agent daemonset: %v", err) } return nil } -func (a *Agent) createAgentDaemonSet(namespace, agentImage string) error { +func (a *Agent) createAgentDaemonSet(namespace, agentImage, serviceAccount string) error { flexvolumeDirPath, source := a.discoverFlexvolumeDir() logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath) @@ -112,7 +79,7 @@ func (a *Agent) createAgentDaemonSet(namespace, agentImage string) error { }, }, Spec: v1.PodSpec{ - ServiceAccountName: agentDaemonsetName, + ServiceAccountName: serviceAccount, Containers: []v1.Container{ { Name: agentDaemonsetName, diff --git a/pkg/operator/ceph/agent/agent_test.go b/pkg/operator/ceph/agent/agent_test.go index d08b80903..f7f674cc0 100644 --- a/pkg/operator/ceph/agent/agent_test.go +++ b/pkg/operator/ceph/agent/agent_test.go @@ -58,27 +58,15 @@ func TestStartAgentDaemonset(t *testing.T) { a := New(clientset) // start a basic cluster - err := a.Start(namespace, "rook/rook:myversion") + err := a.Start(namespace, "rook/rook:myversion", "mysa") assert.Nil(t, err) - // check clusters rbac roles - _, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-ceph-agent", metav1.GetOptions{}) - assert.Nil(t, err) - - role, err := clientset.RbacV1beta1().ClusterRoles().Get("rook-ceph-agent", metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, 4, len(role.Rules)) - - binding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get("rook-ceph-agent", metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, "rook-ceph-agent", binding.Subjects[0].Name) - assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind) - // check daemonset parameters agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-ceph-agent", metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, namespace, agentDS.Namespace) assert.Equal(t, "rook-ceph-agent", agentDS.Name) + assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName) assert.True(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged) volumes := agentDS.Spec.Template.Spec.Volumes assert.Equal(t, 4, len(volumes)) @@ -117,19 +105,12 @@ func TestGetContainerImage(t *testing.T) { clientset.CoreV1().Pods("Default").Create(&pod) // start a basic cluster - image, err := k8sutil.GetContainerImage(clientset, "") + returnPod, err := k8sutil.GetRunningPod(clientset) assert.Nil(t, err) - assert.Equal(t, "rook/test", image) + assert.Equal(t, "mypod", returnPod.Name) } func TestGetContainerImageMultipleContainers(t *testing.T) { - clientset := test.New(3) - - os.Setenv(k8sutil.PodNamespaceEnvVar, "Default") - defer os.Unsetenv(k8sutil.PodNamespaceEnvVar) - - os.Setenv(k8sutil.PodNameEnvVar, "mypod") - defer os.Unsetenv(k8sutil.PodNameEnvVar) pod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -149,11 +130,11 @@ func TestGetContainerImageMultipleContainers(t *testing.T) { }, }, } - clientset.CoreV1().Pods("Default").Create(&pod) // start a basic cluster - _, err := k8sutil.GetContainerImage(clientset, "foo") + container, err := k8sutil.GetContainerImage(&pod, "foo") assert.NotNil(t, err) + assert.Equal(t, "", container) assert.Equal(t, "failed to find image for container foo", err.Error()) } @@ -192,13 +173,14 @@ func TestStartAgentDaemonsetWithToleration(t *testing.T) { a := New(clientset) // start a basic cluster - err := a.Start(namespace, "rook/test") + err := a.Start(namespace, "rook/test", "mysa") assert.Nil(t, err) // check daemonset toleration agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-ceph-agent", metav1.GetOptions{}) assert.Nil(t, err) assert.Equal(t, 1, len(agentDS.Spec.Template.Spec.Tolerations)) + assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName) assert.Equal(t, "NoSchedule", string(agentDS.Spec.Template.Spec.Tolerations[0].Effect)) assert.Equal(t, "example", string(agentDS.Spec.Template.Spec.Tolerations[0].Key)) assert.Equal(t, "Exists", string(agentDS.Spec.Template.Spec.Tolerations[0].Operator)) diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 6079484c8..d5c0a4d3b 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -613,7 +613,7 @@ func (c *cluster) createInstance(rookImage string) error { } // Start the OSDs - c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.Storage, c.Spec.DataDirHostPath, + c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath, cephv1alpha1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1alpha1.GetOSDResources(c.Spec.Resources), c.ownerRef) err = c.osds.Start() if err != nil { diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index 889592c8c..dc6802510 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -33,7 +33,6 @@ import ( "github.com/rook/rook/pkg/util/display" "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" - "k8s.io/api/rbac/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -57,14 +56,6 @@ const ( clusterAvailableSpaceReserve = 0.05 ) -var clusterAccessRules = []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, - }, -} - // Cluster keeps track of the OSDs type Cluster struct { context *clusterd.Context @@ -77,16 +68,18 @@ type Cluster struct { HostNetwork bool resources v1.ResourceRequirements ownerRef metav1.OwnerReference + serviceAccount string } // New creates an instance of the OSD manager -func New(context *clusterd.Context, namespace, version string, storageSpec rookalpha.StorageScopeSpec, +func New(context *clusterd.Context, namespace, version, serviceAccount string, storageSpec rookalpha.StorageScopeSpec, dataDirHostPath string, placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { return &Cluster{ context: context, Namespace: namespace, + serviceAccount: serviceAccount, placement: placement, Version: version, Storage: storageSpec, @@ -106,12 +99,6 @@ type OrchestrationStatus struct { func (c *Cluster) Start() error { logger.Infof("start running osds in namespace %s", c.Namespace) - // create the artifacts for the osd to work with RBAC enabled - err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, &c.ownerRef) - if err != nil { - logger.Warningf("failed to init RBAC for OSDs. %+v", err) - } - if c.Storage.UseAllNodes == false && len(c.Storage.Nodes) == 0 { logger.Warningf("useAllNodes is set to false and no nodes are specified, no OSD pods are going to be created") } @@ -135,7 +122,7 @@ func (c *Cluster) Start() error { // make a daemonset for all nodes in the cluster storeConfig := config.ToStoreConfig(c.Storage.Config) metadataDevice := config.MetadataDevice(c.Storage.Config) - ds := c.makeDaemonSet(c.Storage.Selection, storeConfig, metadataDevice, c.Storage.Location) + ds := c.makeDaemonSet(c.Storage.Selection, storeConfig, c.serviceAccount, metadataDevice, c.Storage.Location) _, err := c.context.Clientset.Extensions().DaemonSets(c.Namespace).Create(ds) if err != nil { if !errors.IsAlreadyExists(err) { @@ -182,7 +169,7 @@ func (c *Cluster) Start() error { logger.Infof("avail devices for node %s: %+v", n.Name, availDev) } // create the replicaSet that will run the OSDs for this node - rs := c.makeReplicaSet(n.Name, devicesToUse, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location) + rs := c.makeReplicaSet(n.Name, devicesToUse, n.Selection, n.Resources, storeConfig, c.serviceAccount, metadataDevice, n.Location) _, err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Create(rs) if err != nil { if !errors.IsAlreadyExists(err) { @@ -239,7 +226,7 @@ func (c *Cluster) Start() error { // trigger orchestration on the removed node by telling it not to use any storage at all. note that the directories are still passed in // so that the pod will be able to mount them and migrate data from them. rs := c.makeReplicaSet(n.Name, nil, rookalpha.Selection{DeviceFilter: "none", Directories: n.Directories}, - v1.ResourceRequirements{}, storeConfig, metadataDevice, n.Location) + v1.ResourceRequirements{}, storeConfig, c.serviceAccount, metadataDevice, n.Location) rs, err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Update(rs) if err != nil { message := fmt.Sprintf("failed to update osd replica set for removed node %s. %+v", n.Name, err) diff --git a/pkg/operator/ceph/cluster/osd/osd_test.go b/pkg/operator/ceph/cluster/osd/osd_test.go index 61a45a7da..8a5a866e7 100644 --- a/pkg/operator/ceph/cluster/osd/osd_test.go +++ b/pkg/operator/ceph/cluster/osd/osd_test.go @@ -39,7 +39,7 @@ import ( func TestStart(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // Start the first time @@ -71,7 +71,7 @@ func TestAddRemoveNode(t *testing.T) { statusMapWatcher := watch.NewFake() clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil)) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine @@ -127,7 +127,7 @@ func TestAddRemoveNode(t *testing.T) { // modify the storage spec to remove the node from the cluster storageSpec.Nodes = []rookalpha.Node{} - c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", + c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // reset the orchestration status watcher @@ -174,7 +174,7 @@ func TestAddNodeFailure(t *testing.T) { return true, nil, fmt.Errorf("mock failed to create replica set") }) - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // kick off the start of the orchestration in a goroutine @@ -195,7 +195,7 @@ func TestAddNodeFailure(t *testing.T) { func TestOrchestrationStatus(t *testing.T) { clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) // status map should not exist yet diff --git a/pkg/operator/ceph/cluster/osd/pod.go b/pkg/operator/ceph/cluster/osd/pod.go index 2526703d7..63ff4a131 100644 --- a/pkg/operator/ceph/cluster/osd/pod.go +++ b/pkg/operator/ceph/cluster/osd/pod.go @@ -41,8 +41,8 @@ const ( osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" ) -func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, storeConfig config.StoreConfig, metadataDevice, location string) *extensions.DaemonSet { - podSpec := c.podTemplateSpec(nil, selection, c.resources, storeConfig, metadataDevice, location) +func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, storeConfig config.StoreConfig, serviceAccount, metadataDevice, location string) *extensions.DaemonSet { + podSpec := c.podTemplateSpec(nil, selection, c.resources, storeConfig, serviceAccount, metadataDevice, location) return &extensions.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: appName, @@ -63,9 +63,9 @@ func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, storeConfig confi } func (c *Cluster) makeReplicaSet(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, metadataDevice, location string) *extensions.ReplicaSet { + storeConfig config.StoreConfig, serviceAccount, metadataDevice, location string) *extensions.ReplicaSet { - podSpec := c.podTemplateSpec(devices, selection, resources, storeConfig, metadataDevice, location) + podSpec := c.podTemplateSpec(devices, selection, resources, storeConfig, serviceAccount, metadataDevice, location) podSpec.Spec.NodeSelector = map[string]string{apis.LabelHostname: nodeName} replicaCount := int32(1) @@ -87,7 +87,7 @@ func (c *Cluster) makeReplicaSet(nodeName string, devices []rookalpha.Device, se } func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, metadataDevice, location string) v1.PodTemplateSpec { + storeConfig config.StoreConfig, serviceAccount, metadataDevice, location string) v1.PodTemplateSpec { // by default, the data/config dir will be an empty volume dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} if c.dataDirHostPath != "" { @@ -119,7 +119,7 @@ func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalph } podSpec := v1.PodSpec{ - ServiceAccountName: appName, + ServiceAccountName: serviceAccount, Containers: []v1.Container{c.osdContainer(devices, selection, resources, storeConfig, metadataDevice, location)}, RestartPolicy: v1.RestartPolicyAlways, Volumes: volumes, diff --git a/pkg/operator/ceph/cluster/osd/pod_test.go b/pkg/operator/ceph/cluster/osd/pod_test.go index 67f50a32a..8f1dbd24c 100644 --- a/pkg/operator/ceph/cluster/osd/pod_test.go +++ b/pkg/operator/ceph/cluster/osd/pod_test.go @@ -34,7 +34,7 @@ import ( func TestPodContainer(t *testing.T) { cluster := &Cluster{Namespace: "myosd", Version: "23"} - c := cluster.podTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "") + c := cluster.podTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", "") assert.NotNil(t, c) assert.Equal(t, 1, len(c.Spec.Containers)) container := c.Spec.Containers[0] @@ -56,16 +56,17 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "mysa", storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) devMountNeeded := deviceFilter != "" || allDevices n := c.resolveNode(storageSpec.Nodes[0]) - replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, c.serviceAccount, "", n.Location) assert.NotNil(t, replicaSet) assert.Equal(t, "rook-ceph-osd-node1", replicaSet.Name) assert.Equal(t, c.Namespace, replicaSet.Namespace) + assert.Equal(t, c.serviceAccount, replicaSet.Spec.Template.Spec.ServiceAccountName) assert.Equal(t, int32(1), *(replicaSet.Spec.Replicas)) assert.Equal(t, "node1", replicaSet.Spec.Template.Spec.NodeSelector[apis.LabelHostname]) assert.Equal(t, v1.RestartPolicyAlways, replicaSet.Spec.Template.Spec.RestartPolicy) @@ -148,11 +149,11 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.resolveNode(storageSpec.Nodes[0]) - replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", n.Location) assert.NotNil(t, replicaSet) // pod spec should have a volume for the given dir @@ -200,13 +201,13 @@ func TestStorageSpecConfig(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.resolveNode(storageSpec.Nodes[0]) storeConfig := config.ToStoreConfig(storageSpec.Nodes[0].Config) metadataDevice := config.MetadataDevice(storageSpec.Nodes[0].Config) - replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, c.Storage.Nodes[0].Resources, storeConfig, metadataDevice, n.Location) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, c.Storage.Nodes[0].Resources, storeConfig, "", metadataDevice, n.Location) assert.NotNil(t, replicaSet) container := replicaSet.Spec.Template.Spec.Containers[0] @@ -245,11 +246,11 @@ func TestHostNetwork(t *testing.T) { } clientset := fake.NewSimpleClientset() - c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.resolveNode(storageSpec.Nodes[0]) - r := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location) + r := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", n.Location) assert.NotNil(t, r) assert.Equal(t, true, r.Spec.Template.Spec.HostNetwork) diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index 1bd7cd19b..ee7965092 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -61,16 +61,17 @@ var provisionerConfigs = map[string]string{ // Operator type for managing storage type Operator struct { - context *clusterd.Context - resources []opkit.CustomResource - rookImage string + context *clusterd.Context + resources []opkit.CustomResource + rookImage string + securityAccount string // The custom resource that is global to the kubernetes cluster. // The cluster is global because you create multiple clusters in k8s clusterController *cluster.ClusterController } // New creates an operator instance -func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage string) *Operator { +func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage, securityAccount string) *Operator { clusterController := cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper) schemes := []opkit.CustomResource{cluster.ClusterResource, pool.PoolResource, object.ObjectStoreResource, @@ -80,6 +81,7 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen clusterController: clusterController, resources: schemes, rookImage: rookImage, + securityAccount: securityAccount, } } @@ -112,12 +114,12 @@ func (o *Operator) Run() error { rookAgent := agent.New(o.context.Clientset) - if err := rookAgent.Start(namespace, o.rookImage); err != nil { + if err := rookAgent.Start(namespace, o.rookImage, o.securityAccount); err != nil { return fmt.Errorf("Error starting agent daemonset: %v", err) } rookDiscover := discover.New(o.context.Clientset) - if err := rookDiscover.Start(namespace, o.rookImage); err != nil { + if err := rookDiscover.Start(namespace, o.rookImage, o.securityAccount); err != nil { return fmt.Errorf("Error starting device discovery daemonset: %v", err) } diff --git a/pkg/operator/ceph/operator_test.go b/pkg/operator/ceph/operator_test.go index 57482d22f..ae295c1a2 100644 --- a/pkg/operator/ceph/operator_test.go +++ b/pkg/operator/ceph/operator_test.go @@ -32,7 +32,7 @@ import ( func TestOperator(t *testing.T) { clientset := test.New(3) context := &clusterd.Context{Clientset: clientset} - o := New(context, &attachment.MockAttachment{}, "") + o := New(context, &attachment.MockAttachment{}, "", "") assert.NotNil(t, o) assert.NotNil(t, o.clusterController) diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index 0af4ba8f4..6c0595b3b 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -67,21 +67,16 @@ func New(clientset kubernetes.Interface) *Discover { } // Start the discover -func (d *Discover) Start(namespace, discoverImage string) error { +func (d *Discover) Start(namespace, discoverImage, securityAccount string) error { - err := k8sutil.MakeRole(d.clientset, namespace, discoverDaemonsetName, accessRules, nil) - if err != nil { - return fmt.Errorf("failed to init RBAC for rook-discover. %+v", err) - } - - err = d.createDiscoverDaemonSet(namespace, discoverImage) + err := d.createDiscoverDaemonSet(namespace, discoverImage, securityAccount) if err != nil { return fmt.Errorf("Error starting discover daemonset: %v", err) } return nil } -func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) error { +func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage, securityAccount string) error { privileged := false ds := &extensions.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ @@ -98,7 +93,7 @@ func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) erro }, }, Spec: v1.PodSpec{ - ServiceAccountName: discoverDaemonsetName, + ServiceAccountName: securityAccount, Containers: []v1.Container{ { Name: discoverDaemonsetName, diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go index 49fef4020..8f7896ffe 100644 --- a/pkg/operator/discover/discover_test.go +++ b/pkg/operator/discover/discover_test.go @@ -46,14 +46,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) { a := New(clientset) // start a basic cluster - err := a.Start(namespace, "rook/rook:myversion") - assert.Nil(t, err) - - // check clusters rbac roles - _, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-discover", metav1.GetOptions{}) - assert.Nil(t, err) - - _, err = clientset.RbacV1beta1().Roles(namespace).Get("rook-discover", metav1.GetOptions{}) + err := a.Start(namespace, "rook/rook:myversion", "mysa") assert.Nil(t, err) // check daemonset parameters @@ -61,6 +54,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) { assert.Nil(t, err) assert.Equal(t, namespace, agentDS.Namespace) assert.Equal(t, "rook-discover", agentDS.Name) + assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName) assert.False(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged) volumes := agentDS.Spec.Template.Spec.Volumes assert.Equal(t, 3, len(volumes)) diff --git a/pkg/operator/k8sutil/pod.go b/pkg/operator/k8sutil/pod.go index 1ff3a0b8b..db2cd6d98 100644 --- a/pkg/operator/k8sutil/pod.go +++ b/pkg/operator/k8sutil/pod.go @@ -93,21 +93,7 @@ func ConfigDirEnvVar() v1.EnvVar { return v1.EnvVar{Name: "ROOK_CONFIG_DIR", Value: DataDir} } -func GetContainerImage(clientset kubernetes.Interface, name string) (string, error) { - - podName := os.Getenv(PodNameEnvVar) - if podName == "" { - return "", fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file") - } - podNamespace := os.Getenv(PodNamespaceEnvVar) - if podName == "" { - return "", fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file") - } - - pod, err := clientset.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) - if err != nil { - return "", err - } +func GetContainerImage(pod *v1.Pod, name string) (string, error) { image, err := GetMatchingContainer(pod.Spec.Containers, name) if err != nil { @@ -116,6 +102,23 @@ func GetContainerImage(clientset kubernetes.Interface, name string) (string, err return image.Image, nil } +func GetRunningPod(clientset kubernetes.Interface) (*v1.Pod, error) { + podName := os.Getenv(PodNameEnvVar) + if podName == "" { + return nil, fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file") + } + podNamespace := os.Getenv(PodNamespaceEnvVar) + if podName == "" { + return nil, fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file") + } + + pod, err := clientset.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return pod, nil +} + func GetMatchingContainer(containers []v1.Container, name string) (v1.Container, error) { var result *v1.Container if len(containers) == 1 { diff --git a/pkg/operator/k8sutil/rbac.go b/pkg/operator/k8sutil/rbac.go deleted file mode 100644 index 9439df6f1..000000000 --- a/pkg/operator/k8sutil/rbac.go +++ /dev/null @@ -1,177 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package k8sutil for Kubernetes helpers. -package k8sutil - -import ( - "fmt" - "os" - - "k8s.io/api/core/v1" - "k8s.io/api/rbac/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -const ( - enableRBACEnv = "RBAC_ENABLED" -) - -func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error { - - err := makeServiceAccount(clientset, namespace, name, ownerRef) - if err != nil { - return err - } - - if !isRBACEnabled() { - return nil - } - - // Create the role if it doesn't yet exist. - // If the role already exists we have to update it. Otherwise if the permissions change during an upgrade, - // the create will fail with an error that we're changing the permissions. - role := &v1beta1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Rules: rules, - } - if ownerRef != nil { - role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef} - } - _, err = clientset.RbacV1beta1().Roles(namespace).Get(role.Name, metav1.GetOptions{}) - if errors.IsNotFound(err) { - logger.Infof("creating role %s in namespace %s", name, namespace) - _, err = clientset.RbacV1beta1().Roles(namespace).Create(role) - } else if err == nil { - logger.Infof("role %s already exists in namespace %s. updating if needed.", name, namespace) - _, err = clientset.RbacV1beta1().Roles(namespace).Update(role) - } - if err != nil { - return fmt.Errorf("failed to create/update role %s in namespace %s. %+v", name, namespace, err) - } - - binding := &v1beta1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - RoleRef: v1beta1.RoleRef{ - Name: name, - Kind: "Role", - APIGroup: "rbac.authorization.k8s.io", - }, - Subjects: []v1beta1.Subject{ - {Kind: "ServiceAccount", Name: name, Namespace: namespace}, - }, - } - if ownerRef != nil { - binding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef} - } - _, err = clientset.RbacV1beta1().RoleBindings(namespace).Create(binding) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create %s role binding in namespace %s. %+v", name, namespace, err) - } - return nil -} - -func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error { - - err := makeServiceAccount(clientset, namespace, name, ownerRef) - if err != nil { - return err - } - - if !isRBACEnabled() { - return nil - } - - // Create the cluster scoped role if it doesn't yet exist. - // If the role already exists we have to update it. Otherwise if the permissions change during an upgrade, - // the create will fail with an error that we're changing the permissions. - role := &v1beta1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Rules: rules, - } - if ownerRef != nil { - role.OwnerReferences = []metav1.OwnerReference{*ownerRef} - } - - _, err = clientset.RbacV1beta1().ClusterRoles().Get(role.Name, metav1.GetOptions{}) - if errors.IsNotFound(err) { - logger.Infof("creating cluster role %s", name) - _, err = clientset.RbacV1beta1().ClusterRoles().Create(role) - } else if err == nil { - logger.Infof("cluster role %s already exists. Updating if needed.", name) - _, err = clientset.RbacV1beta1().ClusterRoles().Update(role) - } - if err != nil { - return fmt.Errorf("failed to create/update cluster role %s. %+v", name, err) - } - - binding := &v1beta1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - RoleRef: v1beta1.RoleRef{ - Name: name, - Kind: "ClusterRole", - APIGroup: "rbac.authorization.k8s.io", - }, - Subjects: []v1beta1.Subject{ - {Kind: "ServiceAccount", Name: name, Namespace: namespace}, - }, - } - if ownerRef != nil { - binding.OwnerReferences = []metav1.OwnerReference{*ownerRef} - } - - _, err = clientset.RbacV1beta1().ClusterRoleBindings().Create(binding) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create cluster role binding %s. %+v", name, err) - } - return nil -} - -func makeServiceAccount(clientset kubernetes.Interface, namespace, name string, ownerRef *metav1.OwnerReference) error { - account := &v1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - if ownerRef != nil { - account.OwnerReferences = []metav1.OwnerReference{*ownerRef} - } - - _, err := clientset.CoreV1().ServiceAccounts(namespace).Create(account) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create %s service account in namespace %s. %+v", name, namespace, err) - } - return nil -} - -func isRBACEnabled() bool { - r := os.Getenv(enableRBACEnv) - if r == "false" { - return false - } - return true -} diff --git a/pkg/operator/k8sutil/rbac_test.go b/pkg/operator/k8sutil/rbac_test.go deleted file mode 100644 index 169908118..000000000 --- a/pkg/operator/k8sutil/rbac_test.go +++ /dev/null @@ -1,228 +0,0 @@ -/* -Copyright 2017 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package k8sutil for Kubernetes helpers. -package k8sutil - -import ( - "os" - "testing" - - "github.com/rook/rook/pkg/operator/test" - "github.com/stretchr/testify/assert" - "k8s.io/api/rbac/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestMakeRole(t *testing.T) { - clientset := test.New(1) - namespace := "myns" - name := "myapp" - - rules := []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"namespaces", "secrets", "pods", "services", "nodes", "configmaps", "events"}, - Verbs: []string{"get", "list", "watch", "create", "update"}, - }, - { - APIGroups: []string{"extensions"}, - Resources: []string{"thirdpartyresources", "deployments", "daemonsets", "replicasets"}, - Verbs: []string{"get", "list", "create", "delete"}, - }, - { - APIGroups: []string{"apiextensions.k8s.io"}, - Resources: []string{"customresourcedefinitions"}, - Verbs: []string{"get", "list", "create"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"get", "list"}, - }, - } - - ownerRef := metav1.OwnerReference{} - err := MakeRole(clientset, namespace, name, rules, &ownerRef) - - role, err := clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, name, role.Name) - assert.Equal(t, 4, len(role.Rules)) - account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, namespace, account.Namespace) - binding, err := clientset.RbacV1beta1().RoleBindings(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, name, binding.RoleRef.Name) - assert.Equal(t, "Role", binding.RoleRef.Kind) - assert.Equal(t, "rbac.authorization.k8s.io", binding.RoleRef.APIGroup) - assert.Equal(t, name, binding.Subjects[0].Name) - assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind) - - // update the rules - newRules := []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list"}, - }, - } - - err = MakeRole(clientset, namespace, name, newRules, &ownerRef) - assert.Nil(t, err) - role, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(role.Rules)) - assert.Equal(t, "", role.Rules[0].APIGroups[0]) - assert.Equal(t, 1, len(role.Rules[0].Resources)) - assert.Equal(t, 2, len(role.Rules[0].Verbs)) -} - -func TestMakeClusterRole(t *testing.T) { - clientset := test.New(1) - namespace := "myns" - name := "myapp" - - rules := []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"get"}, - }, - } - - err := MakeClusterRole(clientset, namespace, name, rules, nil) - - role, err := clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, name, role.Name) - assert.Equal(t, 2, len(role.Rules)) - account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, namespace, account.Namespace) - binding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, name, binding.RoleRef.Name) - assert.Equal(t, "ClusterRole", binding.RoleRef.Kind) - assert.Equal(t, "rbac.authorization.k8s.io", binding.RoleRef.APIGroup) - assert.Equal(t, name, binding.Subjects[0].Name) - assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind) - - // update the rules - newRules := []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list"}, - }, - } - - err = MakeClusterRole(clientset, namespace, name, newRules, nil) - assert.Nil(t, err) - role, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, 1, len(role.Rules)) - assert.Equal(t, "", role.Rules[0].APIGroups[0]) - assert.Equal(t, 1, len(role.Rules[0].Resources)) - assert.Equal(t, 2, len(role.Rules[0].Verbs)) -} - -func TestMakeRoleRBACDisabled(t *testing.T) { - os.Setenv(enableRBACEnv, "false") - defer os.Unsetenv(enableRBACEnv) - clientset := test.New(1) - namespace := "myns" - name := "myapp" - - rules := []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"namespaces", "secrets", "pods", "services", "nodes", "configmaps", "events"}, - Verbs: []string{"get", "list", "watch", "create", "update"}, - }, - { - APIGroups: []string{"extensions"}, - Resources: []string{"thirdpartyresources", "deployments", "daemonsets", "replicasets"}, - Verbs: []string{"get", "list", "create", "delete"}, - }, - { - APIGroups: []string{"apiextensions.k8s.io"}, - Resources: []string{"customresourcedefinitions"}, - Verbs: []string{"get", "list", "create"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"get", "list"}, - }, - } - ownerRef := metav1.OwnerReference{} - err := MakeRole(clientset, namespace, name, rules, &ownerRef) - - _, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) - assert.NotNil(t, err) - assert.True(t, errors.IsNotFound(err)) - - account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, namespace, account.Namespace) - - _, err = clientset.RbacV1beta1().RoleBindings(namespace).Get(name, metav1.GetOptions{}) - assert.NotNil(t, err) - assert.True(t, errors.IsNotFound(err)) -} - -func TestMakeClusterRoleRBACDisabled(t *testing.T) { - os.Setenv(enableRBACEnv, "false") - defer os.Unsetenv(enableRBACEnv) - clientset := test.New(1) - namespace := "myns" - name := "myapp" - - rules := []v1beta1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - Verbs: []string{"get"}, - }, - } - - err := MakeClusterRole(clientset, namespace, name, rules, nil) - assert.Nil(t, err) - _, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{}) - assert.NotNil(t, err) - assert.True(t, errors.IsNotFound(err)) - - account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{}) - assert.Nil(t, err) - assert.Equal(t, namespace, account.Namespace) - - _, err = clientset.RbacV1beta1().ClusterRoleBindings().Get(name, metav1.GetOptions{}) - assert.NotNil(t, err) - assert.True(t, errors.IsNotFound(err)) -} diff --git a/tests/framework/installer/install_data.go b/tests/framework/installer/install_data.go index 63ca39caf..0b96cc8b2 100644 --- a/tests/framework/installer/install_data.go +++ b/tests/framework/installer/install_data.go @@ -29,7 +29,7 @@ func NewK8sInstallData() *InstallData { return &InstallData{} } -func (i *InstallData) GetRookCRDs(namespace string) string { +func (i *InstallData) GetRookCRDs() string { return `apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -110,24 +110,54 @@ metadata: name: ` + namespace + ` --- apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole +kind: Role metadata: - name: rook-ceph-operator + name: rook-ceph-system + namespace: ` + namespace + ` + labels: + operator: rook + storage-backend: ceph +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get + - list + - watch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt + labels: + operator: rook + storage-backend: ceph rules: - apiGroups: - "" resources: - - namespaces - - serviceaccounts - secrets - pods - services - - nodes - - nodes/proxy - configmaps - - events - - persistentvolumes - - persistentvolumeclaims verbs: - get - list @@ -149,17 +179,36 @@ rules: - create - update - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph +rules: - apiGroups: - - rbac.authorization.k8s.io + - "" resources: - - clusterroles - - clusterrolebindings - - roles - - rolebindings + - pods + - nodes + - nodes/proxy verbs: - get - list - watch +- apiGroups: + - "" + resources: + - events + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - patch - create - update - delete @@ -171,7 +220,6 @@ rules: - get - list - watch - - delete - apiGroups: - ceph.rook.io resources: @@ -188,21 +236,44 @@ rules: apiVersion: v1 kind: ServiceAccount metadata: - name: rook-ceph-operator + name: rook-ceph-system + namespace: ` + namespace + ` + labels: + operator: rook + storage-backend: ceph +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-system + namespace: ` + namespace + ` + labels: + operator: rook + storage-backend: ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: +- kind: ServiceAccount + name: rook-ceph-system namespace: ` + namespace + ` --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: rook-ceph-operator + name: rook-ceph-global namespace: ` + namespace + ` + labels: + operator: rook + storage-backend: ceph roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: rook-ceph-operator + name: rook-ceph-global subjects: - kind: ServiceAccount - name: rook-ceph-operator + name: rook-ceph-system namespace: ` + namespace + ` --- apiVersion: apps/v1beta1 @@ -210,6 +281,9 @@ kind: Deployment metadata: name: rook-ceph-operator namespace: ` + namespace + ` + labels: + operator: rook + storage-backend: ceph spec: replicas: 1 template: @@ -217,7 +291,7 @@ spec: labels: app: rook-ceph-operator spec: - serviceAccountName: rook-ceph-operator + serviceAccountName: rook-ceph-system containers: - name: rook-ceph-operator image: rook/ceph:master @@ -243,6 +317,55 @@ spec: fieldPath: metadata.namespace` } +//GetRookCluster returns rook-cluster manifest +func (i *InstallData) GetClusterRoles(namespace, systemNamespace string) string { + return `apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cluster + namespace: ` + namespace + ` +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster + namespace: ` + namespace + ` +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: [ "get", "list", "watch", "create", "update", "delete" ] +--- +# Allow the operator to create resources in this cluster's namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster-mgmt + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-cluster-mgmt +subjects: +- kind: ServiceAccount + name: rook-ceph-system + namespace: ` + systemNamespace + ` +--- +# Allow the pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cluster + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cluster +subjects: +- kind: ServiceAccount + name: rook-ceph-cluster + namespace: ` + namespace +} + //GetRookCluster returns rook-cluster manifest func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string { return `apiVersion: ceph.rook.io/v1alpha1 @@ -251,6 +374,7 @@ metadata: name: ` + namespace + ` namespace: ` + namespace + ` spec: + serviceAccount: rook-ceph-cluster dataDirHostPath: ` + dataDirHostPath + ` network: hostNetwork: false diff --git a/tests/framework/installer/install_helper.go b/tests/framework/installer/install_helper.go index daa52b16a..1066205aa 100644 --- a/tests/framework/installer/install_helper.go +++ b/tests/framework/installer/install_helper.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "k8s.io/api/core/v1" "k8s.io/kubernetes/pkg/kubelet/apis" "flag" @@ -67,11 +68,11 @@ type InstallHelper struct { T func() *testing.T } -func (h *InstallHelper) CreateK8sRookResources(namespace string) (err error) { +func (h *InstallHelper) CreateK8sRookResources() (err error) { var resources string logger.Info("Creating Rook CRD's") - resources = h.installData.GetRookCRDs(namespace) + resources = h.installData.GetRookCRDs() _, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...) @@ -85,7 +86,7 @@ func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) { h.k8shelper.CreateAnonSystemClusterBinding() //creating rook resources - if err = h.CreateK8sRookResources(namespace); err != nil { + if err = h.CreateK8sRookResources(); err != nil { return err } @@ -149,17 +150,26 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) { return nil } -func (h *InstallHelper) CreateK8sRookCluster(namespace string, storeType string) (err error) { - return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", false, +func (h *InstallHelper) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) { + return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, "", false, cephv1alpha1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */) } //CreateK8sRookCluster creates rook cluster via kubectl -func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath string, +func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, dataDirHostPath string, useAllDevices bool, mon cephv1alpha1.MonSpec, startWithAllNodes bool) error { - if err := h.k8shelper.CreateNamespace(namespace); err != nil { - return err + logger.Infof("Creating namespace %s", namespace) + ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + _, err := h.k8shelper.Clientset.CoreV1().Namespaces().Create(ns) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create namespace %s. %+v", namespace, err) + } + + logger.Infof("Creating cluster roles") + roles := h.installData.GetClusterRoles(namespace, systemNamespace) + if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil { + return fmt.Errorf("Failed to create cluster roles. %+v", err) } if h.k8shelper.IsRookClientsetAvailable() { @@ -171,6 +181,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, st Namespace: namespace, }, Spec: v1alpha1.ClusterSpec{ + ServiceAccount: "rook-ceph-cluster", DataDirHostPath: dataDirHostPath, Mon: v1alpha1.MonSpec{ Count: mon.Count, @@ -236,7 +247,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, st } logger.Infof("Rook Cluster started") - err := h.k8shelper.WaitForLabeledPodToRun("app=rook-ceph-osd", namespace) + err = h.k8shelper.WaitForLabeledPodToRun("app=rook-ceph-osd", namespace) return err } @@ -288,7 +299,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT } //Create rook cluster - err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath, + err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, onamespace, storeType, dataDirHostPath, useDevices, cephv1alpha1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes) if err != nil { logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err) @@ -322,13 +333,17 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa for _, namespace := range namespaces { if !h.k8shelper.VersionAtLeast("v1.8.0") { - _, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-osd") - h.checkError(err, "cannot remove serviceaccount rook-ceph-osd") + _, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-cluster") + h.checkError(err, "cannot remove serviceaccount rook-ceph-cluster") assert.NoError(h.T(), err, "%s err -> %v", namespace, err) - err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-osd", namespace) - h.checkError(err, "rook-ceph-osd cluster role and binding cannot be deleted") - assert.NoError(h.T(), err, "rook-ceph-osd cluster role and binding cannot be deleted: %+v", err) + err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-cluster", namespace) + h.checkError(err, "rook-ceph-cluster cluster role and binding cannot be deleted") + assert.NoError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted: %+v", err) + + err = h.k8shelper.DeleteRoleBinding("rook-ceph-cluster-mgmt", namespace) + h.checkError(err, "rook-ceph-cluster-mgmt binding cannot be deleted") + assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err) } _, err = h.k8shelper.DeleteResource("-n", namespace, "cluster.ceph.rook.io", namespace) @@ -357,9 +372,13 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa } h.checkError(err, "cannot uninstall rook-operator") - h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-agent", nil) - h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-agent", nil) - h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) + h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil) + h.k8shelper.Clientset.CoreV1().ServiceAccounts(systemNamespace).Delete("rook-ceph-system", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-cluster-mgmt", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil) + h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil) + logger.Infof("done removing the operator from namespace %s", systemNamespace) } diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index c2ef37c06..3e2412e1c 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -1232,6 +1232,11 @@ func (k8sh *K8sHelper) DeleteRoleAndBindings(name, namespace string) error { return nil } +func (k8sh *K8sHelper) DeleteRoleBinding(name, namespace string) error { + _, err := k8sh.DeleteResource("rolebinding", name, "-n", namespace) + return err +} + func (k8sh *K8sHelper) ScaleStatefulSet(statefulSetName, namespace string, replicationSize int) error { args := []string{"-n", namespace, "scale", "statefulsets", statefulSetName, fmt.Sprintf("--replicas=%d", replicationSize)} _, err := k8sh.Kubectl(args...) diff --git a/tests/integration/multi_cluster_deploy_test.go b/tests/integration/multi_cluster_deploy_test.go index 498a02431..d20b93e2c 100644 --- a/tests/integration/multi_cluster_deploy_test.go +++ b/tests/integration/multi_cluster_deploy_test.go @@ -155,13 +155,14 @@ func (o MCTestOperations) SetUp() { err = o.installer.CreateK8sRookOperator(installer.SystemNamespace(o.namespace1)) require.NoError(o.T(), err) - require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", installer.SystemNamespace(o.namespace1), "Running"), + systemNamespace := installer.SystemNamespace(o.namespace1) + require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", systemNamespace, "Running"), "Make sure rook-operator is in running state") - require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-agent", installer.SystemNamespace(o.namespace1), "Running"), + require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-agent", systemNamespace, "Running"), "Make sure rook-ceph-agent is in running state") - require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", installer.SystemNamespace(o.namespace1), "Running"), + require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", systemNamespace, "Running"), "Make sure rook-discover is in running state") time.Sleep(10 * time.Second) @@ -170,8 +171,8 @@ func (o MCTestOperations) SetUp() { logger.Infof("starting two clusters in parallel") errCh1 := make(chan error, 1) errCh2 := make(chan error, 1) - go o.startCluster(o.namespace1, "bluestore", errCh1) - go o.startCluster(o.namespace2, "filestore", errCh2) + go o.startCluster(o.namespace1, systemNamespace, "bluestore", errCh1) + go o.startCluster(o.namespace2, systemNamespace, "filestore", errCh2) require.NoError(o.T(), <-errCh1) require.NoError(o.T(), <-errCh2) logger.Infof("finished starting clusters") @@ -194,9 +195,9 @@ func (o MCTestOperations) TearDown() { o.installer.UninstallRookFromMultipleNS(false, installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2) } -func (o MCTestOperations) startCluster(namespace, store string, errCh chan error) { +func (o MCTestOperations) startCluster(namespace, systemNamespace, store string, errCh chan error) { logger.Infof("starting cluster %s", namespace) - if err := o.installer.CreateK8sRookCluster(namespace, store); err != nil { + if err := o.installer.CreateK8sRookCluster(namespace, systemNamespace, store); err != nil { errCh <- fmt.Errorf("failed to create cluster %s. %+v", namespace, err) return } diff --git a/tests/longhaul/base_block_test.go b/tests/longhaul/base_block_test.go index 29a7f8898..f512447fb 100644 --- a/tests/longhaul/base_block_test.go +++ b/tests/longhaul/base_block_test.go @@ -163,7 +163,6 @@ func (o BaseLoadTestOperations) SetUp() { true /* startWithAllNodes */) require.NoError(o.T(), err) require.True(o.T(), isRookInstalled) - } // Enable chaos monkey if enable_chaos flag is present From 77cd8783602924f8dd4eac48ac98521712d7a254 Mon Sep 17 00:00:00 2001 From: travisn Date: Wed, 13 Jun 2018 17:07:23 -0700 Subject: [PATCH 2/2] set default service account for the osd Signed-off-by: travisn --- Documentation/ceph-cluster-crd.md | 6 ++++++ cluster/examples/kubernetes/ceph/cluster.yaml | 4 ++-- pkg/apis/ceph.rook.io/v1alpha1/types.go | 2 +- pkg/operator/ceph/cluster/osd/osd.go | 13 ++++++++++--- pkg/operator/ceph/cluster/osd/pod.go | 12 ++++++------ pkg/operator/ceph/cluster/osd/pod_test.go | 10 +++++----- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 70adf0b8f..4e27a51a7 100644 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -23,6 +23,7 @@ Settings can be specified at the global level to apply to the cluster as a whole If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir). - `dashboard`: Settings for the Ceph dashboard. To view the dashboard in your browser see the [dashboard guide](ceph-dashboard.md). - `enabled`: Whether to enable the dashboard to view cluster status +- `serviceAccount`: The service account under which the OSD pods will run that will give access to ConfigMaps in the cluster's namespace. If not set, the default of `rook-ceph-cluster` will be used. - `network`: The network settings for the cluster - `hostNetwork`: uses network of the hosts instead of using the SDN below the containers. - `mon`: contains mon related options [mon settings](#mon-settings) @@ -128,6 +129,7 @@ spec: dataDirHostPath: /var/lib/rook dashboard: enabled: true + serviceAccount: rook-ceph-cluster # cluster level storage configuration and selection storage: useAllNodes: true @@ -157,6 +159,7 @@ metadata: namespace: rook-ceph spec: dataDirHostPath: /var/lib/rook + serviceAccount: rook-ceph-cluster # cluster level storage configuration and selection storage: useAllNodes: false @@ -198,6 +201,7 @@ metadata: namespace: rook-ceph spec: dataDirHostPath: /var/lib/rook + serviceAccount: rook-ceph-cluster # cluster level storage configuration and selection storage: useAllNodes: false @@ -233,6 +237,7 @@ metadata: namespace: rook-ceph spec: dataDirHostPath: /var/lib/rook + serviceAccount: rook-ceph-cluster placement: all: nodeAffinity: @@ -276,6 +281,7 @@ metadata: namespace: rook-ceph spec: dataDirHostPath: /var/lib/rook + serviceAccount: rook-ceph-cluster # cluster level resource requests/limits configuration resources: storage: diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 2d12fe820..9c0280627 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -55,12 +55,12 @@ metadata: name: rook-ceph namespace: rook-ceph spec: - # The service account under which to run the daemon pods in this cluster if the default account is not sufficient (OSDs) - serviceAccount: rook-ceph-cluster # The path on the host where configuration files will be persisted. If not specified, a kubernetes emptyDir will be created (not recommended). # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster. # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment. dataDirHostPath: /var/lib/rook + # The service account under which to run the daemon pods in this cluster if the default account is not sufficient (OSDs) + serviceAccount: rook-ceph-cluster # set the amount of mons to be started mon: count: 3 diff --git a/pkg/apis/ceph.rook.io/v1alpha1/types.go b/pkg/apis/ceph.rook.io/v1alpha1/types.go index a18cf31f2..95d571cc1 100644 --- a/pkg/apis/ceph.rook.io/v1alpha1/types.go +++ b/pkg/apis/ceph.rook.io/v1alpha1/types.go @@ -63,7 +63,7 @@ type ClusterSpec struct { // Resources set resource requests and limits Resources rook.ResourceSpec `json:"resources,omitempty"` - // The service account in which to start the cluster resources + // The service account in which to start the cluster resources if the default is not sufficient (OSD pods) ServiceAccount string `json:"serviceAccount,omitempty"` // The path on the host where config and data can be persisted. diff --git a/pkg/operator/ceph/cluster/osd/osd.go b/pkg/operator/ceph/cluster/osd/osd.go index dc6802510..ffcdb075a 100644 --- a/pkg/operator/ceph/cluster/osd/osd.go +++ b/pkg/operator/ceph/cluster/osd/osd.go @@ -54,6 +54,7 @@ const ( appName = "rook-ceph-osd" appNameFmt = "rook-ceph-osd-%s" clusterAvailableSpaceReserve = 0.05 + defaultServiceAccountName = "rook-ceph-cluster" ) // Cluster keeps track of the OSDs @@ -76,6 +77,12 @@ func New(context *clusterd.Context, namespace, version, serviceAccount string, s dataDirHostPath string, placement rookalpha.Placement, hostNetwork bool, resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster { + if serviceAccount == "" { + // if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient. + serviceAccount = defaultServiceAccountName + logger.Infof("setting the osd pods to use the service account name: %s", serviceAccount) + } + return &Cluster{ context: context, Namespace: namespace, @@ -122,7 +129,7 @@ func (c *Cluster) Start() error { // make a daemonset for all nodes in the cluster storeConfig := config.ToStoreConfig(c.Storage.Config) metadataDevice := config.MetadataDevice(c.Storage.Config) - ds := c.makeDaemonSet(c.Storage.Selection, storeConfig, c.serviceAccount, metadataDevice, c.Storage.Location) + ds := c.makeDaemonSet(c.Storage.Selection, storeConfig, metadataDevice, c.Storage.Location) _, err := c.context.Clientset.Extensions().DaemonSets(c.Namespace).Create(ds) if err != nil { if !errors.IsAlreadyExists(err) { @@ -169,7 +176,7 @@ func (c *Cluster) Start() error { logger.Infof("avail devices for node %s: %+v", n.Name, availDev) } // create the replicaSet that will run the OSDs for this node - rs := c.makeReplicaSet(n.Name, devicesToUse, n.Selection, n.Resources, storeConfig, c.serviceAccount, metadataDevice, n.Location) + rs := c.makeReplicaSet(n.Name, devicesToUse, n.Selection, n.Resources, storeConfig, metadataDevice, n.Location) _, err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Create(rs) if err != nil { if !errors.IsAlreadyExists(err) { @@ -226,7 +233,7 @@ func (c *Cluster) Start() error { // trigger orchestration on the removed node by telling it not to use any storage at all. note that the directories are still passed in // so that the pod will be able to mount them and migrate data from them. rs := c.makeReplicaSet(n.Name, nil, rookalpha.Selection{DeviceFilter: "none", Directories: n.Directories}, - v1.ResourceRequirements{}, storeConfig, c.serviceAccount, metadataDevice, n.Location) + v1.ResourceRequirements{}, storeConfig, metadataDevice, n.Location) rs, err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Update(rs) if err != nil { message := fmt.Sprintf("failed to update osd replica set for removed node %s. %+v", n.Name, err) diff --git a/pkg/operator/ceph/cluster/osd/pod.go b/pkg/operator/ceph/cluster/osd/pod.go index 63ff4a131..fb2d01eaa 100644 --- a/pkg/operator/ceph/cluster/osd/pod.go +++ b/pkg/operator/ceph/cluster/osd/pod.go @@ -41,8 +41,8 @@ const ( osdMetadataDeviceEnvVarName = "ROOK_METADATA_DEVICE" ) -func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, storeConfig config.StoreConfig, serviceAccount, metadataDevice, location string) *extensions.DaemonSet { - podSpec := c.podTemplateSpec(nil, selection, c.resources, storeConfig, serviceAccount, metadataDevice, location) +func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, storeConfig config.StoreConfig, metadataDevice, location string) *extensions.DaemonSet { + podSpec := c.podTemplateSpec(nil, selection, c.resources, storeConfig, metadataDevice, location) return &extensions.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: appName, @@ -63,9 +63,9 @@ func (c *Cluster) makeDaemonSet(selection rookalpha.Selection, storeConfig confi } func (c *Cluster) makeReplicaSet(nodeName string, devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, serviceAccount, metadataDevice, location string) *extensions.ReplicaSet { + storeConfig config.StoreConfig, metadataDevice, location string) *extensions.ReplicaSet { - podSpec := c.podTemplateSpec(devices, selection, resources, storeConfig, serviceAccount, metadataDevice, location) + podSpec := c.podTemplateSpec(devices, selection, resources, storeConfig, metadataDevice, location) podSpec.Spec.NodeSelector = map[string]string{apis.LabelHostname: nodeName} replicaCount := int32(1) @@ -87,7 +87,7 @@ func (c *Cluster) makeReplicaSet(nodeName string, devices []rookalpha.Device, se } func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalpha.Selection, resources v1.ResourceRequirements, - storeConfig config.StoreConfig, serviceAccount, metadataDevice, location string) v1.PodTemplateSpec { + storeConfig config.StoreConfig, metadataDevice, location string) v1.PodTemplateSpec { // by default, the data/config dir will be an empty volume dataDirSource := v1.VolumeSource{EmptyDir: &v1.EmptyDirVolumeSource{}} if c.dataDirHostPath != "" { @@ -119,7 +119,7 @@ func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalph } podSpec := v1.PodSpec{ - ServiceAccountName: serviceAccount, + ServiceAccountName: c.serviceAccount, Containers: []v1.Container{c.osdContainer(devices, selection, resources, storeConfig, metadataDevice, location)}, RestartPolicy: v1.RestartPolicyAlways, Volumes: volumes, diff --git a/pkg/operator/ceph/cluster/osd/pod_test.go b/pkg/operator/ceph/cluster/osd/pod_test.go index 8f1dbd24c..ab86f7887 100644 --- a/pkg/operator/ceph/cluster/osd/pod_test.go +++ b/pkg/operator/ceph/cluster/osd/pod_test.go @@ -34,7 +34,7 @@ import ( func TestPodContainer(t *testing.T) { cluster := &Cluster{Namespace: "myosd", Version: "23"} - c := cluster.podTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", "") + c := cluster.podTemplateSpec([]rookalpha.Device{}, rookalpha.Selection{}, v1.ResourceRequirements{}, config.StoreConfig{}, "", "") assert.NotNil(t, c) assert.Equal(t, 1, len(c.Spec.Containers)) container := c.Spec.Containers[0] @@ -62,7 +62,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool) devMountNeeded := deviceFilter != "" || allDevices n := c.resolveNode(storageSpec.Nodes[0]) - replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, c.serviceAccount, "", n.Location) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location) assert.NotNil(t, replicaSet) assert.Equal(t, "rook-ceph-osd-node1", replicaSet.Name) assert.Equal(t, c.Namespace, replicaSet.Namespace) @@ -153,7 +153,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) { storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.resolveNode(storageSpec.Nodes[0]) - replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", n.Location) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location) assert.NotNil(t, replicaSet) // pod spec should have a volume for the given dir @@ -207,7 +207,7 @@ func TestStorageSpecConfig(t *testing.T) { n := c.resolveNode(storageSpec.Nodes[0]) storeConfig := config.ToStoreConfig(storageSpec.Nodes[0].Config) metadataDevice := config.MetadataDevice(storageSpec.Nodes[0].Config) - replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, c.Storage.Nodes[0].Resources, storeConfig, "", metadataDevice, n.Location) + replicaSet := c.makeReplicaSet(n.Name, n.Devices, n.Selection, c.Storage.Nodes[0].Resources, storeConfig, metadataDevice, n.Location) assert.NotNil(t, replicaSet) container := replicaSet.Spec.Template.Spec.Containers[0] @@ -250,7 +250,7 @@ func TestHostNetwork(t *testing.T) { storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{}) n := c.resolveNode(storageSpec.Nodes[0]) - r := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", "", n.Location) + r := c.makeReplicaSet(n.Name, n.Devices, n.Selection, v1.ResourceRequirements{}, config.StoreConfig{}, "", n.Location) assert.NotNil(t, r) assert.Equal(t, true, r.Spec.Template.Spec.HostNetwork)