Merge pull request #1736 from travisn/min-rbac-security
Refactor cluster roles for minimal rbac access
This commit is contained in:
36 files changed
+606
-651
No files matched your search
@@ -23,6 +23,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
|
||||
If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir).
|
||||
- `dashboard`: Settings for the Ceph dashboard. To view the dashboard in your browser see the [dashboard guide](ceph-dashboard.md).
|
||||
- `enabled`: Whether to enable the dashboard to view cluster status
|
||||
- `serviceAccount`: The service account under which the OSD pods will run that will give access to ConfigMaps in the cluster's namespace. If not set, the default of `rook-ceph-cluster` will be used.
|
||||
- `network`: The network settings for the cluster
|
||||
- `hostNetwork`: uses network of the hosts instead of using the SDN below the containers.
|
||||
- `mon`: contains mon related options [mon settings](#mon-settings)
|
||||
@@ -128,6 +129,7 @@ spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
dashboard:
|
||||
enabled: true
|
||||
serviceAccount: rook-ceph-cluster
|
||||
# cluster level storage configuration and selection
|
||||
storage:
|
||||
useAllNodes: true
|
||||
@@ -157,6 +159,7 @@ metadata:
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
# cluster level storage configuration and selection
|
||||
storage:
|
||||
useAllNodes: false
|
||||
@@ -198,6 +201,7 @@ metadata:
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
# cluster level storage configuration and selection
|
||||
storage:
|
||||
useAllNodes: false
|
||||
@@ -233,6 +237,7 @@ metadata:
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
placement:
|
||||
all:
|
||||
nodeAffinity:
|
||||
@@ -276,6 +281,7 @@ metadata:
|
||||
namespace: rook-ceph
|
||||
spec:
|
||||
dataDirHostPath: /var/lib/rook
|
||||
serviceAccount: rook-ceph-cluster
|
||||
# cluster level resource requests/limits configuration
|
||||
resources:
|
||||
storage:
|
||||
|
||||
@@ -95,18 +95,18 @@ kind: Namespace
|
||||
metadata:
|
||||
name: rook-system
|
||||
---
|
||||
# Allow the rook-ceph-agent serviceAccount to use the privileged PSP
|
||||
# Allow the rook-ceph-system serviceAccount to use the privileged PSP
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: rook-ceph-agent-psp
|
||||
name: rook-ceph-system-psp
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: privileged-psp-user
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-agent
|
||||
name: rook-ceph-system
|
||||
namespace: rook-system
|
||||
```
|
||||
|
||||
@@ -148,6 +148,6 @@ roleRef:
|
||||
name: privileged-psp-user
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-osd
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook-ceph
|
||||
```
|
||||
@@ -145,6 +145,64 @@ kubectl delete clusterrolebindings rook-agent
|
||||
|
||||
Now when the operator is recreated, the agent daemonset will automatically be created again with the new version.
|
||||
|
||||
### Operator Access to Clusters
|
||||
No longer is the operator given privileges across every namespace. The operator will need privileges to manage each Ceph cluster that is configured.
|
||||
The following service account, role, and role bindings are included in `cluster.yaml` when creating a new cluster. Save the following yaml as `cluster-privs.yaml`,
|
||||
modifying the `namespace:` attribute of each resource if you are not using the default `rook` or `rook-system` namespaces.
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
|
||||
---
|
||||
# Allow the operator to create resources in this cluster's namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster-mgmt
|
||||
namespace: rook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-cluster-mgmt
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: rook-system
|
||||
---
|
||||
# Allow the pods in this namespace to work with configmaps
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cluster
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook
|
||||
```
|
||||
|
||||
Now create the objects in the cluster:
|
||||
```bash
|
||||
kubectl create -f cluster-privs.yaml
|
||||
```
|
||||
|
||||
### Operator
|
||||
The Rook operator is the management brains of the cluster, so it should be upgraded first before other components.
|
||||
In the event that the new version requires a migration of metadata or config, the operator is the one that would understand how to perform that migration.
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
- Namespaces: The example namespaces are now backend-specific. Instead of `rook-system` and `rook`, you will see `rook-ceph-system` and `rook-ceph`.
|
||||
- Volume plugins: The dynamic provisioner and flex driver are now based on `ceph.rook.io` instead of `rook.io`
|
||||
- Ceph container images now use CentOS 7 as a base
|
||||
- Minimal privileges are configured with a new cluster role for the operator and Ceph daemons, following the new [security design](design/security-model.md).
|
||||
- A role binding must be defined for each cluster to be managed by the operator.
|
||||
|
||||
### Removal of the API service and rookctl tool
|
||||
|
||||
|
||||
@@ -1,27 +1,20 @@
|
||||
{{- if .Values.rbacEnable }}
|
||||
# The cluster role for managing all the cluster-specific resources in a namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-cluster-mgmt
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- namespaces
|
||||
- serviceaccounts
|
||||
- secrets
|
||||
- pods
|
||||
- services
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- configmaps
|
||||
- events
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
@@ -43,17 +36,40 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
# The cluster role for managing the Rook CRDs
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-global
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- rbac.authorization.k8s.io
|
||||
- ""
|
||||
resources:
|
||||
- clusterroles
|
||||
- clusterrolebindings
|
||||
- roles
|
||||
- rolebindings
|
||||
# Pod access is needed for fencing
|
||||
- pods
|
||||
# Node access is needed for determining nodes where mons should run
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- events
|
||||
# PVs and PVCs are managed by the Rook provisioner
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
@@ -65,7 +81,6 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ceph.rook.io
|
||||
resources:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
{{- if .Values.rbacEnable }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-global
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
@@ -10,17 +11,17 @@ metadata:
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-global
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-system
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- if .Values.pspEnable }}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: rook-ceph-operator-psp-users
|
||||
name: rook-ceph-system-psp-users
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
@@ -28,13 +29,10 @@ metadata:
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-operator-psp-user
|
||||
name: rook-ceph-system-psp-user
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-operator
|
||||
namespace: {{ .Release.Namespace }}
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-agent
|
||||
name: rook-ceph-system
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -78,5 +78,5 @@ spec:
|
||||
resources:
|
||||
{{ toYaml .Values.resources | indent 10 }}
|
||||
{{- if .Values.rbacEnable }}
|
||||
serviceAccountName: rook-ceph-operator
|
||||
serviceAccountName: rook-ceph-system
|
||||
{{- end }}
|
||||
@@ -0,0 +1,35 @@
|
||||
{{- if .Values.rbacEnable }}
|
||||
# The role for the operator to manage resources in the system namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- extensions
|
||||
resources:
|
||||
- daemonsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
{{- end }}
|
||||
@@ -0,0 +1,19 @@
|
||||
{{- if .Values.rbacEnable }}
|
||||
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
@@ -1,8 +1,8 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
|
||||
name: rook-ceph-operator
|
||||
@@ -3,6 +3,52 @@ kind: Namespace
|
||||
metadata:
|
||||
name: rook-ceph
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook-ceph
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook-ceph
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
|
||||
---
|
||||
# Allow the operator to create resources in this cluster's namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster-mgmt
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-cluster-mgmt
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
# Allow the pods in this namespace to work with configmaps
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cluster
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cluster
|
||||
namespace: rook-ceph
|
||||
---
|
||||
apiVersion: ceph.rook.io/v1alpha1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
@@ -13,6 +59,8 @@ spec:
|
||||
# Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
|
||||
# In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
|
||||
dataDirHostPath: /var/lib/rook
|
||||
# The service account under which to run the daemon pods in this cluster if the default account is not sufficient (OSDs)
|
||||
serviceAccount: rook-ceph-cluster
|
||||
# set the amount of mons to be started
|
||||
mon:
|
||||
count: 3
|
||||
|
||||
@@ -83,10 +83,11 @@ spec:
|
||||
scope: Namespaced
|
||||
version: v1alpha2
|
||||
---
|
||||
# The cluster role for managing all the cluster-specific resources in a namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-cluster-mgmt
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
@@ -94,17 +95,10 @@ rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- namespaces
|
||||
- serviceaccounts
|
||||
- secrets
|
||||
- pods
|
||||
- services
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- configmaps
|
||||
- events
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
@@ -126,17 +120,75 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
# The role for the operator to manage resources in the system namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- rbac.authorization.k8s.io
|
||||
- ""
|
||||
resources:
|
||||
- clusterroles
|
||||
- clusterrolebindings
|
||||
- roles
|
||||
- rolebindings
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- extensions
|
||||
resources:
|
||||
- daemonsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
# The cluster role for managing the Rook CRDs
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-global
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
# Pod access is needed for fencing
|
||||
- pods
|
||||
# Node access is needed for determining nodes where mons should run
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- events
|
||||
# PVs and PVCs are managed by the Rook provisioner
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
@@ -148,7 +200,6 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ceph.rook.io
|
||||
resources:
|
||||
@@ -162,19 +213,39 @@ rules:
|
||||
verbs:
|
||||
- "*"
|
||||
---
|
||||
# The rook system service account used by the operator, agent, and discovery pods
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
---
|
||||
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-global
|
||||
namespace: rook-ceph-system
|
||||
labels:
|
||||
operator: rook
|
||||
@@ -182,12 +253,13 @@ metadata:
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-global
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-system
|
||||
namespace: rook-ceph-system
|
||||
---
|
||||
# The deployment for the rook operator
|
||||
apiVersion: apps/v1beta1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@@ -203,7 +275,7 @@ spec:
|
||||
labels:
|
||||
app: rook-ceph-operator
|
||||
spec:
|
||||
serviceAccountName: rook-ceph-operator
|
||||
serviceAccountName: rook-ceph-system
|
||||
containers:
|
||||
- name: rook-ceph-operator
|
||||
image: rook/ceph:master
|
||||
@@ -235,6 +307,9 @@ spec:
|
||||
# which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
|
||||
- name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
|
||||
value: "false"
|
||||
# The logging level for the operator: INFO | DEBUG
|
||||
- name: ROOK_LOG_LEVEL
|
||||
value: "INFO"
|
||||
# The interval to check if every mon is in the quorum.
|
||||
- name: ROOK_MON_HEALTHCHECK_INTERVAL
|
||||
value: "45s"
|
||||
|
||||
@@ -69,13 +69,18 @@ func startOperator(cmd *cobra.Command, args []string) error {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
|
||||
// Using the rook-ceph-operator image to deploy other rook pods
|
||||
rookImage, err := k8sutil.GetContainerImage(clientset, containerName)
|
||||
// Using the current image version to deploy other rook pods
|
||||
pod, err := k8sutil.GetRunningPod(clientset)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
|
||||
}
|
||||
|
||||
rookImage, err := k8sutil.GetContainerImage(pod, containerName)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
|
||||
}
|
||||
|
||||
op := operator.New(context, volumeAttachment, rookImage)
|
||||
op := operator.New(context, volumeAttachment, rookImage, pod.Spec.ServiceAccountName)
|
||||
err = op.Run()
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to run operator. %+v\n", err))
|
||||
|
||||
@@ -58,8 +58,13 @@ func startOperator(cmd *cobra.Command, args []string) error {
|
||||
context.APIExtensionClientset = apiExtClientset
|
||||
context.RookClientset = rookClientset
|
||||
|
||||
// Using the cockroachdb-operator image to deploy other pods
|
||||
rookImage, err := k8sutil.GetContainerImage(clientset, containerName)
|
||||
// Using the current image version to deploy other rook pods
|
||||
pod, err := k8sutil.GetRunningPod(clientset)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
|
||||
}
|
||||
|
||||
rookImage, err := k8sutil.GetContainerImage(pod, containerName)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
|
||||
}
|
||||
|
||||
@@ -61,8 +61,13 @@ func startOperator(cmd *cobra.Command, args []string) error {
|
||||
rook.TerminateFatal(err)
|
||||
}
|
||||
|
||||
// Using the rook-operator image to deploy other rook pods
|
||||
rookImage, err := k8sutil.GetContainerImage(clientset, containerName)
|
||||
// Using the current image version to deploy other rook pods
|
||||
pod, err := k8sutil.GetRunningPod(clientset)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
|
||||
}
|
||||
|
||||
rookImage, err := k8sutil.GetContainerImage(pod, containerName)
|
||||
if err != nil {
|
||||
rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
|
||||
}
|
||||
|
||||
+13
-13
@@ -12,7 +12,7 @@ The Rook operator currently uses a highly privileged service account with permis
|
||||
|
||||
Today the cluster admin creates the rook system namespace, rook-operator service account and RBAC rules as follows:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
@@ -67,7 +67,7 @@ subjects:
|
||||
|
||||
Once the rook operator is up and running it will automatically create the service account for the rook agent and the following RBAC rules:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
@@ -106,7 +106,7 @@ subjects:
|
||||
|
||||
When the cluster admin create a new Rook cluster they do so by adding a namespace and the rook cluster spec:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
@@ -124,7 +124,7 @@ At this point the rook operator will notice that a new rook cluster CRD showed u
|
||||
|
||||
The `rook-api` service account and RBAC rules are as follows:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
@@ -167,7 +167,7 @@ subjects:
|
||||
|
||||
The `rook-ceph-osd` service account and RBAC rules are as follows:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
@@ -203,7 +203,7 @@ subjects:
|
||||
|
||||
Just as we do today the cluster admin is responsible for creating the `rook-system` namespace. I propose we have a single service account in this namespace and call it `rook-system` by default. The names used are inconsequential and can be set to something different by the cluster admin.
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
@@ -218,7 +218,7 @@ metadata:
|
||||
|
||||
The `rook-system` service account is responsible for launching all pods, services, daemonsets, etc. for Rook and should have enough privilege to do and nothing more. I've not audited all the RBAC rules but a good tool to do is [here](https://github.com/liggitt/audit2rbac). For example:
|
||||
|
||||
```
|
||||
```yaml
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
@@ -252,13 +252,13 @@ subjects:
|
||||
namespace: rook-system
|
||||
```
|
||||
|
||||
Notably absent here are privileges to set other RBAC rules read cluster-wide secrets, etc. Because the admin created the `rook-system` namespace and service account they are free to set policies on them using PSP or namespace quotas. Similar to the ones defined [here](https://github.com/rook/rook/blob/master/Documentation/kubernetes.md#rbac-for-podsecuritypolicies)).
|
||||
Notably absent here are privileges to set other RBAC rules and create read cluster-wide secrets and other resources. Because the admin created the `rook-system` namespace and service account they are free to set policies on them using PSP or namespace quotas. Similar to the ones defined [here](https://github.com/rook/rook/blob/master/Documentation/kubernetes.md#rbac-for-podsecuritypolicies)).
|
||||
|
||||
Also note that while we use a `ClusterRole` for rook-system we only use a `RoleBinding` to grant it access to the `rook-system` namespace. It does not have cluster-wide privileges.
|
||||
|
||||
When creating a Rook cluster the cluster admin will continue to define the namespace and cluster CRD as follows:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
@@ -274,7 +274,7 @@ metadata:
|
||||
|
||||
In addition we will require that the cluster-admin define a service account and role binding as follows:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
@@ -284,7 +284,7 @@ metadata:
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-osd
|
||||
name: rook-cluster
|
||||
namespace: mycluster
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
@@ -331,11 +331,11 @@ Also all rook pods except the rook operator pod should run using `rook-cluster`
|
||||
|
||||
Finally, we should support running multiple rook clusters in the same namespaces. While namespaces are a great organizational unit for pods etc. they are also a unit of policy and quotas. While we can force the cluster admin to go to an approach where they need to manage multiple namespaces, we would be better off if we give the option to cluster admin decide how they use namespace.
|
||||
|
||||
For example, it should be possible to run rook-operator, rook-agent, and multiple independent rook clusters in a single namespace. This is going to require setting a prefix for pod names and other resources that could collide. It also means that removing a rook cluster would require enumerating each of the resources and removing them.
|
||||
For example, it should be possible to run rook-operator, rook-agent, and multiple independent rook clusters in a single namespace. This is going to require setting a prefix for pod names and other resources that could collide.
|
||||
|
||||
The following should be possible:
|
||||
|
||||
```
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
|
||||
@@ -63,6 +63,9 @@ type ClusterSpec struct {
|
||||
// Resources set resource requests and limits
|
||||
Resources rook.ResourceSpec `json:"resources,omitempty"`
|
||||
|
||||
// The service account in which to start the cluster resources if the default is not sufficient (OSD pods)
|
||||
ServiceAccount string `json:"serviceAccount,omitempty"`
|
||||
|
||||
// The path on the host where config and data can be persisted.
|
||||
DataDirHostPath string `json:"dataDirHostPath,omitempty"`
|
||||
|
||||
|
||||
@@ -23,14 +23,9 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
cephv1alpha1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1alpha1"
|
||||
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
|
||||
opcluster "github.com/rook/rook/pkg/operator/ceph/cluster"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/api/rbac/v1beta1"
|
||||
kserrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -46,29 +41,6 @@ const (
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-agent")
|
||||
|
||||
var clusterAccessRules = []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes", "nodes/proxy"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{rookv1alpha2.CustomResourceGroup},
|
||||
Resources: []string{attachment.CustomResourceNamePlural},
|
||||
Verbs: []string{"get", "list", "create", "delete", "update"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{cephv1alpha1.CustomResourceGroup},
|
||||
Resources: []string{opcluster.CustomResourceNamePlural},
|
||||
Verbs: []string{"get", "list", "watch"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"storage.k8s.io"},
|
||||
Resources: []string{"storageclasses"},
|
||||
Verbs: []string{"get"},
|
||||
},
|
||||
}
|
||||
|
||||
// New creates an instance of Agent
|
||||
func New(clientset kubernetes.Interface) *Agent {
|
||||
return &Agent{
|
||||
@@ -77,21 +49,16 @@ func New(clientset kubernetes.Interface) *Agent {
|
||||
}
|
||||
|
||||
// Start the agent
|
||||
func (a *Agent) Start(namespace, agentImage string) error {
|
||||
func (a *Agent) Start(namespace, agentImage, serviceAccount string) error {
|
||||
|
||||
err := k8sutil.MakeClusterRole(a.clientset, namespace, agentDaemonsetName, clusterAccessRules, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init RBAC for rook-ceph-agents. %+v", err)
|
||||
}
|
||||
|
||||
err = a.createAgentDaemonSet(namespace, agentImage)
|
||||
err := a.createAgentDaemonSet(namespace, agentImage, serviceAccount)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error starting agent daemonset: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Agent) createAgentDaemonSet(namespace, agentImage string) error {
|
||||
func (a *Agent) createAgentDaemonSet(namespace, agentImage, serviceAccount string) error {
|
||||
|
||||
flexvolumeDirPath, source := a.discoverFlexvolumeDir()
|
||||
logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath)
|
||||
@@ -112,7 +79,7 @@ func (a *Agent) createAgentDaemonSet(namespace, agentImage string) error {
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
ServiceAccountName: agentDaemonsetName,
|
||||
ServiceAccountName: serviceAccount,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: agentDaemonsetName,
|
||||
|
||||
@@ -58,27 +58,15 @@ func TestStartAgentDaemonset(t *testing.T) {
|
||||
a := New(clientset)
|
||||
|
||||
// start a basic cluster
|
||||
err := a.Start(namespace, "rook/rook:myversion")
|
||||
err := a.Start(namespace, "rook/rook:myversion", "mysa")
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check clusters rbac roles
|
||||
_, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-ceph-agent", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
role, err := clientset.RbacV1beta1().ClusterRoles().Get("rook-ceph-agent", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 4, len(role.Rules))
|
||||
|
||||
binding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get("rook-ceph-agent", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "rook-ceph-agent", binding.Subjects[0].Name)
|
||||
assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind)
|
||||
|
||||
// check daemonset parameters
|
||||
agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-ceph-agent", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, namespace, agentDS.Namespace)
|
||||
assert.Equal(t, "rook-ceph-agent", agentDS.Name)
|
||||
assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName)
|
||||
assert.True(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged)
|
||||
volumes := agentDS.Spec.Template.Spec.Volumes
|
||||
assert.Equal(t, 4, len(volumes))
|
||||
@@ -117,19 +105,12 @@ func TestGetContainerImage(t *testing.T) {
|
||||
clientset.CoreV1().Pods("Default").Create(&pod)
|
||||
|
||||
// start a basic cluster
|
||||
image, err := k8sutil.GetContainerImage(clientset, "")
|
||||
returnPod, err := k8sutil.GetRunningPod(clientset)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "rook/test", image)
|
||||
assert.Equal(t, "mypod", returnPod.Name)
|
||||
}
|
||||
|
||||
func TestGetContainerImageMultipleContainers(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
|
||||
os.Setenv(k8sutil.PodNamespaceEnvVar, "Default")
|
||||
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
|
||||
|
||||
os.Setenv(k8sutil.PodNameEnvVar, "mypod")
|
||||
defer os.Unsetenv(k8sutil.PodNameEnvVar)
|
||||
|
||||
pod := v1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -149,11 +130,11 @@ func TestGetContainerImageMultipleContainers(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
clientset.CoreV1().Pods("Default").Create(&pod)
|
||||
|
||||
// start a basic cluster
|
||||
_, err := k8sutil.GetContainerImage(clientset, "foo")
|
||||
container, err := k8sutil.GetContainerImage(&pod, "foo")
|
||||
assert.NotNil(t, err)
|
||||
assert.Equal(t, "", container)
|
||||
assert.Equal(t, "failed to find image for container foo", err.Error())
|
||||
}
|
||||
|
||||
@@ -192,13 +173,14 @@ func TestStartAgentDaemonsetWithToleration(t *testing.T) {
|
||||
a := New(clientset)
|
||||
|
||||
// start a basic cluster
|
||||
err := a.Start(namespace, "rook/test")
|
||||
err := a.Start(namespace, "rook/test", "mysa")
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check daemonset toleration
|
||||
agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-ceph-agent", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(agentDS.Spec.Template.Spec.Tolerations))
|
||||
assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName)
|
||||
assert.Equal(t, "NoSchedule", string(agentDS.Spec.Template.Spec.Tolerations[0].Effect))
|
||||
assert.Equal(t, "example", string(agentDS.Spec.Template.Spec.Tolerations[0].Key))
|
||||
assert.Equal(t, "Exists", string(agentDS.Spec.Template.Spec.Tolerations[0].Operator))
|
||||
|
||||
@@ -613,7 +613,7 @@ func (c *cluster) createInstance(rookImage string) error {
|
||||
}
|
||||
|
||||
// Start the OSDs
|
||||
c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.Storage, c.Spec.DataDirHostPath,
|
||||
c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath,
|
||||
cephv1alpha1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1alpha1.GetOSDResources(c.Spec.Resources), c.ownerRef)
|
||||
err = c.osds.Start()
|
||||
if err != nil {
|
||||
|
||||
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/rook/rook/pkg/util/display"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/api/rbac/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
@@ -55,16 +54,9 @@ const (
|
||||
appName = "rook-ceph-osd"
|
||||
appNameFmt = "rook-ceph-osd-%s"
|
||||
clusterAvailableSpaceReserve = 0.05
|
||||
defaultServiceAccountName = "rook-ceph-cluster"
|
||||
)
|
||||
|
||||
var clusterAccessRules = []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"configmaps"},
|
||||
Verbs: []string{"get", "list", "watch", "create", "update", "delete"},
|
||||
},
|
||||
}
|
||||
|
||||
// Cluster keeps track of the OSDs
|
||||
type Cluster struct {
|
||||
context *clusterd.Context
|
||||
@@ -77,16 +69,24 @@ type Cluster struct {
|
||||
HostNetwork bool
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
serviceAccount string
|
||||
}
|
||||
|
||||
// New creates an instance of the OSD manager
|
||||
func New(context *clusterd.Context, namespace, version string, storageSpec rookalpha.StorageScopeSpec,
|
||||
func New(context *clusterd.Context, namespace, version, serviceAccount string, storageSpec rookalpha.StorageScopeSpec,
|
||||
dataDirHostPath string, placement rookalpha.Placement, hostNetwork bool,
|
||||
resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster {
|
||||
|
||||
if serviceAccount == "" {
|
||||
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
|
||||
serviceAccount = defaultServiceAccountName
|
||||
logger.Infof("setting the osd pods to use the service account name: %s", serviceAccount)
|
||||
}
|
||||
|
||||
return &Cluster{
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
serviceAccount: serviceAccount,
|
||||
placement: placement,
|
||||
Version: version,
|
||||
Storage: storageSpec,
|
||||
@@ -106,12 +106,6 @@ type OrchestrationStatus struct {
|
||||
func (c *Cluster) Start() error {
|
||||
logger.Infof("start running osds in namespace %s", c.Namespace)
|
||||
|
||||
// create the artifacts for the osd to work with RBAC enabled
|
||||
err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, &c.ownerRef)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to init RBAC for OSDs. %+v", err)
|
||||
}
|
||||
|
||||
if c.Storage.UseAllNodes == false && len(c.Storage.Nodes) == 0 {
|
||||
logger.Warningf("useAllNodes is set to false and no nodes are specified, no OSD pods are going to be created")
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ import (
|
||||
|
||||
func TestStart(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// Start the first time
|
||||
@@ -71,7 +71,7 @@ func TestAddRemoveNode(t *testing.T) {
|
||||
statusMapWatcher := watch.NewFake()
|
||||
clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil))
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// kick off the start of the orchestration in a goroutine
|
||||
@@ -127,7 +127,7 @@ func TestAddRemoveNode(t *testing.T) {
|
||||
|
||||
// modify the storage spec to remove the node from the cluster
|
||||
storageSpec.Nodes = []rookalpha.Node{}
|
||||
c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion",
|
||||
c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// reset the orchestration status watcher
|
||||
@@ -174,7 +174,7 @@ func TestAddNodeFailure(t *testing.T) {
|
||||
return true, nil, fmt.Errorf("mock failed to create replica set")
|
||||
})
|
||||
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// kick off the start of the orchestration in a goroutine
|
||||
@@ -195,7 +195,7 @@ func TestAddNodeFailure(t *testing.T) {
|
||||
|
||||
func TestOrchestrationStatus(t *testing.T) {
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// status map should not exist yet
|
||||
|
||||
@@ -119,7 +119,7 @@ func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalph
|
||||
}
|
||||
|
||||
podSpec := v1.PodSpec{
|
||||
ServiceAccountName: appName,
|
||||
ServiceAccountName: c.serviceAccount,
|
||||
Containers: []v1.Container{c.osdContainer(devices, selection, resources, storeConfig, metadataDevice, location)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: volumes,
|
||||
|
||||
@@ -56,7 +56,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool)
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "mysa",
|
||||
storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
devMountNeeded := deviceFilter != "" || allDevices
|
||||
@@ -66,6 +66,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool)
|
||||
assert.NotNil(t, replicaSet)
|
||||
assert.Equal(t, "rook-ceph-osd-node1", replicaSet.Name)
|
||||
assert.Equal(t, c.Namespace, replicaSet.Namespace)
|
||||
assert.Equal(t, c.serviceAccount, replicaSet.Spec.Template.Spec.ServiceAccountName)
|
||||
assert.Equal(t, int32(1), *(replicaSet.Spec.Replicas))
|
||||
assert.Equal(t, "node1", replicaSet.Spec.Template.Spec.NodeSelector[apis.LabelHostname])
|
||||
assert.Equal(t, v1.RestartPolicyAlways, replicaSet.Spec.Template.Spec.RestartPolicy)
|
||||
@@ -148,7 +149,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
n := c.resolveNode(storageSpec.Nodes[0])
|
||||
@@ -200,7 +201,7 @@ func TestStorageSpecConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
n := c.resolveNode(storageSpec.Nodes[0])
|
||||
@@ -245,7 +246,7 @@ func TestHostNetwork(t *testing.T) {
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion",
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
n := c.resolveNode(storageSpec.Nodes[0])
|
||||
|
||||
@@ -61,16 +61,17 @@ var provisionerConfigs = map[string]string{
|
||||
|
||||
// Operator type for managing storage
|
||||
type Operator struct {
|
||||
context *clusterd.Context
|
||||
resources []opkit.CustomResource
|
||||
rookImage string
|
||||
context *clusterd.Context
|
||||
resources []opkit.CustomResource
|
||||
rookImage string
|
||||
securityAccount string
|
||||
// The custom resource that is global to the kubernetes cluster.
|
||||
// The cluster is global because you create multiple clusters in k8s
|
||||
clusterController *cluster.ClusterController
|
||||
}
|
||||
|
||||
// New creates an operator instance
|
||||
func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage string) *Operator {
|
||||
func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage, securityAccount string) *Operator {
|
||||
clusterController := cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper)
|
||||
|
||||
schemes := []opkit.CustomResource{cluster.ClusterResource, pool.PoolResource, object.ObjectStoreResource,
|
||||
@@ -80,6 +81,7 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen
|
||||
clusterController: clusterController,
|
||||
resources: schemes,
|
||||
rookImage: rookImage,
|
||||
securityAccount: securityAccount,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,12 +114,12 @@ func (o *Operator) Run() error {
|
||||
|
||||
rookAgent := agent.New(o.context.Clientset)
|
||||
|
||||
if err := rookAgent.Start(namespace, o.rookImage); err != nil {
|
||||
if err := rookAgent.Start(namespace, o.rookImage, o.securityAccount); err != nil {
|
||||
return fmt.Errorf("Error starting agent daemonset: %v", err)
|
||||
}
|
||||
|
||||
rookDiscover := discover.New(o.context.Clientset)
|
||||
if err := rookDiscover.Start(namespace, o.rookImage); err != nil {
|
||||
if err := rookDiscover.Start(namespace, o.rookImage, o.securityAccount); err != nil {
|
||||
return fmt.Errorf("Error starting device discovery daemonset: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
func TestOperator(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
context := &clusterd.Context{Clientset: clientset}
|
||||
o := New(context, &attachment.MockAttachment{}, "")
|
||||
o := New(context, &attachment.MockAttachment{}, "", "")
|
||||
|
||||
assert.NotNil(t, o)
|
||||
assert.NotNil(t, o.clusterController)
|
||||
|
||||
@@ -67,21 +67,16 @@ func New(clientset kubernetes.Interface) *Discover {
|
||||
}
|
||||
|
||||
// Start the discover
|
||||
func (d *Discover) Start(namespace, discoverImage string) error {
|
||||
func (d *Discover) Start(namespace, discoverImage, securityAccount string) error {
|
||||
|
||||
err := k8sutil.MakeRole(d.clientset, namespace, discoverDaemonsetName, accessRules, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init RBAC for rook-discover. %+v", err)
|
||||
}
|
||||
|
||||
err = d.createDiscoverDaemonSet(namespace, discoverImage)
|
||||
err := d.createDiscoverDaemonSet(namespace, discoverImage, securityAccount)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error starting discover daemonset: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) error {
|
||||
func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage, securityAccount string) error {
|
||||
privileged := false
|
||||
ds := &extensions.DaemonSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -98,7 +93,7 @@ func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) erro
|
||||
},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
ServiceAccountName: discoverDaemonsetName,
|
||||
ServiceAccountName: securityAccount,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: discoverDaemonsetName,
|
||||
|
||||
@@ -46,14 +46,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) {
|
||||
a := New(clientset)
|
||||
|
||||
// start a basic cluster
|
||||
err := a.Start(namespace, "rook/rook:myversion")
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check clusters rbac roles
|
||||
_, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-discover", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
_, err = clientset.RbacV1beta1().Roles(namespace).Get("rook-discover", metav1.GetOptions{})
|
||||
err := a.Start(namespace, "rook/rook:myversion", "mysa")
|
||||
assert.Nil(t, err)
|
||||
|
||||
// check daemonset parameters
|
||||
@@ -61,6 +54,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, namespace, agentDS.Namespace)
|
||||
assert.Equal(t, "rook-discover", agentDS.Name)
|
||||
assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName)
|
||||
assert.False(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged)
|
||||
volumes := agentDS.Spec.Template.Spec.Volumes
|
||||
assert.Equal(t, 3, len(volumes))
|
||||
|
||||
+18
-15
@@ -93,21 +93,7 @@ func ConfigDirEnvVar() v1.EnvVar {
|
||||
return v1.EnvVar{Name: "ROOK_CONFIG_DIR", Value: DataDir}
|
||||
}
|
||||
|
||||
func GetContainerImage(clientset kubernetes.Interface, name string) (string, error) {
|
||||
|
||||
podName := os.Getenv(PodNameEnvVar)
|
||||
if podName == "" {
|
||||
return "", fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file")
|
||||
}
|
||||
podNamespace := os.Getenv(PodNamespaceEnvVar)
|
||||
if podName == "" {
|
||||
return "", fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file")
|
||||
}
|
||||
|
||||
pod, err := clientset.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
func GetContainerImage(pod *v1.Pod, name string) (string, error) {
|
||||
|
||||
image, err := GetMatchingContainer(pod.Spec.Containers, name)
|
||||
if err != nil {
|
||||
@@ -116,6 +102,23 @@ func GetContainerImage(clientset kubernetes.Interface, name string) (string, err
|
||||
return image.Image, nil
|
||||
}
|
||||
|
||||
func GetRunningPod(clientset kubernetes.Interface) (*v1.Pod, error) {
|
||||
podName := os.Getenv(PodNameEnvVar)
|
||||
if podName == "" {
|
||||
return nil, fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file")
|
||||
}
|
||||
podNamespace := os.Getenv(PodNamespaceEnvVar)
|
||||
if podName == "" {
|
||||
return nil, fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file")
|
||||
}
|
||||
|
||||
pod, err := clientset.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
func GetMatchingContainer(containers []v1.Container, name string) (v1.Container, error) {
|
||||
var result *v1.Container
|
||||
if len(containers) == 1 {
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package k8sutil for Kubernetes helpers.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/api/rbac/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
enableRBACEnv = "RBAC_ENABLED"
|
||||
)
|
||||
|
||||
func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error {
|
||||
|
||||
err := makeServiceAccount(clientset, namespace, name, ownerRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !isRBACEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create the role if it doesn't yet exist.
|
||||
// If the role already exists we have to update it. Otherwise if the permissions change during an upgrade,
|
||||
// the create will fail with an error that we're changing the permissions.
|
||||
role := &v1beta1.Role{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Rules: rules,
|
||||
}
|
||||
if ownerRef != nil {
|
||||
role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef}
|
||||
}
|
||||
_, err = clientset.RbacV1beta1().Roles(namespace).Get(role.Name, metav1.GetOptions{})
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Infof("creating role %s in namespace %s", name, namespace)
|
||||
_, err = clientset.RbacV1beta1().Roles(namespace).Create(role)
|
||||
} else if err == nil {
|
||||
logger.Infof("role %s already exists in namespace %s. updating if needed.", name, namespace)
|
||||
_, err = clientset.RbacV1beta1().Roles(namespace).Update(role)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create/update role %s in namespace %s. %+v", name, namespace, err)
|
||||
}
|
||||
|
||||
binding := &v1beta1.RoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
RoleRef: v1beta1.RoleRef{
|
||||
Name: name,
|
||||
Kind: "Role",
|
||||
APIGroup: "rbac.authorization.k8s.io",
|
||||
},
|
||||
Subjects: []v1beta1.Subject{
|
||||
{Kind: "ServiceAccount", Name: name, Namespace: namespace},
|
||||
},
|
||||
}
|
||||
if ownerRef != nil {
|
||||
binding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef}
|
||||
}
|
||||
_, err = clientset.RbacV1beta1().RoleBindings(namespace).Create(binding)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s role binding in namespace %s. %+v", name, namespace, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error {
|
||||
|
||||
err := makeServiceAccount(clientset, namespace, name, ownerRef)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !isRBACEnabled() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create the cluster scoped role if it doesn't yet exist.
|
||||
// If the role already exists we have to update it. Otherwise if the permissions change during an upgrade,
|
||||
// the create will fail with an error that we're changing the permissions.
|
||||
role := &v1beta1.ClusterRole{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Rules: rules,
|
||||
}
|
||||
if ownerRef != nil {
|
||||
role.OwnerReferences = []metav1.OwnerReference{*ownerRef}
|
||||
}
|
||||
|
||||
_, err = clientset.RbacV1beta1().ClusterRoles().Get(role.Name, metav1.GetOptions{})
|
||||
if errors.IsNotFound(err) {
|
||||
logger.Infof("creating cluster role %s", name)
|
||||
_, err = clientset.RbacV1beta1().ClusterRoles().Create(role)
|
||||
} else if err == nil {
|
||||
logger.Infof("cluster role %s already exists. Updating if needed.", name)
|
||||
_, err = clientset.RbacV1beta1().ClusterRoles().Update(role)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create/update cluster role %s. %+v", name, err)
|
||||
}
|
||||
|
||||
binding := &v1beta1.ClusterRoleBinding{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
RoleRef: v1beta1.RoleRef{
|
||||
Name: name,
|
||||
Kind: "ClusterRole",
|
||||
APIGroup: "rbac.authorization.k8s.io",
|
||||
},
|
||||
Subjects: []v1beta1.Subject{
|
||||
{Kind: "ServiceAccount", Name: name, Namespace: namespace},
|
||||
},
|
||||
}
|
||||
if ownerRef != nil {
|
||||
binding.OwnerReferences = []metav1.OwnerReference{*ownerRef}
|
||||
}
|
||||
|
||||
_, err = clientset.RbacV1beta1().ClusterRoleBindings().Create(binding)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create cluster role binding %s. %+v", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeServiceAccount(clientset kubernetes.Interface, namespace, name string, ownerRef *metav1.OwnerReference) error {
|
||||
account := &v1.ServiceAccount{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
if ownerRef != nil {
|
||||
account.OwnerReferences = []metav1.OwnerReference{*ownerRef}
|
||||
}
|
||||
|
||||
_, err := clientset.CoreV1().ServiceAccounts(namespace).Create(account)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s service account in namespace %s. %+v", name, namespace, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isRBACEnabled() bool {
|
||||
r := os.Getenv(enableRBACEnv)
|
||||
if r == "false" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package k8sutil for Kubernetes helpers.
|
||||
package k8sutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/rbac/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestMakeRole(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
namespace := "myns"
|
||||
name := "myapp"
|
||||
|
||||
rules := []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"namespaces", "secrets", "pods", "services", "nodes", "configmaps", "events"},
|
||||
Verbs: []string{"get", "list", "watch", "create", "update"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"extensions"},
|
||||
Resources: []string{"thirdpartyresources", "deployments", "daemonsets", "replicasets"},
|
||||
Verbs: []string{"get", "list", "create", "delete"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"apiextensions.k8s.io"},
|
||||
Resources: []string{"customresourcedefinitions"},
|
||||
Verbs: []string{"get", "list", "create"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"storage.k8s.io"},
|
||||
Resources: []string{"storageclasses"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
}
|
||||
|
||||
ownerRef := metav1.OwnerReference{}
|
||||
err := MakeRole(clientset, namespace, name, rules, &ownerRef)
|
||||
|
||||
role, err := clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, name, role.Name)
|
||||
assert.Equal(t, 4, len(role.Rules))
|
||||
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, namespace, account.Namespace)
|
||||
binding, err := clientset.RbacV1beta1().RoleBindings(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, name, binding.RoleRef.Name)
|
||||
assert.Equal(t, "Role", binding.RoleRef.Kind)
|
||||
assert.Equal(t, "rbac.authorization.k8s.io", binding.RoleRef.APIGroup)
|
||||
assert.Equal(t, name, binding.Subjects[0].Name)
|
||||
assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind)
|
||||
|
||||
// update the rules
|
||||
newRules := []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"namespaces"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
}
|
||||
|
||||
err = MakeRole(clientset, namespace, name, newRules, &ownerRef)
|
||||
assert.Nil(t, err)
|
||||
role, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(role.Rules))
|
||||
assert.Equal(t, "", role.Rules[0].APIGroups[0])
|
||||
assert.Equal(t, 1, len(role.Rules[0].Resources))
|
||||
assert.Equal(t, 2, len(role.Rules[0].Verbs))
|
||||
}
|
||||
|
||||
func TestMakeClusterRole(t *testing.T) {
|
||||
clientset := test.New(1)
|
||||
namespace := "myns"
|
||||
name := "myapp"
|
||||
|
||||
rules := []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"storage.k8s.io"},
|
||||
Resources: []string{"storageclasses"},
|
||||
Verbs: []string{"get"},
|
||||
},
|
||||
}
|
||||
|
||||
err := MakeClusterRole(clientset, namespace, name, rules, nil)
|
||||
|
||||
role, err := clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, name, role.Name)
|
||||
assert.Equal(t, 2, len(role.Rules))
|
||||
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, namespace, account.Namespace)
|
||||
binding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, name, binding.RoleRef.Name)
|
||||
assert.Equal(t, "ClusterRole", binding.RoleRef.Kind)
|
||||
assert.Equal(t, "rbac.authorization.k8s.io", binding.RoleRef.APIGroup)
|
||||
assert.Equal(t, name, binding.Subjects[0].Name)
|
||||
assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind)
|
||||
|
||||
// update the rules
|
||||
newRules := []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"namespaces"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
}
|
||||
|
||||
err = MakeClusterRole(clientset, namespace, name, newRules, nil)
|
||||
assert.Nil(t, err)
|
||||
role, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 1, len(role.Rules))
|
||||
assert.Equal(t, "", role.Rules[0].APIGroups[0])
|
||||
assert.Equal(t, 1, len(role.Rules[0].Resources))
|
||||
assert.Equal(t, 2, len(role.Rules[0].Verbs))
|
||||
}
|
||||
|
||||
func TestMakeRoleRBACDisabled(t *testing.T) {
|
||||
os.Setenv(enableRBACEnv, "false")
|
||||
defer os.Unsetenv(enableRBACEnv)
|
||||
clientset := test.New(1)
|
||||
namespace := "myns"
|
||||
name := "myapp"
|
||||
|
||||
rules := []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"namespaces", "secrets", "pods", "services", "nodes", "configmaps", "events"},
|
||||
Verbs: []string{"get", "list", "watch", "create", "update"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"extensions"},
|
||||
Resources: []string{"thirdpartyresources", "deployments", "daemonsets", "replicasets"},
|
||||
Verbs: []string{"get", "list", "create", "delete"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"apiextensions.k8s.io"},
|
||||
Resources: []string{"customresourcedefinitions"},
|
||||
Verbs: []string{"get", "list", "create"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"storage.k8s.io"},
|
||||
Resources: []string{"storageclasses"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
}
|
||||
ownerRef := metav1.OwnerReference{}
|
||||
err := MakeRole(clientset, namespace, name, rules, &ownerRef)
|
||||
|
||||
_, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.NotNil(t, err)
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
|
||||
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, namespace, account.Namespace)
|
||||
|
||||
_, err = clientset.RbacV1beta1().RoleBindings(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.NotNil(t, err)
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
}
|
||||
|
||||
func TestMakeClusterRoleRBACDisabled(t *testing.T) {
|
||||
os.Setenv(enableRBACEnv, "false")
|
||||
defer os.Unsetenv(enableRBACEnv)
|
||||
clientset := test.New(1)
|
||||
namespace := "myns"
|
||||
name := "myapp"
|
||||
|
||||
rules := []v1beta1.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"},
|
||||
Verbs: []string{"get", "list"},
|
||||
},
|
||||
{
|
||||
APIGroups: []string{"storage.k8s.io"},
|
||||
Resources: []string{"storageclasses"},
|
||||
Verbs: []string{"get"},
|
||||
},
|
||||
}
|
||||
|
||||
err := MakeClusterRole(clientset, namespace, name, rules, nil)
|
||||
assert.Nil(t, err)
|
||||
_, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{})
|
||||
assert.NotNil(t, err)
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
|
||||
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, namespace, account.Namespace)
|
||||
|
||||
_, err = clientset.RbacV1beta1().ClusterRoleBindings().Get(name, metav1.GetOptions{})
|
||||
assert.NotNil(t, err)
|
||||
assert.True(t, errors.IsNotFound(err))
|
||||
}
|
||||
@@ -29,7 +29,7 @@ func NewK8sInstallData() *InstallData {
|
||||
return &InstallData{}
|
||||
}
|
||||
|
||||
func (i *InstallData) GetRookCRDs(namespace string) string {
|
||||
func (i *InstallData) GetRookCRDs() string {
|
||||
return `apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
@@ -110,24 +110,54 @@ metadata:
|
||||
name: ` + namespace + `
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
kind: Role
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-system
|
||||
namespace: ` + namespace + `
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
- apiGroups:
|
||||
- extensions
|
||||
resources:
|
||||
- daemonsets
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-cluster-mgmt
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- namespaces
|
||||
- serviceaccounts
|
||||
- secrets
|
||||
- pods
|
||||
- services
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
- configmaps
|
||||
- events
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
@@ -149,17 +179,36 @@ rules:
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: rook-ceph-global
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- rbac.authorization.k8s.io
|
||||
- ""
|
||||
resources:
|
||||
- clusterroles
|
||||
- clusterrolebindings
|
||||
- roles
|
||||
- rolebindings
|
||||
- pods
|
||||
- nodes
|
||||
- nodes/proxy
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- events
|
||||
- persistentvolumes
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- patch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
@@ -171,7 +220,6 @@ rules:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- delete
|
||||
- apiGroups:
|
||||
- ceph.rook.io
|
||||
resources:
|
||||
@@ -188,21 +236,44 @@ rules:
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-system
|
||||
namespace: ` + namespace + `
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-system
|
||||
namespace: ` + namespace + `
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-global
|
||||
namespace: ` + namespace + `
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-global
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-operator
|
||||
name: rook-ceph-system
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
apiVersion: apps/v1beta1
|
||||
@@ -210,6 +281,9 @@ kind: Deployment
|
||||
metadata:
|
||||
name: rook-ceph-operator
|
||||
namespace: ` + namespace + `
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
@@ -217,7 +291,7 @@ spec:
|
||||
labels:
|
||||
app: rook-ceph-operator
|
||||
spec:
|
||||
serviceAccountName: rook-ceph-operator
|
||||
serviceAccountName: rook-ceph-system
|
||||
containers:
|
||||
- name: rook-ceph-operator
|
||||
image: rook/ceph:master
|
||||
@@ -243,6 +317,55 @@ spec:
|
||||
fieldPath: metadata.namespace`
|
||||
}
|
||||
|
||||
//GetRookCluster returns rook-cluster manifest
|
||||
func (i *InstallData) GetClusterRoles(namespace, systemNamespace string) string {
|
||||
return `apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: ` + namespace + `
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
|
||||
---
|
||||
# Allow the operator to create resources in this cluster's namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster-mgmt
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: rook-ceph-cluster-mgmt
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-system
|
||||
namespace: ` + systemNamespace + `
|
||||
---
|
||||
# Allow the pods in this namespace to work with configmaps
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cluster
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cluster
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cluster
|
||||
namespace: ` + namespace
|
||||
}
|
||||
|
||||
//GetRookCluster returns rook-cluster manifest
|
||||
func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string {
|
||||
return `apiVersion: ceph.rook.io/v1alpha1
|
||||
@@ -251,6 +374,7 @@ metadata:
|
||||
name: ` + namespace + `
|
||||
namespace: ` + namespace + `
|
||||
spec:
|
||||
serviceAccount: rook-ceph-cluster
|
||||
dataDirHostPath: ` + dataDirHostPath + `
|
||||
network:
|
||||
hostNetwork: false
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
|
||||
"flag"
|
||||
@@ -67,11 +68,11 @@ type InstallHelper struct {
|
||||
T func() *testing.T
|
||||
}
|
||||
|
||||
func (h *InstallHelper) CreateK8sRookResources(namespace string) (err error) {
|
||||
func (h *InstallHelper) CreateK8sRookResources() (err error) {
|
||||
var resources string
|
||||
logger.Info("Creating Rook CRD's")
|
||||
|
||||
resources = h.installData.GetRookCRDs(namespace)
|
||||
resources = h.installData.GetRookCRDs()
|
||||
|
||||
_, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...)
|
||||
|
||||
@@ -85,7 +86,7 @@ func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
|
||||
//creating rook resources
|
||||
if err = h.CreateK8sRookResources(namespace); err != nil {
|
||||
if err = h.CreateK8sRookResources(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -149,17 +150,26 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *InstallHelper) CreateK8sRookCluster(namespace string, storeType string) (err error) {
|
||||
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", false,
|
||||
func (h *InstallHelper) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
|
||||
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, "", false,
|
||||
cephv1alpha1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */)
|
||||
}
|
||||
|
||||
//CreateK8sRookCluster creates rook cluster via kubectl
|
||||
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath string,
|
||||
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, dataDirHostPath string,
|
||||
useAllDevices bool, mon cephv1alpha1.MonSpec, startWithAllNodes bool) error {
|
||||
|
||||
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
|
||||
return err
|
||||
logger.Infof("Creating namespace %s", namespace)
|
||||
ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
|
||||
_, err := h.k8shelper.Clientset.CoreV1().Namespaces().Create(ns)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create namespace %s. %+v", namespace, err)
|
||||
}
|
||||
|
||||
logger.Infof("Creating cluster roles")
|
||||
roles := h.installData.GetClusterRoles(namespace, systemNamespace)
|
||||
if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil {
|
||||
return fmt.Errorf("Failed to create cluster roles. %+v", err)
|
||||
}
|
||||
|
||||
if h.k8shelper.IsRookClientsetAvailable() {
|
||||
@@ -171,6 +181,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, st
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: v1alpha1.ClusterSpec{
|
||||
ServiceAccount: "rook-ceph-cluster",
|
||||
DataDirHostPath: dataDirHostPath,
|
||||
Mon: v1alpha1.MonSpec{
|
||||
Count: mon.Count,
|
||||
@@ -236,7 +247,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, st
|
||||
}
|
||||
|
||||
logger.Infof("Rook Cluster started")
|
||||
err := h.k8shelper.WaitForLabeledPodToRun("app=rook-ceph-osd", namespace)
|
||||
err = h.k8shelper.WaitForLabeledPodToRun("app=rook-ceph-osd", namespace)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -288,7 +299,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
|
||||
}
|
||||
|
||||
//Create rook cluster
|
||||
err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath,
|
||||
err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, onamespace, storeType, dataDirHostPath,
|
||||
useDevices, cephv1alpha1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes)
|
||||
if err != nil {
|
||||
logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err)
|
||||
@@ -322,13 +333,17 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
|
||||
for _, namespace := range namespaces {
|
||||
|
||||
if !h.k8shelper.VersionAtLeast("v1.8.0") {
|
||||
_, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-osd")
|
||||
h.checkError(err, "cannot remove serviceaccount rook-ceph-osd")
|
||||
_, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-cluster")
|
||||
h.checkError(err, "cannot remove serviceaccount rook-ceph-cluster")
|
||||
assert.NoError(h.T(), err, "%s err -> %v", namespace, err)
|
||||
|
||||
err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-osd", namespace)
|
||||
h.checkError(err, "rook-ceph-osd cluster role and binding cannot be deleted")
|
||||
assert.NoError(h.T(), err, "rook-ceph-osd cluster role and binding cannot be deleted: %+v", err)
|
||||
err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-cluster", namespace)
|
||||
h.checkError(err, "rook-ceph-cluster cluster role and binding cannot be deleted")
|
||||
assert.NoError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted: %+v", err)
|
||||
|
||||
err = h.k8shelper.DeleteRoleBinding("rook-ceph-cluster-mgmt", namespace)
|
||||
h.checkError(err, "rook-ceph-cluster-mgmt binding cannot be deleted")
|
||||
assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err)
|
||||
}
|
||||
|
||||
_, err = h.k8shelper.DeleteResource("-n", namespace, "cluster.ceph.rook.io", namespace)
|
||||
@@ -357,9 +372,13 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
|
||||
}
|
||||
h.checkError(err, "cannot uninstall rook-operator")
|
||||
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-agent", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-agent", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil)
|
||||
h.k8shelper.Clientset.CoreV1().ServiceAccounts(systemNamespace).Delete("rook-ceph-system", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-cluster-mgmt", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil)
|
||||
h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil)
|
||||
|
||||
logger.Infof("done removing the operator from namespace %s", systemNamespace)
|
||||
}
|
||||
|
||||
|
||||
@@ -1232,6 +1232,11 @@ func (k8sh *K8sHelper) DeleteRoleAndBindings(name, namespace string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k8sh *K8sHelper) DeleteRoleBinding(name, namespace string) error {
|
||||
_, err := k8sh.DeleteResource("rolebinding", name, "-n", namespace)
|
||||
return err
|
||||
}
|
||||
|
||||
func (k8sh *K8sHelper) ScaleStatefulSet(statefulSetName, namespace string, replicationSize int) error {
|
||||
args := []string{"-n", namespace, "scale", "statefulsets", statefulSetName, fmt.Sprintf("--replicas=%d", replicationSize)}
|
||||
_, err := k8sh.Kubectl(args...)
|
||||
|
||||
@@ -155,13 +155,14 @@ func (o MCTestOperations) SetUp() {
|
||||
err = o.installer.CreateK8sRookOperator(installer.SystemNamespace(o.namespace1))
|
||||
require.NoError(o.T(), err)
|
||||
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", installer.SystemNamespace(o.namespace1), "Running"),
|
||||
systemNamespace := installer.SystemNamespace(o.namespace1)
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", systemNamespace, "Running"),
|
||||
"Make sure rook-operator is in running state")
|
||||
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-agent", installer.SystemNamespace(o.namespace1), "Running"),
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-agent", systemNamespace, "Running"),
|
||||
"Make sure rook-ceph-agent is in running state")
|
||||
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", installer.SystemNamespace(o.namespace1), "Running"),
|
||||
require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", systemNamespace, "Running"),
|
||||
"Make sure rook-discover is in running state")
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
@@ -170,8 +171,8 @@ func (o MCTestOperations) SetUp() {
|
||||
logger.Infof("starting two clusters in parallel")
|
||||
errCh1 := make(chan error, 1)
|
||||
errCh2 := make(chan error, 1)
|
||||
go o.startCluster(o.namespace1, "bluestore", errCh1)
|
||||
go o.startCluster(o.namespace2, "filestore", errCh2)
|
||||
go o.startCluster(o.namespace1, systemNamespace, "bluestore", errCh1)
|
||||
go o.startCluster(o.namespace2, systemNamespace, "filestore", errCh2)
|
||||
require.NoError(o.T(), <-errCh1)
|
||||
require.NoError(o.T(), <-errCh2)
|
||||
logger.Infof("finished starting clusters")
|
||||
@@ -194,9 +195,9 @@ func (o MCTestOperations) TearDown() {
|
||||
o.installer.UninstallRookFromMultipleNS(false, installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2)
|
||||
}
|
||||
|
||||
func (o MCTestOperations) startCluster(namespace, store string, errCh chan error) {
|
||||
func (o MCTestOperations) startCluster(namespace, systemNamespace, store string, errCh chan error) {
|
||||
logger.Infof("starting cluster %s", namespace)
|
||||
if err := o.installer.CreateK8sRookCluster(namespace, store); err != nil {
|
||||
if err := o.installer.CreateK8sRookCluster(namespace, systemNamespace, store); err != nil {
|
||||
errCh <- fmt.Errorf("failed to create cluster %s. %+v", namespace, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -163,7 +163,6 @@ func (o BaseLoadTestOperations) SetUp() {
|
||||
true /* startWithAllNodes */)
|
||||
require.NoError(o.T(), err)
|
||||
require.True(o.T(), isRookInstalled)
|
||||
|
||||
}
|
||||
|
||||
// Enable chaos monkey if enable_chaos flag is present
|
||||
|
||||
Reference in new issue
Block a user