Merge pull request #1736 from travisn/min-rbac-security

Refactor cluster roles for minimal rbac access
This commit is contained in:
Travis Nielsen authored and GitHub committed 2018-06-25 12:53:53 -07:00
commit ff136d627e
36 files changed
+606 -651

No files matched your search

+6
View File
@@ -23,6 +23,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir).
- `dashboard`: Settings for the Ceph dashboard. To view the dashboard in your browser see the [dashboard guide](ceph-dashboard.md).
- `enabled`: Whether to enable the dashboard to view cluster status
- `serviceAccount`: The service account under which the OSD pods will run that will give access to ConfigMaps in the cluster's namespace. If not set, the default of `rook-ceph-cluster` will be used.
- `network`: The network settings for the cluster
- `hostNetwork`: uses network of the hosts instead of using the SDN below the containers.
- `mon`: contains mon related options [mon settings](#mon-settings)
@@ -128,6 +129,7 @@ spec:
dataDirHostPath: /var/lib/rook
dashboard:
enabled: true
serviceAccount: rook-ceph-cluster
# cluster level storage configuration and selection
storage:
useAllNodes: true
@@ -157,6 +159,7 @@ metadata:
namespace: rook-ceph
spec:
dataDirHostPath: /var/lib/rook
serviceAccount: rook-ceph-cluster
# cluster level storage configuration and selection
storage:
useAllNodes: false
@@ -198,6 +201,7 @@ metadata:
namespace: rook-ceph
spec:
dataDirHostPath: /var/lib/rook
serviceAccount: rook-ceph-cluster
# cluster level storage configuration and selection
storage:
useAllNodes: false
@@ -233,6 +237,7 @@ metadata:
namespace: rook-ceph
spec:
dataDirHostPath: /var/lib/rook
serviceAccount: rook-ceph-cluster
placement:
all:
nodeAffinity:
@@ -276,6 +281,7 @@ metadata:
namespace: rook-ceph
spec:
dataDirHostPath: /var/lib/rook
serviceAccount: rook-ceph-cluster
# cluster level resource requests/limits configuration
resources:
storage:
+4 -4
View File
@@ -95,18 +95,18 @@ kind: Namespace
metadata:
name: rook-system
---
# Allow the rook-ceph-agent serviceAccount to use the privileged PSP
# Allow the rook-ceph-system serviceAccount to use the privileged PSP
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rook-ceph-agent-psp
name: rook-ceph-system-psp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: privileged-psp-user
subjects:
- kind: ServiceAccount
name: rook-ceph-agent
name: rook-ceph-system
namespace: rook-system
```
@@ -148,6 +148,6 @@ roleRef:
name: privileged-psp-user
subjects:
- kind: ServiceAccount
name: rook-ceph-osd
name: rook-ceph-cluster
namespace: rook-ceph
```
+58
View File
@@ -145,6 +145,64 @@ kubectl delete clusterrolebindings rook-agent
Now when the operator is recreated, the agent daemonset will automatically be created again with the new version.
### Operator Access to Clusters
No longer is the operator given privileges across every namespace. The operator will need privileges to manage each Ceph cluster that is configured.
The following service account, role, and role bindings are included in `cluster.yaml` when creating a new cluster. Save the following yaml as `cluster-privs.yaml`,
modifying the `namespace:` attribute of each resource if you are not using the default `rook` or `rook-system` namespaces.
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cluster
namespace: rook
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster
namespace: rook
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster-mgmt
namespace: rook
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: rook-system
---
# Allow the pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster
namespace: rook
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cluster
subjects:
- kind: ServiceAccount
name: rook-ceph-cluster
namespace: rook
```
Now create the objects in the cluster:
```bash
kubectl create -f cluster-privs.yaml
```
### Operator
The Rook operator is the management brains of the cluster, so it should be upgraded first before other components.
In the event that the new version requires a migration of metadata or config, the operator is the one that would understand how to perform that migration.
+2
View File
@@ -30,6 +30,8 @@
- Namespaces: The example namespaces are now backend-specific. Instead of `rook-system` and `rook`, you will see `rook-ceph-system` and `rook-ceph`.
- Volume plugins: The dynamic provisioner and flex driver are now based on `ceph.rook.io` instead of `rook.io`
- Ceph container images now use CentOS 7 as a base
- Minimal privileges are configured with a new cluster role for the operator and Ceph daemons, following the new [security design](design/security-model.md).
- A role binding must be defined for each cluster to be managed by the operator.
### Removal of the API service and rookctl tool
@@ -1,27 +1,20 @@
{{- if .Values.rbacEnable }}
# The cluster role for managing all the cluster-specific resources in a namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-operator
name: rook-ceph-cluster-mgmt
labels:
operator: rook
storage-backend: ceph
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
rules:
- apiGroups:
- ""
resources:
- namespaces
- serviceaccounts
- secrets
- pods
- services
- nodes
- nodes/proxy
- configmaps
- events
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
@@ -43,17 +36,40 @@ rules:
- create
- update
- delete
---
# The cluster role for managing the Rook CRDs
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- rbac.authorization.k8s.io
- ""
resources:
- clusterroles
- clusterrolebindings
- roles
- rolebindings
# Pod access is needed for fencing
- pods
# Node access is needed for determining nodes where mons should run
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
# PVs and PVCs are managed by the Rook provisioner
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
@@ -65,7 +81,6 @@ rules:
- get
- list
- watch
- delete
- apiGroups:
- ceph.rook.io
resources:
@@ -1,8 +1,9 @@
{{- if .Values.rbacEnable }}
apiVersion: rbac.authorization.k8s.io/v1beta1
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-operator
name: rook-ceph-global
labels:
operator: rook
storage-backend: ceph
@@ -10,17 +11,17 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-operator
name: rook-ceph-global
subjects:
- kind: ServiceAccount
name: rook-ceph-operator
name: rook-ceph-system
namespace: {{ .Release.Namespace }}
{{- if .Values.pspEnable }}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: rook-ceph-operator-psp-users
name: rook-ceph-system-psp-users
labels:
operator: rook
storage-backend: ceph
@@ -28,13 +29,10 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-operator-psp-user
name: rook-ceph-system-psp-user
subjects:
- kind: ServiceAccount
name: rook-ceph-operator
namespace: {{ .Release.Namespace }}
- kind: ServiceAccount
name: rook-ceph-agent
name: rook-ceph-system
namespace: {{ .Release.Namespace }}
{{- end }}
{{- end }}
@@ -78,5 +78,5 @@ spec:
resources:
{{ toYaml .Values.resources | indent 10 }}
{{- if .Values.rbacEnable }}
serviceAccountName: rook-ceph-operator
serviceAccountName: rook-ceph-system
{{- end }}
@@ -0,0 +1,35 @@
{{- if .Values.rbacEnable }}
# The role for the operator to manage resources in the system namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- extensions
resources:
- daemonsets
verbs:
- get
- list
- watch
- create
- update
- delete
{{- end }}
@@ -0,0 +1,19 @@
{{- if .Values.rbacEnable }}
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-system
namespace: {{ .Release.Namespace }}
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-system
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: {{ .Release.Namespace }}
{{- end }}
@@ -1,8 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
name: rook-ceph-operator
@@ -3,6 +3,52 @@ kind: Namespace
metadata:
name: rook-ceph
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cluster
namespace: rook-ceph
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster
namespace: rook-ceph
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster-mgmt
namespace: rook-ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: rook-ceph-system
---
# Allow the pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster
namespace: rook-ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cluster
subjects:
- kind: ServiceAccount
name: rook-ceph-cluster
namespace: rook-ceph
---
apiVersion: ceph.rook.io/v1alpha1
kind: Cluster
metadata:
@@ -13,6 +59,8 @@ spec:
# Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster.
# In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment.
dataDirHostPath: /var/lib/rook
# The service account under which to run the daemon pods in this cluster if the default account is not sufficient (OSDs)
serviceAccount: rook-ceph-cluster
# set the amount of mons to be started
mon:
count: 3
+94 -19
View File
@@ -83,10 +83,11 @@ spec:
scope: Namespaced
version: v1alpha2
---
# The cluster role for managing all the cluster-specific resources in a namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-operator
name: rook-ceph-cluster-mgmt
labels:
operator: rook
storage-backend: ceph
@@ -94,17 +95,10 @@ rules:
- apiGroups:
- ""
resources:
- namespaces
- serviceaccounts
- secrets
- pods
- services
- nodes
- nodes/proxy
- configmaps
- events
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
@@ -126,17 +120,75 @@ rules:
- create
- update
- delete
---
# The role for the operator to manage resources in the system namespace
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: rook-ceph-system
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- rbac.authorization.k8s.io
- ""
resources:
- clusterroles
- clusterrolebindings
- roles
- rolebindings
- pods
- configmaps
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- extensions
resources:
- daemonsets
verbs:
- get
- list
- watch
- create
- update
- delete
---
# The cluster role for managing the Rook CRDs
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
# Pod access is needed for fencing
- pods
# Node access is needed for determining nodes where mons should run
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
# PVs and PVCs are managed by the Rook provisioner
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
@@ -148,7 +200,6 @@ rules:
- get
- list
- watch
- delete
- apiGroups:
- ceph.rook.io
resources:
@@ -162,19 +213,39 @@ rules:
verbs:
- "*"
---
# The rook system service account used by the operator, agent, and discovery pods
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-operator
name: rook-ceph-system
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
---
# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-system
namespace: rook-ceph-system
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-system
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: rook-ceph-system
---
# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-operator
name: rook-ceph-global
namespace: rook-ceph-system
labels:
operator: rook
@@ -182,12 +253,13 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-operator
name: rook-ceph-global
subjects:
- kind: ServiceAccount
name: rook-ceph-operator
name: rook-ceph-system
namespace: rook-ceph-system
---
# The deployment for the rook operator
apiVersion: apps/v1beta1
kind: Deployment
metadata:
@@ -203,7 +275,7 @@ spec:
labels:
app: rook-ceph-operator
spec:
serviceAccountName: rook-ceph-operator
serviceAccountName: rook-ceph-system
containers:
- name: rook-ceph-operator
image: rook/ceph:master
@@ -235,6 +307,9 @@ spec:
# which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
- name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
value: "false"
# The logging level for the operator: INFO | DEBUG
- name: ROOK_LOG_LEVEL
value: "INFO"
# The interval to check if every mon is in the quorum.
- name: ROOK_MON_HEALTHCHECK_INTERVAL
value: "45s"
+8 -3
View File
@@ -69,13 +69,18 @@ func startOperator(cmd *cobra.Command, args []string) error {
rook.TerminateFatal(err)
}
// Using the rook-ceph-operator image to deploy other rook pods
rookImage, err := k8sutil.GetContainerImage(clientset, containerName)
// Using the current image version to deploy other rook pods
pod, err := k8sutil.GetRunningPod(clientset)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
}
rookImage, err := k8sutil.GetContainerImage(pod, containerName)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
}
op := operator.New(context, volumeAttachment, rookImage)
op := operator.New(context, volumeAttachment, rookImage, pod.Spec.ServiceAccountName)
err = op.Run()
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to run operator. %+v\n", err))
+7 -2
View File
@@ -58,8 +58,13 @@ func startOperator(cmd *cobra.Command, args []string) error {
context.APIExtensionClientset = apiExtClientset
context.RookClientset = rookClientset
// Using the cockroachdb-operator image to deploy other pods
rookImage, err := k8sutil.GetContainerImage(clientset, containerName)
// Using the current image version to deploy other rook pods
pod, err := k8sutil.GetRunningPod(clientset)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
}
rookImage, err := k8sutil.GetContainerImage(pod, containerName)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
}
+7 -2
View File
@@ -61,8 +61,13 @@ func startOperator(cmd *cobra.Command, args []string) error {
rook.TerminateFatal(err)
}
// Using the rook-operator image to deploy other rook pods
rookImage, err := k8sutil.GetContainerImage(clientset, containerName)
// Using the current image version to deploy other rook pods
pod, err := k8sutil.GetRunningPod(clientset)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get pod. %+v\n", err))
}
rookImage, err := k8sutil.GetContainerImage(pod, containerName)
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get container image. %+v\n", err))
}
+13 -13
View File
@@ -12,7 +12,7 @@ The Rook operator currently uses a highly privileged service account with permis
Today the cluster admin creates the rook system namespace, rook-operator service account and RBAC rules as follows:
```
```yaml
apiVersion: v1
kind: Namespace
metadata:
@@ -67,7 +67,7 @@ subjects:
Once the rook operator is up and running it will automatically create the service account for the rook agent and the following RBAC rules:
```
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -106,7 +106,7 @@ subjects:
When the cluster admin create a new Rook cluster they do so by adding a namespace and the rook cluster spec:
```
```yaml
apiVersion: v1
kind: Namespace
metadata:
@@ -124,7 +124,7 @@ At this point the rook operator will notice that a new rook cluster CRD showed u
The `rook-api` service account and RBAC rules are as follows:
```
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -167,7 +167,7 @@ subjects:
The `rook-ceph-osd` service account and RBAC rules are as follows:
```
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -203,7 +203,7 @@ subjects:
Just as we do today the cluster admin is responsible for creating the `rook-system` namespace. I propose we have a single service account in this namespace and call it `rook-system` by default. The names used are inconsequential and can be set to something different by the cluster admin.
```
```yaml
apiVersion: v1
kind: Namespace
metadata:
@@ -218,7 +218,7 @@ metadata:
The `rook-system` service account is responsible for launching all pods, services, daemonsets, etc. for Rook and should have enough privilege to do and nothing more. I've not audited all the RBAC rules but a good tool to do is [here](https://github.com/liggitt/audit2rbac). For example:
```
```yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
@@ -252,13 +252,13 @@ subjects:
namespace: rook-system
```
Notably absent here are privileges to set other RBAC rules read cluster-wide secrets, etc. Because the admin created the `rook-system` namespace and service account they are free to set policies on them using PSP or namespace quotas. Similar to the ones defined [here](https://github.com/rook/rook/blob/master/Documentation/kubernetes.md#rbac-for-podsecuritypolicies)).
Notably absent here are privileges to set other RBAC rules and create read cluster-wide secrets and other resources. Because the admin created the `rook-system` namespace and service account they are free to set policies on them using PSP or namespace quotas. Similar to the ones defined [here](https://github.com/rook/rook/blob/master/Documentation/kubernetes.md#rbac-for-podsecuritypolicies)).
Also note that while we use a `ClusterRole` for rook-system we only use a `RoleBinding` to grant it access to the `rook-system` namespace. It does not have cluster-wide privileges.
When creating a Rook cluster the cluster admin will continue to define the namespace and cluster CRD as follows:
```
```yaml
apiVersion: v1
kind: Namespace
metadata:
@@ -274,7 +274,7 @@ metadata:
In addition we will require that the cluster-admin define a service account and role binding as follows:
```
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -284,7 +284,7 @@ metadata:
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
name: rook-cluster
namespace: mycluster
rules:
- apiGroups: [""]
@@ -331,11 +331,11 @@ Also all rook pods except the rook operator pod should run using `rook-cluster`
Finally, we should support running multiple rook clusters in the same namespaces. While namespaces are a great organizational unit for pods etc. they are also a unit of policy and quotas. While we can force the cluster admin to go to an approach where they need to manage multiple namespaces, we would be better off if we give the option to cluster admin decide how they use namespace.
For example, it should be possible to run rook-operator, rook-agent, and multiple independent rook clusters in a single namespace. This is going to require setting a prefix for pod names and other resources that could collide. It also means that removing a rook cluster would require enumerating each of the resources and removing them.
For example, it should be possible to run rook-operator, rook-agent, and multiple independent rook clusters in a single namespace. This is going to require setting a prefix for pod names and other resources that could collide.
The following should be possible:
```
```yaml
apiVersion: v1
kind: Namespace
metadata:
+3
View File
@@ -63,6 +63,9 @@ type ClusterSpec struct {
// Resources set resource requests and limits
Resources rook.ResourceSpec `json:"resources,omitempty"`
// The service account in which to start the cluster resources if the default is not sufficient (OSD pods)
ServiceAccount string `json:"serviceAccount,omitempty"`
// The path on the host where config and data can be persisted.
DataDirHostPath string `json:"dataDirHostPath,omitempty"`
+4 -37
View File
@@ -23,14 +23,9 @@ import (
"os"
"github.com/coreos/pkg/capnslog"
cephv1alpha1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1alpha1"
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
opcluster "github.com/rook/rook/pkg/operator/ceph/cluster"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/api/rbac/v1beta1"
kserrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
@@ -46,29 +41,6 @@ const (
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-agent")
var clusterAccessRules = []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes", "nodes/proxy"},
Verbs: []string{"get", "list"},
},
{
APIGroups: []string{rookv1alpha2.CustomResourceGroup},
Resources: []string{attachment.CustomResourceNamePlural},
Verbs: []string{"get", "list", "create", "delete", "update"},
},
{
APIGroups: []string{cephv1alpha1.CustomResourceGroup},
Resources: []string{opcluster.CustomResourceNamePlural},
Verbs: []string{"get", "list", "watch"},
},
{
APIGroups: []string{"storage.k8s.io"},
Resources: []string{"storageclasses"},
Verbs: []string{"get"},
},
}
// New creates an instance of Agent
func New(clientset kubernetes.Interface) *Agent {
return &Agent{
@@ -77,21 +49,16 @@ func New(clientset kubernetes.Interface) *Agent {
}
// Start the agent
func (a *Agent) Start(namespace, agentImage string) error {
func (a *Agent) Start(namespace, agentImage, serviceAccount string) error {
err := k8sutil.MakeClusterRole(a.clientset, namespace, agentDaemonsetName, clusterAccessRules, nil)
if err != nil {
return fmt.Errorf("failed to init RBAC for rook-ceph-agents. %+v", err)
}
err = a.createAgentDaemonSet(namespace, agentImage)
err := a.createAgentDaemonSet(namespace, agentImage, serviceAccount)
if err != nil {
return fmt.Errorf("Error starting agent daemonset: %v", err)
}
return nil
}
func (a *Agent) createAgentDaemonSet(namespace, agentImage string) error {
func (a *Agent) createAgentDaemonSet(namespace, agentImage, serviceAccount string) error {
flexvolumeDirPath, source := a.discoverFlexvolumeDir()
logger.Infof("discovered flexvolume dir path from source %s. value: %s", source, flexvolumeDirPath)
@@ -112,7 +79,7 @@ func (a *Agent) createAgentDaemonSet(namespace, agentImage string) error {
},
},
Spec: v1.PodSpec{
ServiceAccountName: agentDaemonsetName,
ServiceAccountName: serviceAccount,
Containers: []v1.Container{
{
Name: agentDaemonsetName,
+8 -26
View File
@@ -58,27 +58,15 @@ func TestStartAgentDaemonset(t *testing.T) {
a := New(clientset)
// start a basic cluster
err := a.Start(namespace, "rook/rook:myversion")
err := a.Start(namespace, "rook/rook:myversion", "mysa")
assert.Nil(t, err)
// check clusters rbac roles
_, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-ceph-agent", metav1.GetOptions{})
assert.Nil(t, err)
role, err := clientset.RbacV1beta1().ClusterRoles().Get("rook-ceph-agent", metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, 4, len(role.Rules))
binding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get("rook-ceph-agent", metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, "rook-ceph-agent", binding.Subjects[0].Name)
assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind)
// check daemonset parameters
agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-ceph-agent", metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, namespace, agentDS.Namespace)
assert.Equal(t, "rook-ceph-agent", agentDS.Name)
assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName)
assert.True(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged)
volumes := agentDS.Spec.Template.Spec.Volumes
assert.Equal(t, 4, len(volumes))
@@ -117,19 +105,12 @@ func TestGetContainerImage(t *testing.T) {
clientset.CoreV1().Pods("Default").Create(&pod)
// start a basic cluster
image, err := k8sutil.GetContainerImage(clientset, "")
returnPod, err := k8sutil.GetRunningPod(clientset)
assert.Nil(t, err)
assert.Equal(t, "rook/test", image)
assert.Equal(t, "mypod", returnPod.Name)
}
func TestGetContainerImageMultipleContainers(t *testing.T) {
clientset := test.New(3)
os.Setenv(k8sutil.PodNamespaceEnvVar, "Default")
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
os.Setenv(k8sutil.PodNameEnvVar, "mypod")
defer os.Unsetenv(k8sutil.PodNameEnvVar)
pod := v1.Pod{
ObjectMeta: metav1.ObjectMeta{
@@ -149,11 +130,11 @@ func TestGetContainerImageMultipleContainers(t *testing.T) {
},
},
}
clientset.CoreV1().Pods("Default").Create(&pod)
// start a basic cluster
_, err := k8sutil.GetContainerImage(clientset, "foo")
container, err := k8sutil.GetContainerImage(&pod, "foo")
assert.NotNil(t, err)
assert.Equal(t, "", container)
assert.Equal(t, "failed to find image for container foo", err.Error())
}
@@ -192,13 +173,14 @@ func TestStartAgentDaemonsetWithToleration(t *testing.T) {
a := New(clientset)
// start a basic cluster
err := a.Start(namespace, "rook/test")
err := a.Start(namespace, "rook/test", "mysa")
assert.Nil(t, err)
// check daemonset toleration
agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-ceph-agent", metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, 1, len(agentDS.Spec.Template.Spec.Tolerations))
assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName)
assert.Equal(t, "NoSchedule", string(agentDS.Spec.Template.Spec.Tolerations[0].Effect))
assert.Equal(t, "example", string(agentDS.Spec.Template.Spec.Tolerations[0].Key))
assert.Equal(t, "Exists", string(agentDS.Spec.Template.Spec.Tolerations[0].Operator))
+1 -1
View File
@@ -613,7 +613,7 @@ func (c *cluster) createInstance(rookImage string) error {
}
// Start the OSDs
c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.Storage, c.Spec.DataDirHostPath,
c.osds = osd.New(c.context, c.Namespace, rookImage, c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath,
cephv1alpha1.GetOSDPlacement(c.Spec.Placement), c.Spec.Network.HostNetwork, cephv1alpha1.GetOSDResources(c.Spec.Resources), c.ownerRef)
err = c.osds.Start()
if err != nil {
+10 -16
View File
@@ -33,7 +33,6 @@ import (
"github.com/rook/rook/pkg/util/display"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/api/rbac/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
@@ -55,16 +54,9 @@ const (
appName = "rook-ceph-osd"
appNameFmt = "rook-ceph-osd-%s"
clusterAvailableSpaceReserve = 0.05
defaultServiceAccountName = "rook-ceph-cluster"
)
var clusterAccessRules = []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"configmaps"},
Verbs: []string{"get", "list", "watch", "create", "update", "delete"},
},
}
// Cluster keeps track of the OSDs
type Cluster struct {
context *clusterd.Context
@@ -77,16 +69,24 @@ type Cluster struct {
HostNetwork bool
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
serviceAccount string
}
// New creates an instance of the OSD manager
func New(context *clusterd.Context, namespace, version string, storageSpec rookalpha.StorageScopeSpec,
func New(context *clusterd.Context, namespace, version, serviceAccount string, storageSpec rookalpha.StorageScopeSpec,
dataDirHostPath string, placement rookalpha.Placement, hostNetwork bool,
resources v1.ResourceRequirements, ownerRef metav1.OwnerReference) *Cluster {
if serviceAccount == "" {
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
serviceAccount = defaultServiceAccountName
logger.Infof("setting the osd pods to use the service account name: %s", serviceAccount)
}
return &Cluster{
context: context,
Namespace: namespace,
serviceAccount: serviceAccount,
placement: placement,
Version: version,
Storage: storageSpec,
@@ -106,12 +106,6 @@ type OrchestrationStatus struct {
func (c *Cluster) Start() error {
logger.Infof("start running osds in namespace %s", c.Namespace)
// create the artifacts for the osd to work with RBAC enabled
err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, &c.ownerRef)
if err != nil {
logger.Warningf("failed to init RBAC for OSDs. %+v", err)
}
if c.Storage.UseAllNodes == false && len(c.Storage.Nodes) == 0 {
logger.Warningf("useAllNodes is set to false and no nodes are specified, no OSD pods are going to be created")
}
+5 -5
View File
@@ -39,7 +39,7 @@ import (
func TestStart(t *testing.T) {
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
// Start the first time
@@ -71,7 +71,7 @@ func TestAddRemoveNode(t *testing.T) {
statusMapWatcher := watch.NewFake()
clientset.PrependWatchReactor("configmaps", k8stesting.DefaultWatchReactor(statusMapWatcher, nil))
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "",
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
// kick off the start of the orchestration in a goroutine
@@ -127,7 +127,7 @@ func TestAddRemoveNode(t *testing.T) {
// modify the storage spec to remove the node from the cluster
storageSpec.Nodes = []rookalpha.Node{}
c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion",
c = New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: mockExec}, "ns-add-remove", "myversion", "",
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
// reset the orchestration status watcher
@@ -174,7 +174,7 @@ func TestAddNodeFailure(t *testing.T) {
return true, nil, fmt.Errorf("mock failed to create replica set")
})
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns-add-remove", "myversion", "",
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
// kick off the start of the orchestration in a goroutine
@@ -195,7 +195,7 @@ func TestAddNodeFailure(t *testing.T) {
func TestOrchestrationStatus(t *testing.T) {
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
rookalpha.StorageScopeSpec{}, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
// status map should not exist yet
+1 -1
View File
@@ -119,7 +119,7 @@ func (c *Cluster) podTemplateSpec(devices []rookalpha.Device, selection rookalph
}
podSpec := v1.PodSpec{
ServiceAccountName: appName,
ServiceAccountName: c.serviceAccount,
Containers: []v1.Container{c.osdContainer(devices, selection, resources, storeConfig, metadataDevice, location)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
+5 -4
View File
@@ -56,7 +56,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool)
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "mysa",
storageSpec, dataDir, rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
devMountNeeded := deviceFilter != "" || allDevices
@@ -66,6 +66,7 @@ func testPodDevices(t *testing.T, dataDir, deviceFilter string, allDevices bool)
assert.NotNil(t, replicaSet)
assert.Equal(t, "rook-ceph-osd-node1", replicaSet.Name)
assert.Equal(t, c.Namespace, replicaSet.Namespace)
assert.Equal(t, c.serviceAccount, replicaSet.Spec.Template.Spec.ServiceAccountName)
assert.Equal(t, int32(1), *(replicaSet.Spec.Replicas))
assert.Equal(t, "node1", replicaSet.Spec.Template.Spec.NodeSelector[apis.LabelHostname])
assert.Equal(t, v1.RestartPolicyAlways, replicaSet.Spec.Template.Spec.RestartPolicy)
@@ -148,7 +149,7 @@ func TestStorageSpecDevicesAndDirectories(t *testing.T) {
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
n := c.resolveNode(storageSpec.Nodes[0])
@@ -200,7 +201,7 @@ func TestStorageSpecConfig(t *testing.T) {
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
storageSpec, "", rookalpha.Placement{}, false, v1.ResourceRequirements{}, metav1.OwnerReference{})
n := c.resolveNode(storageSpec.Nodes[0])
@@ -245,7 +246,7 @@ func TestHostNetwork(t *testing.T) {
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion",
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
storageSpec, "", rookalpha.Placement{}, true, v1.ResourceRequirements{}, metav1.OwnerReference{})
n := c.resolveNode(storageSpec.Nodes[0])
+8 -6
View File
@@ -61,16 +61,17 @@ var provisionerConfigs = map[string]string{
// Operator type for managing storage
type Operator struct {
context *clusterd.Context
resources []opkit.CustomResource
rookImage string
context *clusterd.Context
resources []opkit.CustomResource
rookImage string
securityAccount string
// The custom resource that is global to the kubernetes cluster.
// The cluster is global because you create multiple clusters in k8s
clusterController *cluster.ClusterController
}
// New creates an operator instance
func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage string) *Operator {
func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachment, rookImage, securityAccount string) *Operator {
clusterController := cluster.NewClusterController(context, rookImage, volumeAttachmentWrapper)
schemes := []opkit.CustomResource{cluster.ClusterResource, pool.PoolResource, object.ObjectStoreResource,
@@ -80,6 +81,7 @@ func New(context *clusterd.Context, volumeAttachmentWrapper attachment.Attachmen
clusterController: clusterController,
resources: schemes,
rookImage: rookImage,
securityAccount: securityAccount,
}
}
@@ -112,12 +114,12 @@ func (o *Operator) Run() error {
rookAgent := agent.New(o.context.Clientset)
if err := rookAgent.Start(namespace, o.rookImage); err != nil {
if err := rookAgent.Start(namespace, o.rookImage, o.securityAccount); err != nil {
return fmt.Errorf("Error starting agent daemonset: %v", err)
}
rookDiscover := discover.New(o.context.Clientset)
if err := rookDiscover.Start(namespace, o.rookImage); err != nil {
if err := rookDiscover.Start(namespace, o.rookImage, o.securityAccount); err != nil {
return fmt.Errorf("Error starting device discovery daemonset: %v", err)
}
+1 -1
View File
@@ -32,7 +32,7 @@ import (
func TestOperator(t *testing.T) {
clientset := test.New(3)
context := &clusterd.Context{Clientset: clientset}
o := New(context, &attachment.MockAttachment{}, "")
o := New(context, &attachment.MockAttachment{}, "", "")
assert.NotNil(t, o)
assert.NotNil(t, o.clusterController)
+4 -9
View File
@@ -67,21 +67,16 @@ func New(clientset kubernetes.Interface) *Discover {
}
// Start the discover
func (d *Discover) Start(namespace, discoverImage string) error {
func (d *Discover) Start(namespace, discoverImage, securityAccount string) error {
err := k8sutil.MakeRole(d.clientset, namespace, discoverDaemonsetName, accessRules, nil)
if err != nil {
return fmt.Errorf("failed to init RBAC for rook-discover. %+v", err)
}
err = d.createDiscoverDaemonSet(namespace, discoverImage)
err := d.createDiscoverDaemonSet(namespace, discoverImage, securityAccount)
if err != nil {
return fmt.Errorf("Error starting discover daemonset: %v", err)
}
return nil
}
func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) error {
func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage, securityAccount string) error {
privileged := false
ds := &extensions.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
@@ -98,7 +93,7 @@ func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) erro
},
},
Spec: v1.PodSpec{
ServiceAccountName: discoverDaemonsetName,
ServiceAccountName: securityAccount,
Containers: []v1.Container{
{
Name: discoverDaemonsetName,
+2 -8
View File
@@ -46,14 +46,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) {
a := New(clientset)
// start a basic cluster
err := a.Start(namespace, "rook/rook:myversion")
assert.Nil(t, err)
// check clusters rbac roles
_, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-discover", metav1.GetOptions{})
assert.Nil(t, err)
_, err = clientset.RbacV1beta1().Roles(namespace).Get("rook-discover", metav1.GetOptions{})
err := a.Start(namespace, "rook/rook:myversion", "mysa")
assert.Nil(t, err)
// check daemonset parameters
@@ -61,6 +54,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, namespace, agentDS.Namespace)
assert.Equal(t, "rook-discover", agentDS.Name)
assert.Equal(t, "mysa", agentDS.Spec.Template.Spec.ServiceAccountName)
assert.False(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged)
volumes := agentDS.Spec.Template.Spec.Volumes
assert.Equal(t, 3, len(volumes))
+18 -15
View File
@@ -93,21 +93,7 @@ func ConfigDirEnvVar() v1.EnvVar {
return v1.EnvVar{Name: "ROOK_CONFIG_DIR", Value: DataDir}
}
func GetContainerImage(clientset kubernetes.Interface, name string) (string, error) {
podName := os.Getenv(PodNameEnvVar)
if podName == "" {
return "", fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file")
}
podNamespace := os.Getenv(PodNamespaceEnvVar)
if podName == "" {
return "", fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file")
}
pod, err := clientset.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{})
if err != nil {
return "", err
}
func GetContainerImage(pod *v1.Pod, name string) (string, error) {
image, err := GetMatchingContainer(pod.Spec.Containers, name)
if err != nil {
@@ -116,6 +102,23 @@ func GetContainerImage(clientset kubernetes.Interface, name string) (string, err
return image.Image, nil
}
func GetRunningPod(clientset kubernetes.Interface) (*v1.Pod, error) {
podName := os.Getenv(PodNameEnvVar)
if podName == "" {
return nil, fmt.Errorf("cannot detect the pod name. Please provide it using the downward API in the manifest file")
}
podNamespace := os.Getenv(PodNamespaceEnvVar)
if podName == "" {
return nil, fmt.Errorf("cannot detect the pod namespace. Please provide it using the downward API in the manifest file")
}
pod, err := clientset.CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{})
if err != nil {
return nil, err
}
return pod, nil
}
func GetMatchingContainer(containers []v1.Container, name string) (v1.Container, error) {
var result *v1.Container
if len(containers) == 1 {
-177
View File
@@ -1,177 +0,0 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package k8sutil for Kubernetes helpers.
package k8sutil
import (
"fmt"
"os"
"k8s.io/api/core/v1"
"k8s.io/api/rbac/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
const (
enableRBACEnv = "RBAC_ENABLED"
)
func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error {
err := makeServiceAccount(clientset, namespace, name, ownerRef)
if err != nil {
return err
}
if !isRBACEnabled() {
return nil
}
// Create the role if it doesn't yet exist.
// If the role already exists we have to update it. Otherwise if the permissions change during an upgrade,
// the create will fail with an error that we're changing the permissions.
role := &v1beta1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Rules: rules,
}
if ownerRef != nil {
role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef}
}
_, err = clientset.RbacV1beta1().Roles(namespace).Get(role.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
logger.Infof("creating role %s in namespace %s", name, namespace)
_, err = clientset.RbacV1beta1().Roles(namespace).Create(role)
} else if err == nil {
logger.Infof("role %s already exists in namespace %s. updating if needed.", name, namespace)
_, err = clientset.RbacV1beta1().Roles(namespace).Update(role)
}
if err != nil {
return fmt.Errorf("failed to create/update role %s in namespace %s. %+v", name, namespace, err)
}
binding := &v1beta1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
RoleRef: v1beta1.RoleRef{
Name: name,
Kind: "Role",
APIGroup: "rbac.authorization.k8s.io",
},
Subjects: []v1beta1.Subject{
{Kind: "ServiceAccount", Name: name, Namespace: namespace},
},
}
if ownerRef != nil {
binding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef}
}
_, err = clientset.RbacV1beta1().RoleBindings(namespace).Create(binding)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s role binding in namespace %s. %+v", name, namespace, err)
}
return nil
}
func MakeClusterRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error {
err := makeServiceAccount(clientset, namespace, name, ownerRef)
if err != nil {
return err
}
if !isRBACEnabled() {
return nil
}
// Create the cluster scoped role if it doesn't yet exist.
// If the role already exists we have to update it. Otherwise if the permissions change during an upgrade,
// the create will fail with an error that we're changing the permissions.
role := &v1beta1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Rules: rules,
}
if ownerRef != nil {
role.OwnerReferences = []metav1.OwnerReference{*ownerRef}
}
_, err = clientset.RbacV1beta1().ClusterRoles().Get(role.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
logger.Infof("creating cluster role %s", name)
_, err = clientset.RbacV1beta1().ClusterRoles().Create(role)
} else if err == nil {
logger.Infof("cluster role %s already exists. Updating if needed.", name)
_, err = clientset.RbacV1beta1().ClusterRoles().Update(role)
}
if err != nil {
return fmt.Errorf("failed to create/update cluster role %s. %+v", name, err)
}
binding := &v1beta1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
RoleRef: v1beta1.RoleRef{
Name: name,
Kind: "ClusterRole",
APIGroup: "rbac.authorization.k8s.io",
},
Subjects: []v1beta1.Subject{
{Kind: "ServiceAccount", Name: name, Namespace: namespace},
},
}
if ownerRef != nil {
binding.OwnerReferences = []metav1.OwnerReference{*ownerRef}
}
_, err = clientset.RbacV1beta1().ClusterRoleBindings().Create(binding)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create cluster role binding %s. %+v", name, err)
}
return nil
}
func makeServiceAccount(clientset kubernetes.Interface, namespace, name string, ownerRef *metav1.OwnerReference) error {
account := &v1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
}
if ownerRef != nil {
account.OwnerReferences = []metav1.OwnerReference{*ownerRef}
}
_, err := clientset.CoreV1().ServiceAccounts(namespace).Create(account)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s service account in namespace %s. %+v", name, namespace, err)
}
return nil
}
func isRBACEnabled() bool {
r := os.Getenv(enableRBACEnv)
if r == "false" {
return false
}
return true
}
-228
View File
@@ -1,228 +0,0 @@
/*
Copyright 2017 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package k8sutil for Kubernetes helpers.
package k8sutil
import (
"os"
"testing"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
"k8s.io/api/rbac/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestMakeRole(t *testing.T) {
clientset := test.New(1)
namespace := "myns"
name := "myapp"
rules := []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"namespaces", "secrets", "pods", "services", "nodes", "configmaps", "events"},
Verbs: []string{"get", "list", "watch", "create", "update"},
},
{
APIGroups: []string{"extensions"},
Resources: []string{"thirdpartyresources", "deployments", "daemonsets", "replicasets"},
Verbs: []string{"get", "list", "create", "delete"},
},
{
APIGroups: []string{"apiextensions.k8s.io"},
Resources: []string{"customresourcedefinitions"},
Verbs: []string{"get", "list", "create"},
},
{
APIGroups: []string{"storage.k8s.io"},
Resources: []string{"storageclasses"},
Verbs: []string{"get", "list"},
},
}
ownerRef := metav1.OwnerReference{}
err := MakeRole(clientset, namespace, name, rules, &ownerRef)
role, err := clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, name, role.Name)
assert.Equal(t, 4, len(role.Rules))
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, namespace, account.Namespace)
binding, err := clientset.RbacV1beta1().RoleBindings(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, name, binding.RoleRef.Name)
assert.Equal(t, "Role", binding.RoleRef.Kind)
assert.Equal(t, "rbac.authorization.k8s.io", binding.RoleRef.APIGroup)
assert.Equal(t, name, binding.Subjects[0].Name)
assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind)
// update the rules
newRules := []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"namespaces"},
Verbs: []string{"get", "list"},
},
}
err = MakeRole(clientset, namespace, name, newRules, &ownerRef)
assert.Nil(t, err)
role, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, 1, len(role.Rules))
assert.Equal(t, "", role.Rules[0].APIGroups[0])
assert.Equal(t, 1, len(role.Rules[0].Resources))
assert.Equal(t, 2, len(role.Rules[0].Verbs))
}
func TestMakeClusterRole(t *testing.T) {
clientset := test.New(1)
namespace := "myns"
name := "myapp"
rules := []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"},
Verbs: []string{"get", "list"},
},
{
APIGroups: []string{"storage.k8s.io"},
Resources: []string{"storageclasses"},
Verbs: []string{"get"},
},
}
err := MakeClusterRole(clientset, namespace, name, rules, nil)
role, err := clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, name, role.Name)
assert.Equal(t, 2, len(role.Rules))
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, namespace, account.Namespace)
binding, err := clientset.RbacV1beta1().ClusterRoleBindings().Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, name, binding.RoleRef.Name)
assert.Equal(t, "ClusterRole", binding.RoleRef.Kind)
assert.Equal(t, "rbac.authorization.k8s.io", binding.RoleRef.APIGroup)
assert.Equal(t, name, binding.Subjects[0].Name)
assert.Equal(t, "ServiceAccount", binding.Subjects[0].Kind)
// update the rules
newRules := []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"namespaces"},
Verbs: []string{"get", "list"},
},
}
err = MakeClusterRole(clientset, namespace, name, newRules, nil)
assert.Nil(t, err)
role, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, 1, len(role.Rules))
assert.Equal(t, "", role.Rules[0].APIGroups[0])
assert.Equal(t, 1, len(role.Rules[0].Resources))
assert.Equal(t, 2, len(role.Rules[0].Verbs))
}
func TestMakeRoleRBACDisabled(t *testing.T) {
os.Setenv(enableRBACEnv, "false")
defer os.Unsetenv(enableRBACEnv)
clientset := test.New(1)
namespace := "myns"
name := "myapp"
rules := []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"namespaces", "secrets", "pods", "services", "nodes", "configmaps", "events"},
Verbs: []string{"get", "list", "watch", "create", "update"},
},
{
APIGroups: []string{"extensions"},
Resources: []string{"thirdpartyresources", "deployments", "daemonsets", "replicasets"},
Verbs: []string{"get", "list", "create", "delete"},
},
{
APIGroups: []string{"apiextensions.k8s.io"},
Resources: []string{"customresourcedefinitions"},
Verbs: []string{"get", "list", "create"},
},
{
APIGroups: []string{"storage.k8s.io"},
Resources: []string{"storageclasses"},
Verbs: []string{"get", "list"},
},
}
ownerRef := metav1.OwnerReference{}
err := MakeRole(clientset, namespace, name, rules, &ownerRef)
_, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{})
assert.NotNil(t, err)
assert.True(t, errors.IsNotFound(err))
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, namespace, account.Namespace)
_, err = clientset.RbacV1beta1().RoleBindings(namespace).Get(name, metav1.GetOptions{})
assert.NotNil(t, err)
assert.True(t, errors.IsNotFound(err))
}
func TestMakeClusterRoleRBACDisabled(t *testing.T) {
os.Setenv(enableRBACEnv, "false")
defer os.Unsetenv(enableRBACEnv)
clientset := test.New(1)
namespace := "myns"
name := "myapp"
rules := []v1beta1.PolicyRule{
{
APIGroups: []string{""},
Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"},
Verbs: []string{"get", "list"},
},
{
APIGroups: []string{"storage.k8s.io"},
Resources: []string{"storageclasses"},
Verbs: []string{"get"},
},
}
err := MakeClusterRole(clientset, namespace, name, rules, nil)
assert.Nil(t, err)
_, err = clientset.RbacV1beta1().ClusterRoles().Get(name, metav1.GetOptions{})
assert.NotNil(t, err)
assert.True(t, errors.IsNotFound(err))
account, err := clientset.CoreV1().ServiceAccounts(namespace).Get(name, metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, namespace, account.Namespace)
_, err = clientset.RbacV1beta1().ClusterRoleBindings().Get(name, metav1.GetOptions{})
assert.NotNil(t, err)
assert.True(t, errors.IsNotFound(err))
}
+145 -21
View File
@@ -29,7 +29,7 @@ func NewK8sInstallData() *InstallData {
return &InstallData{}
}
func (i *InstallData) GetRookCRDs(namespace string) string {
func (i *InstallData) GetRookCRDs() string {
return `apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
@@ -110,24 +110,54 @@ metadata:
name: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
kind: Role
metadata:
name: rook-ceph-operator
name: rook-ceph-system
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- extensions
resources:
- daemonsets
verbs:
- get
- list
- watch
- create
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-cluster-mgmt
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- namespaces
- serviceaccounts
- secrets
- pods
- services
- nodes
- nodes/proxy
- configmaps
- events
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
@@ -149,17 +179,36 @@ rules:
- create
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- rbac.authorization.k8s.io
- ""
resources:
- clusterroles
- clusterrolebindings
- roles
- rolebindings
- pods
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
- persistentvolumes
- persistentvolumeclaims
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
@@ -171,7 +220,6 @@ rules:
- get
- list
- watch
- delete
- apiGroups:
- ceph.rook.io
resources:
@@ -188,21 +236,44 @@ rules:
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-operator
name: rook-ceph-system
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-system
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-system
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + namespace + `
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-operator
name: rook-ceph-global
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-operator
name: rook-ceph-global
subjects:
- kind: ServiceAccount
name: rook-ceph-operator
name: rook-ceph-system
namespace: ` + namespace + `
---
apiVersion: apps/v1beta1
@@ -210,6 +281,9 @@ kind: Deployment
metadata:
name: rook-ceph-operator
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
spec:
replicas: 1
template:
@@ -217,7 +291,7 @@ spec:
labels:
app: rook-ceph-operator
spec:
serviceAccountName: rook-ceph-operator
serviceAccountName: rook-ceph-system
containers:
- name: rook-ceph-operator
image: rook/ceph:master
@@ -243,6 +317,55 @@ spec:
fieldPath: metadata.namespace`
}
//GetRookCluster returns rook-cluster manifest
func (i *InstallData) GetClusterRoles(namespace, systemNamespace string) string {
return `apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cluster
namespace: ` + namespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster
namespace: ` + namespace + `
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster-mgmt
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + systemNamespace + `
---
# Allow the pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cluster
subjects:
- kind: ServiceAccount
name: rook-ceph-cluster
namespace: ` + namespace
}
//GetRookCluster returns rook-cluster manifest
func (i *InstallData) GetRookCluster(namespace, storeType, dataDirHostPath string, useAllDevices bool, mons int) string {
return `apiVersion: ceph.rook.io/v1alpha1
@@ -251,6 +374,7 @@ metadata:
name: ` + namespace + `
namespace: ` + namespace + `
spec:
serviceAccount: rook-ceph-cluster
dataDirHostPath: ` + dataDirHostPath + `
network:
hostNetwork: false
+37 -18
View File
@@ -22,6 +22,7 @@ import (
"testing"
"time"
"k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/kubelet/apis"
"flag"
@@ -67,11 +68,11 @@ type InstallHelper struct {
T func() *testing.T
}
func (h *InstallHelper) CreateK8sRookResources(namespace string) (err error) {
func (h *InstallHelper) CreateK8sRookResources() (err error) {
var resources string
logger.Info("Creating Rook CRD's")
resources = h.installData.GetRookCRDs(namespace)
resources = h.installData.GetRookCRDs()
_, err = h.k8shelper.KubectlWithStdin(resources, createFromStdinArgs...)
@@ -85,7 +86,7 @@ func (h *InstallHelper) CreateK8sRookOperator(namespace string) (err error) {
h.k8shelper.CreateAnonSystemClusterBinding()
//creating rook resources
if err = h.CreateK8sRookResources(namespace); err != nil {
if err = h.CreateK8sRookResources(); err != nil {
return err
}
@@ -149,17 +150,26 @@ func (h *InstallHelper) CreateK8sRookToolbox(namespace string) (err error) {
return nil
}
func (h *InstallHelper) CreateK8sRookCluster(namespace string, storeType string) (err error) {
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, "", false,
func (h *InstallHelper) CreateK8sRookCluster(namespace, systemNamespace string, storeType string) (err error) {
return h.CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, "", false,
cephv1alpha1.MonSpec{Count: 3, AllowMultiplePerNode: true}, true /* startWithAllNodes */)
}
//CreateK8sRookCluster creates rook cluster via kubectl
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath string,
func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, systemNamespace, storeType, dataDirHostPath string,
useAllDevices bool, mon cephv1alpha1.MonSpec, startWithAllNodes bool) error {
if err := h.k8shelper.CreateNamespace(namespace); err != nil {
return err
logger.Infof("Creating namespace %s", namespace)
ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
_, err := h.k8shelper.Clientset.CoreV1().Namespaces().Create(ns)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create namespace %s. %+v", namespace, err)
}
logger.Infof("Creating cluster roles")
roles := h.installData.GetClusterRoles(namespace, systemNamespace)
if _, err := h.k8shelper.KubectlWithStdin(roles, createFromStdinArgs...); err != nil {
return fmt.Errorf("Failed to create cluster roles. %+v", err)
}
if h.k8shelper.IsRookClientsetAvailable() {
@@ -171,6 +181,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, st
Namespace: namespace,
},
Spec: v1alpha1.ClusterSpec{
ServiceAccount: "rook-ceph-cluster",
DataDirHostPath: dataDirHostPath,
Mon: v1alpha1.MonSpec{
Count: mon.Count,
@@ -236,7 +247,7 @@ func (h *InstallHelper) CreateK8sRookClusterWithHostPathAndDevices(namespace, st
}
logger.Infof("Rook Cluster started")
err := h.k8shelper.WaitForLabeledPodToRun("app=rook-ceph-osd", namespace)
err = h.k8shelper.WaitForLabeledPodToRun("app=rook-ceph-osd", namespace)
return err
}
@@ -288,7 +299,7 @@ func (h *InstallHelper) InstallRookOnK8sWithHostPathAndDevices(namespace, storeT
}
//Create rook cluster
err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, storeType, dataDirHostPath,
err = h.CreateK8sRookClusterWithHostPathAndDevices(namespace, onamespace, storeType, dataDirHostPath,
useDevices, cephv1alpha1.MonSpec{Count: mon.Count, AllowMultiplePerNode: mon.AllowMultiplePerNode}, startWithAllNodes)
if err != nil {
logger.Errorf("Rook cluster %s not installed, error -> %v", namespace, err)
@@ -322,13 +333,17 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
for _, namespace := range namespaces {
if !h.k8shelper.VersionAtLeast("v1.8.0") {
_, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-osd")
h.checkError(err, "cannot remove serviceaccount rook-ceph-osd")
_, err = h.k8shelper.DeleteResource("-n", namespace, "serviceaccount", "rook-ceph-cluster")
h.checkError(err, "cannot remove serviceaccount rook-ceph-cluster")
assert.NoError(h.T(), err, "%s err -> %v", namespace, err)
err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-osd", namespace)
h.checkError(err, "rook-ceph-osd cluster role and binding cannot be deleted")
assert.NoError(h.T(), err, "rook-ceph-osd cluster role and binding cannot be deleted: %+v", err)
err = h.k8shelper.DeleteRoleAndBindings("rook-ceph-cluster", namespace)
h.checkError(err, "rook-ceph-cluster cluster role and binding cannot be deleted")
assert.NoError(h.T(), err, "rook-ceph-cluster cluster role and binding cannot be deleted: %+v", err)
err = h.k8shelper.DeleteRoleBinding("rook-ceph-cluster-mgmt", namespace)
h.checkError(err, "rook-ceph-cluster-mgmt binding cannot be deleted")
assert.NoError(h.T(), err, "rook-ceph-cluster-mgmt binding cannot be deleted: %+v", err)
}
_, err = h.k8shelper.DeleteResource("-n", namespace, "cluster.ceph.rook.io", namespace)
@@ -357,9 +372,13 @@ func (h *InstallHelper) UninstallRookFromMultipleNS(helmInstalled bool, systemNa
}
h.checkError(err, "cannot uninstall rook-operator")
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-agent", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-agent", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil)
h.k8shelper.Clientset.CoreV1().ServiceAccounts(systemNamespace).Delete("rook-ceph-system", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-cluster-mgmt", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("rook-ceph-global", nil)
h.k8shelper.Clientset.RbacV1beta1().Roles(systemNamespace).Delete("rook-ceph-system", nil)
logger.Infof("done removing the operator from namespace %s", systemNamespace)
}
+5
View File
@@ -1232,6 +1232,11 @@ func (k8sh *K8sHelper) DeleteRoleAndBindings(name, namespace string) error {
return nil
}
func (k8sh *K8sHelper) DeleteRoleBinding(name, namespace string) error {
_, err := k8sh.DeleteResource("rolebinding", name, "-n", namespace)
return err
}
func (k8sh *K8sHelper) ScaleStatefulSet(statefulSetName, namespace string, replicationSize int) error {
args := []string{"-n", namespace, "scale", "statefulsets", statefulSetName, fmt.Sprintf("--replicas=%d", replicationSize)}
_, err := k8sh.Kubectl(args...)
@@ -155,13 +155,14 @@ func (o MCTestOperations) SetUp() {
err = o.installer.CreateK8sRookOperator(installer.SystemNamespace(o.namespace1))
require.NoError(o.T(), err)
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", installer.SystemNamespace(o.namespace1), "Running"),
systemNamespace := installer.SystemNamespace(o.namespace1)
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-operator", systemNamespace, "Running"),
"Make sure rook-operator is in running state")
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-agent", installer.SystemNamespace(o.namespace1), "Running"),
require.True(o.T(), o.kh.IsPodInExpectedState("rook-ceph-agent", systemNamespace, "Running"),
"Make sure rook-ceph-agent is in running state")
require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", installer.SystemNamespace(o.namespace1), "Running"),
require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", systemNamespace, "Running"),
"Make sure rook-discover is in running state")
time.Sleep(10 * time.Second)
@@ -170,8 +171,8 @@ func (o MCTestOperations) SetUp() {
logger.Infof("starting two clusters in parallel")
errCh1 := make(chan error, 1)
errCh2 := make(chan error, 1)
go o.startCluster(o.namespace1, "bluestore", errCh1)
go o.startCluster(o.namespace2, "filestore", errCh2)
go o.startCluster(o.namespace1, systemNamespace, "bluestore", errCh1)
go o.startCluster(o.namespace2, systemNamespace, "filestore", errCh2)
require.NoError(o.T(), <-errCh1)
require.NoError(o.T(), <-errCh2)
logger.Infof("finished starting clusters")
@@ -194,9 +195,9 @@ func (o MCTestOperations) TearDown() {
o.installer.UninstallRookFromMultipleNS(false, installer.SystemNamespace(o.namespace1), o.namespace1, o.namespace2)
}
func (o MCTestOperations) startCluster(namespace, store string, errCh chan error) {
func (o MCTestOperations) startCluster(namespace, systemNamespace, store string, errCh chan error) {
logger.Infof("starting cluster %s", namespace)
if err := o.installer.CreateK8sRookCluster(namespace, store); err != nil {
if err := o.installer.CreateK8sRookCluster(namespace, systemNamespace, store); err != nil {
errCh <- fmt.Errorf("failed to create cluster %s. %+v", namespace, err)
return
}
-1
View File
@@ -163,7 +163,6 @@ func (o BaseLoadTestOperations) SetUp() {
true /* startWithAllNodes */)
require.NoError(o.T(), err)
require.True(o.T(), isRookInstalled)
}
// Enable chaos monkey if enable_chaos flag is present