feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity
Files
585d7a963dbee783bcd072ec00fa28ca2efb9722
my-rook-config/tests/framework/installer/ceph_manifests.go
T
Travis Nielsen a777c94a08 Merge pull request #3778 from rohantmp/managedPDB 
Reflect pdb rbac in helm template and ceph_manifests.
2019-09-05 14:10:40 -06:00

1838 lines
45 KiB
Go
Raw Blame History

/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"strconv"
"github.com/google/uuid"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
)
type CephManifests interface {
GetRookCRDs() string
GetRookOperator(namespace string) string
GetClusterRoles(namespace, systemNamespace string) string
GetRookCluster(settings *ClusterSettings) string
GetRookToolBox(namespace string) string
GetCleanupPod(node, removalDir string) string
GetBlockPoolDef(poolName string, namespace string, replicaSize string) string
GetBlockStorageClassDef(poolName string, storageClassName string, reclaimPolicy string, namespace string, varClusterName bool) string
GetBlockPvcDef(claimName string, storageClassName string, accessModes string, size string) string
GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, reclaimPolicy string, blockName string, accessMode string) string
GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string, reclaimPolicy string) string
GetFilesystem(namepace, name string, activeCount int) string
GetNFS(namepace, name, pool string, daemonCount int) string
GetObjectStore(namespace, name string, replicaCount, port int) string
GetObjectStoreUser(namespace, name string, displayName string, store string) string
GetBucketStorageClass(namespace string, storeName string, storageClassName string, reclaimPolicy string, region string) string
GetObc(obcName string, storageClassName string, bucketName string, createBucket bool) string
}
type ClusterSettings struct {
Namespace string
StoreType string
DataDirHostPath string
UseAllDevices bool
Mons int
RBDMirrorWorkers int
CephVersion cephv1.CephVersionSpec
}
// CephManifestsMaster wraps rook yaml definitions
type CephManifestsMaster struct {
imageTag string
}
// NewCephManifests gets the manifest type depending on the Rook version desired
func NewCephManifests(version string) CephManifests {
switch version {
case VersionMaster:
return &CephManifestsMaster{imageTag: VersionMaster}
case Version1_0:
return &CephManifestsV1_0{imageTag: Version1_0}
}
panic(fmt.Errorf("unrecognized ceph manifest version: %s", version))
}
func (m *CephManifestsMaster) GetRookCRDs() string {
return `
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephclusters.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephCluster
listKind: CephClusterList
plural: cephclusters
singular: cephcluster
scope: Namespaced
version: v1
validation:
openAPIV3Schema:
properties:
spec:
properties:
annotations: {}
cephVersion:
properties:
allowUnsupported:
type: boolean
image:
type: string
dashboard:
properties:
enabled:
type: boolean
urlPrefix:
type: string
port:
type: integer
minimum: 0
maximum: 65535
ssl:
type: boolean
dataDirHostPath:
pattern: ^/(\S+)
type: string
mon:
properties:
allowMultiplePerNode:
type: boolean
count:
maximum: 9
minimum: 1
type: integer
required:
- count
network:
properties:
hostNetwork:
type: boolean
storage:
properties:
useAllNodes:
type: boolean
nodes:
items:
properties:
name:
type: string
config:
properties:
metadataDevice:
type: string
storeType:
type: string
pattern: ^(filestore|bluestore)$
databaseSizeMB:
type: string
walSizeMB:
type: string
journalSizeMB:
type: string
osdsPerDevice:
type: string
encryptedDevice:
type: string
pattern: ^(true|false)$
useAllDevices:
type: boolean
deviceFilter: {}
directories:
type: array
items:
properties:
path:
type: string
devices:
type: array
items:
properties:
name:
type: string
config: {}
location: {}
resources: {}
type: array
useAllDevices:
type: boolean
deviceFilter: {}
location: {}
directories:
type: array
items:
properties:
path:
type: string
config: {}
topologyAware:
type: boolean
monitoring:
properties:
enabled:
type: boolean
rulesNamespace:
type: string
rbdMirroring:
properties:
workers:
type: integer
placement: {}
resources: {}
required:
- mon
additionalPrinterColumns:
- name: DataDirHostPath
type: string
description: Directory used on the K8s nodes
JSONPath: .spec.dataDirHostPath
- name: MonCount
type: string
description: Number of MONs
JSONPath: .spec.mon.count
- name: Age
type: date
JSONPath: .metadata.creationTimestamp
- name: State
type: string
description: Current State
JSONPath: .status.state
- name: Health
type: string
description: Ceph Health
JSONPath: .status.ceph.health
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephfilesystems.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephFilesystem
listKind: CephFilesystemList
plural: cephfilesystems
singular: cephfilesystem
scope: Namespaced
version: v1
validation:
openAPIV3Schema:
properties:
spec:
properties:
metadataServer:
properties:
activeCount:
type: integer
activeStandby:
type: boolean
annotations: {}
placement: {}
resources: {}
metadataPool:
properties:
failureDomain:
type: string
replicated:
properties:
size:
type: integer
erasureCoded:
properties:
dataChunks:
type: integer
codingChunks:
type: integer
dataPools:
type: array
items:
properties:
failureDomain:
type: string
replicated:
properties:
size:
type: integer
erasureCoded:
properties:
dataChunks:
type: integer
codingChunks:
type: integer
additionalPrinterColumns:
- name: MdsCount
type: string
description: Number of MDSs
JSONPath: .spec.metadataServer.activeCount
- name: Age
type: date
JSONPath: .metadata.creationTimestamp
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephnfses.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephNFS
listKind: CephNFSList
plural: cephnfses
singular: cephnfs
shortNames:
- nfs
scope: Namespaced
version: v1
validation:
openAPIV3Schema:
properties:
spec:
properties:
rados:
properties:
pool:
type: string
namespace:
type: string
server:
properties:
active:
type: integer
annotations: {}
placement: {}
resources: {}
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephobjectstores.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephObjectStore
listKind: CephObjectStoreList
plural: cephobjectstores
singular: cephobjectstore
scope: Namespaced
version: v1
validation:
openAPIV3Schema:
properties:
spec:
properties:
gateway:
properties:
type:
type: string
sslCertificateRef: {}
port:
type: integer
securePort: {}
instances:
type: integer
annotations: {}
placement: {}
resources: {}
metadataPool:
properties:
failureDomain:
type: string
replicated:
properties:
size:
type: integer
erasureCoded:
properties:
dataChunks:
type: integer
codingChunks:
type: integer
dataPool:
properties:
failureDomain:
type: string
replicated:
properties:
size:
type: integer
erasureCoded:
properties:
dataChunks:
type: integer
codingChunks:
type: integer
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephobjectstoreusers.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephObjectStoreUser
listKind: CephObjectStoreUserList
plural: cephobjectstoreusers
singular: cephobjectstoreuser
scope: Namespaced
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephblockpools.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephBlockPool
listKind: CephBlockPoolList
plural: cephblockpools
singular: cephblockpool
scope: Namespaced
version: v1
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: volumes.rook.io
spec:
group: rook.io
names:
kind: Volume
listKind: VolumeList
plural: volumes
singular: volume
scope: Namespaced
version: v1alpha2
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectbuckets.objectbucket.io
spec:
group: objectbucket.io
versions:
- name: v1alpha1
served: true
storage: true
names:
kind: ObjectBucket
listKind: ObjectBucketList
plural: objectbuckets
singular: objectbucket
shortNames:
- ob
- obs
scope: Cluster
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectbucketclaims.objectbucket.io
spec:
versions:
- name: v1alpha1
served: true
storage: true
group: objectbucket.io
names:
kind: ObjectBucketClaim
listKind: ObjectBucketClaimList
plural: objectbucketclaims
singular: objectbucketclaim
shortNames:
- obc
- obcs
scope: Namespaced
subresources:
status: {}`
}
// GetRookOperator returns rook Operator manifest
func (m *CephManifestsMaster) GetRookOperator(namespace string) string {
return `kind: Namespace
apiVersion: v1
metadata:
name: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: rook-ceph-system
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
- services
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- apps
resources:
- daemonsets
- statefulsets
- deployments
verbs:
- get
- list
- watch
- create
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-cluster-mgmt
labels:
operator: rook
storage-backend: ceph
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true"
rules: []
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-cluster-mgmt-rules
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true"
rules:
- apiGroups:
- ""
resources:
- secrets
- pods
- pods/log
- services
- configmaps
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- apps
resources:
- deployments
- daemonsets
verbs:
- get
- list
- watch
- create
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global
labels:
operator: rook
storage-backend: ceph
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true"
rules: []
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global-rules
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true"
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
- persistentvolumes
- persistentvolumeclaims
- endpoints
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- ceph.rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- policy
- apps
resources:
#this is for the clusterdisruption controller
- poddisruptionbudgets
#this is for both clusterdisruption and nodedrain controllers
- deployments
verbs:
- "*"
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-cluster
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-cluster-rules
labels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true"
rules:
- apiGroups:
- ""
resources:
- configmaps
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- list
- get
- watch
---
# Aspects of Rook Ceph Agent that require access to secrets
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-agent-mount
labels:
operator: rook
storage-backend: ceph
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-agent-mount-rules
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true"
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
---
# Aspects of ceph-mgr that require access to the system namespace
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-system
namespace: ` + namespace + `
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-system-rules
namespace: ` + namespace + `
labels:
rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true"
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-system
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-system
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-system
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + namespace + `
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-global
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-global
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + namespace + `
---
# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-cluster
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-mgr-cluster
subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: ` + namespace + `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-plugin-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin-rules
labels:
rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true"
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: rbd-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-rbd-provisioner-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-provisioner-runner
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-provisioner-runner-rules
labels:
rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create", "list", "watch", "delete", "get", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: rbd-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: ` + namespace + `
name: rbd-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role-cfg
namespace: ` + namespace + `
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
namespace: ` + namespace + `
roleRef:
kind: Role
name: rbd-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-cephfs-plugin-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin-rules
labels:
rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true"
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-plugin-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: cephfs-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-csi-cephfs-provisioner-sa
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-external-provisioner-runner
aggregationRule:
clusterRoleSelectors:
- matchLabels:
rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true"
rules: []
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-external-provisioner-runner-rules
labels:
rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: cephfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: ` + namespace + `
name: cephfs-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role-cfg
namespace: ` + namespace + `
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
namespace: ` + namespace + `
roleRef:
kind: Role
name: cephfs-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: rook-privileged
spec:
privileged: true
allowedCapabilities:
# required by CSI
- SYS_ADMIN
# fsGroup - the flexVolume agent has fsGroup capabilities and could potentially be any group
fsGroup:
rule: RunAsAny
# runAsUser, supplementalGroups - Rook needs to run some pods as root
# Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time
runAsUser:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
# seLinux - seLinux context is unknown ahead of time; set if this is well-known
seLinux:
rule: RunAsAny
volumes:
# recommended minimum set
- configMap
- downwardAPI
- emptyDir
- persistentVolumeClaim
- secret
- projected
# required for Rook
- hostPath
- flexVolume
# allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known
# directory-based OSDs make this hard to nail down
# allowedHostPaths:
# - /run/udev # for OSD prep
# - /dev # for OSD prep
# - /var/lib/rook # or whatever the dataDirHostPath value is set to
# Ceph requires host IPC for setting up encrypted devices
hostIPC: true
# Ceph OSDs need to share the same PID namespace
hostPID: true
# hostNetwork can be set to 'false' if host networking isn't used
hostNetwork: true
hostPorts:
# Ceph messenger protocol v1
- min: 6789
max: 6790 # <- support old default port
# Ceph messenger protocol v2
- min: 3300
max: 3300
# Ceph RADOS ports for OSDs, MDSes
- min: 6800
max: 7300
# # Ceph dashboard port HTTP (not recommended)
# - min: 7000
# max: 7000
# Ceph dashboard port HTTPS
- min: 8443
max: 8443
# Ceph mgr Prometheus Metrics
- min: 9283
max: 9283
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: 'psp:rook'
rules:
- apiGroups:
- policy
resources:
- podsecuritypolicies
resourceNames:
- rook-privileged
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rook-ceph-system-psp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: 'psp:rook'
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rook-csi-rbd-provisioner-sa-psp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: 'psp:rook'
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rook-csi-rbd-plugin-sa-psp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: 'psp:rook'
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-plugin-sa
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rook-csi-cephfs-provisioner-sa-psp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: 'psp:rook'
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: rook-csi-cephfs-plugin-sa-psp
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: 'psp:rook'
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-plugin-sa
namespace: ` + namespace + `
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rook-ceph-operator
namespace: ` + namespace + `
labels:
operator: rook
storage-backend: ceph
spec:
replicas: 1
selector:
matchLabels:
app: rook-ceph-operator
template:
metadata:
labels:
app: rook-ceph-operator
spec:
serviceAccountName: rook-ceph-system
containers:
- name: rook-ceph-operator
image: rook/ceph:` + m.imageTag + `
args: ["ceph", "operator"]
env:
- name: ROOK_LOG_LEVEL
value: INFO
- name: ROOK_CEPH_STATUS_CHECK_INTERVAL
value: "5s"
- name: ROOK_MON_HEALTHCHECK_INTERVAL
value: "10s"
- name: ROOK_MON_OUT_TIMEOUT
value: "15s"
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: ROOK_CSI_ENABLE_CEPHFS
value: "true"
- name: ROOK_CSI_ENABLE_RBD
value: "true"
- name: ROOK_CSI_ENABLE_GRPC_METRICS
value: "true"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-object-bucket
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-object-bucket
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-object-bucket
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true"
rules:
- apiGroups:
- ""
verbs:
- "*"
resources:
- secrets
- configmaps
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- "objectbucket.io"
verbs:
- "*"
resources:
- "*"
`
}
// GetClusterRoles returns rook-cluster manifest
func (m *CephManifestsMaster) GetClusterRoles(namespace, systemNamespace string) string {
return `apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-osd
namespace: ` + namespace + `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-mgr
namespace: ` + namespace + `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
namespace: ` + namespace + `
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: [ "get", "list", "watch", "create", "update", "delete" ]
- apiGroups: ["ceph.rook.io"]
resources: ["cephclusters", "cephclusters/finalizers"]
verbs: [ "get", "list", "create", "update", "delete" ]
---
# Aspects of ceph-mgr that operate within the cluster's namespace
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr
namespace: ` + namespace + `
rules:
- apiGroups:
- ""
resources:
- pods
- services
verbs:
- get
- list
- watch
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- ceph.rook.io
resources:
- "*"
verbs:
- "*"
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cluster-mgmt
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-cluster-mgmt
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + systemNamespace + `
---
# Allow the osd pods in this namespace to work with configmaps
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-osd
subjects:
- kind: ServiceAccount
name: rook-ceph-osd
namespace: ` + namespace + `
---
# Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-mgr
subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: ` + namespace + `
---
# Allow the ceph mgr to access the rook system resources necessary for the mgr modules
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-mgr-system ` + namespace + `
namespace: ` + systemNamespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-mgr-system
subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: ` + namespace + `
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rook-ceph-default-psp
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: psp:rook
subjects:
- kind: ServiceAccount
name: default
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rook-ceph-osd-psp
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: psp:rook
subjects:
- kind: ServiceAccount
name: rook-ceph-osd
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rook-ceph-mgr-psp
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: psp:rook
subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rook-ceph-cmd-reporter-psp
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: psp:rook
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
`
}
// GetRookCluster returns rook-cluster manifest
func (m *CephManifestsMaster) GetRookCluster(settings *ClusterSettings) string {
return `apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
name: ` + settings.Namespace + `
namespace: ` + settings.Namespace + `
spec:
cephVersion:
image: ` + settings.CephVersion.Image + `
allowUnsupported: ` + strconv.FormatBool(settings.CephVersion.AllowUnsupported) + `
dataDirHostPath: ` + settings.DataDirHostPath + `
network:
hostNetwork: false
mon:
count: ` + strconv.Itoa(settings.Mons) + `
allowMultiplePerNode: true
dashboard:
enabled: true
rbdMirroring:
workers: ` + strconv.Itoa(settings.RBDMirrorWorkers) + `
metadataDevice:
storage:
useAllNodes: true
useAllDevices: ` + strconv.FormatBool(settings.UseAllDevices) + `
directories:
- path: ` + settings.DataDirHostPath + /* simulate legacy fallback osd behavior so existing tests still work */ `
deviceFilter:
location:
config:
storeType: "` + settings.StoreType + `"
databaseSizeMB: "1024"
journalSizeMB: "1024"`
}
// GetRookToolBox returns rook-toolbox manifest
func (m *CephManifestsMaster) GetRookToolBox(namespace string) string {
return `apiVersion: v1
kind: Pod
metadata:
name: rook-ceph-tools
namespace: ` + namespace + `
spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: rook-ceph-tools
image: rook/ceph:` + m.imageTag + `
imagePullPolicy: IfNotPresent
command: ["/tini"]
args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
env:
- name: ROOK_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: rook-ceph-mon
key: admin-secret
securityContext:
privileged: true
volumeMounts:
- mountPath: /dev
name: dev
- mountPath: /sys/bus
name: sysbus
- mountPath: /lib/modules
name: libmodules
- name: mon-endpoint-volume
mountPath: /etc/rook
hostNetwork: false
volumes:
- name: dev
hostPath:
path: /dev
- name: sysbus
hostPath:
path: /sys/bus
- name: libmodules
hostPath:
path: /lib/modules
- name: mon-endpoint-volume
configMap:
name: rook-ceph-mon-endpoints
items:
- key: data
path: mon-endpoints`
}
// GetCleanupPod gets a cleanup Pod manifest
func (m *CephManifestsMaster) GetCleanupPod(node, removalDir string) string {
return `apiVersion: batch/v1
kind: Job
metadata:
name: rook-cleanup-` + uuid.Must(uuid.NewRandom()).String() + `
spec:
template:
spec:
restartPolicy: Never
containers:
- name: rook-cleaner
image: rook/ceph:` + m.imageTag + `
securityContext:
privileged: true
volumeMounts:
- name: cleaner
mountPath: /scrub
command:
- "sh"
- "-c"
- "rm -rf /scrub/*"
nodeSelector:
kubernetes.io/hostname: ` + node + `
volumes:
- name: cleaner
hostPath:
path: ` + removalDir
}
func (m *CephManifestsMaster) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string {
return `apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: ` + poolName + `
namespace: ` + namespace + `
spec:
replicated:
size: ` + replicaSize
}
func (m *CephManifestsMaster) GetBlockStorageClassDef(poolName string, storageClassName string, reclaimPolicy string, namespace string, varClusterName bool) string {
namespaceParameter := "clusterNamespace"
if varClusterName {
namespaceParameter = "clusterName"
}
return `apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + storageClassName + `
provisioner: ceph.rook.io/block
allowVolumeExpansion: true
reclaimPolicy: ` + reclaimPolicy + `
parameters:
blockPool: ` + poolName + `
` + namespaceParameter + `: ` + namespace
}
func (m *CephManifestsMaster) GetBlockPvcDef(claimName, storageClassName, accessModes, size string) string {
return `apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ` + claimName + `
annotations:
volume.beta.kubernetes.io/storage-class: ` + storageClassName + `
spec:
accessModes:
- ` + accessModes + `
resources:
requests:
storage: ` + size
}
func (m *CephManifestsMaster) GetBlockPoolStorageClassAndPvcDef(namespace string, poolName string, storageClassName string, reclaimPolicy string, blockName string, accessMode string) string {
return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"),
concatYaml(m.GetBlockStorageClassDef(poolName, storageClassName, reclaimPolicy, namespace, false), m.GetBlockPvcDef(blockName, storageClassName, accessMode, "1M")))
}
func (m *CephManifestsMaster) GetBlockPoolStorageClass(namespace string, poolName string, storageClassName string, reclaimPolicy string) string {
return concatYaml(m.GetBlockPoolDef(poolName, namespace, "1"), m.GetBlockStorageClassDef(poolName, storageClassName, reclaimPolicy, namespace, false))
}
// GetFilesystem returns the manifest to create a Rook filesystem resource with the given config.
func (m *CephManifestsMaster) GetFilesystem(namespace, name string, activeCount int) string {
return `apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
name: ` + name + `
namespace: ` + namespace + `
spec:
metadataPool:
replicated:
size: 1
dataPools:
- replicated:
size: 1
metadataServer:
activeCount: ` + strconv.Itoa(activeCount) + `
activeStandby: true`
}
// GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config.
func (m *CephManifestsMaster) GetNFS(namespace, name, pool string, count int) string {
return `apiVersion: ceph.rook.io/v1
kind: CephNFS
metadata:
name: ` + name + `
namespace: ` + namespace + `
spec:
rados:
pool: ` + pool + `
namespace: nfs-ns
server:
active: ` + strconv.Itoa(count)
}
func (m *CephManifestsMaster) GetObjectStore(namespace, name string, replicaCount, port int) string {
return `apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
name: ` + name + `
namespace: ` + namespace + `
spec:
metadataPool:
replicated:
size: 1
dataPool:
replicated:
size: 1
gateway:
type: s3
sslCertificateRef:
port: ` + strconv.Itoa(port) + `
securePort:
instances: ` + strconv.Itoa(replicaCount) + `
allNodes: false
`
}
func (m *CephManifestsMaster) GetObjectStoreUser(namespace, name string, displayName string, store string) string {
return `apiVersion: ceph.rook.io/v1
kind: CephObjectStoreUser
metadata:
name: ` + name + `
namespace: ` + namespace + `
spec:
displayName: ` + displayName + `
store: ` + store
}
//GetBucketStorageClass returns the manifest to create object bucket
func (m *CephManifestsMaster) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string {
return `apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ` + storageClassName + `
provisioner: ceph.rook.io/bucket
reclaimPolicy: ` + reclaimPolicy + `
parameters:
objectStoreName: ` + storeName + `
objectStoreNamespace: ` + storeNameSpace + `
region: ` + region
}
//GetObc returns the manifest to create object bucket claim
func (m *CephManifestsMaster) GetObc(claimName string, storageClassName string, objectBucketName string, varBucketName bool) string {
bucketParameter := "generateBucketName"
if varBucketName {
bucketParameter = "bucketName"
}
return `apiVersion: objectbucket.io/v1alpha1
kind: ObjectBucketClaim
metadata:
name: ` + claimName + `
spec:
` + bucketParameter + `: ` + objectBucketName + `
storageClassName: ` + storageClassName
}
View Git Blame Copy Permalink