diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 2fbf35595..7bf8786ea 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -349,9 +349,13 @@ available with Ceph-CSI v3.0. First apply new resources. This includes slightly modified privileges (RBAC) needed by the Operator and updates to the Custom Resource Definitions (CRDs). +> **IMPORTANT:** If you are using Kubernetes version v1.15 or lower, you will need to manually +> modify the `upgrade-from-v1.4-apply.yaml` file to use +> `rbac.authorization.k8s.io/v1beta1` instead of `rbac.authorization.k8s.io/v1` +> You will also need to apply `pre-k8s-1.16/crd.yaml` instead of `upgrade-from-v1.4-crds.yaml`. + ```sh -kubectl delete -f upgrade-from-v1.3-delete.yaml -kubectl apply -f upgrade-from-v1.3-apply.yaml -f upgrade-from-v1.3-crds.yaml +kubectl apply -f upgrade-from-v1.4-apply.yaml -f upgrade-from-v1.4-crds.yaml ``` ## 3. Update Ceph CSI version to v3.0 diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml index 3a395974b..2b380ec54 100644 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml +++ b/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml @@ -1,76 +1,388 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: cephblockpools.ceph.rook.io + name: cephclusters.ceph.rook.io spec: group: ceph.rook.io names: - kind: CephBlockPool - listKind: CephBlockPoolList - plural: cephblockpools - singular: cephblockpool + kind: CephCluster + plural: cephclusters + singular: cephcluster scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - type: integer - minimum: 0 - maximum: 9 - targetSizeRatio: - type: number - requireSafeReplicaSize: - type: boolean - replicasPerFailureDomain: - type: integer - subFailureDomain: - type: string - erasureCoded: - properties: - dataChunks: - type: integer - minimum: 0 - maximum: 9 - codingChunks: - type: integer - minimum: 0 - maximum: 9 - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - enableRBDStats: - description: EnableRBDStats is used to enable gathering of statistics - for all RBD images in the pool - type: boolean - parameters: + spec: type: object - mirroring: properties: - enabled: - type: boolean - mode: + cephVersion: + type: object + properties: + allowUnsupported: + type: boolean + image: + type: string + dashboard: + type: object + properties: + enabled: + type: boolean + urlPrefix: + type: string + port: + type: integer + minimum: 0 + maximum: 65535 + ssl: + type: boolean + dataDirHostPath: + pattern: ^/(\S+) type: string - enum: - - image - - pool - subresources: - status: {} + disruptionManagement: + type: object + properties: + machineDisruptionBudgetNamespace: + type: string + managePodBudgets: + type: boolean + osdMaintenanceTimeout: + type: integer + manageMachineDisruptionBudgets: + type: boolean + skipUpgradeChecks: + type: boolean + continueUpgradeAfterChecksEvenIfNotHealthy: + type: boolean + mon: + type: object + properties: + allowMultiplePerNode: + type: boolean + count: + maximum: 9 + minimum: 0 + type: integer + volumeClaimTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + mgr: + type: object + properties: + modules: + type: array + items: + type: object + properties: + name: + type: string + enabled: + type: boolean + crashCollector: + type: object + properties: + disable: + type: boolean + network: + type: object + nullable: true + properties: + hostNetwork: + type: boolean + provider: + type: string + selectors: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + ipFamily: + type: string + storage: + type: object + properties: + disruptionManagement: + type: object + nullable: true + properties: + machineDisruptionBudgetNamespace: + type: string + managePodBudgets: + type: boolean + osdMaintenanceTimeout: + type: integer + manageMachineDisruptionBudgets: + type: boolean + useAllNodes: + type: boolean + nodes: + type: array + nullable: true + items: + type: object + properties: + name: + type: string + config: + type: object + nullable: true + properties: + metadataDevice: + type: string + storeType: + type: string + pattern: ^(bluestore)$ + databaseSizeMB: + type: string + walSizeMB: + type: string + journalSizeMB: + type: string + osdsPerDevice: + type: string + encryptedDevice: + type: string + pattern: ^(true|false)$ + useAllDevices: + type: boolean + deviceFilter: + type: string + nullable: true + devicePathFilter: + type: string + devices: + type: array + items: + type: object + properties: + name: + type: string + config: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + useAllDevices: + type: boolean + deviceFilter: + type: string + nullable: true + devicePathFilter: + type: string + config: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + storageClassDeviceSets: + type: array + nullable: true + items: + type: object + properties: + name: + type: string + count: + type: integer + format: int32 + portable: + type: boolean + tuneDeviceClass: + type: boolean + tuneFastDeviceClass: + type: boolean + encrypted: + type: boolean + schedulerName: + type: string + config: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + preparePlacement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + driveGroups: + type: array + nullable: true + items: + type: object + properties: + name: + type: string + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + required: + - name + - spec + monitoring: + type: object + properties: + enabled: + type: boolean + rulesNamespace: + type: string + externalMgrEndpoints: + type: array + items: + type: object + properties: + ip: + type: string + removeOSDsIfOutAndSafeToRemove: + type: boolean + external: + type: object + properties: + enable: + type: boolean + cleanupPolicy: + type: object + properties: + allowUninstallWithVolumes: + type: boolean + confirmation: + type: string + pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + type: object + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 + security: + type: object + properties: + kms: + type: object + properties: + connectionDetails: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + tokenSecretName: + type: string + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + labels: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + healthCheck: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassNames: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: DataDirHostPath + type: string + description: Directory used on the K8s nodes + jsonPath: .spec.dataDirHostPath + - name: MonCount + type: string + description: Number of MONs + jsonPath: .spec.mon.count + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + - name: Phase + type: string + description: Phase + jsonPath: .status.phase + - name: Message + type: string + description: Message + jsonPath: .status.message + - name: Health + type: string + description: Ceph Health + jsonPath: .status.ceph.health + subresources: + status: {} +# OLM: END CEPH CRD +# OLM: BEGIN CEPH CLIENT CRD --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + caps: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH CLIENT CRD +# OLM: BEGIN CEPH RBD MIRROR CRD +--- +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: cephrbdmirrors.ceph.rook.io @@ -82,242 +394,924 @@ spec: plural: cephrbdmirrors singular: cephrbdmirror scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object properties: - count: - type: integer - minimum: 1 - maximum: 100 - peers: + spec: + type: object properties: - secretNames: - type: array - subresources: - status: {} + count: + type: integer + minimum: 1 + maximum: 100 + peers: + type: object + properties: + secretNames: + type: array + items: + type: string + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + +# OLM: END CEPH RBD MIRROR CRD +# OLM: BEGIN CEPH FS CRD --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: cephclusters.ceph.rook.io + name: cephfilesystems.ceph.rook.io spec: group: ceph.rook.io names: - kind: CephCluster - listKind: CephClusterList - plural: cephclusters - singular: cephcluster + kind: CephFilesystem + listKind: CephFilesystemList + plural: cephfilesystems + singular: cephfilesystem scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object properties: - annotations: {} - cephVersion: + spec: + type: object properties: - allowUnsupported: - type: boolean - image: - type: string - dashboard: - properties: - enabled: - type: boolean - urlPrefix: - type: string - port: - type: integer - minimum: 0 - maximum: 65535 - ssl: - type: boolean - dataDirHostPath: - pattern: ^/(\S+) - type: string - disruptionManagement: - properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: - type: integer - manageMachineDisruptionBudgets: - type: boolean - skipUpgradeChecks: - type: boolean - continueUpgradeAfterChecksEvenIfNotHealthy: - type: boolean - mon: - properties: - allowMultiplePerNode: - type: boolean - count: - maximum: 9 - minimum: 0 - type: integer - volumeClaimTemplate: {} - mgr: - properties: - modules: - items: - properties: - name: - type: string - enabled: - type: boolean - network: - properties: - hostNetwork: - type: boolean - provider: - type: string - selectors: {} - storage: - properties: - disruptionManagement: + metadataServer: + type: object properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: + activeCount: + minimum: 1 + maximum: 10 type: integer - manageMachineDisruptionBudgets: + activeStandby: type: boolean - useAllNodes: - type: boolean - nodes: - items: - properties: - name: - type: string - config: - properties: - metadataDevice: - type: string - storeType: - type: string - pattern: ^(bluestore)$ - databaseSizeMB: - type: string - walSizeMB: - type: string - journalSizeMB: - type: string - osdsPerDevice: - type: string - encryptedDevice: - type: string - pattern: ^(true|false)$ - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - devices: - type: array - items: - properties: - name: - type: string - config: {} - resources: {} - type: array - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - config: {} - storageClassDeviceSets: {} - driveGroups: - type: array - nullable: true - items: - properties: - name: - type: string - spec: {} - placement: {} - required: - - name - - spec - monitoring: - properties: - enabled: - type: boolean - rulesNamespace: - type: string - externalMgrEndpoints: - type: array - items: - properties: - ip: - type: string - removeOSDsIfOutAndSafeToRemove: - type: boolean - external: - properties: - enable: - type: boolean - cleanupPolicy: - properties: - confirmation: - type: string - pattern: ^$|^yes-really-destroy-data$ - sanitizeDisks: + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + labels: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + metadataPool: + type: object + nullable: true properties: - method: + failureDomain: type: string - pattern: ^(complete|quick)$ - dataSource: + deviceClass: type: string - pattern: ^(zero|random)$ - iteration: + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + minimum: 0 + maximum: 10 + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + parameters: + type: object + x-kubernetes-preserve-unknown-fields: true + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + minimum: 0 + maximum: 10 + type: integer + codingChunks: + minimum: 0 + maximum: 10 + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + dataPools: + type: array + nullable: true + items: + type: object + properties: + failureDomain: + type: string + deviceClass: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + minimum: 0 + maximum: 10 + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + minimum: 0 + maximum: 10 + type: integer + codingChunks: + minimum: 0 + maximum: 10 + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + preservePoolsOnDelete: + type: boolean + preserveFilesystemOnDelete: + type: boolean + status: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: ActiveMDS + type: string + description: Number of desired active MDS daemons + jsonPath: .spec.metadataServer.activeCount + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} +# OLM: END CEPH FS CRD +# OLM: BEGIN CEPH NFS CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephnfses.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNFS + listKind: CephNFSList + plural: cephnfses + singular: cephnfs + shortNames: + - nfs + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + rados: + type: object + properties: + pool: + type: string + namespace: + type: string + server: + type: object + properties: + active: + type: integer + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH NFS CRD +# OLM: BEGIN CEPH OBJECT STORE CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectstores.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStore + listKind: CephObjectStoreList + plural: cephobjectstores + singular: cephobjectstore + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + gateway: + type: object + properties: + type: + type: string + sslCertificateRef: + type: string + nullable: true + port: + type: integer + minimum: 0 + maximum: 65535 + securePort: + type: integer + minimum: 0 + maximum: 65535 + instances: + type: integer + externalRgwEndpoints: + type: array + nullable: true + items: + type: object + properties: + ip: + type: string + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + labels: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + metadataPool: + type: object + nullable: true + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + zone: + type: object + properties: + name: + type: string + dataPool: + type: object + nullable: true + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + x-kubernetes-preserve-unknown-fields: true + preservePoolsOnDelete: + type: boolean + healthCheck: + type: object + nullable: true + properties: + bucket: + type: object + nullable: true + properties: + enabled: + type: boolean + interval: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + +# OLM: END CEPH OBJECT STORE CRD +# OLM: BEGIN CEPH OBJECT STORE USERS CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectstoreusers.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStoreUser + listKind: CephObjectStoreUserList + plural: cephobjectstoreusers + singular: cephobjectstoreuser + shortNames: + - rcou + - objectuser + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + store: + type: string + displayName: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT STORE USERS CRD +# OLM: BEGIN CEPH OBJECT REALM CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectrealms.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectRealm + listKind: CephObjectRealmList + plural: cephobjectrealms + singular: cephobjectrealm + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + pull: + type: object + properties: + endpoint: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT REALM CRD +# OLM: BEGIN CEPH OBJECT ZONEGROUP CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzonegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZoneGroup + listKind: CephObjectZoneGroupList + plural: cephobjectzonegroups + singular: cephobjectzonegroup + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + realm: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT ZONEGROUP CRD +# OLM: BEGIN CEPH OBJECT ZONE CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzones.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZone + listKind: CephObjectZoneList + plural: cephobjectzones + singular: cephobjectzone + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + zoneGroup: + type: string + metadataPool: + type: object + nullable: true + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + dataPool: + type: object + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + preservePoolsOnDelete: + type: boolean + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT ZONE CRD +# OLM: BEGIN CEPH BLOCK POOL CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephblockpools.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPool + listKind: CephBlockPoolList + plural: cephblockpools + singular: cephblockpool + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + failureDomain: + type: string + deviceClass: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + minimum: 0 + maximum: 9 + targetSizeRatio: + type: number + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + minimum: 0 + maximum: 9 + codingChunks: + type: integer + minimum: 0 + maximum: 9 + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + enableRBDStats: + description: + EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + mirroring: + type: object + nullable: true + properties: + enabled: + type: boolean + mode: + type: string + enum: + - image + - pool + statusCheck: + type: object + x-kubernetes-preserve-unknown-fields: true + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + +# OLM: END CEPH BLOCK POOL CRD +# OLM: BEGIN CEPH VOLUME POOL CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumes.rook.io +spec: + group: rook.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + singular: volume + shortNames: + - rv + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + attachments: + type: array + items: + type: object + properties: + node: + type: string + podNamespace: + type: string + podName: + type: string + clusterName: + type: string + mountDir: + type: string + readOnly: + type: boolean + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH VOLUME POOL CRD +# OLM: BEGIN OBJECTBUCKET CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: objectbuckets.objectbucket.io +spec: + group: objectbucket.io + names: + kind: ObjectBucket + listKind: ObjectBucketList + plural: objectbuckets + singular: objectbucket + shortNames: + - ob + - obs + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + storageClassName: + type: string + endpoint: + type: object + nullable: true + properties: + bucketHost: + type: string + bucketPort: type: integer format: int32 - security: - properties: - kms: - properties: - connectionDetails: - type: object - tokenSecretName: + bucketName: type: string - placement: {} - resources: {} - healthCheck: {} - subresources: - status: {} - additionalPrinterColumns: - - name: DataDirHostPath - type: string - description: Directory used on the K8s nodes - JSONPath: .spec.dataDirHostPath - - name: MonCount - type: string - description: Number of MONs - JSONPath: .spec.mon.count - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: Phase - type: string - description: Phase - JSONPath: .status.phase - - name: Message - type: string - description: Message - JSONPath: .status.message - - name: Health - type: string - description: Ceph Health - JSONPath: .status.ceph.health + region: + type: string + subRegion: + type: string + additionalConfig: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + authentication: + type: object + nullable: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalState: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + reclaimPolicy: + type: string + claimRef: + type: object + nullable: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END OBJECTBUCKET CRD +# OLM: BEGIN OBJECTBUCKETCLAIM CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: objectbucketclaims.objectbucket.io +spec: + group: objectbucket.io + names: + kind: ObjectBucketClaim + listKind: ObjectBucketClaimList + plural: objectbucketclaims + singular: objectbucketclaim + shortNames: + - obc + - obcs + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + storageClassName: + type: string + bucketName: + type: string + generateBucketName: + type: string + additionalConfig: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END OBJECTBUCKETCLAIM CRD diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index a5f10106b..e034f5755 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -516,7 +516,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion) - if !utils.IsPlatformOpenShift() && rookVersion != Version1_2 && h.k8shelper.VersionAtLeast("v1.15.0") { + if !utils.IsPlatformOpenShift() && h.k8shelper.VersionAtLeast("v1.15.0") { if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") { assert.Fail(h.T(), "admission controller is not running") } @@ -843,7 +843,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH k8sVersion: version.String(), CephVersion: cephVersion, cleanupHost: cleanupHost, - changeHostnames: rookVersion != Version1_2 && k8shelp.VersionAtLeast("v1.13.0"), + changeHostnames: k8shelp.VersionAtLeast("v1.18.0"), T: t, } flag.Parse() diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 0f61b5887..8a182427a 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -26,10 +26,8 @@ import ( ) const ( - // Version1_2 rook version 1.2 - Version1_2 = "v1.2.7" - // Version1_3 rook version 1.3 - Version1_3 = "v1.3.8" + // Version1_4 rook version 1.4 + Version1_4 = "v1.4.7" ) type CephManifests interface { @@ -89,8 +87,8 @@ func NewCephManifests(version string) CephManifests { switch version { case VersionMaster: return &CephManifestsMaster{imageTag: VersionMaster} - case Version1_2: - return &CephManifestsV1_2{imageTag: Version1_2} + case Version1_4: + return &CephManifestsV1_4{imageTag: Version1_4} } panic(fmt.Errorf("unrecognized ceph manifest version: %s", version)) } diff --git a/tests/framework/installer/ceph_manifests_v1.2.go b/tests/framework/installer/ceph_manifests_v1.4.go similarity index 75% rename from tests/framework/installer/ceph_manifests_v1.2.go rename to tests/framework/installer/ceph_manifests_v1.4.go index 2850fcda1..89de292aa 100644 --- a/tests/framework/installer/ceph_manifests_v1.2.go +++ b/tests/framework/installer/ceph_manifests_v1.4.go @@ -21,12 +21,12 @@ import ( "strings" ) -// CephManifestsV1_2 wraps rook yaml definitions -type CephManifestsV1_2 struct { +// CephManifestsV1_4 wraps rook yaml definitions +type CephManifestsV1_4 struct { imageTag string } -func (m *CephManifestsV1_2) GetRookCRDs(v1ExtensionsSupported bool) string { +func (m *CephManifestsV1_4) GetRookCRDs(v1ExtensionsSupported bool) string { return ` apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -68,14 +68,27 @@ spec: dataDirHostPath: pattern: ^/(\S+) type: string + disruptionManagement: + properties: + machineDisruptionBudgetNamespace: + type: string + managePodBudgets: + type: boolean + osdMaintenanceTimeout: + type: integer + manageMachineDisruptionBudgets: + type: boolean + skipUpgradeChecks: + type: boolean mon: properties: allowMultiplePerNode: type: boolean count: maximum: 9 - minimum: 1 + minimum: 0 type: integer + volumeClaimTemplate: {} mgr: properties: modules: @@ -89,10 +102,15 @@ spec: properties: hostNetwork: type: boolean + provider: + type: string + selectors: {} storage: properties: disruptionManagement: properties: + machineDisruptionBudgetNamespace: + type: string managePodBudgets: type: boolean osdMaintenanceTimeout: @@ -112,7 +130,7 @@ spec: type: string storeType: type: string - pattern: ^(filestore|bluestore)$ + pattern: ^(bluestore)$ databaseSizeMB: type: string walSizeMB: @@ -127,12 +145,6 @@ spec: useAllDevices: type: boolean deviceFilter: {} - directories: - type: array - items: - properties: - path: - type: string devices: type: array items: @@ -140,22 +152,24 @@ spec: name: type: string config: {} - location: {} resources: {} type: array useAllDevices: type: boolean deviceFilter: {} - location: {} - directories: - type: array - items: - properties: - path: - type: string config: {} - topologyAware: - type: boolean + storageClassDeviceSets: {} + driveGroups: + type: array + items: + properties: + name: + type: string + spec: {} + placement: {} + required: + - name + - spec monitoring: properties: enabled: @@ -166,10 +180,30 @@ spec: properties: workers: type: integer + removeOSDsIfOutAndSafeToRemove: + type: boolean + external: + properties: + enable: + type: boolean + cleanupPolicy: + properties: + confirmation: + type: string + pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 placement: {} resources: {} - required: - - mon additionalPrinterColumns: - name: DataDirHostPath type: string @@ -190,6 +224,8 @@ spec: type: string description: Ceph Health JSONPath: .status.ceph.health + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -224,36 +260,72 @@ spec: properties: failureDomain: type: string + crushRoot: + type: string replicated: properties: size: - minimum: 1 + minimum: 0 maximum: 10 type: integer + requireSafeReplicaSize: + type: boolean erasureCoded: properties: dataChunks: + minimum: 0 + maximum: 10 type: integer codingChunks: + minimum: 0 + maximum: 10 type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force dataPools: type: array items: properties: failureDomain: type: string + crushRoot: + type: string replicated: properties: size: - minimum: 1 + minimum: 0 maximum: 10 type: integer + requireSafeReplicaSize: + type: boolean erasureCoded: properties: dataChunks: + minimum: 0 + maximum: 10 type: integer codingChunks: + minimum: 0 + maximum: 10 type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + preservePoolsOnDelete: + type: boolean additionalPrinterColumns: - name: ActiveMDS type: string @@ -262,6 +334,8 @@ spec: - name: Age type: date JSONPath: .metadata.creationTimestamp + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -296,6 +370,8 @@ spec: annotations: {} placement: {} resources: {} + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -322,8 +398,9 @@ spec: sslCertificateRef: {} port: type: integer - securePort: - type: integer + minimum: 1 + maximum: 65535 + securePort: {} instances: type: integer annotations: {} @@ -333,30 +410,71 @@ spec: properties: failureDomain: type: string - replicated: - properties: - size: - type: integer - erasureCoded: - properties: - dataChunks: - type: integer - codingChunks: - type: integer - dataPool: - properties: - failureDomain: + crushRoot: type: string replicated: properties: size: type: integer + requireSafeReplicaSize: + type: boolean erasureCoded: properties: dataChunks: type: integer codingChunks: type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + dataPool: + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + erasureCoded: + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + preservePoolsOnDelete: + type: boolean + healthCheck: + properties: + bucket: + properties: + enabled: + type: boolean + interval: + type: string + subresources: + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -369,6 +487,62 @@ spec: listKind: CephObjectStoreUserList plural: cephobjectstoreusers singular: cephobjectstoreuser + shortNames: + - rcou + - objectuser + scope: Namespaced + version: v1 + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectrealms.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectRealm + listKind: CephObjectRealmList + plural: cephobjectrealms + singular: cephobjectrealm + shortNames: + - realm + - realms + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzonegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZoneGroup + listKind: CephObjectZoneGroupList + plural: cephobjectzonegroups + singular: cephobjectzonegroup + shortNames: + - zonegroup + - zonegroups + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzones.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZone + listKind: CephObjectZoneList + plural: cephobjectzones + singular: cephobjectzone + shortNames: + - zone + - zones scope: Namespaced version: v1 --- @@ -385,6 +559,51 @@ spec: singular: cephblockpool scope: Namespaced version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + properties: + size: + type: integer + minimum: 0 + maximum: 9 + targetSizeRatio: + type: number + requireSafeReplicaSize: + type: boolean + erasureCoded: + properties: + dataChunks: + type: integer + minimum: 0 + maximum: 9 + codingChunks: + type: integer + minimum: 0 + maximum: 9 + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + parameters: + type: object + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -397,8 +616,12 @@ spec: listKind: VolumeList plural: volumes singular: volume + shortNames: + - rv scope: Namespaced version: v1alpha2 + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -441,14 +664,68 @@ spec: - obc - obcs scope: Namespaced + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + caps: + properties: + mon: + type: string + osd: + type: string + mds: + type: string + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephrbdmirrors.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephRBDMirror + listKind: CephRBDMirrorList + plural: cephrbdmirrors + singular: cephrbdmirror + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + count: + type: integer + minimum: 1 + maximum: 100 subresources: status: {}` } // GetRookOperator returns rook Operator manifest -func (m *CephManifestsV1_2) GetRookOperator(namespace string) string { +func (m *CephManifestsV1_4) GetRookOperator(namespace string) string { return ` -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: name: rook-ceph-system @@ -483,27 +760,13 @@ rules: verbs: - get --- -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: rook-ceph-cluster-mgmt labels: operator: rook storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" rules: - apiGroups: - "" @@ -535,27 +798,13 @@ rules: - update - delete --- -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: rook-ceph-global labels: operator: rook storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" rules: - apiGroups: - "" @@ -660,21 +909,9 @@ rules: - delete --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-cluster -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster-rules - labels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -698,7 +935,7 @@ rules: - watch --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd rules: @@ -711,27 +948,13 @@ rules: - list --- # Aspects of Rook Ceph Agent that require access to secrets -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: rook-ceph-agent-mount labels: operator: rook storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-agent-mount-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" rules: - apiGroups: - "" @@ -742,21 +965,9 @@ rules: --- # Aspects of ceph-mgr that require access to the system namespace kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system-rules - labels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" rules: - apiGroups: - "" @@ -768,13 +979,12 @@ rules: - watch --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-object-bucket labels: operator: rook storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -799,7 +1009,7 @@ rules: - "*" --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-object-bucket roleRef: @@ -821,7 +1031,7 @@ metadata: storage-backend: ceph --- kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-system namespace: ` + namespace + ` @@ -838,7 +1048,7 @@ subjects: namespace: ` + namespace + ` --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-global labels: @@ -863,18 +1073,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] @@ -918,18 +1116,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] @@ -948,18 +1134,15 @@ rules: verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] + resources: ["volumesnapshotcontents/status"] verbs: ["update"] - apiGroups: [""] resources: ["nodes"] @@ -1024,18 +1207,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] @@ -1077,18 +1248,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] @@ -1161,7 +1320,10 @@ roleRef: apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: - name: rook-privileged + name: 00-rook-privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: true allowedCapabilities: @@ -1191,7 +1353,6 @@ spec: - hostPath - flexVolume # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # directory-based OSDs make this hard to nail down # allowedHostPaths: # - /run/udev # for OSD prep # - /dev # for OSD prep @@ -1232,7 +1393,7 @@ rules: resources: - podsecuritypolicies resourceNames: - - rook-privileged + - 00-rook-privileged verbs: - use --- @@ -1301,6 +1462,35 @@ subjects: name: rook-csi-cephfs-plugin-sa namespace: ` + namespace + ` --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-admission-controller + namespace: ` + namespace + ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-role +rules: + - apiGroups: ["ceph.rook.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-rolebinding +subjects: + - kind: ServiceAccount + name: rook-ceph-admission-controller + apiGroup: "" + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rook-ceph-admission-controller-role + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -1327,12 +1517,6 @@ spec: env: - name: ROOK_LOG_LEVEL value: INFO - - name: ROOK_CEPH_STATUS_CHECK_INTERVAL - value: "5s" - - name: ROOK_MON_HEALTHCHECK_INTERVAL - value: "10s" - - name: ROOK_MON_OUT_TIMEOUT - value: "15s" - name: NODE_NAME valueFrom: fieldRef: @@ -1359,11 +1543,12 @@ data: ROOK_CSI_ENABLE_CEPHFS: "true" ROOK_CSI_ENABLE_RBD: "true" ROOK_CSI_ENABLE_GRPC_METRICS: "true" + ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" ` } // GetClusterRoles returns rook-cluster manifest -func (m *CephManifestsV1_2) GetClusterRoles(namespace, systemNamespace string) string { +func (m *CephManifestsV1_4) GetClusterRoles(namespace, systemNamespace string) string { return `apiVersion: v1 kind: ServiceAccount metadata: @@ -1383,7 +1568,7 @@ metadata: namespace: ` + namespace + ` --- kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd namespace: ` + namespace + ` @@ -1397,7 +1582,7 @@ rules: --- # Aspects of ceph-mgr that operate within the cluster's namespace kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr namespace: ` + namespace + ` @@ -1433,7 +1618,7 @@ rules: --- # Allow the ceph osd to access cluster-wide resources necessary for determining their topology location kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd-` + namespace + ` roleRef: @@ -1447,7 +1632,7 @@ subjects: --- # Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-cluster-` + namespace + ` roleRef: @@ -1460,7 +1645,7 @@ subjects: namespace: ` + namespace + ` --- kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1480,7 +1665,7 @@ rules: --- # Allow the operator to create resources in this cluster's namespace kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cluster-mgmt namespace: ` + namespace + ` @@ -1495,7 +1680,7 @@ subjects: --- # Allow the osd pods in this namespace to work with configmaps kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd namespace: ` + namespace + ` @@ -1510,7 +1695,7 @@ subjects: --- # Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr namespace: ` + namespace + ` @@ -1525,7 +1710,7 @@ subjects: --- # Allow the ceph mgr to access the rook system resources necessary for the mgr modules kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system ` + namespace + ` namespace: ` + systemNamespace + ` @@ -1539,7 +1724,7 @@ subjects: namespace: ` + namespace + ` --- kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1611,7 +1796,7 @@ subjects: } // GetRookCluster returns rook-cluster manifest -func (m *CephManifestsV1_2) GetRookCluster(settings *clusterSettings) string { +func (m *CephManifestsV1_4) GetRookCluster(settings *clusterSettings) string { store := "# storeType not specified; Rook will use default store types" if settings.StoreType != "" { store = `storeType: "` + settings.StoreType + `"` @@ -1621,6 +1806,7 @@ func (m *CephManifestsV1_2) GetRookCluster(settings *clusterSettings) string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: + # set the name to something different from the namespace name: ` + settings.ClusterName + ` namespace: ` + settings.Namespace + ` spec: @@ -1641,8 +1827,6 @@ spec: continueUpgradeAfterChecksEvenIfNotHealthy: false dashboard: enabled: true - rbdMirroring: - workers: ` + strconv.Itoa(settings.RBDMirrorWorkers) + ` network: hostNetwork: false crashCollector: @@ -1653,6 +1837,7 @@ spec: count: 1 portable: false tuneDeviceClass: true + encrypted: false volumeClaimTemplates: - metadata: name: data @@ -1691,9 +1876,9 @@ spec: skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: true - useAllDevices: true - deviceFilter: + useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + deviceFilter: '' config: ` + store + ` databaseSizeMB: "1024" @@ -1701,11 +1886,22 @@ spec: mgr: modules: - name: pg_autoscaler - enabled: true` + enabled: true + - name: rook + enabled: true + healthCheck: + daemonHealth: + mon: + interval: 10s + timeout: 15s + osd: + interval: 10s + status: + interval: 5s` } // GetRookToolBox returns rook-toolbox manifest -func (m *CephManifestsV1_2) GetRookToolBox(namespace string) string { +func (m *CephManifestsV1_4) GetRookToolBox(namespace string) string { return `apiVersion: v1 kind: Pod metadata: @@ -1720,16 +1916,38 @@ spec: command: ["/tini"] args: ["-g", "--", "/usr/local/bin/toolbox.sh"] env: - - name: ROOK_ADMIN_SECRET + - name: ROOK_CEPH_USERNAME valueFrom: secretKeyRef: name: rook-ceph-mon - key: admin-secret + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + securityContext: + privileged: true volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules - name: mon-endpoint-volume mountPath: /etc/rook hostNetwork: false volumes: + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules - name: mon-endpoint-volume configMap: name: rook-ceph-mon-endpoints @@ -1738,7 +1956,7 @@ spec: path: mon-endpoints` } -func (m *CephManifestsV1_2) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { +func (m *CephManifestsV1_4) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: @@ -1749,12 +1967,10 @@ spec: size: ` + replicaSize + ` targetSizeRatio: .5 requireSafeReplicaSize: false - compressionMode: aggressive - poolProperties: - pg_autoscale_mode: on` + compressionMode: aggressive` } -func (m *CephManifestsV1_2) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string { +func (m *CephManifestsV1_4) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string { // Create a CSI driver storage class if csi { return ` @@ -1763,6 +1979,7 @@ kind: StorageClass metadata: name: ` + storageClassName + ` provisioner: ` + systemNamespace + `.rbd.csi.ceph.com +reclaimPolicy: ` + reclaimPolicy + ` parameters: pool: ` + poolName + ` clusterID: ` + namespace + ` @@ -1784,8 +2001,7 @@ parameters: blockPool: ` + poolName + ` clusterNamespace: ` + namespace } - -func (m *CephManifestsV1_2) GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace string) string { +func (m *CephManifestsV1_4) GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace string) string { // Create a CSI driver storage class csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file @@ -1806,15 +2022,14 @@ parameters: ` } -func (m *CephManifestsV1_2) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { +func (m *CephManifestsV1_4) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { return `apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ` + claimName + ` namespace: ` + namespace + ` - annotations: - volume.beta.kubernetes.io/storage-class: ` + storageClassName + ` spec: + storageClassName: ` + storageClassName + ` accessModes: - ` + accessModes + ` resources: @@ -1822,8 +2037,53 @@ spec: storage: ` + size } +func (m *CephManifestsV1_4) GetBlockSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { + return "" +} + +func (m *CephManifestsV1_4) GetFileStorageSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { + return "" +} + +func (m *CephManifestsV1_4) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { + return "" +} + +func (m *CephManifestsV1_4) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { + return "" +} + +func (m *CephManifestsV1_4) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { + return "" +} + +func (m *CephManifestsV1_4) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { + return ` +apiVersion: v1 +kind: Pod +metadata: + name: ` + podName + ` + namespace: ` + namespace + ` +spec: + containers: + - name: ` + podName + ` + image: busybox + command: ["/bin/sleep", "infinity"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: ` + mountPath + ` + name: csivol + volumes: + - name: csivol + persistentVolumeClaim: + claimName: ` + claimName + ` + readOnly: ` + strconv.FormatBool(readOnly) + ` + restartPolicy: Never +` +} + // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. -func (m *CephManifestsV1_2) GetFilesystem(namespace, name string, activeCount int) string { +func (m *CephManifestsV1_4) GetFilesystem(namespace, name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: @@ -1833,16 +2093,19 @@ spec: metadataPool: replicated: size: 1 + requireSafeReplicaSize: false dataPools: - replicated: size: 1 + requireSafeReplicaSize: false + compressionMode: none metadataServer: activeCount: ` + strconv.Itoa(activeCount) + ` activeStandby: true` } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsV1_2) GetNFS(namespace, name, pool string, count int) string { +func (m *CephManifestsV1_4) GetNFS(namespace, name, pool string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: @@ -1856,7 +2119,7 @@ spec: active: ` + strconv.Itoa(count) } -func (m *CephManifestsV1_2) GetObjectStore(namespace, name string, replicaCount, port int) string { +func (m *CephManifestsV1_4) GetObjectStore(namespace, name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: @@ -1866,18 +2129,27 @@ spec: metadataPool: replicated: size: 1 + requireSafeReplicaSize: false + compressionMode: passive dataPool: replicated: size: 1 + requireSafeReplicaSize: false gateway: type: s3 sslCertificateRef: port: ` + strconv.Itoa(port) + ` + securePort: instances: ` + strconv.Itoa(replicaCount) + ` + allNodes: false + healthCheck: + bucket: + disabled: false + interval: 10s ` } -func (m *CephManifestsV1_2) GetObjectStoreUser(namespace, name string, displayName string, store string) string { +func (m *CephManifestsV1_4) GetObjectStoreUser(namespace, name string, displayName string, store string) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: @@ -1889,12 +2161,12 @@ spec: } //GetBucketStorageClass returns the manifest to create object bucket -func (m *CephManifestsV1_2) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { +func (m *CephManifestsV1_4) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { return `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ceph.rook.io/bucket +provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + ` @@ -1903,7 +2175,7 @@ parameters: } //GetObc returns the manifest to create object bucket claim -func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { +func (m *CephManifestsV1_4) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1919,7 +2191,7 @@ spec: maxObjects: "` + maxObject + `"` } -func (m *CephManifestsV1_2) GetClient(namespace string, claimName string, caps map[string]string) string { +func (m *CephManifestsV1_4) GetClient(namespace string, claimName string, caps map[string]string) string { clientCaps := []string{} for name, cap := range caps { str := name + ": " + cap @@ -1935,10 +2207,10 @@ spec: ` + strings.Join(clientCaps, "\n ") } -func (m *CephManifestsV1_2) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { +func (m *CephManifestsV1_4) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { return `apiVersion: v1 kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cluster-mgmt namespace: ` + namespace + ` @@ -1952,7 +2224,7 @@ subjects: namespace: ` + firstClusterNamespace + ` --- kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1972,7 +2244,7 @@ metadata: namespace: ` + namespace + ` --- kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1991,7 +2263,7 @@ rules: - delete` } -func (m *CephManifestsV1_2) GetRookExternalCluster(settings *clusterExternalSettings) string { +func (m *CephManifestsV1_4) GetRookExternalCluster(settings *clusterExternalSettings) string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -2003,36 +2275,13 @@ spec: dataDirHostPath: ` + settings.DataDirHostPath + `` } -func (m *CephManifestsV1_2) GetRBDMirror(namespace, name string, count int) string { - // Not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetBlockSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { - // block snapshot was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetFileStorageSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { - // filesystem snapshot was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { - // snapshot restore to a new PVC was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { - // clone pvc to a new PVC was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { - // pvc snapshot was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { - return "" +// GetRBDMirror returns the manifest to create a Rook Ceph RBD Mirror resource with the given config. +func (m *CephManifestsV1_4) GetRBDMirror(namespace, name string, count int) string { + return `apiVersion: ceph.rook.io/v1 +kind: CephRBDMirror +metadata: + name: ` + name + ` + namespace: ` + namespace + ` +spec: + count: ` + strconv.Itoa(count) } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 21a39c946..d96a393c4 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -85,7 +85,7 @@ func (s *UpgradeSuite) SetupSuite() { rookCephCleanup: false, skipOSDCreation: false, minimalMatrixK8sVersion: upgradeMinimalTestVersion, - rookVersion: installer.Version1_2, + rookVersion: installer.Version1_4, cephVersion: installer.NautilusVersion, } @@ -117,18 +117,28 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { requireBlockImagesRemoved := false defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved) - // Create the filesystem, but wait to create the file test client until we upgrade to nautilus + // Create the filesystem logger.Infof("Initializing file before the upgrade") filesystemName := "upgrade-test-fs" activeCount := 1 createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount) + // Start the file test client + fsStorageClass := "file-upgrade" + assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass)) + useCSI := true + createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI) + defer func() { + cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName) + cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName) + }() + logger.Infof("Initializing object before the upgrade") objectStoreName := "upgraded-object" runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false) // verify that we're actually running the right pre-upgrade image - s.verifyOperatorImage(installer.Version1_2) + s.verifyOperatorImage(installer.Version1_4) message := "my simple message" preFilename := "pre-upgrade-file" @@ -147,41 +157,20 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { require.NotEqual(s.T(), 0, numOSDs) // - // Upgrade Rook from v1.2 to v1.3 + // Upgrade Rook from v1.4 to master // - logger.Infof("*** UPGRADING ROOK FROM v1.2 to v1.3 ***") - s.gatherLogs(systemNamespace, "_before_1.3_upgrade") - s.upgradeToV1_3() - - s.verifyOperatorImage(installer.Version1_3) - s.verifyRookUpgrade(numOSDs) - logger.Infof("Done with automatic upgrade from v1.2 to v1.3") - // Start the file test client now that the CSI driver is supported on nautilus - fsStorageClass := "file-upgrade" - assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass)) - useCSI := true - createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI) - defer func() { - cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName) - cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName) - }() - logger.Infof("Verified upgrade from v1.2 to v1.3") - - // - // Upgrade Rook from v1.3 to master - // - logger.Infof("*** UPGRADING ROOK FROM v1.3 to master ***") + logger.Infof("*** UPGRADING ROOK FROM v1.4 to master ***") s.gatherLogs(systemNamespace, "_before_master_upgrade") s.upgradeToMaster() s.verifyOperatorImage(installer.VersionMaster) s.verifyRookUpgrade(numOSDs) - logger.Infof("Done with automatic upgrade from v1.3 to master") - newFile := "post-upgrade-1_3-to-master-file" + logger.Infof("Done with automatic upgrade from v1.4 to master") + newFile := "post-upgrade-1_4-to-master-file" s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead) rbdFilesToRead = append(rbdFilesToRead, newFile) cephfsFilesToRead = append(cephfsFilesToRead, newFile) - logger.Infof("Verified upgrade from v1.3 to master") + logger.Infof("Verified upgrade from v1.4 to master") // // Upgrade from nautilus to octopus @@ -192,8 +181,6 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Verify reading and writing to the test clients newFile = "post-octopus-upgrade-file" s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead) - _ = append(rbdFilesToRead, newFile) - _ = append(cephfsFilesToRead, newFile) logger.Infof("Verified upgrade from nautilus to octopus") } @@ -211,9 +198,6 @@ func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) { require.NoError(s.T(), err) oldCephVersion := osdDepList.Items[0].Labels["ceph-version"] // upgraded OSDs should not have this version label - // - // Upgrade Ceph version, for example from Mimic to Nautilus. - // _, err = s.k8sh.Kubectl("-n", s.namespace, "patch", "CephCluster", s.namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec": {"cephVersion": {"image": "%s"}}}`, newCephImage)) @@ -326,405 +310,39 @@ func (s *UpgradeSuite) verifyFilesAfterUpgrade(fsName, newFileToWrite, messageFo } } -// UpgradeToV1_3 performs the steps necessary to upgrade a Rook v1.2 cluster to v1.3. It does not -// verify the upgrade but merely starts the upgrade process. -func (s *UpgradeSuite) upgradeToV1_3() { - require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestTo1_3(s.namespace))) - - require.NoError(s.T(), - s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.Version1_3)) -} - -// UpgradeToMaster performs the steps necessary to upgrade a Rook v1.2 cluster to master. It does not +// UpgradeToMaster performs the steps necessary to upgrade a Rook v1.4 cluster to master. It does not // verify the upgrade but merely starts the upgrade process. func (s *UpgradeSuite) upgradeToMaster() { - require.NoError(s.T(), s.k8sh.DeleteResource("ClusterRole", - "cephfs-csi-nodeplugin-rules", "cephfs-external-provisioner-runner-rules", - "rbd-csi-nodeplugin-rules", "rbd-external-provisioner-runner-rules", - "rook-ceph-cluster-mgmt-rules", "rook-ceph-global-rules", - "rook-ceph-mgr-cluster-rules", "rook-ceph-mgr-system-rules", - "cephfs-csi-nodeplugin", "cephfs-external-provisioner-runner", - "rbd-csi-nodeplugin", "rbd-external-provisioner-runner", - "rook-ceph-cluster-mgmt", "rook-ceph-global", - "rook-ceph-mgr-cluster", "rook-ceph-mgr-system")) - require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace))) + v1ExtensionsSupported := false + if s.k8sh.VersionAtLeast("v1.16.0") { + v1ExtensionsSupported = true + } + require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace, v1ExtensionsSupported))) require.NoError(s.T(), s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.VersionMaster)) } -func upgradeManifestToMaster(namespace string) string { - return ` -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephrbdmirrors.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephRBDMirror - listKind: CephRBDMirrorList - plural: cephrbdmirrors - singular: cephrbdmirror - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - count: - type: integer - minimum: 1 - maximum: 100 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectrealms.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectRealm - listKind: CephObjectRealmList - plural: cephobjectrealms - singular: cephobjectrealm - shortNames: - - realm - - realms - scope: Namespaced - version: v1 ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectzonegroups.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectZoneGroup - listKind: CephObjectZoneGroupList - plural: cephobjectzonegroups - singular: cephobjectzonegroup - shortNames: - - zonegroup - - zonegroups - scope: Namespaced - version: v1 ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectzones.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectZone - listKind: CephObjectZoneList - plural: cephobjectzones - singular: cephobjectzone - shortNames: - - zone - - zones - scope: Namespaced - version: v1 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - secrets - - pods - - pods/log - - services - - configmaps - - deployments - - daemonsets - verbs: - - get - - list - - watch - - patch - - create - - update - - delete +func upgradeManifestToMaster(namespace string, useV1Extensions bool) string { + rbacAuthVersion := "rbac.authorization.k8s.io/v1beta1" + if useV1Extensions { + rbacAuthVersion = "rbac.authorization.k8s.io/v1" + } + + // Always apply CRDs for the latest master + m := installer.NewCephManifests(installer.VersionMaster) + manifests := m.GetRookCRDs(useV1Extensions) + + manifests += ` --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: ` + rbacAuthVersion + ` metadata: name: rbd-external-provisioner-runner rules: - apiGroups: [""] resources: ["secrets"] - verbs: ["get", "list"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] @@ -734,6 +352,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] @@ -764,194 +385,22 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] +--- +kind: Role +apiVersion: ` + rbacAuthVersion + ` +metadata: + namespace: ` + namespace + ` + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] ` -} - -func upgradeManifestTo1_3(namespace string) string { - return ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete` + return manifests }