From 2f034824cbd3affc0d1048698bbbe599108d435e Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Wed, 4 Nov 2020 15:53:55 -0700 Subject: [PATCH] ceph: upgrade test Rook only from v1.4 to master Since Rook no longer supports legacy filestore devices, there is no need to keep testing upgrades from v1.2 all the way to master. We can now just test v1.4 to master. Signed-off-by: Blaine Gardner --- Documentation/ceph-upgrade.md | 8 +- .../ceph/upgrade-from-v1.4-crds.yaml | 1556 ++++++++++++++--- tests/framework/installer/ceph_installer.go | 4 +- tests/framework/installer/ceph_manifests.go | 10 +- ...nifests_v1.2.go => ceph_manifests_v1.4.go} | 763 +++++--- tests/integration/ceph_upgrade_test.go | 665 +------ 6 files changed, 1850 insertions(+), 1156 deletions(-) rename tests/framework/installer/{ceph_manifests_v1.2.go => ceph_manifests_v1.4.go} (75%) diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 2fbf35595..7bf8786ea 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -349,9 +349,13 @@ available with Ceph-CSI v3.0. First apply new resources. This includes slightly modified privileges (RBAC) needed by the Operator and updates to the Custom Resource Definitions (CRDs). +> **IMPORTANT:** If you are using Kubernetes version v1.15 or lower, you will need to manually +> modify the `upgrade-from-v1.4-apply.yaml` file to use +> `rbac.authorization.k8s.io/v1beta1` instead of `rbac.authorization.k8s.io/v1` +> You will also need to apply `pre-k8s-1.16/crd.yaml` instead of `upgrade-from-v1.4-crds.yaml`. + ```sh -kubectl delete -f upgrade-from-v1.3-delete.yaml -kubectl apply -f upgrade-from-v1.3-apply.yaml -f upgrade-from-v1.3-crds.yaml +kubectl apply -f upgrade-from-v1.4-apply.yaml -f upgrade-from-v1.4-crds.yaml ``` ## 3. Update Ceph CSI version to v3.0 diff --git a/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml b/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml index 3a395974b..2b380ec54 100644 --- a/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml +++ b/cluster/examples/kubernetes/ceph/upgrade-from-v1.4-crds.yaml @@ -1,76 +1,388 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: cephblockpools.ceph.rook.io + name: cephclusters.ceph.rook.io spec: group: ceph.rook.io names: - kind: CephBlockPool - listKind: CephBlockPoolList - plural: cephblockpools - singular: cephblockpool + kind: CephCluster + plural: cephclusters + singular: cephcluster scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object properties: - failureDomain: - type: string - crushRoot: - type: string - replicated: - properties: - size: - type: integer - minimum: 0 - maximum: 9 - targetSizeRatio: - type: number - requireSafeReplicaSize: - type: boolean - replicasPerFailureDomain: - type: integer - subFailureDomain: - type: string - erasureCoded: - properties: - dataChunks: - type: integer - minimum: 0 - maximum: 9 - codingChunks: - type: integer - minimum: 0 - maximum: 9 - compressionMode: - type: string - enum: - - "" - - none - - passive - - aggressive - - force - enableRBDStats: - description: EnableRBDStats is used to enable gathering of statistics - for all RBD images in the pool - type: boolean - parameters: + spec: type: object - mirroring: properties: - enabled: - type: boolean - mode: + cephVersion: + type: object + properties: + allowUnsupported: + type: boolean + image: + type: string + dashboard: + type: object + properties: + enabled: + type: boolean + urlPrefix: + type: string + port: + type: integer + minimum: 0 + maximum: 65535 + ssl: + type: boolean + dataDirHostPath: + pattern: ^/(\S+) type: string - enum: - - image - - pool - subresources: - status: {} + disruptionManagement: + type: object + properties: + machineDisruptionBudgetNamespace: + type: string + managePodBudgets: + type: boolean + osdMaintenanceTimeout: + type: integer + manageMachineDisruptionBudgets: + type: boolean + skipUpgradeChecks: + type: boolean + continueUpgradeAfterChecksEvenIfNotHealthy: + type: boolean + mon: + type: object + properties: + allowMultiplePerNode: + type: boolean + count: + maximum: 9 + minimum: 0 + type: integer + volumeClaimTemplate: + type: object + x-kubernetes-preserve-unknown-fields: true + mgr: + type: object + properties: + modules: + type: array + items: + type: object + properties: + name: + type: string + enabled: + type: boolean + crashCollector: + type: object + properties: + disable: + type: boolean + network: + type: object + nullable: true + properties: + hostNetwork: + type: boolean + provider: + type: string + selectors: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + ipFamily: + type: string + storage: + type: object + properties: + disruptionManagement: + type: object + nullable: true + properties: + machineDisruptionBudgetNamespace: + type: string + managePodBudgets: + type: boolean + osdMaintenanceTimeout: + type: integer + manageMachineDisruptionBudgets: + type: boolean + useAllNodes: + type: boolean + nodes: + type: array + nullable: true + items: + type: object + properties: + name: + type: string + config: + type: object + nullable: true + properties: + metadataDevice: + type: string + storeType: + type: string + pattern: ^(bluestore)$ + databaseSizeMB: + type: string + walSizeMB: + type: string + journalSizeMB: + type: string + osdsPerDevice: + type: string + encryptedDevice: + type: string + pattern: ^(true|false)$ + useAllDevices: + type: boolean + deviceFilter: + type: string + nullable: true + devicePathFilter: + type: string + devices: + type: array + items: + type: object + properties: + name: + type: string + config: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + useAllDevices: + type: boolean + deviceFilter: + type: string + nullable: true + devicePathFilter: + type: string + config: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + storageClassDeviceSets: + type: array + nullable: true + items: + type: object + properties: + name: + type: string + count: + type: integer + format: int32 + portable: + type: boolean + tuneDeviceClass: + type: boolean + tuneFastDeviceClass: + type: boolean + encrypted: + type: boolean + schedulerName: + type: string + config: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + preparePlacement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + driveGroups: + type: array + nullable: true + items: + type: object + properties: + name: + type: string + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + required: + - name + - spec + monitoring: + type: object + properties: + enabled: + type: boolean + rulesNamespace: + type: string + externalMgrEndpoints: + type: array + items: + type: object + properties: + ip: + type: string + removeOSDsIfOutAndSafeToRemove: + type: boolean + external: + type: object + properties: + enable: + type: boolean + cleanupPolicy: + type: object + properties: + allowUninstallWithVolumes: + type: boolean + confirmation: + type: string + pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + type: object + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 + security: + type: object + properties: + kms: + type: object + properties: + connectionDetails: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + tokenSecretName: + type: string + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + labels: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + healthCheck: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassNames: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: DataDirHostPath + type: string + description: Directory used on the K8s nodes + jsonPath: .spec.dataDirHostPath + - name: MonCount + type: string + description: Number of MONs + jsonPath: .spec.mon.count + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + - name: Phase + type: string + description: Phase + jsonPath: .status.phase + - name: Message + type: string + description: Message + jsonPath: .status.message + - name: Health + type: string + description: Ceph Health + jsonPath: .status.ceph.health + subresources: + status: {} +# OLM: END CEPH CRD +# OLM: BEGIN CEPH CLIENT CRD --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + caps: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH CLIENT CRD +# OLM: BEGIN CEPH RBD MIRROR CRD +--- +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: cephrbdmirrors.ceph.rook.io @@ -82,242 +394,924 @@ spec: plural: cephrbdmirrors singular: cephrbdmirror scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object properties: - count: - type: integer - minimum: 1 - maximum: 100 - peers: + spec: + type: object properties: - secretNames: - type: array - subresources: - status: {} + count: + type: integer + minimum: 1 + maximum: 100 + peers: + type: object + properties: + secretNames: + type: array + items: + type: string + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + +# OLM: END CEPH RBD MIRROR CRD +# OLM: BEGIN CEPH FS CRD --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: cephclusters.ceph.rook.io + name: cephfilesystems.ceph.rook.io spec: group: ceph.rook.io names: - kind: CephCluster - listKind: CephClusterList - plural: cephclusters - singular: cephcluster + kind: CephFilesystem + listKind: CephFilesystemList + plural: cephfilesystems + singular: cephfilesystem scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object properties: - annotations: {} - cephVersion: + spec: + type: object properties: - allowUnsupported: - type: boolean - image: - type: string - dashboard: - properties: - enabled: - type: boolean - urlPrefix: - type: string - port: - type: integer - minimum: 0 - maximum: 65535 - ssl: - type: boolean - dataDirHostPath: - pattern: ^/(\S+) - type: string - disruptionManagement: - properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: - type: integer - manageMachineDisruptionBudgets: - type: boolean - skipUpgradeChecks: - type: boolean - continueUpgradeAfterChecksEvenIfNotHealthy: - type: boolean - mon: - properties: - allowMultiplePerNode: - type: boolean - count: - maximum: 9 - minimum: 0 - type: integer - volumeClaimTemplate: {} - mgr: - properties: - modules: - items: - properties: - name: - type: string - enabled: - type: boolean - network: - properties: - hostNetwork: - type: boolean - provider: - type: string - selectors: {} - storage: - properties: - disruptionManagement: + metadataServer: + type: object properties: - machineDisruptionBudgetNamespace: - type: string - managePodBudgets: - type: boolean - osdMaintenanceTimeout: + activeCount: + minimum: 1 + maximum: 10 type: integer - manageMachineDisruptionBudgets: + activeStandby: type: boolean - useAllNodes: - type: boolean - nodes: - items: - properties: - name: - type: string - config: - properties: - metadataDevice: - type: string - storeType: - type: string - pattern: ^(bluestore)$ - databaseSizeMB: - type: string - walSizeMB: - type: string - journalSizeMB: - type: string - osdsPerDevice: - type: string - encryptedDevice: - type: string - pattern: ^(true|false)$ - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - devices: - type: array - items: - properties: - name: - type: string - config: {} - resources: {} - type: array - useAllDevices: - type: boolean - deviceFilter: - type: string - devicePathFilter: - type: string - config: {} - storageClassDeviceSets: {} - driveGroups: - type: array - nullable: true - items: - properties: - name: - type: string - spec: {} - placement: {} - required: - - name - - spec - monitoring: - properties: - enabled: - type: boolean - rulesNamespace: - type: string - externalMgrEndpoints: - type: array - items: - properties: - ip: - type: string - removeOSDsIfOutAndSafeToRemove: - type: boolean - external: - properties: - enable: - type: boolean - cleanupPolicy: - properties: - confirmation: - type: string - pattern: ^$|^yes-really-destroy-data$ - sanitizeDisks: + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + labels: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + metadataPool: + type: object + nullable: true properties: - method: + failureDomain: type: string - pattern: ^(complete|quick)$ - dataSource: + deviceClass: type: string - pattern: ^(zero|random)$ - iteration: + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + minimum: 0 + maximum: 10 + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + parameters: + type: object + x-kubernetes-preserve-unknown-fields: true + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + minimum: 0 + maximum: 10 + type: integer + codingChunks: + minimum: 0 + maximum: 10 + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + dataPools: + type: array + nullable: true + items: + type: object + properties: + failureDomain: + type: string + deviceClass: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + minimum: 0 + maximum: 10 + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + minimum: 0 + maximum: 10 + type: integer + codingChunks: + minimum: 0 + maximum: 10 + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + preservePoolsOnDelete: + type: boolean + preserveFilesystemOnDelete: + type: boolean + status: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - name: ActiveMDS + type: string + description: Number of desired active MDS daemons + jsonPath: .spec.metadataServer.activeCount + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} +# OLM: END CEPH FS CRD +# OLM: BEGIN CEPH NFS CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephnfses.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephNFS + listKind: CephNFSList + plural: cephnfses + singular: cephnfs + shortNames: + - nfs + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + rados: + type: object + properties: + pool: + type: string + namespace: + type: string + server: + type: object + properties: + active: + type: integer + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH NFS CRD +# OLM: BEGIN CEPH OBJECT STORE CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectstores.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStore + listKind: CephObjectStoreList + plural: cephobjectstores + singular: cephobjectstore + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + gateway: + type: object + properties: + type: + type: string + sslCertificateRef: + type: string + nullable: true + port: + type: integer + minimum: 0 + maximum: 65535 + securePort: + type: integer + minimum: 0 + maximum: 65535 + instances: + type: integer + externalRgwEndpoints: + type: array + nullable: true + items: + type: object + properties: + ip: + type: string + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + placement: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + resources: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + labels: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + type: string + metadataPool: + type: object + nullable: true + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + zone: + type: object + properties: + name: + type: string + dataPool: + type: object + nullable: true + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + x-kubernetes-preserve-unknown-fields: true + preservePoolsOnDelete: + type: boolean + healthCheck: + type: object + nullable: true + properties: + bucket: + type: object + nullable: true + properties: + enabled: + type: boolean + interval: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + +# OLM: END CEPH OBJECT STORE CRD +# OLM: BEGIN CEPH OBJECT STORE USERS CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectstoreusers.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectStoreUser + listKind: CephObjectStoreUserList + plural: cephobjectstoreusers + singular: cephobjectstoreuser + shortNames: + - rcou + - objectuser + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + store: + type: string + displayName: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT STORE USERS CRD +# OLM: BEGIN CEPH OBJECT REALM CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectrealms.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectRealm + listKind: CephObjectRealmList + plural: cephobjectrealms + singular: cephobjectrealm + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + pull: + type: object + properties: + endpoint: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT REALM CRD +# OLM: BEGIN CEPH OBJECT ZONEGROUP CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzonegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZoneGroup + listKind: CephObjectZoneGroupList + plural: cephobjectzonegroups + singular: cephobjectzonegroup + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + realm: + type: string + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT ZONEGROUP CRD +# OLM: BEGIN CEPH OBJECT ZONE CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzones.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZone + listKind: CephObjectZoneList + plural: cephobjectzones + singular: cephobjectzone + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + zoneGroup: + type: string + metadataPool: + type: object + nullable: true + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + dataPool: + type: object + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + preservePoolsOnDelete: + type: boolean + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH OBJECT ZONE CRD +# OLM: BEGIN CEPH BLOCK POOL CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cephblockpools.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPool + listKind: CephBlockPoolList + plural: cephblockpools + singular: cephblockpool + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + failureDomain: + type: string + deviceClass: + type: string + crushRoot: + type: string + replicated: + type: object + nullable: true + properties: + size: + type: integer + minimum: 0 + maximum: 9 + targetSizeRatio: + type: number + requireSafeReplicaSize: + type: boolean + replicasPerFailureDomain: + type: integer + subFailureDomain: + type: string + erasureCoded: + type: object + nullable: true + properties: + dataChunks: + type: integer + minimum: 0 + maximum: 9 + codingChunks: + type: integer + minimum: 0 + maximum: 9 + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + enableRBDStats: + description: + EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + parameters: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + mirroring: + type: object + nullable: true + properties: + enabled: + type: boolean + mode: + type: string + enum: + - image + - pool + statusCheck: + type: object + x-kubernetes-preserve-unknown-fields: true + annotations: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + +# OLM: END CEPH BLOCK POOL CRD +# OLM: BEGIN CEPH VOLUME POOL CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: volumes.rook.io +spec: + group: rook.io + names: + kind: Volume + listKind: VolumeList + plural: volumes + singular: volume + shortNames: + - rv + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + attachments: + type: array + items: + type: object + properties: + node: + type: string + podNamespace: + type: string + podName: + type: string + clusterName: + type: string + mountDir: + type: string + readOnly: + type: boolean + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END CEPH VOLUME POOL CRD +# OLM: BEGIN OBJECTBUCKET CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: objectbuckets.objectbucket.io +spec: + group: objectbucket.io + names: + kind: ObjectBucket + listKind: ObjectBucketList + plural: objectbuckets + singular: objectbucket + shortNames: + - ob + - obs + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + storageClassName: + type: string + endpoint: + type: object + nullable: true + properties: + bucketHost: + type: string + bucketPort: type: integer format: int32 - security: - properties: - kms: - properties: - connectionDetails: - type: object - tokenSecretName: + bucketName: type: string - placement: {} - resources: {} - healthCheck: {} - subresources: - status: {} - additionalPrinterColumns: - - name: DataDirHostPath - type: string - description: Directory used on the K8s nodes - JSONPath: .spec.dataDirHostPath - - name: MonCount - type: string - description: Number of MONs - JSONPath: .spec.mon.count - - name: Age - type: date - JSONPath: .metadata.creationTimestamp - - name: Phase - type: string - description: Phase - JSONPath: .status.phase - - name: Message - type: string - description: Message - JSONPath: .status.message - - name: Health - type: string - description: Ceph Health - JSONPath: .status.ceph.health + region: + type: string + subRegion: + type: string + additionalConfig: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + authentication: + type: object + nullable: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalState: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + reclaimPolicy: + type: string + claimRef: + type: object + nullable: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END OBJECTBUCKET CRD +# OLM: BEGIN OBJECTBUCKETCLAIM CRD +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: objectbucketclaims.objectbucket.io +spec: + group: objectbucket.io + names: + kind: ObjectBucketClaim + listKind: ObjectBucketClaimList + plural: objectbucketclaims + singular: objectbucketclaim + shortNames: + - obc + - obcs + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + storageClassName: + type: string + bucketName: + type: string + generateBucketName: + type: string + additionalConfig: + type: object + nullable: true + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} +# OLM: END OBJECTBUCKETCLAIM CRD diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index a5f10106b..e034f5755 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -516,7 +516,7 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion) - if !utils.IsPlatformOpenShift() && rookVersion != Version1_2 && h.k8shelper.VersionAtLeast("v1.15.0") { + if !utils.IsPlatformOpenShift() && h.k8shelper.VersionAtLeast("v1.15.0") { if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") { assert.Fail(h.T(), "admission controller is not running") } @@ -843,7 +843,7 @@ func NewCephInstaller(t func() *testing.T, clientset *kubernetes.Clientset, useH k8sVersion: version.String(), CephVersion: cephVersion, cleanupHost: cleanupHost, - changeHostnames: rookVersion != Version1_2 && k8shelp.VersionAtLeast("v1.13.0"), + changeHostnames: k8shelp.VersionAtLeast("v1.18.0"), T: t, } flag.Parse() diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 0f61b5887..8a182427a 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -26,10 +26,8 @@ import ( ) const ( - // Version1_2 rook version 1.2 - Version1_2 = "v1.2.7" - // Version1_3 rook version 1.3 - Version1_3 = "v1.3.8" + // Version1_4 rook version 1.4 + Version1_4 = "v1.4.7" ) type CephManifests interface { @@ -89,8 +87,8 @@ func NewCephManifests(version string) CephManifests { switch version { case VersionMaster: return &CephManifestsMaster{imageTag: VersionMaster} - case Version1_2: - return &CephManifestsV1_2{imageTag: Version1_2} + case Version1_4: + return &CephManifestsV1_4{imageTag: Version1_4} } panic(fmt.Errorf("unrecognized ceph manifest version: %s", version)) } diff --git a/tests/framework/installer/ceph_manifests_v1.2.go b/tests/framework/installer/ceph_manifests_v1.4.go similarity index 75% rename from tests/framework/installer/ceph_manifests_v1.2.go rename to tests/framework/installer/ceph_manifests_v1.4.go index 2850fcda1..89de292aa 100644 --- a/tests/framework/installer/ceph_manifests_v1.2.go +++ b/tests/framework/installer/ceph_manifests_v1.4.go @@ -21,12 +21,12 @@ import ( "strings" ) -// CephManifestsV1_2 wraps rook yaml definitions -type CephManifestsV1_2 struct { +// CephManifestsV1_4 wraps rook yaml definitions +type CephManifestsV1_4 struct { imageTag string } -func (m *CephManifestsV1_2) GetRookCRDs(v1ExtensionsSupported bool) string { +func (m *CephManifestsV1_4) GetRookCRDs(v1ExtensionsSupported bool) string { return ` apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -68,14 +68,27 @@ spec: dataDirHostPath: pattern: ^/(\S+) type: string + disruptionManagement: + properties: + machineDisruptionBudgetNamespace: + type: string + managePodBudgets: + type: boolean + osdMaintenanceTimeout: + type: integer + manageMachineDisruptionBudgets: + type: boolean + skipUpgradeChecks: + type: boolean mon: properties: allowMultiplePerNode: type: boolean count: maximum: 9 - minimum: 1 + minimum: 0 type: integer + volumeClaimTemplate: {} mgr: properties: modules: @@ -89,10 +102,15 @@ spec: properties: hostNetwork: type: boolean + provider: + type: string + selectors: {} storage: properties: disruptionManagement: properties: + machineDisruptionBudgetNamespace: + type: string managePodBudgets: type: boolean osdMaintenanceTimeout: @@ -112,7 +130,7 @@ spec: type: string storeType: type: string - pattern: ^(filestore|bluestore)$ + pattern: ^(bluestore)$ databaseSizeMB: type: string walSizeMB: @@ -127,12 +145,6 @@ spec: useAllDevices: type: boolean deviceFilter: {} - directories: - type: array - items: - properties: - path: - type: string devices: type: array items: @@ -140,22 +152,24 @@ spec: name: type: string config: {} - location: {} resources: {} type: array useAllDevices: type: boolean deviceFilter: {} - location: {} - directories: - type: array - items: - properties: - path: - type: string config: {} - topologyAware: - type: boolean + storageClassDeviceSets: {} + driveGroups: + type: array + items: + properties: + name: + type: string + spec: {} + placement: {} + required: + - name + - spec monitoring: properties: enabled: @@ -166,10 +180,30 @@ spec: properties: workers: type: integer + removeOSDsIfOutAndSafeToRemove: + type: boolean + external: + properties: + enable: + type: boolean + cleanupPolicy: + properties: + confirmation: + type: string + pattern: ^$|^yes-really-destroy-data$ + sanitizeDisks: + properties: + method: + type: string + pattern: ^(complete|quick)$ + dataSource: + type: string + pattern: ^(zero|random)$ + iteration: + type: integer + format: int32 placement: {} resources: {} - required: - - mon additionalPrinterColumns: - name: DataDirHostPath type: string @@ -190,6 +224,8 @@ spec: type: string description: Ceph Health JSONPath: .status.ceph.health + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -224,36 +260,72 @@ spec: properties: failureDomain: type: string + crushRoot: + type: string replicated: properties: size: - minimum: 1 + minimum: 0 maximum: 10 type: integer + requireSafeReplicaSize: + type: boolean erasureCoded: properties: dataChunks: + minimum: 0 + maximum: 10 type: integer codingChunks: + minimum: 0 + maximum: 10 type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force dataPools: type: array items: properties: failureDomain: type: string + crushRoot: + type: string replicated: properties: size: - minimum: 1 + minimum: 0 maximum: 10 type: integer + requireSafeReplicaSize: + type: boolean erasureCoded: properties: dataChunks: + minimum: 0 + maximum: 10 type: integer codingChunks: + minimum: 0 + maximum: 10 type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + preservePoolsOnDelete: + type: boolean additionalPrinterColumns: - name: ActiveMDS type: string @@ -262,6 +334,8 @@ spec: - name: Age type: date JSONPath: .metadata.creationTimestamp + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -296,6 +370,8 @@ spec: annotations: {} placement: {} resources: {} + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -322,8 +398,9 @@ spec: sslCertificateRef: {} port: type: integer - securePort: - type: integer + minimum: 1 + maximum: 65535 + securePort: {} instances: type: integer annotations: {} @@ -333,30 +410,71 @@ spec: properties: failureDomain: type: string - replicated: - properties: - size: - type: integer - erasureCoded: - properties: - dataChunks: - type: integer - codingChunks: - type: integer - dataPool: - properties: - failureDomain: + crushRoot: type: string replicated: properties: size: type: integer + requireSafeReplicaSize: + type: boolean erasureCoded: properties: dataChunks: type: integer codingChunks: type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + dataPool: + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + properties: + size: + type: integer + requireSafeReplicaSize: + type: boolean + erasureCoded: + properties: + dataChunks: + type: integer + codingChunks: + type: integer + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + parameters: + type: object + preservePoolsOnDelete: + type: boolean + healthCheck: + properties: + bucket: + properties: + enabled: + type: boolean + interval: + type: string + subresources: + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -369,6 +487,62 @@ spec: listKind: CephObjectStoreUserList plural: cephobjectstoreusers singular: cephobjectstoreuser + shortNames: + - rcou + - objectuser + scope: Namespaced + version: v1 + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectrealms.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectRealm + listKind: CephObjectRealmList + plural: cephobjectrealms + singular: cephobjectrealm + shortNames: + - realm + - realms + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzonegroups.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZoneGroup + listKind: CephObjectZoneGroupList + plural: cephobjectzonegroups + singular: cephobjectzonegroup + shortNames: + - zonegroup + - zonegroups + scope: Namespaced + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephobjectzones.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephObjectZone + listKind: CephObjectZoneList + plural: cephobjectzones + singular: cephobjectzone + shortNames: + - zone + - zones scope: Namespaced version: v1 --- @@ -385,6 +559,51 @@ spec: singular: cephblockpool scope: Namespaced version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + failureDomain: + type: string + crushRoot: + type: string + replicated: + properties: + size: + type: integer + minimum: 0 + maximum: 9 + targetSizeRatio: + type: number + requireSafeReplicaSize: + type: boolean + erasureCoded: + properties: + dataChunks: + type: integer + minimum: 0 + maximum: 9 + codingChunks: + type: integer + minimum: 0 + maximum: 9 + compressionMode: + type: string + enum: + - "" + - none + - passive + - aggressive + - force + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics + for all RBD images in the pool + type: boolean + parameters: + type: object + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -397,8 +616,12 @@ spec: listKind: VolumeList plural: volumes singular: volume + shortNames: + - rv scope: Namespaced version: v1alpha2 + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -441,14 +664,68 @@ spec: - obc - obcs scope: Namespaced + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + caps: + properties: + mon: + type: string + osd: + type: string + mds: + type: string + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephrbdmirrors.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephRBDMirror + listKind: CephRBDMirrorList + plural: cephrbdmirrors + singular: cephrbdmirror + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + count: + type: integer + minimum: 1 + maximum: 100 subresources: status: {}` } // GetRookOperator returns rook Operator manifest -func (m *CephManifestsV1_2) GetRookOperator(namespace string) string { +func (m *CephManifestsV1_4) GetRookOperator(namespace string) string { return ` -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: name: rook-ceph-system @@ -483,27 +760,13 @@ rules: verbs: - get --- -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: rook-ceph-cluster-mgmt labels: operator: rook storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" rules: - apiGroups: - "" @@ -535,27 +798,13 @@ rules: - update - delete --- -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: rook-ceph-global labels: operator: rook storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" rules: - apiGroups: - "" @@ -660,21 +909,9 @@ rules: - delete --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-cluster -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster-rules - labels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -698,7 +935,7 @@ rules: - watch --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd rules: @@ -711,27 +948,13 @@ rules: - list --- # Aspects of Rook Ceph Agent that require access to secrets -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: rook-ceph-agent-mount labels: operator: rook storage-backend: ceph -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-agent-mount-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-agent-mount: "true" rules: - apiGroups: - "" @@ -742,21 +965,9 @@ rules: --- # Aspects of ceph-mgr that require access to the system namespace kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system-rules - labels: - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-system: "true" rules: - apiGroups: - "" @@ -768,13 +979,12 @@ rules: - watch --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-object-bucket labels: operator: rook storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" rules: - apiGroups: - "" @@ -799,7 +1009,7 @@ rules: - "*" --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-object-bucket roleRef: @@ -821,7 +1031,7 @@ metadata: storage-backend: ceph --- kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-system namespace: ` + namespace + ` @@ -838,7 +1048,7 @@ subjects: namespace: ` + namespace + ` --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-global labels: @@ -863,18 +1073,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] @@ -918,18 +1116,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] @@ -948,18 +1134,15 @@ rules: verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotclasses"] verbs: ["get", "list", "watch"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] + resources: ["volumesnapshotcontents/status"] verbs: ["update"] - apiGroups: [""] resources: ["nodes"] @@ -1024,18 +1207,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-csi-nodeplugin: "true" rules: - apiGroups: [""] resources: ["nodes"] @@ -1077,18 +1248,6 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" -rules: [] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" rules: - apiGroups: [""] resources: ["secrets"] @@ -1161,7 +1320,10 @@ roleRef: apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: - name: rook-privileged + name: 00-rook-privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' spec: privileged: true allowedCapabilities: @@ -1191,7 +1353,6 @@ spec: - hostPath - flexVolume # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # directory-based OSDs make this hard to nail down # allowedHostPaths: # - /run/udev # for OSD prep # - /dev # for OSD prep @@ -1232,7 +1393,7 @@ rules: resources: - podsecuritypolicies resourceNames: - - rook-privileged + - 00-rook-privileged verbs: - use --- @@ -1301,6 +1462,35 @@ subjects: name: rook-csi-cephfs-plugin-sa namespace: ` + namespace + ` --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-admission-controller + namespace: ` + namespace + ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-role +rules: + - apiGroups: ["ceph.rook.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-rolebinding +subjects: + - kind: ServiceAccount + name: rook-ceph-admission-controller + apiGroup: "" + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rook-ceph-admission-controller-role + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -1327,12 +1517,6 @@ spec: env: - name: ROOK_LOG_LEVEL value: INFO - - name: ROOK_CEPH_STATUS_CHECK_INTERVAL - value: "5s" - - name: ROOK_MON_HEALTHCHECK_INTERVAL - value: "10s" - - name: ROOK_MON_OUT_TIMEOUT - value: "15s" - name: NODE_NAME valueFrom: fieldRef: @@ -1359,11 +1543,12 @@ data: ROOK_CSI_ENABLE_CEPHFS: "true" ROOK_CSI_ENABLE_RBD: "true" ROOK_CSI_ENABLE_GRPC_METRICS: "true" + ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true" ` } // GetClusterRoles returns rook-cluster manifest -func (m *CephManifestsV1_2) GetClusterRoles(namespace, systemNamespace string) string { +func (m *CephManifestsV1_4) GetClusterRoles(namespace, systemNamespace string) string { return `apiVersion: v1 kind: ServiceAccount metadata: @@ -1383,7 +1568,7 @@ metadata: namespace: ` + namespace + ` --- kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd namespace: ` + namespace + ` @@ -1397,7 +1582,7 @@ rules: --- # Aspects of ceph-mgr that operate within the cluster's namespace kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr namespace: ` + namespace + ` @@ -1433,7 +1618,7 @@ rules: --- # Allow the ceph osd to access cluster-wide resources necessary for determining their topology location kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd-` + namespace + ` roleRef: @@ -1447,7 +1632,7 @@ subjects: --- # Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-cluster-` + namespace + ` roleRef: @@ -1460,7 +1645,7 @@ subjects: namespace: ` + namespace + ` --- kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1480,7 +1665,7 @@ rules: --- # Allow the operator to create resources in this cluster's namespace kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cluster-mgmt namespace: ` + namespace + ` @@ -1495,7 +1680,7 @@ subjects: --- # Allow the osd pods in this namespace to work with configmaps kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-osd namespace: ` + namespace + ` @@ -1510,7 +1695,7 @@ subjects: --- # Allow the ceph mgr to access the cluster-specific resources necessary for the mgr modules kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr namespace: ` + namespace + ` @@ -1525,7 +1710,7 @@ subjects: --- # Allow the ceph mgr to access the rook system resources necessary for the mgr modules kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-mgr-system ` + namespace + ` namespace: ` + systemNamespace + ` @@ -1539,7 +1724,7 @@ subjects: namespace: ` + namespace + ` --- kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1611,7 +1796,7 @@ subjects: } // GetRookCluster returns rook-cluster manifest -func (m *CephManifestsV1_2) GetRookCluster(settings *clusterSettings) string { +func (m *CephManifestsV1_4) GetRookCluster(settings *clusterSettings) string { store := "# storeType not specified; Rook will use default store types" if settings.StoreType != "" { store = `storeType: "` + settings.StoreType + `"` @@ -1621,6 +1806,7 @@ func (m *CephManifestsV1_2) GetRookCluster(settings *clusterSettings) string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: + # set the name to something different from the namespace name: ` + settings.ClusterName + ` namespace: ` + settings.Namespace + ` spec: @@ -1641,8 +1827,6 @@ spec: continueUpgradeAfterChecksEvenIfNotHealthy: false dashboard: enabled: true - rbdMirroring: - workers: ` + strconv.Itoa(settings.RBDMirrorWorkers) + ` network: hostNetwork: false crashCollector: @@ -1653,6 +1837,7 @@ spec: count: 1 portable: false tuneDeviceClass: true + encrypted: false volumeClaimTemplates: - metadata: name: data @@ -1691,9 +1876,9 @@ spec: skipUpgradeChecks: true metadataDevice: storage: - useAllNodes: true - useAllDevices: true - deviceFilter: + useAllNodes: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + useAllDevices: ` + strconv.FormatBool(!settings.skipOSDCreation) + ` + deviceFilter: '' config: ` + store + ` databaseSizeMB: "1024" @@ -1701,11 +1886,22 @@ spec: mgr: modules: - name: pg_autoscaler - enabled: true` + enabled: true + - name: rook + enabled: true + healthCheck: + daemonHealth: + mon: + interval: 10s + timeout: 15s + osd: + interval: 10s + status: + interval: 5s` } // GetRookToolBox returns rook-toolbox manifest -func (m *CephManifestsV1_2) GetRookToolBox(namespace string) string { +func (m *CephManifestsV1_4) GetRookToolBox(namespace string) string { return `apiVersion: v1 kind: Pod metadata: @@ -1720,16 +1916,38 @@ spec: command: ["/tini"] args: ["-g", "--", "/usr/local/bin/toolbox.sh"] env: - - name: ROOK_ADMIN_SECRET + - name: ROOK_CEPH_USERNAME valueFrom: secretKeyRef: name: rook-ceph-mon - key: admin-secret + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + securityContext: + privileged: true volumeMounts: + - mountPath: /dev + name: dev + - mountPath: /sys/bus + name: sysbus + - mountPath: /lib/modules + name: libmodules - name: mon-endpoint-volume mountPath: /etc/rook hostNetwork: false volumes: + - name: dev + hostPath: + path: /dev + - name: sysbus + hostPath: + path: /sys/bus + - name: libmodules + hostPath: + path: /lib/modules - name: mon-endpoint-volume configMap: name: rook-ceph-mon-endpoints @@ -1738,7 +1956,7 @@ spec: path: mon-endpoints` } -func (m *CephManifestsV1_2) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { +func (m *CephManifestsV1_4) GetBlockPoolDef(poolName string, namespace string, replicaSize string) string { return `apiVersion: ceph.rook.io/v1 kind: CephBlockPool metadata: @@ -1749,12 +1967,10 @@ spec: size: ` + replicaSize + ` targetSizeRatio: .5 requireSafeReplicaSize: false - compressionMode: aggressive - poolProperties: - pg_autoscale_mode: on` + compressionMode: aggressive` } -func (m *CephManifestsV1_2) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string { +func (m *CephManifestsV1_4) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string { // Create a CSI driver storage class if csi { return ` @@ -1763,6 +1979,7 @@ kind: StorageClass metadata: name: ` + storageClassName + ` provisioner: ` + systemNamespace + `.rbd.csi.ceph.com +reclaimPolicy: ` + reclaimPolicy + ` parameters: pool: ` + poolName + ` clusterID: ` + namespace + ` @@ -1784,8 +2001,7 @@ parameters: blockPool: ` + poolName + ` clusterNamespace: ` + namespace } - -func (m *CephManifestsV1_2) GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace string) string { +func (m *CephManifestsV1_4) GetFileStorageClassDef(fsName, storageClassName, systemNamespace, namespace string) string { // Create a CSI driver storage class csiCephFSNodeSecret := "rook-csi-cephfs-node" //nolint:gosec // We safely suppress gosec in tests file csiCephFSProvisionerSecret := "rook-csi-cephfs-provisioner" //nolint:gosec // We safely suppress gosec in tests file @@ -1806,15 +2022,14 @@ parameters: ` } -func (m *CephManifestsV1_2) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { +func (m *CephManifestsV1_4) GetPVC(claimName, namespace, storageClassName, accessModes, size string) string { return `apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ` + claimName + ` namespace: ` + namespace + ` - annotations: - volume.beta.kubernetes.io/storage-class: ` + storageClassName + ` spec: + storageClassName: ` + storageClassName + ` accessModes: - ` + accessModes + ` resources: @@ -1822,8 +2037,53 @@ spec: storage: ` + size } +func (m *CephManifestsV1_4) GetBlockSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { + return "" +} + +func (m *CephManifestsV1_4) GetFileStorageSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { + return "" +} + +func (m *CephManifestsV1_4) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { + return "" +} + +func (m *CephManifestsV1_4) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { + return "" +} + +func (m *CephManifestsV1_4) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { + return "" +} + +func (m *CephManifestsV1_4) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { + return ` +apiVersion: v1 +kind: Pod +metadata: + name: ` + podName + ` + namespace: ` + namespace + ` +spec: + containers: + - name: ` + podName + ` + image: busybox + command: ["/bin/sleep", "infinity"] + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: ` + mountPath + ` + name: csivol + volumes: + - name: csivol + persistentVolumeClaim: + claimName: ` + claimName + ` + readOnly: ` + strconv.FormatBool(readOnly) + ` + restartPolicy: Never +` +} + // GetFilesystem returns the manifest to create a Rook filesystem resource with the given config. -func (m *CephManifestsV1_2) GetFilesystem(namespace, name string, activeCount int) string { +func (m *CephManifestsV1_4) GetFilesystem(namespace, name string, activeCount int) string { return `apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: @@ -1833,16 +2093,19 @@ spec: metadataPool: replicated: size: 1 + requireSafeReplicaSize: false dataPools: - replicated: size: 1 + requireSafeReplicaSize: false + compressionMode: none metadataServer: activeCount: ` + strconv.Itoa(activeCount) + ` activeStandby: true` } // GetFilesystem returns the manifest to create a Rook Ceph NFS resource with the given config. -func (m *CephManifestsV1_2) GetNFS(namespace, name, pool string, count int) string { +func (m *CephManifestsV1_4) GetNFS(namespace, name, pool string, count int) string { return `apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: @@ -1856,7 +2119,7 @@ spec: active: ` + strconv.Itoa(count) } -func (m *CephManifestsV1_2) GetObjectStore(namespace, name string, replicaCount, port int) string { +func (m *CephManifestsV1_4) GetObjectStore(namespace, name string, replicaCount, port int) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStore metadata: @@ -1866,18 +2129,27 @@ spec: metadataPool: replicated: size: 1 + requireSafeReplicaSize: false + compressionMode: passive dataPool: replicated: size: 1 + requireSafeReplicaSize: false gateway: type: s3 sslCertificateRef: port: ` + strconv.Itoa(port) + ` + securePort: instances: ` + strconv.Itoa(replicaCount) + ` + allNodes: false + healthCheck: + bucket: + disabled: false + interval: 10s ` } -func (m *CephManifestsV1_2) GetObjectStoreUser(namespace, name string, displayName string, store string) string { +func (m *CephManifestsV1_4) GetObjectStoreUser(namespace, name string, displayName string, store string) string { return `apiVersion: ceph.rook.io/v1 kind: CephObjectStoreUser metadata: @@ -1889,12 +2161,12 @@ spec: } //GetBucketStorageClass returns the manifest to create object bucket -func (m *CephManifestsV1_2) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { +func (m *CephManifestsV1_4) GetBucketStorageClass(storeNameSpace string, storeName string, storageClassName string, reclaimPolicy string, region string) string { return `apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ` + storageClassName + ` -provisioner: ceph.rook.io/bucket +provisioner: ` + storeNameSpace + `.ceph.rook.io/bucket reclaimPolicy: ` + reclaimPolicy + ` parameters: objectStoreName: ` + storeName + ` @@ -1903,7 +2175,7 @@ parameters: } //GetObc returns the manifest to create object bucket claim -func (m *CephManifestsV1_2) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { +func (m *CephManifestsV1_4) GetObc(claimName string, storageClassName string, objectBucketName string, maxObject string, varBucketName bool) string { bucketParameter := "generateBucketName" if varBucketName { bucketParameter = "bucketName" @@ -1919,7 +2191,7 @@ spec: maxObjects: "` + maxObject + `"` } -func (m *CephManifestsV1_2) GetClient(namespace string, claimName string, caps map[string]string) string { +func (m *CephManifestsV1_4) GetClient(namespace string, claimName string, caps map[string]string) string { clientCaps := []string{} for name, cap := range caps { str := name + ": " + cap @@ -1935,10 +2207,10 @@ spec: ` + strings.Join(clientCaps, "\n ") } -func (m *CephManifestsV1_2) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { +func (m *CephManifestsV1_4) GetClusterExternalRoles(namespace, firstClusterNamespace string) string { return `apiVersion: v1 kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cluster-mgmt namespace: ` + namespace + ` @@ -1952,7 +2224,7 @@ subjects: namespace: ` + firstClusterNamespace + ` --- kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1972,7 +2244,7 @@ metadata: namespace: ` + namespace + ` --- kind: Role -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: rook-ceph-cmd-reporter namespace: ` + namespace + ` @@ -1991,7 +2263,7 @@ rules: - delete` } -func (m *CephManifestsV1_2) GetRookExternalCluster(settings *clusterExternalSettings) string { +func (m *CephManifestsV1_4) GetRookExternalCluster(settings *clusterExternalSettings) string { return `apiVersion: ceph.rook.io/v1 kind: CephCluster metadata: @@ -2003,36 +2275,13 @@ spec: dataDirHostPath: ` + settings.DataDirHostPath + `` } -func (m *CephManifestsV1_2) GetRBDMirror(namespace, name string, count int) string { - // Not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetBlockSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { - // block snapshot was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetFileStorageSnapshotClass(snapshotClassName, namespace, systemNamespace, reclaimPolicy string) string { - // filesystem snapshot was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetPVCRestore(claimName, snapshotName, namespace, storageClassName, accessModes, size string) string { - // snapshot restore to a new PVC was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetPVCClone(cloneClaimName, parentClaimName, namespace, storageClassName, accessModes, size string) string { - // clone pvc to a new PVC was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetSnapshot(snapshotName, claimName, snapshotClassName, namespace string) string { - // pvc snapshot was not implemented in v1.2 - return "" -} - -func (m *CephManifestsV1_2) GetPod(podName, claimName, namespace, mountPath string, readOnly bool) string { - return "" +// GetRBDMirror returns the manifest to create a Rook Ceph RBD Mirror resource with the given config. +func (m *CephManifestsV1_4) GetRBDMirror(namespace, name string, count int) string { + return `apiVersion: ceph.rook.io/v1 +kind: CephRBDMirror +metadata: + name: ` + name + ` + namespace: ` + namespace + ` +spec: + count: ` + strconv.Itoa(count) } diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index 21a39c946..d96a393c4 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -85,7 +85,7 @@ func (s *UpgradeSuite) SetupSuite() { rookCephCleanup: false, skipOSDCreation: false, minimalMatrixK8sVersion: upgradeMinimalTestVersion, - rookVersion: installer.Version1_2, + rookVersion: installer.Version1_4, cephVersion: installer.NautilusVersion, } @@ -117,18 +117,28 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { requireBlockImagesRemoved := false defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved) - // Create the filesystem, but wait to create the file test client until we upgrade to nautilus + // Create the filesystem logger.Infof("Initializing file before the upgrade") filesystemName := "upgrade-test-fs" activeCount := 1 createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount) + // Start the file test client + fsStorageClass := "file-upgrade" + assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass)) + useCSI := true + createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI) + defer func() { + cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName) + cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName) + }() + logger.Infof("Initializing object before the upgrade") objectStoreName := "upgraded-object" runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false) // verify that we're actually running the right pre-upgrade image - s.verifyOperatorImage(installer.Version1_2) + s.verifyOperatorImage(installer.Version1_4) message := "my simple message" preFilename := "pre-upgrade-file" @@ -147,41 +157,20 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { require.NotEqual(s.T(), 0, numOSDs) // - // Upgrade Rook from v1.2 to v1.3 + // Upgrade Rook from v1.4 to master // - logger.Infof("*** UPGRADING ROOK FROM v1.2 to v1.3 ***") - s.gatherLogs(systemNamespace, "_before_1.3_upgrade") - s.upgradeToV1_3() - - s.verifyOperatorImage(installer.Version1_3) - s.verifyRookUpgrade(numOSDs) - logger.Infof("Done with automatic upgrade from v1.2 to v1.3") - // Start the file test client now that the CSI driver is supported on nautilus - fsStorageClass := "file-upgrade" - assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, systemNamespace, s.namespace, fsStorageClass)) - useCSI := true - createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI) - defer func() { - cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName) - cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName) - }() - logger.Infof("Verified upgrade from v1.2 to v1.3") - - // - // Upgrade Rook from v1.3 to master - // - logger.Infof("*** UPGRADING ROOK FROM v1.3 to master ***") + logger.Infof("*** UPGRADING ROOK FROM v1.4 to master ***") s.gatherLogs(systemNamespace, "_before_master_upgrade") s.upgradeToMaster() s.verifyOperatorImage(installer.VersionMaster) s.verifyRookUpgrade(numOSDs) - logger.Infof("Done with automatic upgrade from v1.3 to master") - newFile := "post-upgrade-1_3-to-master-file" + logger.Infof("Done with automatic upgrade from v1.4 to master") + newFile := "post-upgrade-1_4-to-master-file" s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead) rbdFilesToRead = append(rbdFilesToRead, newFile) cephfsFilesToRead = append(cephfsFilesToRead, newFile) - logger.Infof("Verified upgrade from v1.3 to master") + logger.Infof("Verified upgrade from v1.4 to master") // // Upgrade from nautilus to octopus @@ -192,8 +181,6 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Verify reading and writing to the test clients newFile = "post-octopus-upgrade-file" s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead) - _ = append(rbdFilesToRead, newFile) - _ = append(cephfsFilesToRead, newFile) logger.Infof("Verified upgrade from nautilus to octopus") } @@ -211,9 +198,6 @@ func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) { require.NoError(s.T(), err) oldCephVersion := osdDepList.Items[0].Labels["ceph-version"] // upgraded OSDs should not have this version label - // - // Upgrade Ceph version, for example from Mimic to Nautilus. - // _, err = s.k8sh.Kubectl("-n", s.namespace, "patch", "CephCluster", s.namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec": {"cephVersion": {"image": "%s"}}}`, newCephImage)) @@ -326,405 +310,39 @@ func (s *UpgradeSuite) verifyFilesAfterUpgrade(fsName, newFileToWrite, messageFo } } -// UpgradeToV1_3 performs the steps necessary to upgrade a Rook v1.2 cluster to v1.3. It does not -// verify the upgrade but merely starts the upgrade process. -func (s *UpgradeSuite) upgradeToV1_3() { - require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestTo1_3(s.namespace))) - - require.NoError(s.T(), - s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.Version1_3)) -} - -// UpgradeToMaster performs the steps necessary to upgrade a Rook v1.2 cluster to master. It does not +// UpgradeToMaster performs the steps necessary to upgrade a Rook v1.4 cluster to master. It does not // verify the upgrade but merely starts the upgrade process. func (s *UpgradeSuite) upgradeToMaster() { - require.NoError(s.T(), s.k8sh.DeleteResource("ClusterRole", - "cephfs-csi-nodeplugin-rules", "cephfs-external-provisioner-runner-rules", - "rbd-csi-nodeplugin-rules", "rbd-external-provisioner-runner-rules", - "rook-ceph-cluster-mgmt-rules", "rook-ceph-global-rules", - "rook-ceph-mgr-cluster-rules", "rook-ceph-mgr-system-rules", - "cephfs-csi-nodeplugin", "cephfs-external-provisioner-runner", - "rbd-csi-nodeplugin", "rbd-external-provisioner-runner", - "rook-ceph-cluster-mgmt", "rook-ceph-global", - "rook-ceph-mgr-cluster", "rook-ceph-mgr-system")) - require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace))) + v1ExtensionsSupported := false + if s.k8sh.VersionAtLeast("v1.16.0") { + v1ExtensionsSupported = true + } + require.NoError(s.T(), s.k8sh.ResourceOperation("apply", upgradeManifestToMaster(s.namespace, v1ExtensionsSupported))) require.NoError(s.T(), s.k8sh.SetDeploymentVersion(installer.SystemNamespace(s.namespace), operatorContainer, operatorContainer, installer.VersionMaster)) } -func upgradeManifestToMaster(namespace string) string { - return ` -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephrbdmirrors.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephRBDMirror - listKind: CephRBDMirrorList - plural: cephrbdmirrors - singular: cephrbdmirror - scope: Namespaced - version: v1 - validation: - openAPIV3Schema: - properties: - spec: - properties: - count: - type: integer - minimum: 1 - maximum: 100 - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectrealms.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectRealm - listKind: CephObjectRealmList - plural: cephobjectrealms - singular: cephobjectrealm - shortNames: - - realm - - realms - scope: Namespaced - version: v1 ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectzonegroups.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectZoneGroup - listKind: CephObjectZoneGroupList - plural: cephobjectzonegroups - singular: cephobjectzonegroup - shortNames: - - zonegroup - - zonegroups - scope: Namespaced - version: v1 ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: cephobjectzones.ceph.rook.io -spec: - group: ceph.rook.io - names: - kind: CephObjectZone - listKind: CephObjectZoneList - plural: cephobjectzones - singular: cephobjectzone - shortNames: - - zone - - zones - scope: Namespaced - version: v1 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - - extensions - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-cluster-mgmt - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - - apps - - extensions - resources: - - secrets - - pods - - pods/log - - services - - configmaps - - deployments - - daemonsets - verbs: - - get - - list - - watch - - patch - - create - - update - - delete +func upgradeManifestToMaster(namespace string, useV1Extensions bool) string { + rbacAuthVersion := "rbac.authorization.k8s.io/v1beta1" + if useV1Extensions { + rbacAuthVersion = "rbac.authorization.k8s.io/v1" + } + + // Always apply CRDs for the latest master + m := installer.NewCephManifests(installer.VersionMaster) + manifests := m.GetRookCRDs(useV1Extensions) + + manifests += ` --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-cluster - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - resources: - - configmaps - - nodes - - nodes/proxy - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - list - - get - - watch ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-object-bucket - labels: - operator: rook - storage-backend: ceph -rules: -- apiGroups: - - "" - verbs: - - "*" - resources: - - secrets - - configmaps -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - "objectbucket.io" - verbs: - - "*" - resources: - - "*" ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-ceph-mgr-system -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: ` + rbacAuthVersion + ` metadata: name: rbd-external-provisioner-runner rules: - apiGroups: [""] resources: ["secrets"] - verbs: ["get", "list"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] @@ -734,6 +352,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] @@ -764,194 +385,22 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims/status"] verbs: ["update", "patch"] +--- +kind: Role +apiVersion: ` + rbacAuthVersion + ` +metadata: + namespace: ` + namespace + ` + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] ` -} - -func upgradeManifestTo1_3(namespace string) string { - return ` -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rbd-external-provisioner-runner-rules - labels: - rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-ceph-global-rules - labels: - operator: rook - storage-backend: ceph - rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" -rules: -- apiGroups: - - "" - resources: - # Pod access is needed for fencing - - pods - # Node access is needed for determining nodes where mons should run - - nodes - - nodes/proxy - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - # PVs and PVCs are managed by the Rook provisioner - - persistentvolumes - - persistentvolumeclaims - - endpoints - verbs: - - get - - list - - watch - - patch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - ceph.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - policy - - apps - resources: - # This is for the clusterdisruption controller - - poddisruptionbudgets - # This is for both clusterdisruption and nodedrain controllers - - deployments - - replicasets - verbs: - - "*" -- apiGroups: - - healthchecking.openshift.io - resources: - - machinedisruptionbudgets - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - machine.openshift.io - resources: - - machines - verbs: - - get - - list - - watch - - create - - update - - delete -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - verbs: - - create - - delete` + return manifests }