@@ -2553,6 +2684,32 @@ int64
+
COSIDeploymentStrategy
+(string alias)
+
+(Appears on:CephCOSIDriverSpec)
+
+
+
COSIDeploymentStrategy represents the strategy to use to deploy the Ceph COSI driver
+
+
+
+
+| Value |
+Description |
+
+
+"Always" |
+Always means the Ceph COSI driver will be deployed even if the object store is not present
+ |
+
"Auto" |
+Auto means the Ceph COSI driver will be deployed automatically if object store is present
+ |
+
"Never" |
+Never means the Ceph COSI driver will never deployed
+ |
+
+
Capacity
@@ -2789,6 +2946,90 @@ int64
+
CephCOSIDriverSpec
+
+
+(Appears on:CephCOSIDriver)
+
+
+
CephCOSIDriverSpec represents the specification of a Ceph COSI Driver
+
+
+
+
+| Field |
+Description |
+
+
+
+
+
+image
+
+string
+
+ |
+
+(Optional)
+ Image is the container image to run the Ceph COSI driver
+ |
+
+
+
+objectProvisionerImage
+
+string
+
+ |
+
+(Optional)
+ ObjectProvisionerImage is the container image to run the COSI driver sidecar
+ |
+
+
+
+deploymentStrategy
+
+
+COSIDeploymentStrategy
+
+
+ |
+
+(Optional)
+ DeploymentStrategy is the strategy to use to deploy the COSI driver.
+ |
+
+
+
+placement
+
+
+Placement
+
+
+ |
+
+(Optional)
+ Placement is the placement strategy to use for the COSI driver
+ |
+
+
+
+resources
+
+
+Kubernetes core/v1.ResourceRequirements
+
+
+ |
+
+(Optional)
+ Resources is the resource requirements for the COSI driver
+ |
+
+
+
CephClientStatus
@@ -8933,7 +9174,7 @@ string
Placement
-(Appears on:FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, RBDMirroringSpec, StorageClassDeviceSet)
+(Appears on:CephCOSIDriverSpec, FilesystemMirroringSpec, GaneshaServerSpec, GatewaySpec, MetadataServerSpec, RBDMirroringSpec, StorageClassDeviceSet)
Placement is the placement for an object
diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md
new file mode 100644
index 000000000..aa2b30775
--- /dev/null
+++ b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md
@@ -0,0 +1,211 @@
+---
+title: Container Object Storage Interface (COSI)
+---
+
+
+## Ceph COSI Driver
+
+The Ceph COSI driver will be started automatically with default settings when first CephObjectStore gets created. The driver will be deleted when Rook operator is uninstalled. The driver will be deployed in the same namespace as Rook operator. The [COSI controller](https://github.com/kubernetes-sigs/container-object-storage-interface-controller#readme) must be running as a prerequisite. The COSI controller can be deployed by following the commands below:
+
+```bash
+kubectl apply -k github.com/kubernetes-sigs/container-object-storage-interface-api
+kubectl apply -k github.com/kubernetes-sigs/container-object-storage-interface-controller
+```
+
+## Admin Operations
+
+### Create a Ceph Object Store User
+
+First admin need to create CephObjectStoreUser use the following command, this is required for BucketClass and BucketAccessClass:
+
+```bash
+kubectl -n rook-ceph create -f - <
matches that of any node on which a pod of the set of pods is running
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ type: object
+ podAntiAffinity:
+ description: PodAntiAffinity is a group of inter pod anti affinity scheduling rules
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ type: object
+ tolerations:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator
+ items:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .
+ properties:
+ effect:
+ description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ type: string
+ key:
+ description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ type: string
+ operator:
+ description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
+ type: string
+ tolerationSeconds:
+ description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
+ format: int64
+ type: integer
+ value:
+ description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
+ type: string
+ type: object
+ type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraint specifies how to spread matching pods among the given topology
+ items:
+ description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ description: "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. MatchLabelKeys cannot be set when LabelSelector isn't set. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector. \n This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default)."
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ maxSkew:
+ description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It''s a required field. Default value is 1 and 0 is not allowed.'
+ format: int32
+ type: integer
+ minDomains:
+ description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. \n This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default)."
+ format: int32
+ type: integer
+ nodeAffinityPolicy:
+ description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. \n If this value is nil, the behavior is equivalent to the Honor policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+ type: string
+ nodeTaintsPolicy:
+ description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. \n If this value is nil, the behavior is equivalent to the Ignore policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+ type: string
+ topologyKey:
+ description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. It's a required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ type: object
+ resources:
+ description: Resources is the resource requirements for the COSI driver
+ properties:
+ claims:
+ description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
+ items:
+ description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+ properties:
+ name:
+ description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
+ type: string
+ required:
+ - name
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - name
+ x-kubernetes-list-type: map
+ limits:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ requests:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ type: object
+ type: object
+ required:
+ - metadata
+ - spec
+ type: object
+ served: true
+ storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.11.3
diff --git a/deploy/charts/rook-ceph/templates/serviceaccount.yaml b/deploy/charts/rook-ceph/templates/serviceaccount.yaml
index 28be9c70b..f286d7dfb 100644
--- a/deploy/charts/rook-ceph/templates/serviceaccount.yaml
+++ b/deploy/charts/rook-ceph/templates/serviceaccount.yaml
@@ -59,3 +59,15 @@ metadata:
name: rook-csi-rbd-provisioner-sa
namespace: {{ .Release.Namespace }} # namespace:operator
{{ template "library.imagePullSecrets" . }}
+---
+# Service account for Ceph COSI driver
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: objectstorage-provisioner
+ namespace: {{ .Release.Namespace }} # namespace:operator
+ labels:
+ app.kubernetes.io/part-of: container-object-storage-interface
+ app.kubernetes.io/component: driver-ceph
+ app.kubernetes.io/name: cosi-driver-ceph
+{{ template "library.imagePullSecrets" . }}
diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml
index 587cdc328..ce030b7aa 100644
--- a/deploy/examples/common.yaml
+++ b/deploy/examples/common.yaml
@@ -69,6 +69,25 @@ rules:
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: objectstorage-provisioner-role
+ labels:
+ app.kubernetes.io/part-of: container-object-storage-interface
+ app.kubernetes.io/component: driver-ceph
+ app.kubernetes.io/name: cosi-driver-ceph
+rules:
+ - apiGroups: ["objectstorage.k8s.io"]
+ resources: ["buckets", "bucketaccesses", "bucketclaims", "bucketaccessclasses", "buckets/status", "bucketaccesses/status", "bucketclaims/status", "bucketaccessclasses/status"]
+ verbs: ["get", "list", "watch", "update", "create", "delete"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "watch", "list", "delete", "update", "create"]
+ - apiGroups: [""]
+ resources: ["secrets", "events"]
+ verbs: ["get", "delete", "update", "create"]
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin
labels:
@@ -280,6 +299,7 @@ rules:
- cephfilesystemmirrors
- cephfilesystemsubvolumegroups
- cephblockpoolradosnamespaces
+ - cephcosidrivers
verbs:
- get
- list
@@ -581,6 +601,24 @@ roleRef:
name: cephfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
+# RBAC for ceph cosi driver service account
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: objectstorage-provisioner-role-binding
+ labels:
+ app.kubernetes.io/part-of: container-object-storage-interface
+ app.kubernetes.io/component: driver-ceph
+ app.kubernetes.io/name: cosi-driver-ceph
+subjects:
+ - kind: ServiceAccount
+ name: objectstorage-provisioner
+ namespace: rook-ceph # namespace:operator
+roleRef:
+ kind: ClusterRole
+ name: objectstorage-provisioner-role
+ apiGroup: rbac.authorization.k8s.io
+---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
@@ -786,6 +824,7 @@ rules:
- cephfilesystemmirrors
- cephfilesystemsubvolumegroups
- cephblockpoolradosnamespaces
+ - cephcosidrivers
verbs:
- get
- list
@@ -1091,6 +1130,19 @@ subjects:
name: rook-ceph-system
namespace: rook-ceph # namespace:operator
---
+# Service account for Ceph COSI driver
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: objectstorage-provisioner
+ namespace: rook-ceph # namespace:operator
+ labels:
+ app.kubernetes.io/part-of: container-object-storage-interface
+ app.kubernetes.io/component: driver-ceph
+ app.kubernetes.io/name: cosi-driver-ceph
+# imagePullSecrets:
+# - name: my-registry-secret
+---
# Service account for the job that reports the Ceph version in an image
apiVersion: v1
kind: ServiceAccount
diff --git a/deploy/examples/cosi/bucketaccess.yaml b/deploy/examples/cosi/bucketaccess.yaml
new file mode 100644
index 000000000..5b835cc26
--- /dev/null
+++ b/deploy/examples/cosi/bucketaccess.yaml
@@ -0,0 +1,10 @@
+kind: BucketAccess
+apiVersion: objectstorage.k8s.io/v1alpha1
+metadata:
+ name: sample-access
+ namespace: default # any namespace
+spec:
+ bucketClaimName: sample-bucket
+ bucketAccessClassName: sample-bac
+ credentialsSecretName: sample-access-secret
+ protocol: s3
diff --git a/deploy/examples/cosi/bucketaccessclass.yaml b/deploy/examples/cosi/bucketaccessclass.yaml
new file mode 100644
index 000000000..e01171666
--- /dev/null
+++ b/deploy/examples/cosi/bucketaccessclass.yaml
@@ -0,0 +1,9 @@
+kind: BucketAccessClass
+apiVersion: objectstorage.k8s.io/v1alpha1
+metadata:
+ name: sample-bac
+driverName: ceph.objectstorage.k8s.io
+authenticationType: KEY
+parameters:
+ objectStoreUserSecretName: rook-ceph-object-user-my-store-cosi
+ objectStoreUserSecretNamespace: rook-ceph
diff --git a/deploy/examples/cosi/bucketclaim.yaml b/deploy/examples/cosi/bucketclaim.yaml
new file mode 100644
index 000000000..4d6b7406f
--- /dev/null
+++ b/deploy/examples/cosi/bucketclaim.yaml
@@ -0,0 +1,9 @@
+kind: BucketClaim
+apiVersion: objectstorage.k8s.io/v1alpha1
+metadata:
+ name: sample-bucket
+ namespace: default # any namespace
+spec:
+ bucketClassName: sample-bcc
+ protocols:
+ - s3
diff --git a/deploy/examples/cosi/bucketclass.yaml b/deploy/examples/cosi/bucketclass.yaml
new file mode 100644
index 000000000..52f0263da
--- /dev/null
+++ b/deploy/examples/cosi/bucketclass.yaml
@@ -0,0 +1,9 @@
+kind: BucketClass
+apiVersion: objectstorage.k8s.io/v1alpha1
+metadata:
+ name: sample-bcc
+driverName: ceph.objectstorage.k8s.io
+deletionPolicy: Delete
+parameters:
+ objectStoreUserSecretName: rook-ceph-object-user-my-store-cosi
+ objectStoreUserSecretNamespace: rook-ceph
diff --git a/deploy/examples/cosi/cephcosidriver.yaml b/deploy/examples/cosi/cephcosidriver.yaml
new file mode 100644
index 000000000..85eef8fbd
--- /dev/null
+++ b/deploy/examples/cosi/cephcosidriver.yaml
@@ -0,0 +1,7 @@
+apiVersion: ceph.rook.io/v1
+kind: CephCOSIDriver
+metadata:
+ name: ceph-cosi-driver
+ namespace: rook-ceph
+spec:
+ deploymentStrategy: "auto"
diff --git a/deploy/examples/cosi/cosi-user.yaml b/deploy/examples/cosi/cosi-user.yaml
new file mode 100644
index 000000000..9574d7aea
--- /dev/null
+++ b/deploy/examples/cosi/cosi-user.yaml
@@ -0,0 +1,16 @@
+#################################################################################################################
+# Create an object store user for access to the ceph cosi driver.
+# kubectl create -f cosi-user.yaml
+#################################################################################################################
+
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+ name: cosi
+ namespace: rook-ceph # namespace:cluster
+spec:
+ store: my-store
+ displayName: "user for cosi"
+ capabilities:
+ user: "*"
+ bucket: "*"
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 2fbca1617..d25a739b8 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -4743,6 +4743,661 @@ spec:
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.11.3
+ creationTimestamp: null
+ name: cephcosidrivers.ceph.rook.io
+spec:
+ group: ceph.rook.io
+ names:
+ kind: CephCOSIDriver
+ listKind: CephCOSIDriverList
+ plural: cephcosidrivers
+ shortNames:
+ - cephcosi
+ singular: cephcosidriver
+ scope: Namespaced
+ versions:
+ - name: v1
+ schema:
+ openAPIV3Schema:
+ description: CephCOSIDriver represents the CRD for the Ceph COSI Driver Deployment
+ properties:
+ apiVersion:
+ description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+ type: string
+ kind:
+ description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: Spec represents the specification of a Ceph COSI Driver
+ properties:
+ deploymentStrategy:
+ description: DeploymentStrategy is the strategy to use to deploy the COSI driver.
+ enum:
+ - Never
+ - Auto
+ - Always
+ type: string
+ image:
+ description: Image is the container image to run the Ceph COSI driver
+ type: string
+ objectProvisionerImage:
+ description: ObjectProvisionerImage is the container image to run the COSI driver sidecar
+ type: string
+ placement:
+ description: Placement is the placement strategy to use for the COSI driver
+ properties:
+ nodeAffinity:
+ description: NodeAffinity is a group of node affinity scheduling rules
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.
+ items:
+ description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
+ properties:
+ preference:
+ description: A node selector term, associated with the corresponding weight.
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ type: object
+ x-kubernetes-map-type: atomic
+ weight:
+ description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
+ format: int32
+ type: integer
+ required:
+ - preference
+ - weight
+ type: object
+ type: array
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.
+ properties:
+ nodeSelectorTerms:
+ description: Required. A list of node selector terms. The terms are ORed.
+ items:
+ description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
+ properties:
+ matchExpressions:
+ description: A list of node selector requirements by node's labels.
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchFields:
+ description: A list of node selector requirements by node's fields.
+ items:
+ description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: The label key that the selector applies to.
+ type: string
+ operator:
+ description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
+ type: string
+ values:
+ description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ type: object
+ x-kubernetes-map-type: atomic
+ type: array
+ required:
+ - nodeSelectorTerms
+ type: object
+ x-kubernetes-map-type: atomic
+ type: object
+ podAffinity:
+ description: PodAffinity is a group of inter pod affinity scheduling rules
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ type: object
+ podAntiAffinity:
+ description: PodAntiAffinity is a group of inter pod anti affinity scheduling rules
+ properties:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.
+ items:
+ description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
+ properties:
+ podAffinityTerm:
+ description: Required. A pod affinity term, associated with the corresponding weight.
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ weight:
+ description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100.
+ format: int32
+ type: integer
+ required:
+ - podAffinityTerm
+ - weight
+ type: object
+ type: array
+ requiredDuringSchedulingIgnoredDuringExecution:
+ description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.
+ items:
+ description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running
+ properties:
+ labelSelector:
+ description: A label query over a set of resources, in this case pods.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaceSelector:
+ description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ namespaces:
+ description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace".
+ items:
+ type: string
+ type: array
+ topologyKey:
+ description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.
+ type: string
+ required:
+ - topologyKey
+ type: object
+ type: array
+ type: object
+ tolerations:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator
+ items:
+ description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator .
+ properties:
+ effect:
+ description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+ type: string
+ key:
+ description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+ type: string
+ operator:
+ description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.
+ type: string
+ tolerationSeconds:
+ description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.
+ format: int64
+ type: integer
+ value:
+ description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.
+ type: string
+ type: object
+ type: array
+ topologySpreadConstraints:
+ description: TopologySpreadConstraint specifies how to spread matching pods among the given topology
+ items:
+ description: TopologySpreadConstraint specifies how to spread matching pods among the given topology.
+ properties:
+ labelSelector:
+ description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.
+ properties:
+ matchExpressions:
+ description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+ items:
+ description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+ properties:
+ key:
+ description: key is the label key that the selector applies to.
+ type: string
+ operator:
+ description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+ type: string
+ values:
+ description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+ items:
+ type: string
+ type: array
+ required:
+ - key
+ - operator
+ type: object
+ type: array
+ matchLabels:
+ additionalProperties:
+ type: string
+ description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+ type: object
+ type: object
+ x-kubernetes-map-type: atomic
+ matchLabelKeys:
+ description: "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. MatchLabelKeys cannot be set when LabelSelector isn't set. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector. \n This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default)."
+ items:
+ type: string
+ type: array
+ x-kubernetes-list-type: atomic
+ maxSkew:
+ description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It''s a required field. Default value is 1 and 0 is not allowed.'
+ format: int32
+ type: integer
+ minDomains:
+ description: "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. \n For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew. \n This is a beta field and requires the MinDomainsInPodTopologySpread feature gate to be enabled (enabled by default)."
+ format: int32
+ type: integer
+ nodeAffinityPolicy:
+ description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. \n If this value is nil, the behavior is equivalent to the Honor policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+ type: string
+ nodeTaintsPolicy:
+ description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. \n If this value is nil, the behavior is equivalent to the Ignore policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag."
+ type: string
+ topologyKey:
+ description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. It's a required field.
+ type: string
+ whenUnsatisfiable:
+ description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.'
+ type: string
+ required:
+ - maxSkew
+ - topologyKey
+ - whenUnsatisfiable
+ type: object
+ type: array
+ type: object
+ resources:
+ description: Resources is the resource requirements for the COSI driver
+ properties:
+ claims:
+ description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers."
+ items:
+ description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+ properties:
+ name:
+ description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.
+ type: string
+ required:
+ - name
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - name
+ x-kubernetes-list-type: map
+ limits:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ requests:
+ additionalProperties:
+ anyOf:
+ - type: integer
+ - type: string
+ pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+ x-kubernetes-int-or-string: true
+ description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+ type: object
+ type: object
+ type: object
+ required:
+ - metadata
+ - spec
+ type: object
+ served: true
+ storage: true
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.11.3
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index 27f0f605f..06e02e44e 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -1,4 +1,6 @@
+ gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995
quay.io/ceph/ceph:v17.2.6
+ quay.io/ceph/cosi:v0.1.0
quay.io/cephcsi/cephcsi:v3.9.0
quay.io/csiaddons/k8s-sidecar:v0.7.0
registry.k8s.io/sig-storage/csi-attacher:v4.3.0
diff --git a/images/ceph/Makefile b/images/ceph/Makefile
index 85450cd75..38fcc8ca9 100755
--- a/images/ceph/Makefile
+++ b/images/ceph/Makefile
@@ -135,6 +135,8 @@ list-image: ## Create a list of images for offline installation
cut -d: -f2- | tee $(IMAGE_TMP)
awk '/quay.io/ || /registry.k8s.io/ {print $3}' ../../pkg/operator/ceph/csi/spec.go | \
cut -d= -f2- | tr -d '"' | tee -a $(IMAGE_TMP)
+ awk '/quay.io/ || /gcr.io/ {print $4}' ../../pkg/operator/ceph/object/cosi/spec.go | \
+ cut -d= -f2- | tr -d '"' | tee -a $(IMAGE_TMP)
rm -f $(MANIFESTS_DIR)/images.txt
cat $(IMAGE_TMP) | sort -h | uniq | tee $(MANIFESTS_DIR)/images.txt
rm -f $(IMAGE_TMP)
diff --git a/pkg/apis/ceph.rook.io/v1/register.go b/pkg/apis/ceph.rook.io/v1/register.go
index 928fd1aff..1eebf7949 100644
--- a/pkg/apis/ceph.rook.io/v1/register.go
+++ b/pkg/apis/ceph.rook.io/v1/register.go
@@ -87,6 +87,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
&CephFilesystemSubVolumeGroupList{},
&CephBlockPoolRadosNamespace{},
&CephBlockPoolRadosNamespaceList{},
+ &CephCOSIDriver{},
+ &CephCOSIDriverList{},
)
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
scheme.AddKnownTypes(bktv1alpha1.SchemeGroupVersion,
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index f1b107357..6a4f576e8 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -2824,3 +2824,56 @@ type ConfigFileVolumeSource struct {
// projected items for all in one resources secrets, configmaps, and downward API
Projected *v1.ProjectedVolumeSource `json:"projected,omitempty" protobuf:"bytes,26,opt,name=projected"`
}
+
+// +genclient
+// +genclient:noStatus
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// CephCOSIDriver represents the CRD for the Ceph COSI Driver Deployment
+// +kubebuilder:resource:shortName=cephcosi
+type CephCOSIDriver struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ObjectMeta `json:"metadata"`
+ // Spec represents the specification of a Ceph COSI Driver
+ Spec CephCOSIDriverSpec `json:"spec"`
+}
+
+// CephCOSIDriverList represents a list of Ceph COSI Driver
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type CephCOSIDriverList struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ListMeta `json:"metadata"`
+ Items []CephCOSIDriver `json:"items"`
+}
+
+// CephCOSIDriverSpec represents the specification of a Ceph COSI Driver
+type CephCOSIDriverSpec struct {
+ // Image is the container image to run the Ceph COSI driver
+ // +optional
+ Image string `json:"image,omitempty"`
+ // ObjectProvisionerImage is the container image to run the COSI driver sidecar
+ // +optional
+ ObjectProvisionerImage string `json:"objectProvisionerImage,omitempty"`
+ // DeploymentStrategy is the strategy to use to deploy the COSI driver.
+ // +optional
+ // +kubebuilder:validation:Enum=Never;Auto;Always
+ DeploymentStrategy COSIDeploymentStrategy `json:"deploymentStrategy,omitempty"`
+ // Placement is the placement strategy to use for the COSI driver
+ // +optional
+ Placement Placement `json:"placement,omitempty"`
+ // Resources is the resource requirements for the COSI driver
+ // +optional
+ Resources v1.ResourceRequirements `json:"resources,omitempty"`
+}
+
+// COSIDeploymentStrategy represents the strategy to use to deploy the Ceph COSI driver
+type COSIDeploymentStrategy string
+
+const (
+ // Never means the Ceph COSI driver will never deployed
+ COSIDeploymentStrategyNever COSIDeploymentStrategy = "Never"
+ // Auto means the Ceph COSI driver will be deployed automatically if object store is present
+ COSIDeploymentStrategyAuto COSIDeploymentStrategy = "Auto"
+ // Always means the Ceph COSI driver will be deployed even if the object store is not present
+ COSIDeploymentStrategyAlways COSIDeploymentStrategy = "Always"
+)
diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
index ae7787076..042e278d9 100644
--- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
@@ -521,6 +521,84 @@ func (in *CephBucketTopicList) DeepCopyObject() runtime.Object {
return nil
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CephCOSIDriver) DeepCopyInto(out *CephCOSIDriver) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+ in.Spec.DeepCopyInto(&out.Spec)
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephCOSIDriver.
+func (in *CephCOSIDriver) DeepCopy() *CephCOSIDriver {
+ if in == nil {
+ return nil
+ }
+ out := new(CephCOSIDriver)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CephCOSIDriver) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CephCOSIDriverList) DeepCopyInto(out *CephCOSIDriverList) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ListMeta.DeepCopyInto(&out.ListMeta)
+ if in.Items != nil {
+ in, out := &in.Items, &out.Items
+ *out = make([]CephCOSIDriver, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephCOSIDriverList.
+func (in *CephCOSIDriverList) DeepCopy() *CephCOSIDriverList {
+ if in == nil {
+ return nil
+ }
+ out := new(CephCOSIDriverList)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *CephCOSIDriverList) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CephCOSIDriverSpec) DeepCopyInto(out *CephCOSIDriverSpec) {
+ *out = *in
+ in.Placement.DeepCopyInto(&out.Placement)
+ in.Resources.DeepCopyInto(&out.Resources)
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CephCOSIDriverSpec.
+func (in *CephCOSIDriverSpec) DeepCopy() *CephCOSIDriverSpec {
+ if in == nil {
+ return nil
+ }
+ out := new(CephCOSIDriverSpec)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CephClient) DeepCopyInto(out *CephClient) {
*out = *in
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go
index 394dc08e1..59f25fd0d 100644
--- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/ceph.rook.io_client.go
@@ -30,6 +30,7 @@ type CephV1Interface interface {
CephBlockPoolRadosNamespacesGetter
CephBucketNotificationsGetter
CephBucketTopicsGetter
+ CephCOSIDriversGetter
CephClientsGetter
CephClustersGetter
CephFilesystemsGetter
@@ -65,6 +66,10 @@ func (c *CephV1Client) CephBucketTopics(namespace string) CephBucketTopicInterfa
return newCephBucketTopics(c, namespace)
}
+func (c *CephV1Client) CephCOSIDrivers(namespace string) CephCOSIDriverInterface {
+ return newCephCOSIDrivers(c, namespace)
+}
+
func (c *CephV1Client) CephClients(namespace string) CephClientInterface {
return newCephClients(c, namespace)
}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephcosidriver.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephcosidriver.go
new file mode 100644
index 000000000..7e084f521
--- /dev/null
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/cephcosidriver.go
@@ -0,0 +1,178 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ "context"
+ "time"
+
+ v1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ scheme "github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ types "k8s.io/apimachinery/pkg/types"
+ watch "k8s.io/apimachinery/pkg/watch"
+ rest "k8s.io/client-go/rest"
+)
+
+// CephCOSIDriversGetter has a method to return a CephCOSIDriverInterface.
+// A group's client should implement this interface.
+type CephCOSIDriversGetter interface {
+ CephCOSIDrivers(namespace string) CephCOSIDriverInterface
+}
+
+// CephCOSIDriverInterface has methods to work with CephCOSIDriver resources.
+type CephCOSIDriverInterface interface {
+ Create(ctx context.Context, cephCOSIDriver *v1.CephCOSIDriver, opts metav1.CreateOptions) (*v1.CephCOSIDriver, error)
+ Update(ctx context.Context, cephCOSIDriver *v1.CephCOSIDriver, opts metav1.UpdateOptions) (*v1.CephCOSIDriver, error)
+ Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
+ DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
+ Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.CephCOSIDriver, error)
+ List(ctx context.Context, opts metav1.ListOptions) (*v1.CephCOSIDriverList, error)
+ Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
+ Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.CephCOSIDriver, err error)
+ CephCOSIDriverExpansion
+}
+
+// cephCOSIDrivers implements CephCOSIDriverInterface
+type cephCOSIDrivers struct {
+ client rest.Interface
+ ns string
+}
+
+// newCephCOSIDrivers returns a CephCOSIDrivers
+func newCephCOSIDrivers(c *CephV1Client, namespace string) *cephCOSIDrivers {
+ return &cephCOSIDrivers{
+ client: c.RESTClient(),
+ ns: namespace,
+ }
+}
+
+// Get takes name of the cephCOSIDriver, and returns the corresponding cephCOSIDriver object, and an error if there is any.
+func (c *cephCOSIDrivers) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.CephCOSIDriver, err error) {
+ result = &v1.CephCOSIDriver{}
+ err = c.client.Get().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ Name(name).
+ VersionedParams(&options, scheme.ParameterCodec).
+ Do(ctx).
+ Into(result)
+ return
+}
+
+// List takes label and field selectors, and returns the list of CephCOSIDrivers that match those selectors.
+func (c *cephCOSIDrivers) List(ctx context.Context, opts metav1.ListOptions) (result *v1.CephCOSIDriverList, err error) {
+ var timeout time.Duration
+ if opts.TimeoutSeconds != nil {
+ timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+ }
+ result = &v1.CephCOSIDriverList{}
+ err = c.client.Get().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ VersionedParams(&opts, scheme.ParameterCodec).
+ Timeout(timeout).
+ Do(ctx).
+ Into(result)
+ return
+}
+
+// Watch returns a watch.Interface that watches the requested cephCOSIDrivers.
+func (c *cephCOSIDrivers) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
+ var timeout time.Duration
+ if opts.TimeoutSeconds != nil {
+ timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+ }
+ opts.Watch = true
+ return c.client.Get().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ VersionedParams(&opts, scheme.ParameterCodec).
+ Timeout(timeout).
+ Watch(ctx)
+}
+
+// Create takes the representation of a cephCOSIDriver and creates it. Returns the server's representation of the cephCOSIDriver, and an error, if there is any.
+func (c *cephCOSIDrivers) Create(ctx context.Context, cephCOSIDriver *v1.CephCOSIDriver, opts metav1.CreateOptions) (result *v1.CephCOSIDriver, err error) {
+ result = &v1.CephCOSIDriver{}
+ err = c.client.Post().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ VersionedParams(&opts, scheme.ParameterCodec).
+ Body(cephCOSIDriver).
+ Do(ctx).
+ Into(result)
+ return
+}
+
+// Update takes the representation of a cephCOSIDriver and updates it. Returns the server's representation of the cephCOSIDriver, and an error, if there is any.
+func (c *cephCOSIDrivers) Update(ctx context.Context, cephCOSIDriver *v1.CephCOSIDriver, opts metav1.UpdateOptions) (result *v1.CephCOSIDriver, err error) {
+ result = &v1.CephCOSIDriver{}
+ err = c.client.Put().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ Name(cephCOSIDriver.Name).
+ VersionedParams(&opts, scheme.ParameterCodec).
+ Body(cephCOSIDriver).
+ Do(ctx).
+ Into(result)
+ return
+}
+
+// Delete takes name of the cephCOSIDriver and deletes it. Returns an error if one occurs.
+func (c *cephCOSIDrivers) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
+ return c.client.Delete().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ Name(name).
+ Body(&opts).
+ Do(ctx).
+ Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *cephCOSIDrivers) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
+ var timeout time.Duration
+ if listOpts.TimeoutSeconds != nil {
+ timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
+ }
+ return c.client.Delete().
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ VersionedParams(&listOpts, scheme.ParameterCodec).
+ Timeout(timeout).
+ Body(&opts).
+ Do(ctx).
+ Error()
+}
+
+// Patch applies the patch and returns the patched cephCOSIDriver.
+func (c *cephCOSIDrivers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.CephCOSIDriver, err error) {
+ result = &v1.CephCOSIDriver{}
+ err = c.client.Patch(pt).
+ Namespace(c.ns).
+ Resource("cephcosidrivers").
+ Name(name).
+ SubResource(subresources...).
+ VersionedParams(&opts, scheme.ParameterCodec).
+ Body(data).
+ Do(ctx).
+ Into(result)
+ return
+}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go
index c92c4e768..cb7ee4075 100644
--- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_ceph.rook.io_client.go
@@ -44,6 +44,10 @@ func (c *FakeCephV1) CephBucketTopics(namespace string) v1.CephBucketTopicInterf
return &FakeCephBucketTopics{c, namespace}
}
+func (c *FakeCephV1) CephCOSIDrivers(namespace string) v1.CephCOSIDriverInterface {
+ return &FakeCephCOSIDrivers{c, namespace}
+}
+
func (c *FakeCephV1) CephClients(namespace string) v1.CephClientInterface {
return &FakeCephClients{c, namespace}
}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephcosidriver.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephcosidriver.go
new file mode 100644
index 000000000..0a5c7490b
--- /dev/null
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/fake/fake_cephcosidriver.go
@@ -0,0 +1,130 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+ "context"
+
+ cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ labels "k8s.io/apimachinery/pkg/labels"
+ schema "k8s.io/apimachinery/pkg/runtime/schema"
+ types "k8s.io/apimachinery/pkg/types"
+ watch "k8s.io/apimachinery/pkg/watch"
+ testing "k8s.io/client-go/testing"
+)
+
+// FakeCephCOSIDrivers implements CephCOSIDriverInterface
+type FakeCephCOSIDrivers struct {
+ Fake *FakeCephV1
+ ns string
+}
+
+var cephcosidriversResource = schema.GroupVersionResource{Group: "ceph.rook.io", Version: "v1", Resource: "cephcosidrivers"}
+
+var cephcosidriversKind = schema.GroupVersionKind{Group: "ceph.rook.io", Version: "v1", Kind: "CephCOSIDriver"}
+
+// Get takes name of the cephCOSIDriver, and returns the corresponding cephCOSIDriver object, and an error if there is any.
+func (c *FakeCephCOSIDrivers) Get(ctx context.Context, name string, options v1.GetOptions) (result *cephrookiov1.CephCOSIDriver, err error) {
+ obj, err := c.Fake.
+ Invokes(testing.NewGetAction(cephcosidriversResource, c.ns, name), &cephrookiov1.CephCOSIDriver{})
+
+ if obj == nil {
+ return nil, err
+ }
+ return obj.(*cephrookiov1.CephCOSIDriver), err
+}
+
+// List takes label and field selectors, and returns the list of CephCOSIDrivers that match those selectors.
+func (c *FakeCephCOSIDrivers) List(ctx context.Context, opts v1.ListOptions) (result *cephrookiov1.CephCOSIDriverList, err error) {
+ obj, err := c.Fake.
+ Invokes(testing.NewListAction(cephcosidriversResource, cephcosidriversKind, c.ns, opts), &cephrookiov1.CephCOSIDriverList{})
+
+ if obj == nil {
+ return nil, err
+ }
+
+ label, _, _ := testing.ExtractFromListOptions(opts)
+ if label == nil {
+ label = labels.Everything()
+ }
+ list := &cephrookiov1.CephCOSIDriverList{ListMeta: obj.(*cephrookiov1.CephCOSIDriverList).ListMeta}
+ for _, item := range obj.(*cephrookiov1.CephCOSIDriverList).Items {
+ if label.Matches(labels.Set(item.Labels)) {
+ list.Items = append(list.Items, item)
+ }
+ }
+ return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested cephCOSIDrivers.
+func (c *FakeCephCOSIDrivers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+ return c.Fake.
+ InvokesWatch(testing.NewWatchAction(cephcosidriversResource, c.ns, opts))
+
+}
+
+// Create takes the representation of a cephCOSIDriver and creates it. Returns the server's representation of the cephCOSIDriver, and an error, if there is any.
+func (c *FakeCephCOSIDrivers) Create(ctx context.Context, cephCOSIDriver *cephrookiov1.CephCOSIDriver, opts v1.CreateOptions) (result *cephrookiov1.CephCOSIDriver, err error) {
+ obj, err := c.Fake.
+ Invokes(testing.NewCreateAction(cephcosidriversResource, c.ns, cephCOSIDriver), &cephrookiov1.CephCOSIDriver{})
+
+ if obj == nil {
+ return nil, err
+ }
+ return obj.(*cephrookiov1.CephCOSIDriver), err
+}
+
+// Update takes the representation of a cephCOSIDriver and updates it. Returns the server's representation of the cephCOSIDriver, and an error, if there is any.
+func (c *FakeCephCOSIDrivers) Update(ctx context.Context, cephCOSIDriver *cephrookiov1.CephCOSIDriver, opts v1.UpdateOptions) (result *cephrookiov1.CephCOSIDriver, err error) {
+ obj, err := c.Fake.
+ Invokes(testing.NewUpdateAction(cephcosidriversResource, c.ns, cephCOSIDriver), &cephrookiov1.CephCOSIDriver{})
+
+ if obj == nil {
+ return nil, err
+ }
+ return obj.(*cephrookiov1.CephCOSIDriver), err
+}
+
+// Delete takes name of the cephCOSIDriver and deletes it. Returns an error if one occurs.
+func (c *FakeCephCOSIDrivers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+ _, err := c.Fake.
+ Invokes(testing.NewDeleteAction(cephcosidriversResource, c.ns, name), &cephrookiov1.CephCOSIDriver{})
+
+ return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeCephCOSIDrivers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+ action := testing.NewDeleteCollectionAction(cephcosidriversResource, c.ns, listOpts)
+
+ _, err := c.Fake.Invokes(action, &cephrookiov1.CephCOSIDriverList{})
+ return err
+}
+
+// Patch applies the patch and returns the patched cephCOSIDriver.
+func (c *FakeCephCOSIDrivers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *cephrookiov1.CephCOSIDriver, err error) {
+ obj, err := c.Fake.
+ Invokes(testing.NewPatchSubresourceAction(cephcosidriversResource, c.ns, name, pt, data, subresources...), &cephrookiov1.CephCOSIDriver{})
+
+ if obj == nil {
+ return nil, err
+ }
+ return obj.(*cephrookiov1.CephCOSIDriver), err
+}
diff --git a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go
index bfb06dbcd..ca470183b 100644
--- a/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go
+++ b/pkg/client/clientset/versioned/typed/ceph.rook.io/v1/generated_expansion.go
@@ -26,6 +26,8 @@ type CephBucketNotificationExpansion interface{}
type CephBucketTopicExpansion interface{}
+type CephCOSIDriverExpansion interface{}
+
type CephClientExpansion interface{}
type CephClusterExpansion interface{}
diff --git a/pkg/client/informers/externalversions/ceph.rook.io/v1/cephcosidriver.go b/pkg/client/informers/externalversions/ceph.rook.io/v1/cephcosidriver.go
new file mode 100644
index 000000000..35ce75ff6
--- /dev/null
+++ b/pkg/client/informers/externalversions/ceph.rook.io/v1/cephcosidriver.go
@@ -0,0 +1,90 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ "context"
+ time "time"
+
+ cephrookiov1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ versioned "github.com/rook/rook/pkg/client/clientset/versioned"
+ internalinterfaces "github.com/rook/rook/pkg/client/informers/externalversions/internalinterfaces"
+ v1 "github.com/rook/rook/pkg/client/listers/ceph.rook.io/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ runtime "k8s.io/apimachinery/pkg/runtime"
+ watch "k8s.io/apimachinery/pkg/watch"
+ cache "k8s.io/client-go/tools/cache"
+)
+
+// CephCOSIDriverInformer provides access to a shared informer and lister for
+// CephCOSIDrivers.
+type CephCOSIDriverInformer interface {
+ Informer() cache.SharedIndexInformer
+ Lister() v1.CephCOSIDriverLister
+}
+
+type cephCOSIDriverInformer struct {
+ factory internalinterfaces.SharedInformerFactory
+ tweakListOptions internalinterfaces.TweakListOptionsFunc
+ namespace string
+}
+
+// NewCephCOSIDriverInformer constructs a new informer for CephCOSIDriver type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewCephCOSIDriverInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+ return NewFilteredCephCOSIDriverInformer(client, namespace, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredCephCOSIDriverInformer constructs a new informer for CephCOSIDriver type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredCephCOSIDriverInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+ return cache.NewSharedIndexInformer(
+ &cache.ListWatch{
+ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+ if tweakListOptions != nil {
+ tweakListOptions(&options)
+ }
+ return client.CephV1().CephCOSIDrivers(namespace).List(context.TODO(), options)
+ },
+ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+ if tweakListOptions != nil {
+ tweakListOptions(&options)
+ }
+ return client.CephV1().CephCOSIDrivers(namespace).Watch(context.TODO(), options)
+ },
+ },
+ &cephrookiov1.CephCOSIDriver{},
+ resyncPeriod,
+ indexers,
+ )
+}
+
+func (f *cephCOSIDriverInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+ return NewFilteredCephCOSIDriverInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *cephCOSIDriverInformer) Informer() cache.SharedIndexInformer {
+ return f.factory.InformerFor(&cephrookiov1.CephCOSIDriver{}, f.defaultInformer)
+}
+
+func (f *cephCOSIDriverInformer) Lister() v1.CephCOSIDriverLister {
+ return v1.NewCephCOSIDriverLister(f.Informer().GetIndexer())
+}
diff --git a/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go b/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go
index c63b242b4..922853644 100644
--- a/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go
+++ b/pkg/client/informers/externalversions/ceph.rook.io/v1/interface.go
@@ -32,6 +32,8 @@ type Interface interface {
CephBucketNotifications() CephBucketNotificationInformer
// CephBucketTopics returns a CephBucketTopicInformer.
CephBucketTopics() CephBucketTopicInformer
+ // CephCOSIDrivers returns a CephCOSIDriverInformer.
+ CephCOSIDrivers() CephCOSIDriverInformer
// CephClients returns a CephClientInformer.
CephClients() CephClientInformer
// CephClusters returns a CephClusterInformer.
@@ -89,6 +91,11 @@ func (v *version) CephBucketTopics() CephBucketTopicInformer {
return &cephBucketTopicInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
}
+// CephCOSIDrivers returns a CephCOSIDriverInformer.
+func (v *version) CephCOSIDrivers() CephCOSIDriverInformer {
+ return &cephCOSIDriverInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
+}
+
// CephClients returns a CephClientInformer.
func (v *version) CephClients() CephClientInformer {
return &cephClientInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go
index 9a0e6bc28..1fcf03c8f 100644
--- a/pkg/client/informers/externalversions/generic.go
+++ b/pkg/client/informers/externalversions/generic.go
@@ -61,6 +61,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephBucketNotifications().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("cephbuckettopics"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephBucketTopics().Informer()}, nil
+ case v1.SchemeGroupVersion.WithResource("cephcosidrivers"):
+ return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephCOSIDrivers().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("cephclients"):
return &genericInformer{resource: resource.GroupResource(), informer: f.Ceph().V1().CephClients().Informer()}, nil
case v1.SchemeGroupVersion.WithResource("cephclusters"):
diff --git a/pkg/client/listers/ceph.rook.io/v1/cephcosidriver.go b/pkg/client/listers/ceph.rook.io/v1/cephcosidriver.go
new file mode 100644
index 000000000..fff30dd0a
--- /dev/null
+++ b/pkg/client/listers/ceph.rook.io/v1/cephcosidriver.go
@@ -0,0 +1,99 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ v1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/client-go/tools/cache"
+)
+
+// CephCOSIDriverLister helps list CephCOSIDrivers.
+// All objects returned here must be treated as read-only.
+type CephCOSIDriverLister interface {
+ // List lists all CephCOSIDrivers in the indexer.
+ // Objects returned here must be treated as read-only.
+ List(selector labels.Selector) (ret []*v1.CephCOSIDriver, err error)
+ // CephCOSIDrivers returns an object that can list and get CephCOSIDrivers.
+ CephCOSIDrivers(namespace string) CephCOSIDriverNamespaceLister
+ CephCOSIDriverListerExpansion
+}
+
+// cephCOSIDriverLister implements the CephCOSIDriverLister interface.
+type cephCOSIDriverLister struct {
+ indexer cache.Indexer
+}
+
+// NewCephCOSIDriverLister returns a new CephCOSIDriverLister.
+func NewCephCOSIDriverLister(indexer cache.Indexer) CephCOSIDriverLister {
+ return &cephCOSIDriverLister{indexer: indexer}
+}
+
+// List lists all CephCOSIDrivers in the indexer.
+func (s *cephCOSIDriverLister) List(selector labels.Selector) (ret []*v1.CephCOSIDriver, err error) {
+ err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+ ret = append(ret, m.(*v1.CephCOSIDriver))
+ })
+ return ret, err
+}
+
+// CephCOSIDrivers returns an object that can list and get CephCOSIDrivers.
+func (s *cephCOSIDriverLister) CephCOSIDrivers(namespace string) CephCOSIDriverNamespaceLister {
+ return cephCOSIDriverNamespaceLister{indexer: s.indexer, namespace: namespace}
+}
+
+// CephCOSIDriverNamespaceLister helps list and get CephCOSIDrivers.
+// All objects returned here must be treated as read-only.
+type CephCOSIDriverNamespaceLister interface {
+ // List lists all CephCOSIDrivers in the indexer for a given namespace.
+ // Objects returned here must be treated as read-only.
+ List(selector labels.Selector) (ret []*v1.CephCOSIDriver, err error)
+ // Get retrieves the CephCOSIDriver from the indexer for a given namespace and name.
+ // Objects returned here must be treated as read-only.
+ Get(name string) (*v1.CephCOSIDriver, error)
+ CephCOSIDriverNamespaceListerExpansion
+}
+
+// cephCOSIDriverNamespaceLister implements the CephCOSIDriverNamespaceLister
+// interface.
+type cephCOSIDriverNamespaceLister struct {
+ indexer cache.Indexer
+ namespace string
+}
+
+// List lists all CephCOSIDrivers in the indexer for a given namespace.
+func (s cephCOSIDriverNamespaceLister) List(selector labels.Selector) (ret []*v1.CephCOSIDriver, err error) {
+ err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
+ ret = append(ret, m.(*v1.CephCOSIDriver))
+ })
+ return ret, err
+}
+
+// Get retrieves the CephCOSIDriver from the indexer for a given namespace and name.
+func (s cephCOSIDriverNamespaceLister) Get(name string) (*v1.CephCOSIDriver, error) {
+ obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
+ if err != nil {
+ return nil, err
+ }
+ if !exists {
+ return nil, errors.NewNotFound(v1.Resource("cephcosidriver"), name)
+ }
+ return obj.(*v1.CephCOSIDriver), nil
+}
diff --git a/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go b/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go
index ebeeb58a8..e675b9167 100644
--- a/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go
+++ b/pkg/client/listers/ceph.rook.io/v1/expansion_generated.go
@@ -50,6 +50,14 @@ type CephBucketTopicListerExpansion interface{}
// CephBucketTopicNamespaceLister.
type CephBucketTopicNamespaceListerExpansion interface{}
+// CephCOSIDriverListerExpansion allows custom methods to be added to
+// CephCOSIDriverLister.
+type CephCOSIDriverListerExpansion interface{}
+
+// CephCOSIDriverNamespaceListerExpansion allows custom methods to be added to
+// CephCOSIDriverNamespaceLister.
+type CephCOSIDriverNamespaceListerExpansion interface{}
+
// CephClientListerExpansion allows custom methods to be added to
// CephClientLister.
type CephClientListerExpansion interface{}
diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go
index ec02e1aa5..5ed9acfda 100644
--- a/pkg/operator/ceph/cr_manager.go
+++ b/pkg/operator/ceph/cr_manager.go
@@ -35,6 +35,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/nfs"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/ceph/object/bucket"
+ "github.com/rook/rook/pkg/operator/ceph/object/cosi"
"github.com/rook/rook/pkg/operator/ceph/object/notification"
"github.com/rook/rook/pkg/operator/ceph/object/realm"
"github.com/rook/rook/pkg/operator/ceph/object/topic"
@@ -104,6 +105,7 @@ var AddToManagerFuncs = []func(manager.Manager, *clusterd.Context, context.Conte
notification.Add,
subvolumegroup.Add,
radosnamespace.Add,
+ cosi.Add,
}
// AddToManagerOpFunc is a list of functions to add all Controllers to the Manager (entrypoint for
diff --git a/pkg/operator/ceph/object/cosi/controller.go b/pkg/operator/ceph/object/cosi/controller.go
new file mode 100644
index 000000000..6876ea711
--- /dev/null
+++ b/pkg/operator/ceph/object/cosi/controller.go
@@ -0,0 +1,203 @@
+/*
+Copyright 2023 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cosi implements the controller for the Ceph COSI Driver.
+package cosi
+
+import (
+ "context"
+ "os"
+ "reflect"
+ "time"
+
+ "github.com/coreos/pkg/capnslog"
+ "github.com/pkg/errors"
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ "github.com/rook/rook/pkg/clusterd"
+ opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
+ "github.com/rook/rook/pkg/operator/ceph/reporting"
+ "github.com/rook/rook/pkg/operator/k8sutil"
+ appsv1 "k8s.io/api/apps/v1"
+ kapiv1 "k8s.io/api/core/v1"
+ kerrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/tools/record"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller"
+ "sigs.k8s.io/controller-runtime/pkg/handler"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+ "sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+const (
+ packageName = "ceph-cosi"
+ controllerName = "ceph-cosi-controller"
+ CephCOSIDriverName = "ceph-cosi-driver"
+ COSISideCarName = "objectstorage-provisioner-sidecar"
+ cosiSocketMountPath = "/var/lib/cosi"
+ DefaultServiceAccountName = "objectstorage-provisioner"
+ cosiSocketVolumeName = "socket"
+)
+
+var (
+ logger = capnslog.NewPackageLogger("github.com/rook/rook", controllerName)
+ waitForRequeueObjectStoreNotPresent = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
+)
+
+// ReconcileCephCOSIDriver reconciles the Ceph COSI Driver
+type ReconcileCephCOSIDriver struct {
+ client client.Client
+ context *clusterd.Context
+ scheme *runtime.Scheme
+ opManagerContext context.Context
+ recorder record.EventRecorder
+}
+
+// Add creates a new CephCOSIDriver Controller and adds it to the Manager. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
+ return add(mgr, newReconciler(mgr, context, opManagerContext))
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context) reconcile.Reconciler {
+ return &ReconcileCephCOSIDriver{
+ client: mgr.GetClient(),
+ context: context,
+ scheme: mgr.GetScheme(),
+ opManagerContext: opManagerContext,
+ recorder: mgr.GetEventRecorderFor(controllerName),
+ }
+}
+
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+ // Create a new controller
+ controller, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
+ if err != nil {
+ return errors.Wrapf(err, "failed to create %s controller", controllerName)
+ }
+
+ logger.Info("successfully started")
+ // Watch for changes to CephCOSIDriver
+ err = controller.Watch(source.Kind(mgr.GetCache(), &cephv1.CephCOSIDriver{}), &handler.EnqueueRequestForObject{}, opcontroller.WatchControllerPredicate())
+ if err != nil {
+ return errors.Wrap(err, "failed to watch for CephCOSIDriver object changes")
+ }
+
+ // Watch for changes to CephObjectStore as arbitrary resource and predicate functions
+ err = controller.Watch(source.Kind(mgr.GetCache(), &cephv1.CephObjectStore{}), &handler.EnqueueRequestForObject{}, opcontroller.WatchControllerPredicate())
+ if err != nil {
+ return errors.Wrap(err, "failed to watch for CephObjectStore object changes")
+ }
+
+ return nil
+}
+
+// Reconcile reads that state of the cluster for a CephCOSIDriver object and makes changes based on the state read
+// and what is in the CephCOSIDriver.Spec
+func (r *ReconcileCephCOSIDriver) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
+ reconcileResponse, cephCOSIDriver, err := r.reconcile(request)
+
+ return reporting.ReportReconcileResult(logger, r.recorder, request, &cephCOSIDriver, reconcileResponse, err)
+}
+
+func (r *ReconcileCephCOSIDriver) reconcile(request reconcile.Request) (reconcile.Result, cephv1.CephCOSIDriver, error) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{}
+
+ // Fetch the CephCOSIDriver instance
+ err := r.client.Get(r.opManagerContext, request.NamespacedName, cephCOSIDriver)
+ logger.Debugf("CephCOSIDriver: %+v", cephCOSIDriver)
+ if err != nil && client.IgnoreNotFound(err) != nil {
+ return reconcile.Result{}, *cephCOSIDriver, errors.Wrapf(err, "failed to get Ceph COSI Driver %s", request.NamespacedName)
+ }
+
+ cosiDeploymentStrategy := cephv1.COSIDeploymentStrategyAuto
+ if !reflect.DeepEqual(cephCOSIDriver.Spec, cephv1.CephCOSIDriverSpec{}) && cephCOSIDriver.Spec.DeploymentStrategy != "" {
+ cosiDeploymentStrategy = cephCOSIDriver.Spec.DeploymentStrategy
+ }
+ if cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyNever {
+ logger.Info("Ceph COSI Driver is disabled, delete if exists")
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(r.opManagerContext, request.NamespacedName, cephCOSIDriverDeployment)
+ if err != nil && client.IgnoreNotFound(err) != nil {
+ return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to get Ceph COSI Driver Deployment")
+ }
+ err = r.client.Delete(r.opManagerContext, cephCOSIDriverDeployment)
+ if err != nil && client.IgnoreNotFound(err) != nil {
+ return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to delete Ceph COSI Driver Deployment")
+ }
+ return reconcile.Result{}, *cephCOSIDriver, nil
+ }
+
+ r.recorder.Eventf(cephCOSIDriver, kapiv1.EventTypeNormal, "Reconciling", "Reconciling Ceph COSI Driver %q", cephCOSIDriver.Name)
+ // Set the default CephCOSIDriver name if not already set
+ if cephCOSIDriver.Name == "" {
+ cephCOSIDriver.Name = CephCOSIDriverName
+ }
+
+ // Set the default CephCOSIDriver namespace if not already set
+ if cephCOSIDriver.Namespace == "" {
+ cephCOSIDriver.Namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
+ }
+
+ // Check whether object store is running
+ list := &cephv1.CephObjectStoreList{}
+ err = r.client.List(r.opManagerContext, list)
+ if err != nil {
+ return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to list CephObjectStore")
+ }
+
+ if len(list.Items) == 0 && cosiDeploymentStrategy == cephv1.COSIDeploymentStrategyAuto {
+ logger.Info("no object stores found, skipping cosi driver config")
+ return waitForRequeueObjectStoreNotPresent, *cephCOSIDriver, nil
+ }
+
+ // Start the Ceph COSI Driver
+ err = r.startCephCOSIDriver(cephCOSIDriver)
+ if err != nil {
+ return reconcile.Result{}, *cephCOSIDriver, errors.Wrap(err, "failed to start Ceph COSI Driver")
+ }
+
+ return reconcile.Result{}, *cephCOSIDriver, nil
+}
+
+// Start the Ceph COSI Driver
+func (r *ReconcileCephCOSIDriver) startCephCOSIDriver(cephCOSIDriver *cephv1.CephCOSIDriver) error {
+ // Create the Ceph COSI Driver Deployment
+ cephcosidriverDeployment, err := createCephCOSIDriverDeployment(cephCOSIDriver)
+ if err != nil {
+ return errors.Wrap(err, "failed to create Ceph COSI Driver Deployment CRD")
+ }
+
+ logger.Infof("starting Ceph COSI Driver deployment %q in namespace %q", cephcosidriverDeployment.Name, cephcosidriverDeployment.Namespace)
+
+ err = r.client.Create(r.opManagerContext, cephcosidriverDeployment)
+ if err != nil {
+ if !kerrors.IsAlreadyExists(err) {
+ return errors.Wrap(err, "failed to create Ceph COSI Driver deployment")
+ }
+ logger.Info("Ceph COSI Driver deployment already exists, updating")
+ err = r.client.Update(r.opManagerContext, cephcosidriverDeployment)
+ if err != nil {
+ return errors.Wrap(err, "failed to update Ceph COSI Driver deployment")
+ }
+ } else {
+ logger.Info("Ceph COSI Driver deployment started")
+ }
+
+ return nil
+}
diff --git a/pkg/operator/ceph/object/cosi/controller_test.go b/pkg/operator/ceph/object/cosi/controller_test.go
new file mode 100644
index 000000000..de8b977a2
--- /dev/null
+++ b/pkg/operator/ceph/object/cosi/controller_test.go
@@ -0,0 +1,255 @@
+package cosi
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/coreos/pkg/capnslog"
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake"
+ "github.com/rook/rook/pkg/client/clientset/versioned/scheme"
+ "github.com/rook/rook/pkg/clusterd"
+ "github.com/rook/rook/pkg/operator/test"
+ exectest "github.com/rook/rook/pkg/util/exec/test"
+ "github.com/stretchr/testify/assert"
+ appsv1 "k8s.io/api/apps/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/tools/record"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+ cephObjectStoreName = "my-store"
+ namespace = "rook-ceph"
+)
+
+func TestCephCOSIDriverController(t *testing.T) {
+ ctx := context.TODO()
+ capnslog.SetGlobalLogLevel(capnslog.DEBUG)
+ os.Setenv("ROOK_LOG_LEVEL", "DEBUG")
+ os.Setenv("POD_NAMESPACE", namespace)
+
+ setupNewEnvironment := func(objects ...runtime.Object) *ReconcileCephCOSIDriver {
+ executor := &exectest.MockExecutor{
+ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
+ if args[0] == "status" {
+ return `{"fsid":"c47cac40-9bee-4d52-823b-ccd803ba5bfe","health":{"checks":{},"status":"HEALTH_ERR"},"pgmap":{"num_pgs":100,"pgs_by_state":[{"state_name":"active+clean","count":100}]}}`, nil
+ }
+ return "", nil
+ },
+ }
+ clientset := test.New(t, 3)
+ c := &clusterd.Context{
+ Executor: executor,
+ RookClientset: rookfake.NewSimpleClientset(),
+ Clientset: clientset,
+ }
+
+ // Register operator types with the runtime scheme.
+ s := scheme.Scheme
+ err := appsv1.AddToScheme(s)
+ if err != nil {
+ assert.Fail(t, "failed to build scheme")
+ }
+
+ s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{})
+ s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCOSIDriver{})
+ s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStoreList{})
+
+ // Create a fake client to mock API calls.
+ cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
+
+ return &ReconcileCephCOSIDriver{
+ client: cl,
+ scheme: s,
+ context: c,
+ recorder: &record.FakeRecorder{},
+ opManagerContext: ctx,
+ }
+ }
+
+ req := reconcile.Request{
+ NamespacedName: types.NamespacedName{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ }
+ t.Run("requeue no object store exists", func(t *testing.T) {
+ r := setupNewEnvironment()
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, true, res.Requeue)
+ })
+
+ t.Run("object store exists", func(t *testing.T) {
+ objectStore := &cephv1.CephObjectStore{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cephObjectStoreName,
+ Namespace: namespace,
+ },
+ }
+ r := setupNewEnvironment(objectStore)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, false, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: CephCOSIDriverName, Namespace: namespace}, cephCOSIDriverDeployment)
+ assert.NoError(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with disabled mode without any object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ DeploymentStrategy: cephv1.COSIDeploymentStrategyNever,
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, false, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.Error(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with disabled mode with object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ DeploymentStrategy: cephv1.COSIDeploymentStrategyNever,
+ },
+ }
+ objectStore := &cephv1.CephObjectStore{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cephObjectStoreName,
+ Namespace: namespace,
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver, objectStore)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, false, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.Error(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with enforced mode without any object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ DeploymentStrategy: cephv1.COSIDeploymentStrategyAlways,
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, false, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.NoError(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with auto mode without object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ DeploymentStrategy: cephv1.COSIDeploymentStrategyAuto,
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, true, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.Error(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with auto mode with object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ DeploymentStrategy: cephv1.COSIDeploymentStrategyAuto,
+ },
+ }
+ objectStore := &cephv1.CephObjectStore{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cephObjectStoreName,
+ Namespace: namespace,
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver, objectStore)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, false, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.NoError(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with custom values and no object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ Image: "quay.io/ceph/cosi:custom",
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, true, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.Error(t, err)
+ })
+
+ t.Run("ceph cosi driver CRD with custom values and object stores", func(t *testing.T) {
+ cephCOSIDriver := &cephv1.CephCOSIDriver{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Namespace: namespace,
+ },
+ Spec: cephv1.CephCOSIDriverSpec{
+ Image: "quay.io/ceph/cosi:custom",
+ },
+ }
+ objectStore := &cephv1.CephObjectStore{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cephObjectStoreName,
+ Namespace: namespace,
+ },
+ }
+ r := setupNewEnvironment(cephCOSIDriver, objectStore)
+ res, err := r.Reconcile(ctx, req)
+ assert.NoError(t, err)
+ assert.Equal(t, false, res.Requeue)
+ cephCOSIDriverDeployment := &appsv1.Deployment{}
+ err = r.client.Get(ctx, types.NamespacedName{Name: cephCOSIDriver.Name, Namespace: cephCOSIDriver.Namespace}, cephCOSIDriverDeployment)
+ assert.NoError(t, err)
+ assert.Equal(t, "quay.io/ceph/cosi:custom", cephCOSIDriverDeployment.Spec.Template.Spec.Containers[0].Image)
+ })
+}
diff --git a/pkg/operator/ceph/object/cosi/spec.go b/pkg/operator/ceph/object/cosi/spec.go
new file mode 100644
index 000000000..9289614d2
--- /dev/null
+++ b/pkg/operator/ceph/object/cosi/spec.go
@@ -0,0 +1,143 @@
+/*
+Copyright 2023 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cosi
+
+import (
+ "github.com/pkg/errors"
+ cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+ "github.com/rook/rook/pkg/operator/ceph/controller"
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+ defaultCOSISideCarImage = "gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20230130-v0.1.0-24-gc0cf995"
+ defaultCephCOSIDriverImage = "quay.io/ceph/cosi:v0.1.0"
+)
+
+func createCephCOSIDriverDeployment(cephCOSIDriver *cephv1.CephCOSIDriver) (*appsv1.Deployment, error) {
+ cosiPodSpec, err := createCOSIPodSpec(cephCOSIDriver)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to create cosi pod spec")
+ }
+ strategy := appsv1.DeploymentStrategy{
+ Type: appsv1.RecreateDeploymentStrategyType,
+ }
+
+ replica := int32(1)
+ minReadySeconds := int32(30)
+ progressDeadlineSeconds := int32(600)
+ revisionHistoryLimit := int32(3)
+
+ cephcosidriverDeployment := &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cephCOSIDriver.Name,
+ Namespace: cephCOSIDriver.Namespace,
+ Labels: getCOSILabels(cephCOSIDriver.Name, cephCOSIDriver.Namespace),
+ },
+ Spec: appsv1.DeploymentSpec{
+ Replicas: &replica,
+ Selector: &metav1.LabelSelector{
+ MatchLabels: getCOSILabels(cephCOSIDriver.Name, cephCOSIDriver.Namespace),
+ },
+ Template: cosiPodSpec,
+ Strategy: strategy,
+ MinReadySeconds: minReadySeconds,
+ ProgressDeadlineSeconds: &progressDeadlineSeconds,
+ RevisionHistoryLimit: &revisionHistoryLimit,
+ },
+ }
+
+ return cephcosidriverDeployment, nil
+}
+
+func setCOSILabels(label map[string]string) {
+ label["app.kubernetes.io/part-of"] = CephCOSIDriverName
+ label["app.kubernetes.io/name"] = CephCOSIDriverName
+ label["app.kubernetes.io/component"] = CephCOSIDriverName
+}
+
+func getCOSILabels(name, namespace string) map[string]string {
+ label := controller.AppLabels(name, namespace)
+ setCOSILabels(label)
+ return label
+}
+
+func createCOSIPodSpec(cephCOSIDriver *cephv1.CephCOSIDriver) (corev1.PodTemplateSpec, error) {
+ cosiDriverContainer := createCOSIDriverContainer(cephCOSIDriver)
+ cosiSideCarContainer := createCOSISideCarContainer(cephCOSIDriver)
+
+ podSpec := corev1.PodSpec{
+ Containers: []corev1.Container{
+ cosiDriverContainer,
+ cosiSideCarContainer,
+ },
+ ServiceAccountName: DefaultServiceAccountName,
+ Volumes: []corev1.Volume{
+ {Name: cosiSocketVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}},
+ },
+ }
+
+ cephCOSIDriver.Spec.Placement.ApplyToPodSpec(&podSpec)
+
+ podTemplateSpec := corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: CephCOSIDriverName,
+ Labels: getCOSILabels(cephCOSIDriver.Name, cephCOSIDriver.Namespace),
+ },
+ Spec: podSpec,
+ }
+
+ return podTemplateSpec, nil
+}
+
+func createCOSIDriverContainer(cephCOSIDriver *cephv1.CephCOSIDriver) corev1.Container {
+ cephCOSIDriveImage := defaultCephCOSIDriverImage
+ if cephCOSIDriver.Spec.Image != "" {
+ cephCOSIDriveImage = cephCOSIDriver.Spec.Image
+ }
+ return corev1.Container{
+ Name: CephCOSIDriverName,
+ Image: cephCOSIDriveImage,
+ Env: []corev1.EnvVar{
+ {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}},
+ VolumeMounts: []corev1.VolumeMount{
+ {Name: cosiSocketVolumeName, MountPath: cosiSocketMountPath},
+ },
+ Resources: cephCOSIDriver.Spec.Resources,
+ }
+}
+
+func createCOSISideCarContainer(cephCOSIDriver *cephv1.CephCOSIDriver) corev1.Container {
+ defaultCOSISideCarImage := defaultCOSISideCarImage
+ if cephCOSIDriver.Spec.ObjectProvisionerImage != "" {
+ defaultCOSISideCarImage = cephCOSIDriver.Spec.ObjectProvisionerImage
+ }
+ return corev1.Container{
+ Name: COSISideCarName,
+ Image: defaultCOSISideCarImage,
+ Args: []string{
+ "--v=5",
+ },
+ Env: []corev1.EnvVar{
+ {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}},
+ VolumeMounts: []corev1.VolumeMount{
+ {Name: cosiSocketVolumeName, MountPath: cosiSocketMountPath},
+ },
+ }
+}
diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go
index 7e093c96e..6b86aca37 100644
--- a/tests/framework/clients/bucket.go
+++ b/tests/framework/clients/bucket.go
@@ -74,9 +74,10 @@ func (b *BucketOperation) UpdateObcNotificationRemove(obcName string, storageCla
// CheckOBC, returns true if the obc, secret and configmap are all in the "check" state,
// and returns false if any of these resources are not in the "check" state.
// Check state values:
-// "created", all must exist,
-// "bound", all must exist and OBC in Bound phase
-// "deleted", all must be missing.
+//
+// "created", all must exist,
+// "bound", all must exist and OBC in Bound phase
+// "deleted", all must be missing.
func (b *BucketOperation) CheckOBC(obcName, check string) bool {
resources := []string{"obc", "secret", "configmap"}
shouldBeBound := (check == "bound")
diff --git a/tests/framework/clients/cosi.go b/tests/framework/clients/cosi.go
new file mode 100644
index 000000000..b03a7131a
--- /dev/null
+++ b/tests/framework/clients/cosi.go
@@ -0,0 +1,39 @@
+package clients
+
+import (
+ "github.com/rook/rook/tests/framework/installer"
+ "github.com/rook/rook/tests/framework/utils"
+)
+
+type COSIOperation struct {
+ k8sh *utils.K8sHelper
+ manifests installer.CephManifests
+}
+
+func CreateCOSIOperation(k8sh *utils.K8sHelper, manifests installer.CephManifests) *COSIOperation {
+ return &COSIOperation{k8sh, manifests}
+}
+
+func (c *COSIOperation) CreateCOSI() error {
+ return c.k8sh.ResourceOperation("create", c.manifests.GetCOSIDriver())
+}
+
+func (c *COSIOperation) DeleteCOSI() error {
+ return c.k8sh.ResourceOperation("delete", c.manifests.GetCOSIDriver())
+}
+
+func (c *COSIOperation) CreateBucketClass(name, objectStoreUserSecretName, deletionPolicy string) error {
+ return c.k8sh.ResourceOperation("create", c.manifests.GetBucketClass(name, objectStoreUserSecretName, deletionPolicy))
+}
+
+func (c *COSIOperation) DeleteBucketClass(name, objectStoreUserSecretName, deletionPolicy string) error {
+ return c.k8sh.ResourceOperation("delete", c.manifests.GetBucketClass(name, objectStoreUserSecretName, deletionPolicy))
+}
+
+func (c *COSIOperation) CreateBucketClaim(name, bucketClassName string) error {
+ return c.k8sh.ResourceOperation("create", c.manifests.GetBucketClaim(name, bucketClassName))
+}
+
+func (c *COSIOperation) DeleteBucketClaim(name, bucketClassName string) error {
+ return c.k8sh.ResourceOperation("delete", c.manifests.GetBucketClaim(name, bucketClassName))
+}
diff --git a/tests/framework/clients/test_client.go b/tests/framework/clients/test_client.go
index b59359539..e7bcae229 100644
--- a/tests/framework/clients/test_client.go
+++ b/tests/framework/clients/test_client.go
@@ -37,6 +37,7 @@ type TestClient struct {
RBDMirrorClient *RBDMirrorOperation
TopicClient *TopicOperation
NotificationClient *NotificationOperation
+ COSIClient *COSIOperation
k8sh *utils.K8sHelper
}
@@ -54,6 +55,7 @@ func CreateTestClient(k8sHelper *utils.K8sHelper, manifests installer.CephManife
CreateRBDMirrorOperation(k8sHelper, manifests),
CreateTopicOperation(k8sHelper, manifests),
CreateNotificationOperation(k8sHelper, manifests),
+ CreateCOSIOperation(k8sHelper, manifests),
k8sHelper,
}
}
diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go
index b0e34b0e6..f40109e8b 100644
--- a/tests/framework/installer/ceph_manifests.go
+++ b/tests/framework/installer/ceph_manifests.go
@@ -21,6 +21,7 @@ import (
"strconv"
"strings"
+ "github.com/rook/rook/pkg/operator/ceph/object/cosi"
"github.com/rook/rook/tests/framework/utils"
)
@@ -54,6 +55,9 @@ type CephManifests interface {
GetBucketTopic(topicName string, storeName string, httpEndpointService string) string
GetClient(name string, caps map[string]string) string
GetFilesystemSubvolumeGroup(fsName, groupName string) string
+ GetCOSIDriver() string
+ GetBucketClass(name, objectstoreUserName, deletionPolicy string) string
+ GetBucketClaim(claimName, className string) string
}
// CephManifestsMaster wraps rook yaml definitions
@@ -504,7 +508,9 @@ spec:
maxObjects: ` + strconv.Itoa(maxobjects) + `
maxSize: ` + maxsize + `
capabilities:
- user: ` + usercaps
+ user: "` + usercaps + `"
+ bucket: "` + usercaps + `"
+`
}
// GetBucketStorageClass returns the manifest to create object bucket
@@ -635,3 +641,39 @@ metadata:
spec:
filesystemName: ` + fsName
}
+
+func (m *CephManifestsMaster) GetCOSIDriver() string {
+ // TODO: use the official image once it is available
+ return `apiVersion: ceph.rook.io/v1
+kind: CephCOSIDriver
+metadata:
+ name: ` + cosi.CephCOSIDriverName + `
+ namespace: ` + m.settings.OperatorNamespace + `
+spec:
+ deploymentStrategy: Auto `
+}
+
+func (m *CephManifestsMaster) GetBucketClass(name, objectStoreUserSecretName, deletionPolicy string) string {
+ return `apiVersion: objectstorage.k8s.io/v1alpha1
+kind: BucketClass
+metadata:
+ name: ` + name + `
+ namespace: ` + m.settings.OperatorNamespace + `
+driverName: ceph.objectstorage.k8s.io
+deletionPolicy: ` + deletionPolicy + `
+parameters:
+ objectStoreUserSecretName: ` + objectStoreUserSecretName + `
+ objectStoreUserSecretNamespace: ` + m.settings.Namespace
+}
+
+func (m *CephManifestsMaster) GetBucketClaim(name, bucketClassName string) string {
+ return `apiVersion: objectstorage.k8s.io/v1alpha1
+kind: BucketClaim
+metadata:
+ name: ` + name + `
+ namespace: ` + m.settings.OperatorNamespace + `
+spec:
+ bucketClassName: ` + bucketClassName + `
+ protocols:
+ - s3 `
+}
diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go
index 5815701b8..895a512fc 100644
--- a/tests/framework/installer/ceph_manifests_previous.go
+++ b/tests/framework/installer/ceph_manifests_previous.go
@@ -180,3 +180,15 @@ func (m *CephManifestsPreviousVersion) GetRBDMirror(name string, count int) stri
func (m *CephManifestsPreviousVersion) GetFilesystemSubvolumeGroup(fsName, groupName string) string {
return m.latest.GetFilesystemSubvolumeGroup(fsName, groupName)
}
+
+func (m *CephManifestsPreviousVersion) GetCOSIDriver() string {
+ return m.latest.GetCOSIDriver()
+}
+
+func (m *CephManifestsPreviousVersion) GetBucketClass(name, objectStoreUserSecretName, deletionPolicy string) string {
+ return m.latest.GetBucketClass(name, objectStoreUserSecretName, deletionPolicy)
+}
+
+func (m *CephManifestsPreviousVersion) GetBucketClaim(name, bucketClassName string) string {
+ return m.latest.GetBucketClaim(name, bucketClassName)
+}
diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go
index aa0d7819a..cf80e474a 100644
--- a/tests/integration/ceph_base_object_test.go
+++ b/tests/integration/ceph_base_object_test.go
@@ -60,7 +60,7 @@ var (
bucketStorageClassName = "rook-smoke-delete-bucket"
maxBucket = 1
maxSize = "100000"
- userCap = "read"
+ userCap = "*"
)
// Test Object StoreCreation on Rook that was installed via helm
diff --git a/tests/integration/ceph_cosi_test.go b/tests/integration/ceph_cosi_test.go
new file mode 100644
index 000000000..f56e50537
--- /dev/null
+++ b/tests/integration/ceph_cosi_test.go
@@ -0,0 +1,146 @@
+package integration
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/rook/rook/pkg/daemon/ceph/client"
+ rgw "github.com/rook/rook/pkg/operator/ceph/object"
+ "github.com/rook/rook/tests/framework/clients"
+ "github.com/rook/rook/tests/framework/installer"
+ "github.com/rook/rook/tests/framework/utils"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/suite"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+ objectStoreCOSI = "cosi-store"
+ cosiUser = "cosi"
+ bucketClassName = "cosi-bucketclass"
+ deletionPolicy = "Delete"
+ cosiBucketName = "cosi-bucket"
+)
+
+func testCOSIDriver(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, cephinstaller *installer.CephInstaller, namespace string) {
+ if utils.IsPlatformOpenShift() {
+ s.T().Skip("ceph cosi driver tests skipped on openshift")
+ }
+
+ t := s.T()
+
+ t.Run("COSI Controller and CRD installation", func(t *testing.T) {
+ _, err := k8sh.Kubectl("create", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-api")
+ assert.NoError(t, err, "failed to create COSI CRDs")
+ _, err = k8sh.Kubectl("create", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-controller")
+ assert.NoError(t, err, "failed to create COSI controller")
+ })
+
+ createCephObjectStore(s.T(), helper, k8sh, cephinstaller, namespace, objectStoreCOSI, 1, false)
+
+ t.Run("Creating CephCOSIDriver CRD", func(t *testing.T) {
+ err := helper.COSIClient.CreateCOSI()
+ assert.NoError(t, err, "failed to create Ceph COSI Driver CRD")
+
+ })
+
+ operatorNamespace := cephinstaller.Manifests.Settings().OperatorNamespace
+ t.Run("Check ceph cosi driver running", func(t *testing.T) {
+ for i := 24; i < 24 && k8sh.CheckPodCountAndState("ceph-cosi-driver", operatorNamespace, 1, "Running") == false; i++ {
+ logger.Infof("ceph-cosi-driver is not running, trying again")
+ k8sh.CheckPodCountAndState("ceph-cosi-driver", namespace, 1, "Running")
+ }
+ assert.True(t, k8sh.CheckPodCountAndState("ceph-cosi-driver", operatorNamespace, 1, "Running"))
+ assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=ceph-cosi-driver", operatorNamespace))
+ })
+
+ createCephObjectUser(s, helper, k8sh, namespace, objectStoreCOSI, cosiUser, true, true)
+ objectStoreUserSecretName := "rook-ceph-object-user" + "-" + objectStoreCOSI + "-" + cosiUser
+ t.Run("Creating BucketClass", func(t *testing.T) {
+ err := helper.COSIClient.CreateBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy)
+ assert.NoError(t, err, "failed to create BucketClass")
+ })
+
+ t.Run("Creating BucketClaim", func(t *testing.T) {
+ err := helper.COSIClient.CreateBucketClaim(cosiBucketName, bucketClassName)
+ assert.NoError(t, err, "failed to create BucketClaim")
+ })
+
+ var cosiBucket string
+ t.Run("Check Bucket is ready", func(t *testing.T) {
+ _, err := k8sh.GetResource("bucketclaim", "-n", operatorNamespace, cosiBucketName)
+ assert.NoError(t, err, "failed to get BucketClaim")
+ cosiBucket, err = k8sh.GetResource("bucketclaim", "-n", operatorNamespace, cosiBucketName, "-o", "jsonpath={.status.bucketName}")
+ assert.NoError(t, err, "failed to get Bucket name")
+ i := 0
+ var bucketReady string
+ for i = 0; i < 4; i++ {
+ bucketReady, err = k8sh.GetResource("bucket", cosiBucket, "-o", "jsonpath={.status.bucketReady}")
+ if bucketReady == "true" && err == nil {
+ break
+ }
+ logger.Warningf("bucket %q is not ready, retrying... bucketReady=%q, err=%v", cosiBucket, bucketReady, err)
+ time.Sleep(5 * time.Second)
+ }
+ assert.NotEqual(t, 4, i)
+ assert.Equal(t, "true", bucketReady, "Bucket is not ready")
+ })
+
+ t.Run("Check Bucket is created in Backend Ceph", func(t *testing.T) {
+ ctx := context.TODO()
+ // check if bucket is created in the backend
+ context := k8sh.MakeContext()
+ clusterInfo := client.AdminTestClusterInfo(namespace)
+ objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, objectStoreCOSI, metav1.GetOptions{})
+ assert.Nil(t, err)
+ rgwcontext, err := rgw.NewMultisiteContext(context, clusterInfo, objectStore)
+ assert.Nil(t, err)
+ var bkt rgw.ObjectBucket
+ i := 0
+ for i = 0; i < 4; i++ {
+ b, code, err := rgw.GetBucket(rgwcontext, cosiBucket)
+ if b != nil && err == nil {
+ bkt = *b
+ break
+ }
+ logger.Warningf("cannot get bucket %q, retrying... bucket: %v. code: %d, err: %v", cosiBucket, b, code, err)
+ logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
+ time.Sleep(5 * time.Second)
+ }
+ assert.NotEqual(t, 4, i)
+ assert.Equal(t, cosiBucket, bkt.Name)
+ })
+
+ t.Run("Deleting BucketClaim", func(t *testing.T) {
+ err := helper.COSIClient.DeleteBucketClaim(cosiBucketName, bucketClassName)
+ assert.NoError(t, err, "failed to delete BucketClaim")
+ })
+
+ t.Run("Deleting BucketClass", func(t *testing.T) {
+ err := helper.COSIClient.DeleteBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy)
+ assert.NoError(t, err, "failed to delete BucketClass")
+ })
+
+ t.Run("Deleting object user for cosi", func(t *testing.T) {
+ err := helper.ObjectUserClient.Delete(namespace, cosiUser)
+ assert.NoError(t, err, "failed to delete cosi user")
+ })
+
+ t.Run("delete CephObjectStore", func(t *testing.T) {
+ deleteObjectStore(t, k8sh, namespace, objectStoreCOSI)
+ assertObjectStoreDeletion(t, k8sh, namespace, objectStoreCOSI)
+ })
+
+ t.Run("delete CephCOSIDriver CRD", func(t *testing.T) {
+ err := helper.COSIClient.DeleteCOSI()
+ assert.NoError(t, err, "failed to delete Ceph COSI Driver CRD")
+ })
+
+ t.Run("delete COSI Controller", func(t *testing.T) {
+ _, err := k8sh.Kubectl("delete", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-api")
+ assert.NoError(t, err, "failed to create COSI CRDs")
+ _, err = k8sh.Kubectl("delete", "-k", "github.com/kubernetes-sigs/container-object-storage-interface-controller")
+ assert.NoError(t, err, "failed to create COSI controller")
+ })
+}
diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go
index a26754c57..e420d8841 100644
--- a/tests/integration/ceph_object_test.go
+++ b/tests/integration/ceph_object_test.go
@@ -145,6 +145,13 @@ func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, install
bucketNotificationTestStoreName := "bucket-notification-" + storeName
createCephObjectStore(s.T(), helper, k8sh, installer, namespace, bucketNotificationTestStoreName, 1, tlsEnable)
testBucketNotifications(s, helper, k8sh, namespace, bucketNotificationTestStoreName)
+ if !tlsEnable {
+ // TODO : need to fix COSI driver to support TLS
+ logger.Info("Testing COSI driver")
+ testCOSIDriver(s, helper, k8sh, installer, namespace)
+ } else {
+ logger.Info("Skipping COSI driver test as TLS is enabled")
+ }
}
func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go
index 21a370a7a..af9ff3982 100644
--- a/tests/integration/ceph_upgrade_test.go
+++ b/tests/integration/ceph_upgrade_test.go
@@ -460,7 +460,7 @@ func (s *UpgradeSuite) upgradeToMaster() {
return
}
- require.NoError(s.T(), s.k8sh.ResourceOperation("replace", s.installer.Manifests.GetCRDs(s.k8sh)))
+ require.NoError(s.T(), s.k8sh.ResourceOperation("apply", s.installer.Manifests.GetCRDs(s.k8sh)))
require.NoError(s.T(), s.k8sh.ResourceOperation("apply", s.installer.Manifests.GetCommon()))
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 2ce1e4c7a..311bee7f3 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -208,6 +208,10 @@ function validate_yaml() {
keda_url="https://github.com/kedacore/keda/releases/download/v${keda_version}/keda-${keda_version}.yaml"
kubectl apply -f "${keda_url}"
+ #create the COSI CRDS
+ cosi_crd_url="github.com/kubernetes-sigs/container-object-storage-interface-api"
+ kubectl create -k "${cosi_crd_url}"
+
# skipping folders and some yamls that are only for openshift.
manifests="$(find . -maxdepth 1 -type f -name '*.yaml' -and -not -name '*openshift*' -and -not -name 'scc*' -and -not -name 'psp*' -and -not -name 'kustomization*')"
with_f_arg="$(echo "$manifests" | awk '{printf " -f %s",$1}')" # don't add newline