From 6ad9190f5560709532047ddb416957dcdc56f5c7 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 15 Jul 2021 16:00:26 -0600 Subject: [PATCH 1/3] ceph: factor out test manifest helpers for providers The ceph provider had been refactored to read the crds and operator manifests from a file instead of copying the manifests into the test code. Now the helpers are refactored to allow the other providers to also reduce the manifest duplication in tests. Signed-off-by: Travis Nielsen --- tests/framework/installer/ceph_settings.go | 45 ++------------ tests/framework/installer/settings.go | 69 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 41 deletions(-) create mode 100644 tests/framework/installer/settings.go diff --git a/tests/framework/installer/ceph_settings.go b/tests/framework/installer/ceph_settings.go index ff3219201..d8b25b441 100644 --- a/tests/framework/installer/ceph_settings.go +++ b/tests/framework/installer/ceph_settings.go @@ -18,16 +18,10 @@ package installer import ( "fmt" - "io/ioutil" - "net/http" "os" - "path" "strings" - "time" - "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/tests/framework/utils" ) // TestCephSettings struct for handling panic and test suite tear down @@ -68,17 +62,8 @@ func (s *TestCephSettings) ApplyEnvVars() { } func (s *TestCephSettings) readManifest(filename string) string { - rootDir, err := utils.FindRookRoot() - if err != nil { - panic(err) - } - manifest := path.Join(rootDir, "cluster/examples/kubernetes/ceph", filename) - logger.Infof("Reading manifest: %s", manifest) - contents, err := ioutil.ReadFile(manifest) - if err != nil { - panic(errors.Wrapf(err, "failed to read manifest at %s", manifest)) - } - return replaceNamespaces(manifest, string(contents), s.OperatorNamespace, s.Namespace) + manifest := readManifest("ceph", filename) + return replaceNamespaces(manifest, manifest, s.OperatorNamespace, s.Namespace) } func (s *TestCephSettings) readManifestFromGithub(filename string) string { @@ -86,30 +71,8 @@ func (s *TestCephSettings) readManifestFromGithub(filename string) string { } func (s *TestCephSettings) readManifestFromGithubWithClusterNamespace(filename, clusterNamespace string) string { - url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/ceph/%s", s.RookVersion, filename) - logger.Infof("Retrieving manifest: %s", url) - var response *http.Response - var err error - for i := 1; i <= 3; i++ { - // #nosec G107 This is only test code and is expected to read from a url - response, err = http.Get(url) - if err != nil { - if i == 3 { - panic(errors.Wrapf(err, "failed to read manifest from %s", url)) - } - logger.Warningf("failed to read manifest from %s. retrying in 1sec. %v", url, err) - time.Sleep(time.Second) - continue - } - break - } - defer response.Body.Close() - - content, err := ioutil.ReadAll(response.Body) - if err != nil { - panic(errors.Wrapf(err, "failed to read content from %s", url)) - } - return replaceNamespaces(url, string(content), s.OperatorNamespace, clusterNamespace) + manifest := readManifestFromGithub(s.RookVersion, "ceph", filename) + return replaceNamespaces(filename, manifest, s.OperatorNamespace, clusterNamespace) } func (s *TestCephSettings) replaceOperatorSettings(manifest string) string { diff --git a/tests/framework/installer/settings.go b/tests/framework/installer/settings.go new file mode 100644 index 000000000..033cf957a --- /dev/null +++ b/tests/framework/installer/settings.go @@ -0,0 +1,69 @@ +/* +Copyright 2021 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "io/ioutil" + "net/http" + "path" + "time" + + "github.com/pkg/errors" + "github.com/rook/rook/tests/framework/utils" +) + +func readManifest(provider, filename string) string { + rootDir, err := utils.FindRookRoot() + if err != nil { + panic(err) + } + manifest := path.Join(rootDir, "cluster/examples/kubernetes/", provider, filename) + logger.Infof("Reading manifest: %s", manifest) + contents, err := ioutil.ReadFile(manifest) + if err != nil { + panic(errors.Wrapf(err, "failed to read manifest at %s", manifest)) + } + return string(contents) +} + +func readManifestFromGithub(rookVersion, provider, filename string) string { + url := fmt.Sprintf("https://raw.githubusercontent.com/rook/rook/%s/cluster/examples/kubernetes/%s/%s", rookVersion, provider, filename) + logger.Infof("Retrieving manifest: %s", url) + var response *http.Response + var err error + for i := 1; i <= 3; i++ { + // #nosec G107 This is only test code and is expected to read from a url + response, err = http.Get(url) + if err != nil { + if i == 3 { + panic(errors.Wrapf(err, "failed to read manifest from %s", url)) + } + logger.Warningf("failed to read manifest from %s. retrying in 1sec. %v", url, err) + time.Sleep(time.Second) + continue + } + break + } + defer response.Body.Close() + + content, err := ioutil.ReadAll(response.Body) + if err != nil { + panic(errors.Wrapf(err, "failed to read content from %s", url)) + } + return string(content) +} From 093cf6dbbd0dab19e6a7c46b7614fb5c63d73eb8 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Thu, 15 Jul 2021 16:04:05 -0600 Subject: [PATCH 2/3] cassandra: generate the crds from types for v1 The CRDs v1 requires the full schema for all settings, so we now generate the CRDs for cassandra for full fidelity of all the settings. Signed-off-by: Travis Nielsen --- .../integration-test-cassandra-suite.yaml | 2 +- Documentation/cassandra.md | 4 +- PendingReleaseNotes.md | 6 + build/crds/build-crds.sh | 35 +- .../examples/kubernetes/cassandra/crds.yaml | 894 ++++++++++++++++++ .../kubernetes/cassandra/operator.yaml | 84 +- pkg/apis/cassandra.rook.io/v1alpha1/types.go | 38 +- .../installer/cassandra_manifests.go | 147 +-- 8 files changed, 967 insertions(+), 243 deletions(-) create mode 100644 cluster/examples/kubernetes/cassandra/crds.yaml diff --git a/.github/workflows/integration-test-cassandra-suite.yaml b/.github/workflows/integration-test-cassandra-suite.yaml index 56527b9bd..e76e5f23c 100644 --- a/.github/workflows/integration-test-cassandra-suite.yaml +++ b/.github/workflows/integration-test-cassandra-suite.yaml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - kubernetes-versions : ['v1.15.12', 'v1.21.0'] + kubernetes-versions : ['v1.16.15', 'v1.21.0'] steps: - name: checkout uses: actions/checkout@v2 diff --git a/Documentation/cassandra.md b/Documentation/cassandra.md index 31f539e32..91abe3112 100644 --- a/Documentation/cassandra.md +++ b/Documentation/cassandra.md @@ -13,7 +13,7 @@ indent: true ## Prerequisites -A Kubernetes cluster is necessary to run the Rook Cassandra operator. +A Kubernetes cluster (v1.16 or higher) is necessary to run the Rook Cassandra operator. To make sure you have a Kubernetes cluster that is ready for `Rook`, you can [follow these instructions](k8s-pre-reqs.md) (the flexvolume plugin is not necessary for Cassandra) ## Deploy Cassandra Operator @@ -23,6 +23,7 @@ First deploy the Rook Cassandra Operator using the following commands: ```console $ git clone --single-branch --branch {{ branchName }} https://github.com/rook/rook.git cd rook/cluster/examples/kubernetes/cassandra +kubectl apply -f crds.yaml kubectl apply -f operator.yaml ``` @@ -133,6 +134,7 @@ To clean up all resources associated with this walk-through, you can run the com ```console kubectl delete -f cluster.yaml kubectl delete -f operator.yaml +kubectl delete -f crds.yaml ``` ## Troubleshooting diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index d7e2bb33d..df2d70e17 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -34,3 +34,9 @@ v1.7... - Add support for Kubernetes TLS secret for referring TLS certs needed for ceph RGW server. - Stretch clusters are considered stable - Ceph v16.2.5 or greater is required for stretch clusters + +### Cassandra + +- CRDs converted from v1beta1 to v1 + - Schema is generated from the internal types for more complete validation + - Minimum K8s version for the v1 CRDs is K8s 1.16 diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 266a8bb42..8a23bca3e 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -24,9 +24,10 @@ YQ_BIN_PATH=$2 # allowDangerousTypes is used to accept float64 CRD_OPTIONS="crd:maxDescLen=$MAX_DESC_LEN,trivialVersions=true,generateEmbeddedObjectMeta=true,allowDangerousTypes=true" OLM_CATALOG_DIR="${SCRIPT_ROOT}/cluster/olm/ceph/deploy/crds" -CRDS_FILE_PATH="${SCRIPT_ROOT}/cluster/examples/kubernetes/ceph/crds.yaml" -HELM_CRDS_FILE_PATH="${SCRIPT_ROOT}/cluster/charts/rook-ceph/templates/resources.yaml" -CRDS_BEFORE_1_16_FILE_PATH="${SCRIPT_ROOT}/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml" +CEPH_CRDS_FILE_PATH="${SCRIPT_ROOT}/cluster/examples/kubernetes/ceph/crds.yaml" +CEPH_HELM_CRDS_FILE_PATH="${SCRIPT_ROOT}/cluster/charts/rook-ceph/templates/resources.yaml" +CEPH_CRDS_BEFORE_1_16_FILE_PATH="${SCRIPT_ROOT}/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml" +CASSANDRA_CRDS_DIR="${SCRIPT_ROOT}/cluster/examples/kubernetes/cassandra" ############# # FUNCTIONS # @@ -39,10 +40,20 @@ copy_ob_obc_crds() { } generating_crds_v1() { - echo "Generating v1 in crds.yaml" + echo "Generating ceph crds" "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/ceph.rook.io/v1" output:crd:artifacts:config="$OLM_CATALOG_DIR" # the csv upgrade is failing on the volumeClaimTemplate.metadata.annotations.crushDeviceClass unless we preserve the annotations as an unknown field $YQ_BIN_PATH w -i cluster/olm/ceph/deploy/crds/ceph.rook.io_cephclusters.yaml spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields true + + echo "Generating cassandra crds" + "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/cassandra.rook.io/v1alpha1" output:crd:artifacts:config="$CASSANDRA_CRDS_DIR" + # Format with yq for consistent whitespace + $YQ_BIN_PATH read $CASSANDRA_CRDS_DIR/cassandra.rook.io_clusters.yaml > $CASSANDRA_CRDS_DIR/crds.yaml + rm -f $CASSANDRA_CRDS_DIR/cassandra.rook.io_clusters.yaml + + echo "Generating nfs crds" + "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/nfs.rook.io/v1alpha1" output:crd:artifacts:config="$NFS_CRDS_DIR" + mv $NFS_CRDS_DIR/nfs.rook.io_nfsservers.yaml $NFS_CRDS_DIR/crds.yaml } generating_crds_v1alpha2() { @@ -60,9 +71,9 @@ generate_vol_rep_crds() { } generating_main_crd() { - true > "$CRDS_FILE_PATH" - true > "$HELM_CRDS_FILE_PATH" -cat < "$CRDS_FILE_PATH" + true > "$CEPH_CRDS_FILE_PATH" + true > "$CEPH_HELM_CRDS_FILE_PATH" +cat < "$CEPH_CRDS_FILE_PATH" ############################################################################## # Create the CRDs that are necessary before creating your Rook cluster. # These resources *must* be created before the cluster.yaml or their variants. @@ -78,18 +89,18 @@ build_helm_resources() { echo "{{- if semverCompare \">=1.16.0\" .Capabilities.KubeVersion.GitVersion }}" # Add helm annotations to all CRDS and skip the first 4 lines of crds.yaml - "$YQ_BIN_PATH" w -d'*' "$CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep | tail -n +5 + "$YQ_BIN_PATH" w -d'*' "$CEPH_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep | tail -n +5 # add else echo "{{- else }}" # add footer - cat "$CRDS_BEFORE_1_16_FILE_PATH" + cat "$CEPH_CRDS_BEFORE_1_16_FILE_PATH" # DO NOT REMOVE the empty line, it is necessary echo "" echo "{{- end }}" echo "{{- end }}" - } >>"$HELM_CRDS_FILE_PATH" + } >>"$CEPH_HELM_CRDS_FILE_PATH" } ######## @@ -108,10 +119,10 @@ generate_vol_rep_crds generating_main_crd for crd in "$OLM_CATALOG_DIR/"*.yaml; do - echo "---" >> "$CRDS_FILE_PATH" # yq doesn't output doc separators + echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output doc separators # Process each intermediate CRD file with yq to enforce consistent formatting in the final product # regardless of whether yq was used in previous steps to alter CRD intermediate files. - $YQ_BIN_PATH read "$crd" >> "$CRDS_FILE_PATH" + $YQ_BIN_PATH read "$crd" >> "$CEPH_CRDS_FILE_PATH" done build_helm_resources diff --git a/cluster/examples/kubernetes/cassandra/crds.yaml b/cluster/examples/kubernetes/cassandra/crds.yaml new file mode 100644 index 000000000..d619be299 --- /dev/null +++ b/cluster/examples/kubernetes/cassandra/crds.yaml @@ -0,0 +1,894 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c + creationTimestamp: null + name: clusters.cassandra.rook.io +spec: + group: cassandra.rook.io + names: + kind: Cluster + listKind: ClusterList + plural: clusters + singular: cluster + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterSpec is the desired state for a Cassandra Cluster. + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on each Pod related object. + nullable: true + type: object + datacenter: + description: Datacenter that will make up this cluster. + nullable: true + properties: + name: + description: Name of the Cassandra Datacenter. Used in the cassandra-rackdc.properties file. + type: string + racks: + description: Racks of the specific Datacenter. + items: + description: RackSpec is the desired state for a Cassandra Rack. + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on each Pod related object. + nullable: true + type: object + configMapName: + description: User-provided ConfigMap applied to the specific statefulset. + nullable: true + type: string + jmxExporterConfigMapName: + description: User-provided ConfigMap for jmx prometheus exporter + nullable: true + type: string + members: + description: Members is the number of Cassandra instances in this rack. + format: int32 + type: integer + name: + description: Name of the Cassandra Rack. Used in the cassandra-rackdc.properties file. + type: string + placement: + description: Placement describes restrictions for the nodes Cassandra is scheduled on. + nullable: true + properties: + nodeAffinity: + description: NodeAffinity is a group of node affinity scheduling rules + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: PodAffinity is a group of inter pod affinity scheduling rules + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: PodAntiAffinity is a group of inter pod anti affinity scheduling rules + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. This field is alpha-level and is only honored when PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator + items: + description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It''s a required field. Default value is 1 and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for an incoming pod if and only if every possible node assigment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + resources: + description: Resources the Cassandra Pods will use. + nullable: true + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + storage: + description: Storage describes the underlying storage that Cassandra will consume. + properties: + nodes: + items: + description: Node is a storage nodes + properties: + name: + type: string + type: object + nullable: true + type: array + volumeClaimTemplates: + description: PersistentVolumeClaims to use as storage + items: + description: PersistentVolumeClaim is a user's request for and claim to a persistent volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: 'Spec defines the desired characteristics of a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Represents the actual resources of the underlying volume. + type: object + conditions: + description: Current Condition of persistent volume claim. If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails details about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, this should be a short, machine understandable string that gives the reason for condition's last transition. If it reports "ResizeStarted" that means the underlying persistent volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: array + type: object + required: + - members + - name + type: object + type: array + required: + - name + - racks + type: object + mode: + description: Mode selects an operating mode. + type: string + repository: + description: Repository to pull the image from. + nullable: true + type: string + sidecarImage: + description: User-provided image for the sidecar that replaces default. + nullable: true + properties: + repository: + description: Repository to pull the image from. + type: string + version: + description: Version of the image. + type: string + required: + - version + type: object + version: + description: Version of Cassandra to use. + type: string + required: + - version + type: object + status: + description: ClusterStatus is the status of a Cassandra Cluster + nullable: true + properties: + racks: + additionalProperties: + description: RackStatus is the status of a Cassandra Rack + properties: + conditions: + description: Conditions are the latest available observations of a rack's state. + items: + description: RackCondition is an observation about the state of a rack. + properties: + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array + members: + description: Members is the current number of members requested in the specific Rack + format: int32 + type: integer + readyMembers: + description: ReadyMembers is the number of ready members in the specific Rack + format: int32 + type: integer + required: + - members + - readyMembers + type: object + type: object + type: object + required: + - metadata + - spec + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/cluster/examples/kubernetes/cassandra/operator.yaml b/cluster/examples/kubernetes/cassandra/operator.yaml index 807352870..cc406664e 100644 --- a/cluster/examples/kubernetes/cassandra/operator.yaml +++ b/cluster/examples/kubernetes/cassandra/operator.yaml @@ -2,83 +2,7 @@ apiVersion: v1 kind: Namespace metadata: - name: rook-cassandra-system - ---- -# Cassandra Cluster CRD -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusters.cassandra.rook.io -spec: - group: cassandra.rook.io - names: - kind: Cluster - listKind: ClusterList - plural: clusters - singular: cluster - scope: Namespaced - version: v1alpha1 - validation: - openAPIV3Schema: - properties: - spec: - type: object - properties: - version: - type: string - description: "Version of Cassandra" - datacenter: - type: object - properties: - name: - type: string - description: "Datacenter Name" - racks: - type: array - properties: - name: - type: string - members: - type: integer - configMapName: - type: string - jmxExporterConfigMapName: - type: string - storage: - type: object - properties: - volumeClaimTemplates: - type: object - # TODO: Check if we can ref the already existing schema - required: - - "volumeClaimTemplates" - placement: - type: object - resources: - type: object - properties: - # TODO: Check if we can ref the already existing schema - cassandra: - type: object - sidecar: - type: object - required: - - "cassandra" - - "sidecar" - sidecarImage: - type: object - required: - - "name" - - "members" - - "storage" - - "resources" - required: - - "name" - required: - - "version" - - "datacenter" - + name: rook-cassandra-system # namespace:operator --- # ClusterRole for cassandra-operator. apiVersion: rbac.authorization.k8s.io/v1 @@ -147,7 +71,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: rook-cassandra-operator - namespace: rook-cassandra-system + namespace: rook-cassandra-system # namespace:operator --- # Bind cassandra-operator ServiceAccount with ClusterRole. kind: ClusterRoleBinding @@ -161,14 +85,14 @@ roleRef: subjects: - kind: ServiceAccount name: rook-cassandra-operator - namespace: rook-cassandra-system + namespace: rook-cassandra-system # namespace:operator --- # cassandra-operator StatefulSet. apiVersion: apps/v1 kind: StatefulSet metadata: name: rook-cassandra-operator - namespace: rook-cassandra-system + namespace: rook-cassandra-system # namespace:operator labels: app: rook-cassandra-operator spec: diff --git a/pkg/apis/cassandra.rook.io/v1alpha1/types.go b/pkg/apis/cassandra.rook.io/v1alpha1/types.go index 0d61240bf..99881d30f 100644 --- a/pkg/apis/cassandra.rook.io/v1alpha1/types.go +++ b/pkg/apis/cassandra.rook.io/v1alpha1/types.go @@ -52,8 +52,10 @@ const ( type Cluster struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata"` - Spec ClusterSpec `json:"spec"` - Status ClusterStatus `json:"status"` + Spec ClusterSpec `json:"spec"` + // +optional + // +nullable + Status ClusterStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -67,16 +69,25 @@ type ClusterList struct { // ClusterSpec is the desired state for a Cassandra Cluster. type ClusterSpec struct { // The annotations-related configuration to add/set on each Pod related object. - Annotations rook.Annotations `json:"annotations"` + // +optional + // +nullable + Annotations rook.Annotations `json:"annotations,omitempty"` // Version of Cassandra to use. Version string `json:"version"` // Repository to pull the image from. + // +optional + // +nullable Repository *string `json:"repository,omitempty"` // Mode selects an operating mode. + // +optional Mode ClusterMode `json:"mode,omitempty"` // Datacenter that will make up this cluster. - Datacenter DatacenterSpec `json:"datacenter"` + // +optional + // +nullable + Datacenter DatacenterSpec `json:"datacenter,omitempty"` // User-provided image for the sidecar that replaces default. + // +optional + // +nullable SidecarImage *ImageSpec `json:"sidecarImage,omitempty"` } @@ -102,17 +113,27 @@ type RackSpec struct { // Members is the number of Cassandra instances in this rack. Members int32 `json:"members"` // User-provided ConfigMap applied to the specific statefulset. + // +optional + // +nullable ConfigMapName *string `json:"configMapName,omitempty"` // User-provided ConfigMap for jmx prometheus exporter + // +optional + // +nullable JMXExporterConfigMapName *string `json:"jmxExporterConfigMapName,omitempty"` // Storage describes the underlying storage that Cassandra will consume. - Storage StorageScopeSpec `json:"storage"` + Storage StorageScopeSpec `json:"storage,omitempty"` // The annotations-related configuration to add/set on each Pod related object. - Annotations map[string]string `json:"annotations"` + // +optional + // +nullable + Annotations map[string]string `json:"annotations,omitempty"` // Placement describes restrictions for the nodes Cassandra is scheduled on. + // +optional + // +nullable Placement *Placement `json:"placement,omitempty"` // Resources the Cassandra Pods will use. - Resources corev1.ResourceRequirements `json:"resources"` + // +optional + // +nullable + Resources corev1.ResourceRequirements `json:"resources,omitempty"` } // ImageSpec is the desired state for a container image. @@ -120,7 +141,8 @@ type ImageSpec struct { // Version of the image. Version string `json:"version"` // Repository to pull the image from. - Repository string `json:"repository"` + // +optional + Repository string `json:"repository,omitempty"` } // ClusterStatus is the status of a Cassandra Cluster diff --git a/tests/framework/installer/cassandra_manifests.go b/tests/framework/installer/cassandra_manifests.go index f55a8eeaa..413c0a23d 100644 --- a/tests/framework/installer/cassandra_manifests.go +++ b/tests/framework/installer/cassandra_manifests.go @@ -18,6 +18,7 @@ package installer import ( "fmt" + "strings" cassandrav1alpha1 "github.com/rook/rook/pkg/apis/cassandra.rook.io/v1alpha1" ) @@ -25,151 +26,15 @@ import ( type CassandraManifests struct{} func (i *CassandraManifests) GetCassandraCRDs() string { - return `apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: clusters.cassandra.rook.io -spec: - group: cassandra.rook.io - names: - kind: Cluster - listKind: ClusterList - plural: clusters - singular: cluster - scope: Namespaced - version: v1alpha1` + manifest := readManifest("cassandra", "crds.yaml") + return manifest } func (i *CassandraManifests) GetCassandraOperator(namespace string) string { + manifest := readManifest("cassandra", "operator.yaml") + manifest = strings.ReplaceAll(manifest, "rook-cassandra-system # namespace:operator", namespace) - return fmt.Sprintf(` -# Namespace where Cassandra Operator will live -apiVersion: v1 -kind: Namespace -metadata: - name: %[1]s - ---- -# ClusterRole for cassandra-operator. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-cassandra-operator -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - delete - - apiGroups: - - "" - resources: - - services - - serviceaccounts - verbs: - - get - - list - - watch - - create - - apiGroups: - - "" - resources: - - persistentvolumes - - persistentvolumeclaims - verbs: - - get - - delete - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - apps - resources: - - statefulsets - verbs: - - "*" - - apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - create - - apiGroups: - - cassandra.rook.io - resources: - - "*" - verbs: - - "*" - - apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - patch ---- -# ServiceAccount for cassandra-operator. Serves as its authorization identity. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-cassandra-operator - namespace: %[1]s ---- -# Bind cassandra-operator ServiceAccount with ClusterRole. -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-cassandra-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-cassandra-operator -subjects: -- kind: ServiceAccount - name: rook-cassandra-operator - namespace: %[1]s ---- -# cassandra-operator StatefulSet. - apiVersion: apps/v1 - kind: StatefulSet - metadata: - name: rook-cassandra-operator - namespace: %[1]s - spec: - replicas: 1 - serviceName: "non-existent-service" - selector: - matchLabels: - app: rook-cassandra-operator - template: - metadata: - labels: - app: rook-cassandra-operator - spec: - serviceAccountName: rook-cassandra-operator - containers: - - name: rook-cassandra-operator - image: rook/cassandra:master - imagePullPolicy: "IfNotPresent" - args: ["cassandra", "operator"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -`, namespace) + return manifest } func (i *CassandraManifests) GetCassandraCluster(namespace string, count int, mode cassandrav1alpha1.ClusterMode) string { From f114db952b5c8215baa311a551dcb3a73dd17de9 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Mon, 19 Jul 2021 13:51:21 -0600 Subject: [PATCH 3/3] nfs: generate the crds from types for v1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CRDs v1 requires the full schema for all settings, so we now generate the CRDs for nfs for full fidelity of all settings. Co-authored-by: Nicolaj Græsholt Signed-off-by: Travis Nielsen --- .../workflows/integration-test-nfs-suite.yaml | 2 +- Documentation/k8s-pre-reqs.md | 5 +- Documentation/nfs.md | 6 +- PendingReleaseNotes.md | 6 + build/crds/build-crds.sh | 5 +- cluster/examples/kubernetes/nfs/common.yaml | 177 ---------- cluster/examples/kubernetes/nfs/crds.yaml | 141 ++++++++ cluster/examples/kubernetes/nfs/operator.yaml | 11 +- tests/framework/installer/nfs_manifests.go | 309 +----------------- 9 files changed, 178 insertions(+), 484 deletions(-) delete mode 100644 cluster/examples/kubernetes/nfs/common.yaml create mode 100644 cluster/examples/kubernetes/nfs/crds.yaml diff --git a/.github/workflows/integration-test-nfs-suite.yaml b/.github/workflows/integration-test-nfs-suite.yaml index d7cfffc69..70ef77ecf 100644 --- a/.github/workflows/integration-test-nfs-suite.yaml +++ b/.github/workflows/integration-test-nfs-suite.yaml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - kubernetes-versions : ['v1.15.12', 'v1.21.0'] + kubernetes-versions : ['v1.16.15', 'v1.21.0'] steps: - name: checkout uses: actions/checkout@v2 diff --git a/Documentation/k8s-pre-reqs.md b/Documentation/k8s-pre-reqs.md index 75c4a1835..df45dc19f 100644 --- a/Documentation/k8s-pre-reqs.md +++ b/Documentation/k8s-pre-reqs.md @@ -12,9 +12,10 @@ you can quickly set one up using [Minikube](#minikube), [Kubeadm](#kubeadm) or [ ## Minimum Version -Kubernetes **v1.11** or higher is supported by Rook. +Kubernetes **v1.11** or higher is supported for the Ceph operator. +Kubernetes **v1.16** or higher is supported for the Cassandra and NFS operators. -**Important** If you are using K8s 1.15 or older, you will need to create a different version of the Rook CRDs. Create the `crds.yaml` found in the [pre-k8s-1.16](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/pre-k8s-1.16) subfolder of the example manifests. +**Important** If you are using K8s 1.15 or older, you will need to create a different version of the Ceph CRDs. Create the `crds.yaml` found in the [pre-k8s-1.16](https://github.com/rook/rook/blob/{{ branchName }}/cluster/examples/kubernetes/ceph/pre-k8s-1.16) subfolder of the example manifests. ## Ceph Prerequisites diff --git a/Documentation/nfs.md b/Documentation/nfs.md index 76abbfaf5..e8227e823 100644 --- a/Documentation/nfs.md +++ b/Documentation/nfs.md @@ -11,7 +11,7 @@ NFS allows remote hosts to mount filesystems over a network and interact with th ## Prerequisites -1. A Kubernetes cluster is necessary to run the Rook NFS operator. To make sure you have a Kubernetes cluster that is ready for `Rook`, you can [follow these instructions](k8s-pre-reqs.md). +1. A Kubernetes cluster (v1.16 or higher) is necessary to run the Rook NFS operator. To make sure you have a Kubernetes cluster that is ready for `Rook`, you can [follow these instructions](k8s-pre-reqs.md). 2. The desired volume to export needs to be attached to the NFS server pod via a [PVC](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims). Any type of PVC can be attached and exported, such as Host Path, AWS Elastic Block Store, GCP Persistent Disk, CephFS, Ceph RBD, etc. The limitations of these volumes also apply while they are shared by NFS. @@ -25,7 +25,7 @@ First deploy the Rook NFS operator using the following commands: ```console $ git clone --single-branch --branch {{ branchName }} https://github.com/rook/rook.git cd rook/cluster/examples/kubernetes/nfs -kubectl create -f common.yaml +kubectl create -f crds.yaml kubectl create -f operator.yaml ``` @@ -590,7 +590,7 @@ kubectl delete -f psp.yaml kubectl delete -f scc.yaml # if deployed kubectl delete -f operator.yaml kubectl delete -f webhook.yaml # if deployed -kubectl delete -f common.yaml +kubectl delete -f crds.yaml ``` ## Troubleshooting diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index df2d70e17..ad0b4133a 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -40,3 +40,9 @@ v1.7... - CRDs converted from v1beta1 to v1 - Schema is generated from the internal types for more complete validation - Minimum K8s version for the v1 CRDs is K8s 1.16 + +### NFS + +- CRDs converted from v1beta1 to v1 + - Schema is generated from the internal types for more complete validation + - Minimum K8s version for the v1 CRDs is K8s 1.16 diff --git a/build/crds/build-crds.sh b/build/crds/build-crds.sh index 8a23bca3e..70bdd12d1 100755 --- a/build/crds/build-crds.sh +++ b/build/crds/build-crds.sh @@ -28,6 +28,7 @@ CEPH_CRDS_FILE_PATH="${SCRIPT_ROOT}/cluster/examples/kubernetes/ceph/crds.yaml" CEPH_HELM_CRDS_FILE_PATH="${SCRIPT_ROOT}/cluster/charts/rook-ceph/templates/resources.yaml" CEPH_CRDS_BEFORE_1_16_FILE_PATH="${SCRIPT_ROOT}/cluster/examples/kubernetes/ceph/pre-k8s-1.16/crds.yaml" CASSANDRA_CRDS_DIR="${SCRIPT_ROOT}/cluster/examples/kubernetes/cassandra" +NFS_CRDS_DIR="${SCRIPT_ROOT}/cluster/examples/kubernetes/nfs" ############# # FUNCTIONS # @@ -53,7 +54,9 @@ generating_crds_v1() { echo "Generating nfs crds" "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/nfs.rook.io/v1alpha1" output:crd:artifacts:config="$NFS_CRDS_DIR" - mv $NFS_CRDS_DIR/nfs.rook.io_nfsservers.yaml $NFS_CRDS_DIR/crds.yaml + # Format with yq for consistent whitespace + $YQ_BIN_PATH read $NFS_CRDS_DIR/nfs.rook.io_nfsservers.yaml > $NFS_CRDS_DIR/crds.yaml + rm -f $NFS_CRDS_DIR/nfs.rook.io_nfsservers.yaml } generating_crds_v1alpha2() { diff --git a/cluster/examples/kubernetes/nfs/common.yaml b/cluster/examples/kubernetes/nfs/common.yaml deleted file mode 100644 index 619ca7ee4..000000000 --- a/cluster/examples/kubernetes/nfs/common.yaml +++ /dev/null @@ -1,177 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: rook-nfs-system ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.3.0 - creationTimestamp: null - name: nfsservers.nfs.rook.io -spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - name: AGE - type: date - - JSONPath: .status.state - description: NFS Server instance state - name: State - type: string - group: nfs.rook.io - names: - kind: NFSServer - listKind: NFSServerList - plural: nfsservers - singular: nfsserver - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: NFSServer is the Schema for the nfsservers API - properties: - apiVersion: - description: - "APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" - type: string - kind: - description: - "Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - type: string - metadata: - type: object - spec: - description: NFSServerSpec represents the spec of NFS daemon - properties: - annotations: - additionalProperties: - type: string - description: - The annotations-related configuration to add/set on each - Pod related object. - type: object - exports: - description: The parameters to configure the NFS export - items: - description: ExportsSpec represents the spec of NFS exports - properties: - name: - description: Name of the export - type: string - persistentVolumeClaim: - description: PVC from which the NFS daemon gets storage for sharing - properties: - claimName: - description: - "ClaimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims" - type: string - readOnly: - description: - Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - server: - description: The NFS server configuration - properties: - accessMode: - description: - Reading and Writing permissions on the export - Valid values are "ReadOnly", "ReadWrite" and "none" - enum: - - ReadOnly - - ReadWrite - - none - type: string - allowedClients: - description: The clients allowed to access the NFS export - items: - description: - AllowedClientsSpec represents the client specs - for accessing the NFS export - properties: - accessMode: - description: - Reading and Writing permissions for the - client to access the NFS export Valid values are "ReadOnly", - "ReadWrite" and "none" Gets overridden when ServerSpec.accessMode - is specified - enum: - - ReadOnly - - ReadWrite - - none - type: string - clients: - description: - The clients that can access the share Values - can be hostname, ip address, netgroup, CIDR network - address, or all - items: - type: string - type: array - name: - description: Name of the clients group - type: string - squash: - description: - Squash options for clients Valid values - are "none", "rootid", "root", and "all" Gets overridden - when ServerSpec.squash is specified - enum: - - none - - rootid - - root - - all - type: string - type: object - type: array - squash: - description: - This prevents the root users connected remotely - from having root privileges Valid values are "none", "rootid", - "root", and "all" - enum: - - none - - rootid - - root - - all - type: string - type: object - type: object - type: array - replicas: - description: Replicas of the NFS daemon - type: integer - type: object - status: - description: NFSServerStatus defines the observed state of NFSServer - properties: - message: - type: string - reason: - type: string - state: - type: string - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/cluster/examples/kubernetes/nfs/crds.yaml b/cluster/examples/kubernetes/nfs/crds.yaml new file mode 100644 index 000000000..f47ffe197 --- /dev/null +++ b/cluster/examples/kubernetes/nfs/crds.yaml @@ -0,0 +1,141 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.1-0.20210420220833-f284e2e8098c + creationTimestamp: null + name: nfsservers.nfs.rook.io +spec: + group: nfs.rook.io + names: + kind: NFSServer + listKind: NFSServerList + plural: nfsservers + singular: nfsserver + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - description: NFS Server instance state + jsonPath: .status.state + name: State + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NFSServer is the Schema for the nfsservers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NFSServerSpec represents the spec of NFS daemon + properties: + annotations: + additionalProperties: + type: string + description: The annotations-related configuration to add/set on each Pod related object. + type: object + exports: + description: The parameters to configure the NFS export + items: + description: ExportsSpec represents the spec of NFS exports + properties: + name: + description: Name of the export + type: string + persistentVolumeClaim: + description: PVC from which the NFS daemon gets storage for sharing + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + server: + description: The NFS server configuration + properties: + accessMode: + description: Reading and Writing permissions on the export Valid values are "ReadOnly", "ReadWrite" and "none" + enum: + - ReadOnly + - ReadWrite + - none + type: string + allowedClients: + description: The clients allowed to access the NFS export + items: + description: AllowedClientsSpec represents the client specs for accessing the NFS export + properties: + accessMode: + description: Reading and Writing permissions for the client to access the NFS export Valid values are "ReadOnly", "ReadWrite" and "none" Gets overridden when ServerSpec.accessMode is specified + enum: + - ReadOnly + - ReadWrite + - none + type: string + clients: + description: The clients that can access the share Values can be hostname, ip address, netgroup, CIDR network address, or all + items: + type: string + type: array + name: + description: Name of the clients group + type: string + squash: + description: Squash options for clients Valid values are "none", "rootid", "root", and "all" Gets overridden when ServerSpec.squash is specified + enum: + - none + - rootid + - root + - all + type: string + type: object + type: array + squash: + description: This prevents the root users connected remotely from having root privileges Valid values are "none", "rootid", "root", and "all" + enum: + - none + - rootid + - root + - all + type: string + type: object + type: object + type: array + replicas: + description: Replicas of the NFS daemon + type: integer + type: object + status: + description: NFSServerStatus defines the observed state of NFSServer + properties: + message: + type: string + reason: + type: string + state: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index f515e0ad5..b28990977 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -1,8 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: rook-nfs-system # namespace:operator +--- +apiVersion: v1 kind: ServiceAccount metadata: name: rook-nfs-operator - namespace: rook-nfs-system + namespace: rook-nfs-system # namespace:operator --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -15,7 +20,7 @@ roleRef: subjects: - kind: ServiceAccount name: rook-nfs-operator - namespace: rook-nfs-system + namespace: rook-nfs-system # namespace:operator --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -101,7 +106,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: rook-nfs-operator - namespace: rook-nfs-system + namespace: rook-nfs-system # namespace:operator labels: app: rook-nfs-operator spec: diff --git a/tests/framework/installer/nfs_manifests.go b/tests/framework/installer/nfs_manifests.go index 1f134820e..b5a270271 100644 --- a/tests/framework/installer/nfs_manifests.go +++ b/tests/framework/installer/nfs_manifests.go @@ -18,313 +18,28 @@ package installer import ( "strconv" + "strings" ) type NFSManifests struct { } // GetNFSServerCRDs returns NFSServer CRD definition -func (i *NFSManifests) GetNFSServerCRDs() string { - return `apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: nfsservers.nfs.rook.io -spec: - additionalPrinterColumns: - - JSONPath: .status.state - description: NFS Server instance state - name: State - type: string - group: nfs.rook.io - names: - kind: NFSServer - listKind: NFSServerList - plural: nfsservers - singular: nfsserver - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: NFSServer is the Schema for the nfsservers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NFSServerSpec defines the desired state of NFSServer - properties: - annotations: - additionalProperties: - type: string - description: The annotations-related configuration to add/set on each - Pod related object. - type: object - exports: - description: The parameters to configure the NFS export - items: - description: ExportsSpec represents the spec of NFS exports - properties: - name: - description: Name of the export - type: string - persistentVolumeClaim: - description: PVC from which the NFS daemon gets storage for sharing - properties: - claimName: - description: 'ClaimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - server: - description: The NFS server configuration - properties: - accessMode: - description: Reading and Writing permissions on the export - Valid values are "ReadOnly", "ReadWrite" and "none" - enum: - - ReadOnly - - ReadWrite - - none - type: string - allowedClients: - description: The clients allowed to access the NFS export - items: - description: AllowedClientsSpec represents the client specs - for accessing the NFS export - properties: - accessMode: - description: Reading and Writing permissions for the - client to access the NFS export Valid values are "ReadOnly", - "ReadWrite" and "none" Gets overridden when ServerSpec.accessMode - is specified - enum: - - ReadOnly - - ReadWrite - - none - type: string - clients: - description: The clients that can access the share Values - can be hostname, ip address, netgroup, CIDR network - address, or all - items: - type: string - type: array - name: - description: Name of the clients group - type: string - squash: - description: Squash options for clients Valid values - are "none", "rootid", "root", and "all" Gets overridden - when ServerSpec.squash is specified - enum: - - none - - rootid - - root - - all - type: string - type: object - type: array - squash: - description: This prevents the root users connected remotely - from having root privileges Valid values are "none", "rootid", - "root", and "all" - enum: - - none - - rootid - - root - - all - type: string - type: object - type: object - type: array - replicas: - description: Replicas of the NFS daemon - type: integer - type: object - status: - description: NFSServerStatus defines the observed state of NFSServer - properties: - message: - type: string - reason: - type: string - state: - type: string - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] -` +func (n *NFSManifests) GetNFSServerCRDs() string { + manifest := readManifest("nfs", "crds.yaml") + logger.Info(manifest) + return manifest } // GetNFSServerOperator returns the NFSServer operator definition -func (i *NFSManifests) GetNFSServerOperator(namespace string) string { - return `apiVersion: v1 -kind: Namespace -metadata: - name: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: rook-nfs-operator -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - get - - watch - - create -- apiGroups: - - "" - resources: - - services - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - create - - get - - list - - patch - - update - - watch -- apiGroups: - - nfs.rook.io - resources: - - nfsservers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - nfs.rook.io - resources: - - nfsservers/status - verbs: - - get - - patch - - update ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-nfs-operator - namespace: ` + namespace + ` ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: rook-nfs-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-nfs-operator -subjects: -- kind: ServiceAccount - name: rook-nfs-operator - namespace: ` + namespace + ` ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-nfs-operator - namespace: ` + namespace + ` - labels: - app: rook-nfs-operator -spec: - selector: - matchLabels: - app: rook-nfs-operator - replicas: 1 - template: - metadata: - labels: - app: rook-nfs-operator - spec: - serviceAccountName: rook-nfs-operator - containers: - - name: rook-nfs-operator - imagePullPolicy: IfNotPresent - image: rook/nfs:master - args: ["nfs", "operator"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace -` +func (n *NFSManifests) GetNFSServerOperator(namespace string) string { + manifest := readManifest("nfs", "operator.yaml") + manifest = strings.ReplaceAll(manifest, "rook-nfs-system # namespace:operator", namespace) + return manifest } // GetNFSServerPV returns NFSServer PV definition -func (i *NFSManifests) GetNFSServerPV(namespace string, clusterIP string) string { +func (n *NFSManifests) GetNFSServerPV(namespace string, clusterIP string) string { return `apiVersion: v1 kind: PersistentVolume metadata: @@ -362,7 +77,7 @@ spec: } // GetNFSServerPVC returns NFSServer PVC definition -func (i *NFSManifests) GetNFSServerPVC(namespace string) string { +func (n *NFSManifests) GetNFSServerPVC(namespace string) string { return ` --- apiVersion: storage.k8s.io/v1 @@ -420,7 +135,7 @@ spec: } // GetNFSServer returns NFSServer CRD instance definition -func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string { +func (n *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string { return ` apiVersion: v1 kind: ServiceAccount