diff --git a/Documentation/cockroachdb-cluster-crd.md b/Documentation/cockroachdb-cluster-crd.md index 02ba8b5bc..776c2c8f6 100644 --- a/Documentation/cockroachdb-cluster-crd.md +++ b/Documentation/cockroachdb-cluster-crd.md @@ -21,6 +21,15 @@ metadata: spec: scope: nodeCount: 3 + volumeClaimTemplates: + - spec: + accessModes: [ "ReadWriteOnce" ] + # Uncomment and specify your StorageClass, otherwise + # the cluster admin defined default StorageClass will be used. + #storageClassName: "my-storage-class" + resources: + requests: + storage: "100Gi" network: ports: - name: http @@ -28,7 +37,6 @@ spec: - name: grpc port: 26257 secure: false - volumeSize: 100Gi cachePercent: 25 maxSQLMemoryPercent: 25 ``` @@ -40,7 +48,6 @@ spec: The settings below are specific to CockroachDB database clusters: * `secure`: `true` to create a secure cluster installation using certificates and encryption. `false` to create an insecure installation (strongly discouraged for production usage). Currently, only insecure is supported. -* `volumeSize`: Each database instance will get an underlying persistent data volume created to store its instance data using the default storage class. This value represents the size of the volume that will be created. The value should be expressed in the [standard Kubernetes resource format](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory). * `cachePercent`: The total size used for caches, expressed as a percentage of total physical memory. * `maxSQLMemoryPercent`: The maximum memory capacity available to store temporary data for SQL clients, expressed as a percentage of total physical memory. @@ -50,6 +57,7 @@ Under the `scope` field, a `StorageScopeSpec` can be specified to influence the These properties are currently supported: * `nodeCount`: The number of CockroachDB instances to create. Some of these instances may be scheduled on the same nodes, but exactly this many instances will be created and included in the cluster. +* `volumeClaimTemplates`: A list of PersistentVolumeClaim templates which must contain only **one or no** PersistentVolumeClaim. If no PersistentVolumeClaim is given an `emptyDir` will be given, meaning the instance data will be lost when a Pod is restarted. For an example of how PersistentVolumeClaim template should look, please look at the above [sample](#sample). ### Network diff --git a/Documentation/minio-object-store-crd.md b/Documentation/minio-object-store-crd.md index e9644052d..22df4caf0 100644 --- a/Documentation/minio-object-store-crd.md +++ b/Documentation/minio-object-store-crd.md @@ -18,6 +18,29 @@ metadata: spec: scope: nodeCount: 4 + # You can have multiple PersistentVolumeClaims in the volumeClaimTemplates list. + # Be aware though that all PersistentVolumeClaim Templates will be used for each intance (see nodeCount). + volumeClaimTemplates: + - metadata: + name: rook-minio-data1 + spec: + accessModes: [ "ReadWriteOnce" ] + # Uncomment and specify your StorageClass, otherwise + # the cluster admin defined default StorageClass will be used. + #storageClassName: "your-cluster-storageclass" + resources: + requests: + storage: "8Gi" + #- metadata: + # name: rook-minio-data2 + # spec: + # accessModes: [ "ReadWriteOnce" ] + # # Uncomment and specify your StorageClass, otherwise + # # the cluster admin defined default StorageClass will be used. + # #storageClassName: "my-storage-class" + # resources: + # requests: + # storage: "8Gi" placement: tolerations: nodeAffinity: @@ -26,7 +49,6 @@ spec: credentials: name: access-keys namespace: rook-minio - storageAmount: "10G" clusterDomain: ``` @@ -45,3 +67,4 @@ The settings below are specific to Minio object stores: Under the `scope` field, a `StorageScopeSpec` can be specified to influence the scope or boundaries of storage that the cluster will use for its underlying storage. These properties are currently supported: * `nodeCount`: The number of Minio instances to create. Some of these instances may be scheduled on the same nodes, but exactly this many instances will be created and included in the cluster. +* `volumeClaimTemplates`: A list of one or more PersistentVolumeClaim templates to use for each Minio repliace. For an example of how the list should look like, please look at the above [sample](#sample). diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index 368730109..1be467b2d 100755 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -222,7 +222,7 @@ spec: # osd: storage: # cluster level storage configuration and selection useAllNodes: true - useAllDevices: false + useAllDevices: true deviceFilter: location: config: diff --git a/cluster/examples/kubernetes/cockroachdb/cluster.yaml b/cluster/examples/kubernetes/cockroachdb/cluster.yaml index 2eb2fa082..0f08b3e6e 100644 --- a/cluster/examples/kubernetes/cockroachdb/cluster.yaml +++ b/cluster/examples/kubernetes/cockroachdb/cluster.yaml @@ -13,6 +13,18 @@ spec: # https://rook.io/docs/rook/master/cockroachdb-cluster-crd.html scope: nodeCount: 3 + # You can only have one PersistentVolumeClaim in this list! + volumeClaimTemplates: + - metadata: + name: rook-cockroachdb-data + spec: + accessModes: [ "ReadWriteOnce" ] + # Uncomment and specify your StorageClass, otherwise + # the cluster admin defined default StorageClass will be used. + #storageClassName: "your-cluster-storageclass" + resources: + requests: + storage: "1Gi" network: ports: - name: http @@ -20,6 +32,5 @@ spec: - name: grpc port: 26257 secure: false - volumeSize: 1Gi cachePercent: 25 maxSQLMemoryPercent: 25 diff --git a/cluster/examples/kubernetes/minio/object-store.yaml b/cluster/examples/kubernetes/minio/object-store.yaml index 83ab93518..7578a4be0 100644 --- a/cluster/examples/kubernetes/minio/object-store.yaml +++ b/cluster/examples/kubernetes/minio/object-store.yaml @@ -23,6 +23,28 @@ metadata: spec: scope: nodeCount: 4 + # You can have multiple PersistentVolumeClaims in the volumeClaimTemplates list. + # Be aware though that all PersistentVolumeClaim Templates will be used for each intance (see nodeCount). + volumeClaimTemplates: + - metadata: + name: rook-minio-data1 + spec: + accessModes: [ "ReadWriteOnce" ] + # Set the storage class that will be used, otherwise Kubernetes' default storage class will be used. + #storageClassName: "my-storage-class" + resources: + requests: + storage: "8Gi" + #- metadata: + # name: rook-minio-data2 + # spec: + # accessModes: [ "ReadWriteOnce" ] + # # Uncomment and specify your StorageClass, otherwise + # # the cluster admin defined default StorageClass will be used. + # #storageClassName: "your-cluster-storageclass" + # resources: + # requests: + # storage: "8Gi" placement: tolerations: nodeAffinity: @@ -31,7 +53,6 @@ spec: credentials: name: access-keys namespace: rook-minio - storageAmount: "10G" clusterDomain: --- apiVersion: v1 diff --git a/pkg/apis/cockroachdb.rook.io/v1alpha1/types.go b/pkg/apis/cockroachdb.rook.io/v1alpha1/types.go index 0e0e1ed17..5dec6d03b 100644 --- a/pkg/apis/cockroachdb.rook.io/v1alpha1/types.go +++ b/pkg/apis/cockroachdb.rook.io/v1alpha1/types.go @@ -16,7 +16,6 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" @@ -50,7 +49,6 @@ type ClusterSpec struct { Storage rook.StorageScopeSpec `json:"scope,omitempty"` Network rook.NetworkSpec `json:"network,omitempty"` Secure bool `json:"secure,omitempty"` - VolumeSize resource.Quantity `json:"volumeSize,omitempty"` CachePercent int `json:"cachePercent,omitempty"` MaxSQLMemoryPercent int `json:"maxSQLMemoryPercent,omitempty"` } diff --git a/pkg/apis/cockroachdb.rook.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cockroachdb.rook.io/v1alpha1/zz_generated.deepcopy.go index 505253f56..42c258183 100644 --- a/pkg/apis/cockroachdb.rook.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cockroachdb.rook.io/v1alpha1/zz_generated.deepcopy.go @@ -89,7 +89,6 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = *in in.Storage.DeepCopyInto(&out.Storage) in.Network.DeepCopyInto(&out.Network) - out.VolumeSize = in.VolumeSize.DeepCopy() return } diff --git a/pkg/operator/cockroachdb/controller.go b/pkg/operator/cockroachdb/controller.go index 2d6ca59ec..157be28b0 100644 --- a/pkg/operator/cockroachdb/controller.go +++ b/pkg/operator/cockroachdb/controller.go @@ -16,7 +16,7 @@ limitations under the License. Portions of this file came from https://github.com/cockroachdb/cockroach, which uses the same license. */ -// Package cluster to manage a cockroachdb cluster. +// Package cockroachdb to manage a cockroachdb cluster. package cockroachdb import ( @@ -56,7 +56,7 @@ const ( httpPortName = "http" grpcPortDefault = int32(26257) grpcPortName = "grpc" - volumeNameDataDir = "datadir" + volumeDataName = "datadir" envVarChannel = "COCKROACH_CHANNEL" envVarValChannelSecure = "kubernetes-secure" envVarValChannelInsecure = "kubernetes-insecure" @@ -113,7 +113,6 @@ func clusterOwnerRef(namespace, clusterID string) metav1.OwnerReference { } func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) error { - resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ AddFunc: c.onAdd, UpdateFunc: c.onUpdate, @@ -332,22 +331,7 @@ func (c *ClusterController) createStatefulSet(cluster *cluster) error { UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{ Type: appsv1beta1.RollingUpdateStatefulSetStrategyType, }, - VolumeClaimTemplates: []v1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: volumeNameDataDir, - Namespace: cluster.namespace, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: cluster.spec.VolumeSize, - }, - }, - }, - }, - }, + VolumeClaimTemplates: cluster.spec.Storage.VolumeClaimTemplates, }, } k8sutil.SetOwnerRef(c.context.Clientset, cluster.namespace, &statefulSet.ObjectMeta, &cluster.ownerRef) @@ -367,6 +351,16 @@ func (c *ClusterController) createStatefulSet(cluster *cluster) error { func createPodSpec(cluster *cluster, containerImage string, httpPort, grpcPort int32) v1.PodSpec { terminationGracePeriodSeconds := int64(60) + volumes := []v1.Volume{} + if len(cluster.spec.Storage.VolumeClaimTemplates) == 0 { + volumes = append(volumes, v1.Volume{ + Name: volumeDataName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }) + } + return v1.PodSpec{ Affinity: &v1.Affinity{ PodAntiAffinity: &v1.PodAntiAffinity{ @@ -390,18 +384,9 @@ func createPodSpec(cluster *cluster, containerImage string, httpPort, grpcPort i }, }, Containers: []v1.Container{createContainer(cluster, containerImage, httpPort, grpcPort)}, - // No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node. + // No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node. TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Volumes: []v1.Volume{ - { - Name: volumeNameDataDir, - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: volumeNameDataDir, - }, - }, - }, - }, + Volumes: volumes, } } @@ -413,6 +398,11 @@ func createContainer(cluster *cluster, containerImage string, httpPort, grpcPort envVarChannelVal = envVarValChannelInsecure } + cockroachDataVolumeName := volumeDataName + if len(cluster.spec.Storage.VolumeClaimTemplates) == 1 { + cockroachDataVolumeName = cluster.spec.Storage.VolumeClaimTemplates[0].GetName() + } + return v1.Container{ Name: appName, Image: containerImage, @@ -450,7 +440,7 @@ func createContainer(cluster *cluster, containerImage string, httpPort, grpcPort }, VolumeMounts: []v1.VolumeMount{ { - Name: volumeNameDataDir, + Name: cockroachDataVolumeName, MountPath: "/cockroach/cockroach-data", }, }, diff --git a/pkg/operator/cockroachdb/controller_test.go b/pkg/operator/cockroachdb/controller_test.go index 075a5db7b..fc73f3730 100644 --- a/pkg/operator/cockroachdb/controller_test.go +++ b/pkg/operator/cockroachdb/controller_test.go @@ -115,14 +115,34 @@ func TestOnAdd(t *testing.T) { Namespace: namespace, }, Spec: cockroachdbv1alpha1.ClusterSpec{ - Storage: rookalpha.StorageScopeSpec{NodeCount: 5}, + Storage: rookalpha.StorageScopeSpec{ + NodeCount: 5, + Selection: rookalpha.Selection{ + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-cockroachdb-test", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Mi"), + }, + }, + }, + }, + }, + }, + }, Network: rookalpha.NetworkSpec{ Ports: []rookalpha.PortSpec{ {Name: "http", Port: 123}, {Name: "grpc", Port: 456}, }, }, - VolumeSize: resource.MustParse("1Mi"), CachePercent: 30, MaxSQLMemoryPercent: 40, }, @@ -186,8 +206,8 @@ func TestOnAdd(t *testing.T) { assert.NotNil(t, ss) assert.Equal(t, int32(5), *ss.Spec.Replicas) assert.Equal(t, 1, len(ss.Spec.VolumeClaimTemplates)) - assert.Equal(t, cluster.Spec.VolumeSize, - ss.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[v1.ResourceStorage]) + assert.Equal(t, cluster.Spec.Storage.Selection.VolumeClaimTemplates, + ss.Spec.VolumeClaimTemplates) assert.Equal(t, 1, len(ss.Spec.Template.Spec.Containers)) container := ss.Spec.Template.Spec.Containers[0] @@ -197,7 +217,7 @@ func TestOnAdd(t *testing.T) { } assert.Equal(t, expectedContainerPorts, container.Ports) - expectedVolumeMounts := []v1.VolumeMount{{Name: "datadir", MountPath: "/cockroach/cockroach-data"}} + expectedVolumeMounts := []v1.VolumeMount{{Name: "rook-cockroachdb-test", MountPath: "/cockroach/cockroach-data"}} assert.Equal(t, expectedVolumeMounts, container.VolumeMounts) expectedEnvVars := []v1.EnvVar{{Name: "COCKROACH_CHANNEL", Value: "kubernetes-insecure"}} diff --git a/pkg/operator/minio/controller.go b/pkg/operator/minio/controller.go index da74c259a..0092aa9f9 100644 --- a/pkg/operator/minio/controller.go +++ b/pkg/operator/minio/controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package to manage a Minio object store. +// Package minio to manage a Minio object store. package minio import ( @@ -29,7 +29,6 @@ import ( "k8s.io/api/apps/v1beta2" "k8s.io/api/core/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/api/resource" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/cache" @@ -38,15 +37,15 @@ import ( // TODO: A lot of these constants are specific to the KubeCon demo. Let's // revisit these and determine what should be specified in the resource spec. const ( - customResourceName = "objectstore" - customResourceNamePlural = "objectstores" - minioCtrName = "minio" - minioLabel = "minio" - minioPVCName = "minio-pvc" - minioServerSuffixFmt = "%s.svc.%s" // namespace.svc.clusterDomain, e.g., default.svc.cluster.local - minioVolumeName = "data" - objectStoreDataDir = "/data" - minioPort = int32(9000) + objectStoreDataDirTemplate = "/data/%s" + objectStoreDataEmptyDirName = "minio-data" + customResourceName = "objectstore" + customResourceNamePlural = "objectstores" + minioCtrName = "minio" + minioLabel = "minio" + minioPVCName = "minio-pvc" + minioServerSuffixFmt = "%s.svc.%s" // namespace.svc.clusterDomain, e.g., default.svc.cluster.local + minioPort = int32(9000) ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "minio-op-object") @@ -110,11 +109,12 @@ func (c *MinioController) makeMinioHeadlessService(name, namespace string, spec return svc, err } -func (c *MinioController) buildMinioCtrArgs(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverCount int32) []string { - +func (c *MinioController) buildMinioCtrArgs(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverCount int32, volumeMounts []v1.VolumeMount) []string { args := []string{"server"} for i := int32(0); i < serverCount; i++ { - args = append(args, makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain, i)) + for _, mount := range volumeMounts { + args = append(args, makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain, i, getPVCDataDir(mount.Name))) + } } logger.Infof("Building Minio container args: %v", args) @@ -122,22 +122,44 @@ func (c *MinioController) buildMinioCtrArgs(statefulSetPrefix, headlessServiceNa } // Generates the full server address for the given server params, e.g., http://my-store-0.my-store.rook-minio.svc.cluster.local/data -func makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverNum int32) string { +func makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverNum int32, pvcDataDir string) string { if clusterDomain == "" { clusterDomain = miniov1alpha1.ClusterDomainDefault } dnsSuffix := fmt.Sprintf(minioServerSuffixFmt, namespace, clusterDomain) - return fmt.Sprintf("http://%s-%d.%s.%s%s", statefulSetPrefix, serverNum, headlessServiceName, dnsSuffix, objectStoreDataDir) + return fmt.Sprintf("http://%s-%d.%s.%s%s", statefulSetPrefix, serverNum, headlessServiceName, dnsSuffix, pvcDataDir) } -func (c *MinioController) makeMinioPodSpec(name, namespace string, ctrName string, ctrImage string, clusterDomain string, envVars map[string]string, numServers int32) v1.PodTemplateSpec { - +func (c *MinioController) makeMinioPodSpec(name, namespace string, ctrName string, ctrImage string, clusterDomain string, envVars map[string]string, numServers int32, volumeClaims []v1.PersistentVolumeClaim) v1.PodTemplateSpec { var env []v1.EnvVar for k, v := range envVars { env = append(env, v1.EnvVar{Name: k, Value: v}) } + volumes := []v1.Volume{} + volumeMounts := []v1.VolumeMount{} + if len(volumeClaims) > 0 { + for i := range volumeClaims { + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: volumeClaims[i].GetName(), + MountPath: getPVCDataDir(volumeClaims[i].GetName()), + }) + } + } else { + volumes = append(volumes, v1.Volume{ + Name: objectStoreDataEmptyDirName, + VolumeSource: v1.VolumeSource{ + // TODO should the size limit be configurable (only) on empty dir? + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }) + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: objectStoreDataEmptyDirName, + MountPath: fmt.Sprintf(objectStoreDataDirTemplate, objectStoreDataEmptyDirName), + }) + } + podSpec := v1.PodTemplateSpec{ ObjectMeta: meta_v1.ObjectMeta{ Name: name, @@ -147,30 +169,16 @@ func (c *MinioController) makeMinioPodSpec(name, namespace string, ctrName strin Spec: v1.PodSpec{ Containers: []v1.Container{ { - Name: ctrName, - Image: ctrImage, - Env: env, - Command: []string{"/usr/bin/minio"}, - Ports: []v1.ContainerPort{{ContainerPort: minioPort}}, - Args: c.buildMinioCtrArgs(name, name, namespace, clusterDomain, numServers), - VolumeMounts: []v1.VolumeMount{ - { - Name: minioVolumeName, - MountPath: objectStoreDataDir, - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: minioVolumeName, - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: minioPVCName, - }, - }, + Name: ctrName, + Image: ctrImage, + Env: env, + Command: []string{"/usr/bin/minio"}, + Ports: []v1.ContainerPort{{ContainerPort: minioPort}}, + Args: c.buildMinioCtrArgs(name, name, namespace, clusterDomain, numServers, volumeMounts), + VolumeMounts: volumeMounts, }, }, + Volumes: volumes, }, } @@ -212,7 +220,7 @@ func (c *MinioController) makeMinioStatefulSet(name, namespace string, spec mini "MINIO_SECRET_KEY": secretKey, } - podSpec := c.makeMinioPodSpec(name, namespace, minioCtrName, c.rookImage, spec.ClusterDomain, envVars, int32(spec.Storage.NodeCount)) + podSpec := c.makeMinioPodSpec(name, namespace, minioCtrName, c.rookImage, spec.ClusterDomain, envVars, int32(spec.Storage.NodeCount), spec.Storage.VolumeClaimTemplates) nodeCount := int32(spec.Storage.NodeCount) ss := v1beta2.StatefulSet{ @@ -226,24 +234,9 @@ func (c *MinioController) makeMinioStatefulSet(name, namespace string, spec mini Selector: &meta_v1.LabelSelector{ MatchLabels: map[string]string{k8sutil.AppAttr: minioLabel}, }, - Template: podSpec, - VolumeClaimTemplates: []v1.PersistentVolumeClaim{ - { - ObjectMeta: meta_v1.ObjectMeta{ - Name: minioVolumeName, - Namespace: namespace, - }, - Spec: v1.PersistentVolumeClaimSpec{ - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceStorage: resource.MustParse(spec.StorageSize), - }, - }, - }, - }, - }, - ServiceName: name, + Template: podSpec, + VolumeClaimTemplates: spec.Storage.VolumeClaimTemplates, + ServiceName: name, // TODO: liveness probe }, } @@ -304,3 +297,7 @@ func (c *MinioController) onDelete(obj interface{}) { // Cleanup is handled by the owner references set in 'onAdd' and the k8s garbage collector. } + +func getPVCDataDir(pvcName string) string { + return fmt.Sprintf(objectStoreDataDirTemplate, pvcName) +} diff --git a/pkg/operator/minio/controller_test.go b/pkg/operator/minio/controller_test.go index b772241c6..175a14c87 100644 --- a/pkg/operator/minio/controller_test.go +++ b/pkg/operator/minio/controller_test.go @@ -24,6 +24,7 @@ import ( testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -37,12 +38,47 @@ func TestOnAdd(t *testing.T) { Namespace: namespace, }, Spec: miniov.ObjectStoreSpec{ - Storage: rookalpha.StorageScopeSpec{NodeCount: 6}, + Storage: rookalpha.StorageScopeSpec{ + NodeCount: 6, + Selection: rookalpha.Selection{ + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-minio-test1", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-minio-test2", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + }, Credentials: v1.SecretReference{ Name: "whatever", Namespace: namespace, }, - StorageSize: "1337G", }, } @@ -78,20 +114,28 @@ func TestOnAdd(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, ss) assert.Equal(t, int32(6), *ss.Spec.Replicas) - assert.Equal(t, 1, len(ss.Spec.VolumeClaimTemplates)) + assert.Equal(t, 2, len(ss.Spec.VolumeClaimTemplates)) assert.Equal(t, 1, len(ss.Spec.Template.Spec.Containers)) container := ss.Spec.Template.Spec.Containers[0] expectedContainerPorts := []v1.ContainerPort{{ContainerPort: minioPort}} assert.Equal(t, expectedContainerPorts, container.Ports) + assert.Equal(t, 2, len(container.VolumeMounts)) + assert.Equal(t, 13, len(container.Args)) expectedContainerArgs := []string{ "server", - "http://some_object_store-0.some_object_store.rook-minio-123.svc.cluster.local/data", - "http://some_object_store-1.some_object_store.rook-minio-123.svc.cluster.local/data", - "http://some_object_store-2.some_object_store.rook-minio-123.svc.cluster.local/data", - "http://some_object_store-3.some_object_store.rook-minio-123.svc.cluster.local/data", - "http://some_object_store-4.some_object_store.rook-minio-123.svc.cluster.local/data", - "http://some_object_store-5.some_object_store.rook-minio-123.svc.cluster.local/data", + "http://some_object_store-0.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1", + "http://some_object_store-0.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2", + "http://some_object_store-1.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1", + "http://some_object_store-1.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2", + "http://some_object_store-2.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1", + "http://some_object_store-2.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2", + "http://some_object_store-3.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1", + "http://some_object_store-3.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2", + "http://some_object_store-4.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1", + "http://some_object_store-4.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2", + "http://some_object_store-5.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1", + "http://some_object_store-5.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2", } assert.Equal(t, expectedContainerArgs, ss.Spec.Template.Spec.Containers[0].Args) } @@ -104,12 +148,32 @@ func TestSpecVerification(t *testing.T) { Namespace: namespace, }, Spec: miniov.ObjectStoreSpec{ - Storage: rookalpha.StorageScopeSpec{NodeCount: 6}, + Storage: rookalpha.StorageScopeSpec{ + NodeCount: 6, + Selection: rookalpha.Selection{ + VolumeClaimTemplates: []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "rook-minio-test", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + }, + }, + }, + }, Credentials: v1.SecretReference{ Name: "whatever", Namespace: namespace, }, - StorageSize: "1337G", }, } @@ -132,10 +196,14 @@ func TestSpecVerification(t *testing.T) { func TestMakeServerAddress(t *testing.T) { // pass empty string for cluster domain, default should be used - serverAddress := makeServerAddress("my-store", "my-store", "rook-minio", "", 3) - assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.cluster.local/data", serverAddress) + serverAddress := makeServerAddress("my-store", "my-store", "rook-minio", "", 3, "/my-cool-data/dir123") + assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.cluster.local/my-cool-data/dir123", serverAddress) // pass custom cluster domain, it should be used - serverAddress = makeServerAddress("my-store", "my-store", "rook-minio", "acme.com", 3) - assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.acme.com/data", serverAddress) + serverAddress = makeServerAddress("my-store", "my-store", "rook-minio", "acme.com", 3, "/data/mydir1") + assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.acme.com/data/mydir1", serverAddress) +} + +func TestGetPVCDataDir(t *testing.T) { + assert.Equal(t, "/data/rook-test123", getPVCDataDir("rook-test123")) } diff --git a/tests/framework/installer/cockroachdb_manifests.go b/tests/framework/installer/cockroachdb_manifests.go index e8327af36..609456e55 100644 --- a/tests/framework/installer/cockroachdb_manifests.go +++ b/tests/framework/installer/cockroachdb_manifests.go @@ -20,9 +20,11 @@ import ( "strconv" ) +// CockroachDBManifests holds the funcs which return the CockroachDB manifests type CockroachDBManifests struct { } +// GetCockroachDBCRDs return the CockroachDB Cluster CRD func (i *CockroachDBManifests) GetCockroachDBCRDs() string { return `apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -40,8 +42,8 @@ spec: ` } +// GetCockroachDBOperator return the CockroachDB operator manifest func (i *CockroachDBManifests) GetCockroachDBOperator(namespace string) string { - return `kind: Namespace apiVersion: v1 metadata: @@ -136,6 +138,7 @@ spec: ` } +// GetCockroachDBCluster return a CockroacDB Cluster object func (i *CockroachDBManifests) GetCockroachDBCluster(namespace string, count int) string { return `apiVersion: cockroachdb.rook.io/v1alpha1 kind: Cluster @@ -145,8 +148,8 @@ metadata: spec: scope: nodeCount: ` + strconv.Itoa(count) + ` + volumeClaimTemplates: [] secure: false - volumeSize: 1Gi cachePercent: 25 maxSQLMemoryPercent: 25 `