Cockroachdb + Minio: Use PersistentVolumeClaimTemplates storage
selection field. Signed-off-by: Alexander Trost <galexrt@googlemail.com>
This commit is contained in:
12 files changed
+259
-121
No files matched your search
@@ -21,6 +21,15 @@ metadata:
|
||||
spec:
|
||||
scope:
|
||||
nodeCount: 3
|
||||
volumeClaimTemplates:
|
||||
- spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
# Uncomment and specify your StorageClass, otherwise
|
||||
# the cluster admin defined default StorageClass will be used.
|
||||
#storageClassName: "my-storage-class"
|
||||
resources:
|
||||
requests:
|
||||
storage: "100Gi"
|
||||
network:
|
||||
ports:
|
||||
- name: http
|
||||
@@ -28,7 +37,6 @@ spec:
|
||||
- name: grpc
|
||||
port: 26257
|
||||
secure: false
|
||||
volumeSize: 100Gi
|
||||
cachePercent: 25
|
||||
maxSQLMemoryPercent: 25
|
||||
```
|
||||
@@ -40,7 +48,6 @@ spec:
|
||||
The settings below are specific to CockroachDB database clusters:
|
||||
|
||||
* `secure`: `true` to create a secure cluster installation using certificates and encryption. `false` to create an insecure installation (strongly discouraged for production usage). Currently, only insecure is supported.
|
||||
* `volumeSize`: Each database instance will get an underlying persistent data volume created to store its instance data using the default storage class. This value represents the size of the volume that will be created. The value should be expressed in the [standard Kubernetes resource format](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory).
|
||||
* `cachePercent`: The total size used for caches, expressed as a percentage of total physical memory.
|
||||
* `maxSQLMemoryPercent`: The maximum memory capacity available to store temporary data for SQL clients, expressed as a percentage of total physical memory.
|
||||
|
||||
@@ -50,6 +57,7 @@ Under the `scope` field, a `StorageScopeSpec` can be specified to influence the
|
||||
These properties are currently supported:
|
||||
|
||||
* `nodeCount`: The number of CockroachDB instances to create. Some of these instances may be scheduled on the same nodes, but exactly this many instances will be created and included in the cluster.
|
||||
* `volumeClaimTemplates`: A list of PersistentVolumeClaim templates which must contain only **one or no** PersistentVolumeClaim. If no PersistentVolumeClaim is given an `emptyDir` will be given, meaning the instance data will be lost when a Pod is restarted. For an example of how PersistentVolumeClaim template should look, please look at the above [sample](#sample).
|
||||
|
||||
### Network
|
||||
|
||||
|
||||
@@ -18,6 +18,29 @@ metadata:
|
||||
spec:
|
||||
scope:
|
||||
nodeCount: 4
|
||||
# You can have multiple PersistentVolumeClaims in the volumeClaimTemplates list.
|
||||
# Be aware though that all PersistentVolumeClaim Templates will be used for each intance (see nodeCount).
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: rook-minio-data1
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
# Uncomment and specify your StorageClass, otherwise
|
||||
# the cluster admin defined default StorageClass will be used.
|
||||
#storageClassName: "your-cluster-storageclass"
|
||||
resources:
|
||||
requests:
|
||||
storage: "8Gi"
|
||||
#- metadata:
|
||||
# name: rook-minio-data2
|
||||
# spec:
|
||||
# accessModes: [ "ReadWriteOnce" ]
|
||||
# # Uncomment and specify your StorageClass, otherwise
|
||||
# # the cluster admin defined default StorageClass will be used.
|
||||
# #storageClassName: "my-storage-class"
|
||||
# resources:
|
||||
# requests:
|
||||
# storage: "8Gi"
|
||||
placement:
|
||||
tolerations:
|
||||
nodeAffinity:
|
||||
@@ -26,7 +49,6 @@ spec:
|
||||
credentials:
|
||||
name: access-keys
|
||||
namespace: rook-minio
|
||||
storageAmount: "10G"
|
||||
clusterDomain:
|
||||
```
|
||||
|
||||
@@ -45,3 +67,4 @@ The settings below are specific to Minio object stores:
|
||||
Under the `scope` field, a `StorageScopeSpec` can be specified to influence the scope or boundaries of storage that the cluster will use for its underlying storage. These properties are currently supported:
|
||||
|
||||
* `nodeCount`: The number of Minio instances to create. Some of these instances may be scheduled on the same nodes, but exactly this many instances will be created and included in the cluster.
|
||||
* `volumeClaimTemplates`: A list of one or more PersistentVolumeClaim templates to use for each Minio repliace. For an example of how the list should look like, please look at the above [sample](#sample).
|
||||
@@ -222,7 +222,7 @@ spec:
|
||||
# osd:
|
||||
storage: # cluster level storage configuration and selection
|
||||
useAllNodes: true
|
||||
useAllDevices: false
|
||||
useAllDevices: true
|
||||
deviceFilter:
|
||||
location:
|
||||
config:
|
||||
|
||||
@@ -13,6 +13,18 @@ spec:
|
||||
# https://rook.io/docs/rook/master/cockroachdb-cluster-crd.html
|
||||
scope:
|
||||
nodeCount: 3
|
||||
# You can only have one PersistentVolumeClaim in this list!
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: rook-cockroachdb-data
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
# Uncomment and specify your StorageClass, otherwise
|
||||
# the cluster admin defined default StorageClass will be used.
|
||||
#storageClassName: "your-cluster-storageclass"
|
||||
resources:
|
||||
requests:
|
||||
storage: "1Gi"
|
||||
network:
|
||||
ports:
|
||||
- name: http
|
||||
@@ -20,6 +32,5 @@ spec:
|
||||
- name: grpc
|
||||
port: 26257
|
||||
secure: false
|
||||
volumeSize: 1Gi
|
||||
cachePercent: 25
|
||||
maxSQLMemoryPercent: 25
|
||||
@@ -23,6 +23,28 @@ metadata:
|
||||
spec:
|
||||
scope:
|
||||
nodeCount: 4
|
||||
# You can have multiple PersistentVolumeClaims in the volumeClaimTemplates list.
|
||||
# Be aware though that all PersistentVolumeClaim Templates will be used for each intance (see nodeCount).
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: rook-minio-data1
|
||||
spec:
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
# Set the storage class that will be used, otherwise Kubernetes' default storage class will be used.
|
||||
#storageClassName: "my-storage-class"
|
||||
resources:
|
||||
requests:
|
||||
storage: "8Gi"
|
||||
#- metadata:
|
||||
# name: rook-minio-data2
|
||||
# spec:
|
||||
# accessModes: [ "ReadWriteOnce" ]
|
||||
# # Uncomment and specify your StorageClass, otherwise
|
||||
# # the cluster admin defined default StorageClass will be used.
|
||||
# #storageClassName: "your-cluster-storageclass"
|
||||
# resources:
|
||||
# requests:
|
||||
# storage: "8Gi"
|
||||
placement:
|
||||
tolerations:
|
||||
nodeAffinity:
|
||||
@@ -31,7 +53,6 @@ spec:
|
||||
credentials:
|
||||
name: access-keys
|
||||
namespace: rook-minio
|
||||
storageAmount: "10G"
|
||||
clusterDomain:
|
||||
---
|
||||
apiVersion: v1
|
||||
|
||||
@@ -16,7 +16,6 @@ limitations under the License.
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
rook "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
@@ -50,7 +49,6 @@ type ClusterSpec struct {
|
||||
Storage rook.StorageScopeSpec `json:"scope,omitempty"`
|
||||
Network rook.NetworkSpec `json:"network,omitempty"`
|
||||
Secure bool `json:"secure,omitempty"`
|
||||
VolumeSize resource.Quantity `json:"volumeSize,omitempty"`
|
||||
CachePercent int `json:"cachePercent,omitempty"`
|
||||
MaxSQLMemoryPercent int `json:"maxSQLMemoryPercent,omitempty"`
|
||||
}
|
||||
@@ -89,7 +89,6 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
|
||||
*out = *in
|
||||
in.Storage.DeepCopyInto(&out.Storage)
|
||||
in.Network.DeepCopyInto(&out.Network)
|
||||
out.VolumeSize = in.VolumeSize.DeepCopy()
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ limitations under the License.
|
||||
Portions of this file came from https://github.com/cockroachdb/cockroach, which uses the same license.
|
||||
*/
|
||||
|
||||
// Package cluster to manage a cockroachdb cluster.
|
||||
// Package cockroachdb to manage a cockroachdb cluster.
|
||||
package cockroachdb
|
||||
|
||||
import (
|
||||
@@ -56,7 +56,7 @@ const (
|
||||
httpPortName = "http"
|
||||
grpcPortDefault = int32(26257)
|
||||
grpcPortName = "grpc"
|
||||
volumeNameDataDir = "datadir"
|
||||
volumeDataName = "datadir"
|
||||
envVarChannel = "COCKROACH_CHANNEL"
|
||||
envVarValChannelSecure = "kubernetes-secure"
|
||||
envVarValChannelInsecure = "kubernetes-insecure"
|
||||
@@ -113,7 +113,6 @@ func clusterOwnerRef(namespace, clusterID string) metav1.OwnerReference {
|
||||
}
|
||||
|
||||
func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) error {
|
||||
|
||||
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onAdd,
|
||||
UpdateFunc: c.onUpdate,
|
||||
@@ -332,22 +331,7 @@ func (c *ClusterController) createStatefulSet(cluster *cluster) error {
|
||||
UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{
|
||||
Type: appsv1beta1.RollingUpdateStatefulSetStrategyType,
|
||||
},
|
||||
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: volumeNameDataDir,
|
||||
Namespace: cluster.namespace,
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: cluster.spec.VolumeSize,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
VolumeClaimTemplates: cluster.spec.Storage.VolumeClaimTemplates,
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, cluster.namespace, &statefulSet.ObjectMeta, &cluster.ownerRef)
|
||||
@@ -367,6 +351,16 @@ func (c *ClusterController) createStatefulSet(cluster *cluster) error {
|
||||
func createPodSpec(cluster *cluster, containerImage string, httpPort, grpcPort int32) v1.PodSpec {
|
||||
terminationGracePeriodSeconds := int64(60)
|
||||
|
||||
volumes := []v1.Volume{}
|
||||
if len(cluster.spec.Storage.VolumeClaimTemplates) == 0 {
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: volumeDataName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return v1.PodSpec{
|
||||
Affinity: &v1.Affinity{
|
||||
PodAntiAffinity: &v1.PodAntiAffinity{
|
||||
@@ -390,18 +384,9 @@ func createPodSpec(cluster *cluster, containerImage string, httpPort, grpcPort i
|
||||
},
|
||||
},
|
||||
Containers: []v1.Container{createContainer(cluster, containerImage, httpPort, grpcPort)},
|
||||
// No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node.
|
||||
// No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node.
|
||||
TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,
|
||||
Volumes: []v1.Volume{
|
||||
{
|
||||
Name: volumeNameDataDir,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: volumeNameDataDir,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: volumes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,6 +398,11 @@ func createContainer(cluster *cluster, containerImage string, httpPort, grpcPort
|
||||
envVarChannelVal = envVarValChannelInsecure
|
||||
}
|
||||
|
||||
cockroachDataVolumeName := volumeDataName
|
||||
if len(cluster.spec.Storage.VolumeClaimTemplates) == 1 {
|
||||
cockroachDataVolumeName = cluster.spec.Storage.VolumeClaimTemplates[0].GetName()
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: appName,
|
||||
Image: containerImage,
|
||||
@@ -450,7 +440,7 @@ func createContainer(cluster *cluster, containerImage string, httpPort, grpcPort
|
||||
},
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{
|
||||
Name: volumeNameDataDir,
|
||||
Name: cockroachDataVolumeName,
|
||||
MountPath: "/cockroach/cockroach-data",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -115,14 +115,34 @@ func TestOnAdd(t *testing.T) {
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: cockroachdbv1alpha1.ClusterSpec{
|
||||
Storage: rookalpha.StorageScopeSpec{NodeCount: 5},
|
||||
Storage: rookalpha.StorageScopeSpec{
|
||||
NodeCount: 5,
|
||||
Selection: rookalpha.Selection{
|
||||
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-cockroachdb-test",
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{
|
||||
v1.ReadWriteOnce,
|
||||
},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: resource.MustParse("1Mi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Network: rookalpha.NetworkSpec{
|
||||
Ports: []rookalpha.PortSpec{
|
||||
{Name: "http", Port: 123},
|
||||
{Name: "grpc", Port: 456},
|
||||
},
|
||||
},
|
||||
VolumeSize: resource.MustParse("1Mi"),
|
||||
CachePercent: 30,
|
||||
MaxSQLMemoryPercent: 40,
|
||||
},
|
||||
@@ -186,8 +206,8 @@ func TestOnAdd(t *testing.T) {
|
||||
assert.NotNil(t, ss)
|
||||
assert.Equal(t, int32(5), *ss.Spec.Replicas)
|
||||
assert.Equal(t, 1, len(ss.Spec.VolumeClaimTemplates))
|
||||
assert.Equal(t, cluster.Spec.VolumeSize,
|
||||
ss.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[v1.ResourceStorage])
|
||||
assert.Equal(t, cluster.Spec.Storage.Selection.VolumeClaimTemplates,
|
||||
ss.Spec.VolumeClaimTemplates)
|
||||
assert.Equal(t, 1, len(ss.Spec.Template.Spec.Containers))
|
||||
container := ss.Spec.Template.Spec.Containers[0]
|
||||
|
||||
@@ -197,7 +217,7 @@ func TestOnAdd(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, expectedContainerPorts, container.Ports)
|
||||
|
||||
expectedVolumeMounts := []v1.VolumeMount{{Name: "datadir", MountPath: "/cockroach/cockroach-data"}}
|
||||
expectedVolumeMounts := []v1.VolumeMount{{Name: "rook-cockroachdb-test", MountPath: "/cockroach/cockroach-data"}}
|
||||
assert.Equal(t, expectedVolumeMounts, container.VolumeMounts)
|
||||
|
||||
expectedEnvVars := []v1.EnvVar{{Name: "COCKROACH_CHANNEL", Value: "kubernetes-insecure"}}
|
||||
|
||||
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package to manage a Minio object store.
|
||||
// Package minio to manage a Minio object store.
|
||||
package minio
|
||||
|
||||
import (
|
||||
@@ -29,7 +29,6 @@ import (
|
||||
"k8s.io/api/apps/v1beta2"
|
||||
"k8s.io/api/core/v1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
@@ -38,15 +37,15 @@ import (
|
||||
// TODO: A lot of these constants are specific to the KubeCon demo. Let's
|
||||
// revisit these and determine what should be specified in the resource spec.
|
||||
const (
|
||||
customResourceName = "objectstore"
|
||||
customResourceNamePlural = "objectstores"
|
||||
minioCtrName = "minio"
|
||||
minioLabel = "minio"
|
||||
minioPVCName = "minio-pvc"
|
||||
minioServerSuffixFmt = "%s.svc.%s" // namespace.svc.clusterDomain, e.g., default.svc.cluster.local
|
||||
minioVolumeName = "data"
|
||||
objectStoreDataDir = "/data"
|
||||
minioPort = int32(9000)
|
||||
objectStoreDataDirTemplate = "/data/%s"
|
||||
objectStoreDataEmptyDirName = "minio-data"
|
||||
customResourceName = "objectstore"
|
||||
customResourceNamePlural = "objectstores"
|
||||
minioCtrName = "minio"
|
||||
minioLabel = "minio"
|
||||
minioPVCName = "minio-pvc"
|
||||
minioServerSuffixFmt = "%s.svc.%s" // namespace.svc.clusterDomain, e.g., default.svc.cluster.local
|
||||
minioPort = int32(9000)
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "minio-op-object")
|
||||
@@ -110,11 +109,12 @@ func (c *MinioController) makeMinioHeadlessService(name, namespace string, spec
|
||||
return svc, err
|
||||
}
|
||||
|
||||
func (c *MinioController) buildMinioCtrArgs(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverCount int32) []string {
|
||||
|
||||
func (c *MinioController) buildMinioCtrArgs(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverCount int32, volumeMounts []v1.VolumeMount) []string {
|
||||
args := []string{"server"}
|
||||
for i := int32(0); i < serverCount; i++ {
|
||||
args = append(args, makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain, i))
|
||||
for _, mount := range volumeMounts {
|
||||
args = append(args, makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain, i, getPVCDataDir(mount.Name)))
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("Building Minio container args: %v", args)
|
||||
@@ -122,22 +122,44 @@ func (c *MinioController) buildMinioCtrArgs(statefulSetPrefix, headlessServiceNa
|
||||
}
|
||||
|
||||
// Generates the full server address for the given server params, e.g., http://my-store-0.my-store.rook-minio.svc.cluster.local/data
|
||||
func makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverNum int32) string {
|
||||
func makeServerAddress(statefulSetPrefix, headlessServiceName, namespace, clusterDomain string, serverNum int32, pvcDataDir string) string {
|
||||
if clusterDomain == "" {
|
||||
clusterDomain = miniov1alpha1.ClusterDomainDefault
|
||||
}
|
||||
|
||||
dnsSuffix := fmt.Sprintf(minioServerSuffixFmt, namespace, clusterDomain)
|
||||
return fmt.Sprintf("http://%s-%d.%s.%s%s", statefulSetPrefix, serverNum, headlessServiceName, dnsSuffix, objectStoreDataDir)
|
||||
return fmt.Sprintf("http://%s-%d.%s.%s%s", statefulSetPrefix, serverNum, headlessServiceName, dnsSuffix, pvcDataDir)
|
||||
}
|
||||
|
||||
func (c *MinioController) makeMinioPodSpec(name, namespace string, ctrName string, ctrImage string, clusterDomain string, envVars map[string]string, numServers int32) v1.PodTemplateSpec {
|
||||
|
||||
func (c *MinioController) makeMinioPodSpec(name, namespace string, ctrName string, ctrImage string, clusterDomain string, envVars map[string]string, numServers int32, volumeClaims []v1.PersistentVolumeClaim) v1.PodTemplateSpec {
|
||||
var env []v1.EnvVar
|
||||
for k, v := range envVars {
|
||||
env = append(env, v1.EnvVar{Name: k, Value: v})
|
||||
}
|
||||
|
||||
volumes := []v1.Volume{}
|
||||
volumeMounts := []v1.VolumeMount{}
|
||||
if len(volumeClaims) > 0 {
|
||||
for i := range volumeClaims {
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{
|
||||
Name: volumeClaims[i].GetName(),
|
||||
MountPath: getPVCDataDir(volumeClaims[i].GetName()),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: objectStoreDataEmptyDirName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
// TODO should the size limit be configurable (only) on empty dir?
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{},
|
||||
},
|
||||
})
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{
|
||||
Name: objectStoreDataEmptyDirName,
|
||||
MountPath: fmt.Sprintf(objectStoreDataDirTemplate, objectStoreDataEmptyDirName),
|
||||
})
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: meta_v1.ObjectMeta{
|
||||
Name: name,
|
||||
@@ -147,30 +169,16 @@ func (c *MinioController) makeMinioPodSpec(name, namespace string, ctrName strin
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: ctrName,
|
||||
Image: ctrImage,
|
||||
Env: env,
|
||||
Command: []string{"/usr/bin/minio"},
|
||||
Ports: []v1.ContainerPort{{ContainerPort: minioPort}},
|
||||
Args: c.buildMinioCtrArgs(name, name, namespace, clusterDomain, numServers),
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{
|
||||
Name: minioVolumeName,
|
||||
MountPath: objectStoreDataDir,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: []v1.Volume{
|
||||
{
|
||||
Name: minioVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: minioPVCName,
|
||||
},
|
||||
},
|
||||
Name: ctrName,
|
||||
Image: ctrImage,
|
||||
Env: env,
|
||||
Command: []string{"/usr/bin/minio"},
|
||||
Ports: []v1.ContainerPort{{ContainerPort: minioPort}},
|
||||
Args: c.buildMinioCtrArgs(name, name, namespace, clusterDomain, numServers, volumeMounts),
|
||||
VolumeMounts: volumeMounts,
|
||||
},
|
||||
},
|
||||
Volumes: volumes,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -212,7 +220,7 @@ func (c *MinioController) makeMinioStatefulSet(name, namespace string, spec mini
|
||||
"MINIO_SECRET_KEY": secretKey,
|
||||
}
|
||||
|
||||
podSpec := c.makeMinioPodSpec(name, namespace, minioCtrName, c.rookImage, spec.ClusterDomain, envVars, int32(spec.Storage.NodeCount))
|
||||
podSpec := c.makeMinioPodSpec(name, namespace, minioCtrName, c.rookImage, spec.ClusterDomain, envVars, int32(spec.Storage.NodeCount), spec.Storage.VolumeClaimTemplates)
|
||||
|
||||
nodeCount := int32(spec.Storage.NodeCount)
|
||||
ss := v1beta2.StatefulSet{
|
||||
@@ -226,24 +234,9 @@ func (c *MinioController) makeMinioStatefulSet(name, namespace string, spec mini
|
||||
Selector: &meta_v1.LabelSelector{
|
||||
MatchLabels: map[string]string{k8sutil.AppAttr: minioLabel},
|
||||
},
|
||||
Template: podSpec,
|
||||
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
|
||||
{
|
||||
ObjectMeta: meta_v1.ObjectMeta{
|
||||
Name: minioVolumeName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: resource.MustParse(spec.StorageSize),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
ServiceName: name,
|
||||
Template: podSpec,
|
||||
VolumeClaimTemplates: spec.Storage.VolumeClaimTemplates,
|
||||
ServiceName: name,
|
||||
// TODO: liveness probe
|
||||
},
|
||||
}
|
||||
@@ -304,3 +297,7 @@ func (c *MinioController) onDelete(obj interface{}) {
|
||||
|
||||
// Cleanup is handled by the owner references set in 'onAdd' and the k8s garbage collector.
|
||||
}
|
||||
|
||||
func getPVCDataDir(pvcName string) string {
|
||||
return fmt.Sprintf(objectStoreDataDirTemplate, pvcName)
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
@@ -37,12 +38,47 @@ func TestOnAdd(t *testing.T) {
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: miniov.ObjectStoreSpec{
|
||||
Storage: rookalpha.StorageScopeSpec{NodeCount: 6},
|
||||
Storage: rookalpha.StorageScopeSpec{
|
||||
NodeCount: 6,
|
||||
Selection: rookalpha.Selection{
|
||||
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-minio-test1",
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{
|
||||
v1.ReadWriteOnce,
|
||||
},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: resource.MustParse("10Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-minio-test2",
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{
|
||||
v1.ReadWriteOnce,
|
||||
},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: resource.MustParse("10Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Credentials: v1.SecretReference{
|
||||
Name: "whatever",
|
||||
Namespace: namespace,
|
||||
},
|
||||
StorageSize: "1337G",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -78,20 +114,28 @@ func TestOnAdd(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, ss)
|
||||
assert.Equal(t, int32(6), *ss.Spec.Replicas)
|
||||
assert.Equal(t, 1, len(ss.Spec.VolumeClaimTemplates))
|
||||
assert.Equal(t, 2, len(ss.Spec.VolumeClaimTemplates))
|
||||
assert.Equal(t, 1, len(ss.Spec.Template.Spec.Containers))
|
||||
container := ss.Spec.Template.Spec.Containers[0]
|
||||
expectedContainerPorts := []v1.ContainerPort{{ContainerPort: minioPort}}
|
||||
assert.Equal(t, expectedContainerPorts, container.Ports)
|
||||
assert.Equal(t, 2, len(container.VolumeMounts))
|
||||
assert.Equal(t, 13, len(container.Args))
|
||||
|
||||
expectedContainerArgs := []string{
|
||||
"server",
|
||||
"http://some_object_store-0.some_object_store.rook-minio-123.svc.cluster.local/data",
|
||||
"http://some_object_store-1.some_object_store.rook-minio-123.svc.cluster.local/data",
|
||||
"http://some_object_store-2.some_object_store.rook-minio-123.svc.cluster.local/data",
|
||||
"http://some_object_store-3.some_object_store.rook-minio-123.svc.cluster.local/data",
|
||||
"http://some_object_store-4.some_object_store.rook-minio-123.svc.cluster.local/data",
|
||||
"http://some_object_store-5.some_object_store.rook-minio-123.svc.cluster.local/data",
|
||||
"http://some_object_store-0.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1",
|
||||
"http://some_object_store-0.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2",
|
||||
"http://some_object_store-1.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1",
|
||||
"http://some_object_store-1.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2",
|
||||
"http://some_object_store-2.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1",
|
||||
"http://some_object_store-2.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2",
|
||||
"http://some_object_store-3.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1",
|
||||
"http://some_object_store-3.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2",
|
||||
"http://some_object_store-4.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1",
|
||||
"http://some_object_store-4.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2",
|
||||
"http://some_object_store-5.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test1",
|
||||
"http://some_object_store-5.some_object_store.rook-minio-123.svc.cluster.local/data/rook-minio-test2",
|
||||
}
|
||||
assert.Equal(t, expectedContainerArgs, ss.Spec.Template.Spec.Containers[0].Args)
|
||||
}
|
||||
@@ -104,12 +148,32 @@ func TestSpecVerification(t *testing.T) {
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: miniov.ObjectStoreSpec{
|
||||
Storage: rookalpha.StorageScopeSpec{NodeCount: 6},
|
||||
Storage: rookalpha.StorageScopeSpec{
|
||||
NodeCount: 6,
|
||||
Selection: rookalpha.Selection{
|
||||
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "rook-minio-test",
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{
|
||||
v1.ReadWriteOnce,
|
||||
},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: resource.MustParse("10Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Credentials: v1.SecretReference{
|
||||
Name: "whatever",
|
||||
Namespace: namespace,
|
||||
},
|
||||
StorageSize: "1337G",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -132,10 +196,14 @@ func TestSpecVerification(t *testing.T) {
|
||||
|
||||
func TestMakeServerAddress(t *testing.T) {
|
||||
// pass empty string for cluster domain, default should be used
|
||||
serverAddress := makeServerAddress("my-store", "my-store", "rook-minio", "", 3)
|
||||
assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.cluster.local/data", serverAddress)
|
||||
serverAddress := makeServerAddress("my-store", "my-store", "rook-minio", "", 3, "/my-cool-data/dir123")
|
||||
assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.cluster.local/my-cool-data/dir123", serverAddress)
|
||||
|
||||
// pass custom cluster domain, it should be used
|
||||
serverAddress = makeServerAddress("my-store", "my-store", "rook-minio", "acme.com", 3)
|
||||
assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.acme.com/data", serverAddress)
|
||||
serverAddress = makeServerAddress("my-store", "my-store", "rook-minio", "acme.com", 3, "/data/mydir1")
|
||||
assert.Equal(t, "http://my-store-3.my-store.rook-minio.svc.acme.com/data/mydir1", serverAddress)
|
||||
}
|
||||
|
||||
func TestGetPVCDataDir(t *testing.T) {
|
||||
assert.Equal(t, "/data/rook-test123", getPVCDataDir("rook-test123"))
|
||||
}
|
||||
@@ -20,9 +20,11 @@ import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// CockroachDBManifests holds the funcs which return the CockroachDB manifests
|
||||
type CockroachDBManifests struct {
|
||||
}
|
||||
|
||||
// GetCockroachDBCRDs return the CockroachDB Cluster CRD
|
||||
func (i *CockroachDBManifests) GetCockroachDBCRDs() string {
|
||||
return `apiVersion: apiextensions.k8s.io/v1beta1
|
||||
kind: CustomResourceDefinition
|
||||
@@ -40,8 +42,8 @@ spec:
|
||||
`
|
||||
}
|
||||
|
||||
// GetCockroachDBOperator return the CockroachDB operator manifest
|
||||
func (i *CockroachDBManifests) GetCockroachDBOperator(namespace string) string {
|
||||
|
||||
return `kind: Namespace
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@@ -136,6 +138,7 @@ spec:
|
||||
`
|
||||
}
|
||||
|
||||
// GetCockroachDBCluster return a CockroacDB Cluster object
|
||||
func (i *CockroachDBManifests) GetCockroachDBCluster(namespace string, count int) string {
|
||||
return `apiVersion: cockroachdb.rook.io/v1alpha1
|
||||
kind: Cluster
|
||||
@@ -145,8 +148,8 @@ metadata:
|
||||
spec:
|
||||
scope:
|
||||
nodeCount: ` + strconv.Itoa(count) + `
|
||||
volumeClaimTemplates: []
|
||||
secure: false
|
||||
volumeSize: 1Gi
|
||||
cachePercent: 25
|
||||
maxSQLMemoryPercent: 25
|
||||
`
|
||||
|
||||
Reference in new issue
Block a user