From d75a281d6e041b61aacc266246230670864679ce Mon Sep 17 00:00:00 2001 From: Anton Skriptsov Date: Wed, 23 Oct 2019 06:15:30 -0700 Subject: [PATCH] edgefs: FailureDomain-SysRepCount validation Additional failureDomain,sysRepCount validation Signed-off-by: Anton Skriptsov --- .../examples/kubernetes/edgefs/cluster.yaml | 2 + pkg/apis/edgefs.rook.io/v1/cluster.go | 49 ++++++++++ .../cluster/cluster_deployment_config.go | 95 ++++++++++++++++--- pkg/operator/edgefs/cluster/utils.go | 1 + 4 files changed, 133 insertions(+), 14 deletions(-) diff --git a/cluster/examples/kubernetes/edgefs/cluster.yaml b/cluster/examples/kubernetes/edgefs/cluster.yaml index caf13e299..3ce87ad1e 100644 --- a/cluster/examples/kubernetes/edgefs/cluster.yaml +++ b/cluster/examples/kubernetes/edgefs/cluster.yaml @@ -142,6 +142,8 @@ spec: # broker: "enp2s0f0" #skipHostPrepare: true #maxContainerCapacity: 132Ti + #sysRepCount: 1 # SystemReplicationCount [1..n](default is 3) + #failureDomain: "device" # Cluster's failureDomain ["device", "host", "zone"] (default is "host") #trlogProcessingInterval: 2 # set transaction log processing interval to 2s to speed up ISGW Link delivery #trlogKeepDays: 2 # keep up to 2 days of transaction log interval batches to reduce local storage overhead #useHostLocalTime: true diff --git a/pkg/apis/edgefs.rook.io/v1/cluster.go b/pkg/apis/edgefs.rook.io/v1/cluster.go index 307bdbd7e..f8589de00 100644 --- a/pkg/apis/edgefs.rook.io/v1/cluster.go +++ b/pkg/apis/edgefs.rook.io/v1/cluster.go @@ -47,6 +47,55 @@ func (deploymentConfig *ClusterDeploymentConfig) GetRtlfsDevices() []RtlfsDevice return rtlfsDevices } +func (deploymentConfig *ClusterDeploymentConfig) GetRtkvsDevicesCount() int { + rtkvsDevicesCount := 0 + for _, devConfig := range deploymentConfig.DevConfig { + if devConfig.IsGatewayNode { + continue + } + rtkvsDevicesCount += len(devConfig.Rtkvs.Devices) + } + return rtkvsDevicesCount +} + +func (deploymentConfig *ClusterDeploymentConfig) GetRtrdDevicesCount() int { + rtrdsDevicesCount := 0 + for _, devConfig := range deploymentConfig.DevConfig { + if devConfig.IsGatewayNode { + continue + } + rtrdsDevicesCount += len(devConfig.Rtrd.Devices) + for _, slave := range devConfig.RtrdSlaves { + rtrdsDevicesCount += len(slave.Devices) + } + } + return rtrdsDevicesCount +} + +func (deploymentConfig *ClusterDeploymentConfig) GetTargetsCount() int { + targets := 0 + for _, devConfig := range deploymentConfig.DevConfig { + if devConfig.IsGatewayNode { + continue + } + targets++ + } + return targets +} + +func (deploymentConfig *ClusterDeploymentConfig) GetRtrdContainersCount() int { + + containers := 0 + for _, devConfig := range deploymentConfig.DevConfig { + if devConfig.IsGatewayNode { + continue + } + // 1 is main target container + containers += 1 + len(devConfig.RtrdSlaves) + } + return containers +} + func (deploymentConfig *ClusterDeploymentConfig) CompatibleWith(newConfig ClusterDeploymentConfig) (bool, error) { if deploymentConfig.DeploymentType != newConfig.DeploymentType { return false, fmt.Errorf("DeploymentType `%s` != `%s` for updated cluster configuration", deploymentConfig.DeploymentType, newConfig.DeploymentType) diff --git a/pkg/operator/edgefs/cluster/cluster_deployment_config.go b/pkg/operator/edgefs/cluster/cluster_deployment_config.go index 46e7c645e..739c45b96 100644 --- a/pkg/operator/edgefs/cluster/cluster_deployment_config.go +++ b/pkg/operator/edgefs/cluster/cluster_deployment_config.go @@ -18,6 +18,7 @@ package cluster import ( "fmt" "os" + "strings" edgefsv1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1" rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" @@ -40,6 +41,7 @@ func (c *cluster) createClusterReconfigurationSpec(existingConfig edgefsv1.Clust if err != nil { return ClusterReconfigureSpec{}, err } + logger.Debugf("ClusterSpec: %s", ToJSON(c.Spec)) transportKey := getClusterTransportKey(deploymentType) @@ -112,6 +114,30 @@ func (c *cluster) getClusterDeploymentType() (string, error) { return "", fmt.Errorf("Can't determine deployment type for [%s] cluster", c.Namespace) } +func (c *cluster) getClusterSysRepCount() int { + if c.Spec.SystemReplicationCount > 0 { + return c.Spec.SystemReplicationCount + } + return 3 +} + +func (c *cluster) getClusterFailureDomain() (string, error) { + failureDomain := "host" + if len(c.Spec.FailureDomain) > 0 { + switch strings.ToLower(c.Spec.FailureDomain) { + case "device": + failureDomain = "device" + case "host": + failureDomain = "host" + case "zone": + failureDomain = "zone" + default: + return "", fmt.Errorf("Unknow failure domain %s, skipped", c.Spec.FailureDomain) + } + } + return failureDomain, nil +} + func (c *cluster) needPrivelege(deploymentType string) bool { needPriveleges := false @@ -285,7 +311,7 @@ func (c *cluster) createDevicesConfig(deploymentType string, node rookv1alpha2.N } // ValidateZones validates all nodes in cluster that each one has valid zone number or all of them has zone == 0 -func validateZones(deploymentConfig edgefsv1.ClusterDeploymentConfig) error { +func validateZones(deploymentConfig edgefsv1.ClusterDeploymentConfig, sysRepCount int, failureDomain string) error { validZonesFound := 0 for _, nodeDevConfig := range deploymentConfig.DevConfig { if nodeDevConfig.Zone > 0 { @@ -297,6 +323,12 @@ func validateZones(deploymentConfig edgefsv1.ClusterDeploymentConfig) error { return fmt.Errorf("Valid Zone number must be propagated to all nodes") } + if failureDomain == "zone" { + if validZonesFound < sysRepCount { + return fmt.Errorf("Assigned cluster zones count =%d should be greater or equal then cluster.Spec.SysRepCount = %d", validZonesFound, sysRepCount) + } + } + return nil } @@ -306,7 +338,13 @@ func (c *cluster) validateDeploymentConfig(deploymentConfig edgefsv1.ClusterDepl return fmt.Errorf("ClusterDeploymentConfig has no valid TransportKey or DeploymentType") } - err := validateZones(deploymentConfig) + sysRepCount := c.getClusterSysRepCount() + failureDomain, err := c.getClusterFailureDomain() + if err != nil { + return err + } + + err = validateZones(deploymentConfig, sysRepCount, failureDomain) if err != nil { return err } @@ -320,32 +358,61 @@ func (c *cluster) validateDeploymentConfig(deploymentConfig edgefsv1.ClusterDepl return fmt.Errorf("RTKVS configuration error: storage.useAllNodes and storage.useAllDevices options aren't allowed." + " A per-node NVME KVSSD disks list is expected") } + + switch failureDomain { + case "device": + rtkvsDevices := deploymentConfig.GetRtkvsDevicesCount() + if rtkvsDevices < sysRepCount { + return fmt.Errorf("Rtkvs devices should be greater or equal then SysRepCount=%d on all nodes summary", sysRepCount) + } + case "host": + targets := deploymentConfig.GetTargetsCount() + if targets < sysRepCount { + return fmt.Errorf("Rtkvs targets=(%d) should be greater or equal then SysRepCount=%d on all nodes summary", targets, sysRepCount) + } + } } else if deploymentConfig.TransportKey == edgefsv1.DeploymentRtlfs { // Check directories devices count on all nodes for autoRtlfs mode - deploymentNodesCount := len(deploymentConfig.DevConfig) - if len(c.Spec.Storage.Directories) > 0 { - if len(c.Spec.Storage.Directories)*deploymentNodesCount < 3 { - return fmt.Errorf("Rtlfs devices should be more then 3 on all nodes summary") + switch failureDomain { + case "device": + deploymentNodesCount := len(deploymentConfig.DevConfig) + if len(c.Spec.Storage.Directories) > 0 { + if len(c.Spec.Storage.Directories)*deploymentNodesCount < sysRepCount { + return fmt.Errorf("Rtlfs devices should be greater or equal then SysRepCount=%d on all nodes summary", sysRepCount) + } + } else { // Autoftlfs case: 4 folders per node + if (4 * deploymentNodesCount) < sysRepCount { + return fmt.Errorf("AutoRtlfs devices should be greater or equal then SysRepCount=%d on all nodes summary", sysRepCount) + } + } + + case "host": + targets := deploymentConfig.GetTargetsCount() + if targets < sysRepCount { + return fmt.Errorf("Rtlfs containers=(%d) should be greater or equal then SysRepCount=%d on all nodes summary", targets, sysRepCount) } } } else if deploymentConfig.TransportKey == edgefsv1.DeploymentRtrd { - // Check all deployment nodes has available disk devices - devicesCount := 0 for nodeName, devCfg := range deploymentConfig.DevConfig { - if devCfg.IsGatewayNode { continue } - if len(devCfg.Rtrd.Devices) == 0 && !dro.NeedToResurrect { return fmt.Errorf("Node %s has no devices to deploy", nodeName) } - devicesCount += devCfg.GetRtrdDeviceCount() } - // Check new deployment devices count - if !dro.NeedToResurrect && devicesCount < 3 { - return fmt.Errorf("At least 3 empty disks required for Edgefs cluster. Those disks should be distributed over all nodes specified for [%s] EdgeFS cluster. Currently there are `%d` nodes cluster and `%d` disks only", c.Namespace, len(deploymentConfig.DevConfig), devicesCount) + switch failureDomain { + case "device": + rtrdDevicesCount := deploymentConfig.GetRtrdDevicesCount() + if rtrdDevicesCount < sysRepCount { + return fmt.Errorf("Rtrd devices=(%d) should be greater or equal then SysRepCount=%d on all nodes summary", rtrdDevicesCount, sysRepCount) + } + case "host": + containers := deploymentConfig.GetRtrdContainersCount() + if containers < sysRepCount { + return fmt.Errorf("Rtrd containers=(%d) should be greater or equal then SysRepCount=%d on all nodes summary", containers, sysRepCount) + } } } return nil diff --git a/pkg/operator/edgefs/cluster/utils.go b/pkg/operator/edgefs/cluster/utils.go index 8ecc4bd05..f1e26d8e8 100644 --- a/pkg/operator/edgefs/cluster/utils.go +++ b/pkg/operator/edgefs/cluster/utils.go @@ -246,6 +246,7 @@ func (c *cluster) PrintDeploymentConfig(deploymentConfig *edgefsv1.ClusterDeploy logger.Infof("\t\tContainer[0] Path: /mnt/disks/disk0") logger.Infof("\t\tContainer[0] Path: /mnt/disks/disk1") logger.Infof("\t\tContainer[0] Path: /mnt/disks/disk2") + logger.Infof("\t\tContainer[0] Path: /mnt/disks/disk3") default: logger.Errorf("[%s] Unknown DeploymentType '%s'", c.Namespace, deploymentConfig.DeploymentType) }