diff --git a/pkg/operator/edgefs/cluster/cluster.go b/pkg/operator/edgefs/cluster/cluster.go new file mode 100644 index 000000000..b15d1b3d2 --- /dev/null +++ b/pkg/operator/edgefs/cluster/cluster.go @@ -0,0 +1,158 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "fmt" + "reflect" + "sort" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/edgefs/cluster/mgr" + "github.com/rook/rook/pkg/operator/edgefs/cluster/target" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + httpPortDefault = int32(8080) + httpPortName = "http" + grpcPortDefault = int32(6789) + grpcPortName = "grpc" + udpTotemPortDefault = int32(5405) + udpTotemPortName = "totem" + volumeNameDataDir = "datadir" + defaultServiceAccountName = "rook-edgefs-cluster" + + /* Volumes definitions */ + configName = "edgefs-config" +) + +type cluster struct { + context *clusterd.Context + Namespace string + Spec edgefsv1alpha1.ClusterSpec + ownerRef metav1.OwnerReference + targets *target.Cluster + mgrs *mgr.Cluster + stopCh chan struct{} +} + +func newCluster(c *edgefsv1alpha1.Cluster, context *clusterd.Context) *cluster { + + return &cluster{ + context: context, + Namespace: c.Namespace, + Spec: c.Spec, + stopCh: make(chan struct{}), + ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)), + } +} + +func (c *cluster) createInstance(rookImage string) error { + + logger.Infof("Cluster spec\n %+v", c.Spec) + + // + // Validate Cluster CRD + // + if err := c.validateClusterSpec(); err != nil { + logger.Errorf("invalid cluster spec: %+v", err) + return err + } + + // Create a configmap for overriding edgefs config settings + // These settings should only be modified by a user after they are initialized + placeholderConfig := map[string]string{ + k8sutil.ConfigOverrideVal: "", + } + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.ConfigOverrideName, + }, + Data: placeholderConfig, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef) + _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err) + } + + // + // Create and start EdgeFS Targets StatefulSet + // + c.targets = target.New(c.context, c.Namespace, "latest", c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.DataVolumeSize, + edgefsv1alpha1.GetTargetPlacement(c.Spec.Placement), c.Spec.Network, c.Spec.Resources, c.ownerRef) + + err = c.targets.Start(rookImage, c.Spec.DevicesResurrectMode) + if err != nil { + return fmt.Errorf("failed to start the targets. %+v", err) + } + + // + // Create and start EdgeFS manager Deployment (gRPC proxy, Prometheus metrics) + // + c.mgrs = mgr.New(c.context, c.Namespace, "latest", c.Spec.ServiceAccount, c.Spec.DataDirHostPath, c.Spec.DataVolumeSize, + edgefsv1alpha1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network, + v1.ResourceRequirements{}, c.ownerRef) + err = c.mgrs.Start(rookImage) + if err != nil { + return fmt.Errorf("failed to start the edgefs mgr. %+v", err) + } + + logger.Infof("Done creating rook instance in namespace %s", c.Namespace) + return nil +} + +func (c *cluster) validateClusterSpec() error { + if len(c.Spec.DataDirHostPath) == 0 && c.Spec.DataVolumeSize.Value() == 0 { + return fmt.Errorf("DataDirHostPath or DataVolumeSize EdgeFS cluster's options not specified.") + } + + if len(c.Spec.DataDirHostPath) > 0 && c.Spec.DataVolumeSize.Value() != 0 { + return fmt.Errorf("Both deployment options DataDirHostPath and DataVolumeSize are specified. Should be only one deployment option in cluster specification.") + } + + if len(c.Spec.Storage.Directories) > 0 && + ((c.Spec.Storage.UseAllDevices != nil && *c.Spec.Storage.UseAllDevices) || + len(c.Spec.Storage.Devices) > 0 || len(c.Spec.Storage.DeviceFilter) > 0) { + return fmt.Errorf("Directories option specified as well as Devices. Remove Directories or Devices option from cluster specification") + } + + logger.Info("Validate cluster spec") + return nil +} + +func clusterChanged(oldCluster, newCluster edgefsv1alpha1.ClusterSpec) bool { + changeFound := false + oldStorage := oldCluster.Storage + newStorage := newCluster.Storage + + // sort the nodes by name then compare to see if there are changes + sort.Sort(rookv1alpha2.NodesByName(oldStorage.Nodes)) + sort.Sort(rookv1alpha2.NodesByName(newStorage.Nodes)) + if !reflect.DeepEqual(oldStorage.Nodes, newStorage.Nodes) { + logger.Infof("The list of nodes has changed") + changeFound = true + } + + return changeFound +} diff --git a/pkg/operator/edgefs/cluster/controller.go b/pkg/operator/edgefs/cluster/controller.go new file mode 100644 index 000000000..25f2822f2 --- /dev/null +++ b/pkg/operator/edgefs/cluster/controller.go @@ -0,0 +1,432 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Portions of this file came from https://github.com/cockroachdb/cockroach, which uses the same license. +*/ + +// Package cluster to manage a edgefs cluster. +package cluster + +import ( + "fmt" + "reflect" + "time" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/discover" + "github.com/rook/rook/pkg/operator/edgefs/iscsi" + "github.com/rook/rook/pkg/operator/edgefs/nfs" + "github.com/rook/rook/pkg/operator/edgefs/s3" + "github.com/rook/rook/pkg/operator/edgefs/s3x" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" +) + +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-cluster") + finalizerName = fmt.Sprintf("%s.%s", ClusterResource.Name, ClusterResource.Group) +) + +const ( + CustomResourceName = "cluster" + CustomResourceNamePlural = "clusters" + appName = "rook-edgefs" + clusterCreateInterval = 6 * time.Second + clusterCreateTimeout = 5 * time.Minute + updateClusterInterval = 30 * time.Second + updateClusterTimeout = 1 * time.Hour + clusterDeleteRetryInterval = 2 //seconds + clusterDeleteMaxRetries = 15 + defaultEdgefsImageName = "edgefs/edgefs:latest" +) + +var ClusterResource = opkit.CustomResource{ + Name: CustomResourceName, + Plural: CustomResourceNamePlural, + Group: edgefsv1alpha1.CustomResourceGroup, + Version: edgefsv1alpha1.Version, + Scope: apiextensionsv1beta1.NamespaceScoped, + Kind: reflect.TypeOf(edgefsv1alpha1.Cluster{}).Name(), +} + +type ClusterController struct { + context *clusterd.Context + containerImage string + devicesInUse bool + clusterMap map[string]*cluster +} + +func NewClusterController(context *clusterd.Context, containerImage string) *ClusterController { + return &ClusterController{ + context: context, + containerImage: containerImage, + //createInitRetryInterval: createInitRetryIntervalDefault, + clusterMap: make(map[string]*cluster), + } +} + +func ClusterOwnerRef(namespace, clusterID string) metav1.OwnerReference { + blockOwner := true + return metav1.OwnerReference{ + APIVersion: ClusterResource.Version, + Kind: ClusterResource.Kind, + Name: namespace, + UID: types.UID(clusterID), + BlockOwnerDeletion: &blockOwner, + } +} + +func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) error { + + resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ + AddFunc: c.onAdd, + UpdateFunc: c.onUpdate, + DeleteFunc: c.onDelete, + } + + logger.Infof("start watching edgefs clusters in all namespaces") + watcher := opkit.NewWatcher(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient()) + go watcher.Watch(&edgefsv1alpha1.Cluster{}, stopCh) + + return nil +} + +func (c *ClusterController) StopWatch() { + for _, cluster := range c.clusterMap { + close(cluster.stopCh) + } + c.clusterMap = make(map[string]*cluster) +} + +func (c *ClusterController) onAdd(obj interface{}) { + clusterObj := obj.(*edgefsv1alpha1.Cluster).DeepCopy() + logger.Infof("new cluster %s added to namespace %s", clusterObj.Name, clusterObj.Namespace) + + cluster := newCluster(clusterObj, c.context) + c.clusterMap[cluster.Namespace] = cluster + + //Override rook containerImage value + c.containerImage = defaultEdgefsImageName + if cluster.Spec.EdgefsImageName != "" { + c.containerImage = cluster.Spec.EdgefsImageName + } + + logger.Infof("starting cluster in namespace %s", cluster.Namespace) + + if c.devicesInUse && cluster.Spec.Storage.AnyUseAllDevices() { + message := "using all devices in more than one namespace not supported" + logger.Error(message) + if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateError, message); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err) + } + return + } + + if cluster.Spec.Storage.AnyUseAllDevices() { + c.devicesInUse = true + } + + // Start the Rook cluster components. Retry several times in case of failure. + err := wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) { + if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateCreating, ""); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err) + return false, nil + } + + err := cluster.createInstance(c.containerImage) + if err != nil { + logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Namespace, err) + return false, nil + } + + // cluster is created, update the cluster CRD status now + if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateCreated, ""); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err) + return false, nil + } + + return true, nil + }) + if err != nil { + message := fmt.Sprintf("giving up creating cluster in namespace %s after %s", cluster.Namespace, clusterCreateTimeout) + logger.Error(message) + if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateError, message); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err) + } + return + } + + logger.Infof("succeeded creating and initializing EdgeFS cluster in namespace %s", cluster.Namespace) + + // Start NFS service CRD watcher + NFSController := nfs.NewNFSController(c.context, c.containerImage, + isHostNetworkDefined(cluster.Spec.Network), + cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize, + edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement), + cluster.Spec.Resources, + cluster.ownerRef) + NFSController.StartWatch(cluster.Namespace, cluster.stopCh) + + // Start S3 service CRD watcher + S3Controller := s3.NewS3Controller(c.context, c.containerImage, + isHostNetworkDefined(cluster.Spec.Network), + cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize, + edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement), + cluster.Spec.Resources, + cluster.ownerRef) + S3Controller.StartWatch(cluster.Namespace, cluster.stopCh) + + // Start S3X service CRD watcher + S3XController := s3x.NewS3XController(c.context, c.containerImage, + isHostNetworkDefined(cluster.Spec.Network), + cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize, + edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement), + cluster.Spec.Resources, + cluster.ownerRef) + S3XController.StartWatch(cluster.Namespace, cluster.stopCh) + + // Start ISCSI service CRD watcher + ISCSIController := iscsi.NewISCSIController(c.context, c.containerImage, + isHostNetworkDefined(cluster.Spec.Network), + cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize, + edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement), + cluster.Spec.Resources, + cluster.ownerRef) + ISCSIController.StartWatch(cluster.Namespace, cluster.stopCh) + + // add the finalizer to the crd + err = c.addFinalizer(clusterObj) + if err != nil { + logger.Errorf("failed to add finalizer to cluster crd. %+v", err) + } +} + +func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { + oldCluster := oldObj.(*edgefsv1alpha1.Cluster).DeepCopy() + newCluster := newObj.(*edgefsv1alpha1.Cluster).DeepCopy() + logger.Infof("update event for cluster %s", newCluster.Namespace) + + // Check if the cluster is being deleted. This code path is called when a finalizer is specified in the crd. + // When a cluster is requested for deletion, K8s will only set the deletion timestamp if there are any finalizers in the list. + // K8s will only delete the crd and child resources when the finalizers have been removed from the crd. + if newCluster.DeletionTimestamp != nil { + logger.Infof("cluster %s has a deletion timestamp", newCluster.Namespace) + err := c.handleDelete(newCluster, time.Duration(clusterDeleteRetryInterval)*time.Second) + if err != nil { + logger.Errorf("failed finalizer for cluster. %+v", err) + return + } + // remove the finalizer from the crd, which indicates to k8s that the resource can safely be deleted + c.removeFinalizer(newCluster) + return + } + + if !clusterChanged(oldCluster.Spec, newCluster.Spec) { + logger.Infof("update event for cluster %s is not supported", newCluster.Namespace) + return + } + + logger.Infof("update event for cluster %s is supported, orchestrating update now", newCluster.Namespace) + logger.Debugf("old cluster: %+v", oldCluster.Spec) + logger.Debugf("new cluster: %+v", newCluster.Spec) + + cluster, ok := c.clusterMap[newCluster.Namespace] + if !ok { + logger.Errorf("Cannot update cluster %s that does not exist", newCluster.Namespace) + return + } + cluster.Spec = newCluster.Spec + + // attempt to update the cluster. note this is done outside of wait.Poll because that function + // will wait for the retry interval before trying for the first time. + done, _ := c.handleUpdate(newCluster, cluster) + if done { + return + } + + err := wait.Poll(updateClusterInterval, updateClusterTimeout, func() (bool, error) { + return c.handleUpdate(newCluster, cluster) + }) + if err != nil { + message := fmt.Sprintf("giving up trying to update cluster in namespace %s after %s", cluster.Namespace, updateClusterTimeout) + logger.Error(message) + if err := c.updateClusterStatus(newCluster.Namespace, newCluster.Name, edgefsv1alpha1.ClusterStateError, message); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", newCluster.Namespace, err) + } + return + } + logger.Infof("cluster %s updated in namespace %s", newCluster.Name, newCluster.Namespace) +} + +func (c *ClusterController) handleUpdate(newClust *edgefsv1alpha1.Cluster, cluster *cluster) (bool, error) { + if err := c.updateClusterStatus(newClust.Namespace, newClust.Name, edgefsv1alpha1.ClusterStateUpdating, ""); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", newClust.Namespace, err) + return false, nil + } + + if err := cluster.createInstance(c.containerImage); err != nil { + logger.Errorf("failed to update cluster in namespace %s. %+v", newClust.Namespace, err) + return false, nil + } + + if err := c.updateClusterStatus(newClust.Namespace, newClust.Name, edgefsv1alpha1.ClusterStateCreated, ""); err != nil { + logger.Errorf("failed to update cluster status in namespace %s: %+v", newClust.Namespace, err) + return false, nil + } + + logger.Infof("succeeded updating cluster in namespace %s", newClust.Namespace) + return true, nil +} + +func (c *ClusterController) onDelete(obj interface{}) { + clust := obj.(*edgefsv1alpha1.Cluster).DeepCopy() + + logger.Infof("delete event for cluster %s in namespace %s", clust.Name, clust.Namespace) + + err := c.handleDelete(clust, time.Duration(clusterDeleteRetryInterval)*time.Second) + if err != nil { + logger.Errorf("failed to delete cluster. %+v", err) + } + if cluster, ok := c.clusterMap[clust.Namespace]; ok { + close(cluster.stopCh) + delete(c.clusterMap, clust.Namespace) + } + if clust.Spec.Storage.AnyUseAllDevices() { + c.devicesInUse = false + } + discover.FreeDevicesByCluster(c.context, clust.Namespace) +} + +func (c *ClusterController) handleDelete(clust *edgefsv1alpha1.Cluster, retryInterval time.Duration) error { + cluster, ok := c.clusterMap[clust.Namespace] + if !ok { + return fmt.Errorf("Cannot delete cluster %s that does not exist", clust.Namespace) + } + + for _, node := range cluster.targets.Storage.Nodes { + k := cluster.Namespace + err := cluster.targets.RemoveLabelOffNode(cluster.context.Clientset, node.Name, []string{k}) + logger.Infof("removed label %s from %s: %+v", k, node.Name, err) + } + + // delete associated node labels + return nil +} + +func (c *ClusterController) updateClusterStatus(namespace, name string, state edgefsv1alpha1.ClusterState, message string) error { + // get the most recent cluster CRD object + cluster, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get cluster from namespace %s prior to updating its status: %+v", namespace, err) + } + + // update the status on the retrieved cluster object + cluster.Status = edgefsv1alpha1.ClusterStatus{State: state, Message: message} + if _, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Update(cluster); err != nil { + return fmt.Errorf("failed to update cluster %s status: %+v", cluster.Namespace, err) + } + + return nil +} + +func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool { + if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 { + return true + } + return false +} + +func (c *ClusterController) addFinalizer(clust *edgefsv1alpha1.Cluster) error { + + // get the latest cluster object since we probably updated it before we got to this point (e.g. by updating its status) + clust, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(clust.Namespace).Get(clust.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + // add the finalizer (cluster.edgefs.rook.io) if it is not yet defined on the cluster CRD + for _, finalizer := range clust.Finalizers { + if finalizer == finalizerName { + logger.Infof("finalizer already set on cluster %s", clust.Namespace) + return nil + } + } + + // adding finalizer to the cluster crd + clust.Finalizers = append(clust.Finalizers, finalizerName) + + // update the crd + _, err = c.context.RookClientset.EdgefsV1alpha1().Clusters(clust.Namespace).Update(clust) + if err != nil { + return fmt.Errorf("failed to add finalizer to cluster. %+v", err) + } + + logger.Infof("added finalizer to cluster %s", clust.Name) + return nil +} + +func (c *ClusterController) removeFinalizer(obj interface{}) { + var fname string + var objectMeta *metav1.ObjectMeta + + // first determine what type/version of cluster we are dealing with + if cl, ok := obj.(*edgefsv1alpha1.Cluster); ok { + fname = finalizerName + objectMeta = &cl.ObjectMeta + } else { + logger.Warningf("cannot remove finalizer from object that is not a cluster: %+v", obj) + return + } + + // remove the finalizer from the slice if it exists + found := false + for i, finalizer := range objectMeta.Finalizers { + if finalizer == fname { + objectMeta.Finalizers = append(objectMeta.Finalizers[:i], objectMeta.Finalizers[i+1:]...) + found = true + break + } + } + if !found { + logger.Infof("finalizer %s not found in the cluster crd '%s'", fname, objectMeta.Name) + return + } + + // update the crd to remove the finalizer for good. retry several times in case of intermittent failures. + maxRetries := 5 + retrySeconds := 5 * time.Second + for i := 0; i < maxRetries; i++ { + var err error + if cluster, ok := obj.(*edgefsv1alpha1.Cluster); ok { + _, err = c.context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Update(cluster) + } + + if err != nil { + logger.Errorf("failed to remove finalizer %s from cluster %s. %+v", fname, objectMeta.Name, err) + time.Sleep(retrySeconds) + continue + } + logger.Infof("removed finalizer %s from cluster %s", fname, objectMeta.Name) + return + } + + logger.Warningf("giving up from removing the %s cluster finalizer", fname) +} diff --git a/pkg/operator/edgefs/cluster/controller_test.go b/pkg/operator/edgefs/cluster/controller_test.go new file mode 100644 index 000000000..0ffc19c3e --- /dev/null +++ b/pkg/operator/edgefs/cluster/controller_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cluster + +import ( + "os" + "testing" + "time" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + testop "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestClusterDelete(t *testing.T) { + //nodeName := "node841" + clusterName := "cluster684" + //pvName := "pvc-540" + rookSystemNamespace := "rook-system-6413" + + os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace) + defer os.Unsetenv(k8sutil.PodNamespaceEnvVar) + + clientset := testop.New(3) + context := &clusterd.Context{ + Clientset: clientset, + } + + // create the cluster controller and tell it that the cluster has been deleted + controller := NewClusterController(context, "") + clusterToDelete := &edgefsv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}} + controller.handleDelete(clusterToDelete, time.Microsecond) +} + +func TestClusterChanged(t *testing.T) { + // a new node added, should be a change + old := edgefsv1alpha1.ClusterSpec{ + Storage: rookalpha.StorageScopeSpec{ + Nodes: []rookalpha.Node{ + {Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + }, + }, + } + new := edgefsv1alpha1.ClusterSpec{ + Storage: rookalpha.StorageScopeSpec{ + Nodes: []rookalpha.Node{ + {Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + {Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + }, + }, + } + assert.True(t, clusterChanged(old, new)) + + // a node was removed, should be a change + old.Storage.Nodes = []rookalpha.Node{ + {Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + {Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + } + new.Storage.Nodes = []rookalpha.Node{ + {Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + } + assert.True(t, clusterChanged(old, new)) + + // the nodes being in a different order should not be a change + old.Storage.Nodes = []rookalpha.Node{ + {Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + {Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + } + new.Storage.Nodes = []rookalpha.Node{ + {Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + {Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}}, + } + assert.False(t, clusterChanged(old, new)) +} + +func TestRemoveFinalizer(t *testing.T) { + clientset := testop.New(3) + context := &clusterd.Context{ + Clientset: clientset, + RookClientset: rookfake.NewSimpleClientset(), + } + controller := NewClusterController(context, "") + + // ***************************************** + // start with a current version edgefs cluster + // ***************************************** + cluster := &edgefsv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1893", + Namespace: "namespace-6551", + Finalizers: []string{finalizerName}, + }, + } + + // create the cluster initially so it exists in the k8s api + cluster, err := context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Create(cluster) + assert.NoError(t, err) + assert.Len(t, cluster.Finalizers, 1) + + // remove the finalizer from the cluster object + controller.removeFinalizer(cluster) + + // verify the finalier was removed + cluster, err = context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Get(cluster.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.NotNil(t, cluster) + assert.Len(t, cluster.Finalizers, 0) +} diff --git a/pkg/operator/edgefs/cluster/mgr/mgr.go b/pkg/operator/edgefs/cluster/mgr/mgr.go new file mode 100644 index 000000000..f24b06526 --- /dev/null +++ b/pkg/operator/edgefs/cluster/mgr/mgr.go @@ -0,0 +1,295 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mgr for the Edgefs manager. +package mgr + +import ( + "fmt" + "strconv" + + "github.com/coreos/pkg/capnslog" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-mgr") + +const ( + appName = "rook-edgefs-mgr" + defaultServiceAccountName = "rook-edgefs-cluster" + defaultGrpcPort = 6789 + defaultMetricsPort = 8881 + + /* Volumes definitions */ + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" +) + +// Cluster is the edgefs mgr manager +type Cluster struct { + Namespace string + Version string + serviceAccount string + Replicas int + dataDirHostPath string + dataVolumeSize resource.Quantity + placement rookalpha.Placement + context *clusterd.Context + HostNetworkSpec edgefsv1alpha1.NetworkSpec + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference +} + +// New creates an instance of the mgr +func New( + context *clusterd.Context, namespace, version string, + serviceAccount string, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, HostNetworkSpec edgefsv1alpha1.NetworkSpec, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *Cluster { + + if serviceAccount == "" { + // if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient. + serviceAccount = defaultServiceAccountName + logger.Infof("setting the mgr pod to use the service account name: %s", serviceAccount) + } + + return &Cluster{ + context: context, + Namespace: namespace, + serviceAccount: serviceAccount, + placement: placement, + Version: version, + Replicas: 1, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + HostNetworkSpec: HostNetworkSpec, + resources: resources, + ownerRef: ownerRef, + } +} + +func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool { + if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 { + return true + } + return false +} + +// Start the mgr instance +func (c *Cluster) Start(rookImage string) error { + logger.Infof("start running mgr") + + logger.Infof("Mgr Image is %s", rookImage) + // start the deployment + deployment := c.makeDeployment(appName, c.Namespace, rookImage, 1) + if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s deployment. %+v", appName, err) + } + logger.Infof("%s deployment already exists", appName) + } else { + logger.Infof("%s deployment started", appName) + } + + // create the mgr service + service := c.makeMgrService(appName) + if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(service); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create mgr service. %+v", err) + } + logger.Infof("mgr mgr service already exists") + } else { + logger.Infof("mgr mgr service started") + } + + return nil +} + +func (c *Cluster) makeMgrService(name string) *v1.Service { + labels := c.getLabels() + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.Namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeClusterIP, + Ports: []v1.ServicePort{ + { + Name: "mgr", + Port: int32(defaultGrpcPort), + Protocol: v1.ProtocolTCP, + }, + { + Name: "http-metrics", + Port: int32(defaultMetricsPort), + Protocol: v1.ProtocolTCP, + }, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *Cluster) makeDeployment(name, clusterName, rookImage string, replicas int32) *extensions.Deployment { + + volumes := []v1.Volume{} + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + // dataDir case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + }, + }, + }) + } + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: c.getDaemonLabels(clusterName), + Annotations: map[string]string{"prometheus.io/scrape": "true", + "prometheus.io/port": strconv.Itoa(defaultMetricsPort)}, + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.serviceAccount, + Containers: []v1.Container{c.mgrContainer(name, rookImage)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + HostIPC: true, + HostNetwork: isHostNetworkDefined(c.HostNetworkSpec), + NodeSelector: map[string]string{c.Namespace: "cluster"}, + }, + } + if isHostNetworkDefined(c.HostNetworkSpec) { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + c.placement.ApplyToPodSpec(&podSpec.Spec) + + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: c.Namespace, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas}, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *Cluster) mgrContainer(name string, containerImage string) v1.Container { + + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc.target", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"mgmt"}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + SecurityContext: securityContext, + Resources: c.resources, + Ports: []v1.ContainerPort{ + { + Name: "mgr", + ContainerPort: int32(defaultGrpcPort), + Protocol: v1.ProtocolTCP, + }, + { + Name: "http-metrics", + ContainerPort: int32(defaultMetricsPort), + Protocol: v1.ProtocolTCP, + }, + }, + VolumeMounts: volumeMounts, + } +} + +func (c *Cluster) getLabels() map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: c.Namespace, + } +} + +func (c *Cluster) getDaemonLabels(clusterName string) map[string]string { + labels := c.getLabels() + labels["instance"] = clusterName + return labels +} diff --git a/pkg/operator/edgefs/cluster/mgr/mgr_test.go b/pkg/operator/edgefs/cluster/mgr/mgr_test.go new file mode 100644 index 000000000..fa53fc4d7 --- /dev/null +++ b/pkg/operator/edgefs/cluster/mgr/mgr_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package mgr + +import ( + "io/ioutil" + "os" + "strconv" + "testing" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + testop "github.com/rook/rook/pkg/operator/test" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestStartMGR(t *testing.T) { + executor := &exectest.MockExecutor{ + MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) { + return "{\"key\":\"mysecurekey\"}", nil + }, + } + + configDir, _ := ioutil.TempDir("", "") + defer os.RemoveAll(configDir) + context := &clusterd.Context{ + Executor: executor, + ConfigDir: configDir, + Clientset: testop.New(3)} + volSize := resource.NewQuantity(100000.0, resource.BinarySI) + c := New(context, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + // start a basic service + err := c.Start("edgefs") + assert.Nil(t, err) + validateStart(t, c) + + // starting again with more replicas + c.Replicas = 3 + err = c.Start("edgefs") + assert.Nil(t, err) + validateStart(t, c) +} + +func validateStart(t *testing.T, c *Cluster) { + + _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("rook-edgefs-mgr", metav1.GetOptions{}) + assert.Nil(t, err) + + _, err = c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-edgefs-mgr", metav1.GetOptions{}) + assert.Nil(t, err) +} + +func TestPodSpec(t *testing.T) { + volSize := resource.NewQuantity(100000.0, resource.BinarySI) + c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "rook/rook:myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), + }, + }, metav1.OwnerReference{}) + + d := c.makeDeployment("mgr-a", "rook-edgefs", "edgefs", 1) + assert.NotNil(t, d) + assert.Equal(t, "mgr-a", d.Name) + assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) + assert.Equal(t, 1, len(d.Spec.Template.Spec.Volumes)) + + //check ports + assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers[0].Ports)) + assert.Equal(t, int32(6789), d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort) + assert.Equal(t, int32(8881), d.Spec.Template.Spec.Containers[0].Ports[1].ContainerPort) + + assert.Equal(t, "edgefs-datadir", d.Spec.Template.Spec.Volumes[0].Name) + assert.Equal(t, "mgr-a", d.ObjectMeta.Name) + assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"]) + assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"]) + assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) + + assert.Equal(t, 2, len(d.Spec.Template.ObjectMeta.Annotations)) + assert.Equal(t, "true", d.Spec.Template.ObjectMeta.Annotations["prometheus.io/scrape"]) + assert.Equal(t, strconv.Itoa(defaultMetricsPort), d.Spec.Template.ObjectMeta.Annotations["prometheus.io/port"]) + + cont := d.Spec.Template.Spec.Containers[0] + assert.Equal(t, 2, len(cont.VolumeMounts)) + assert.Equal(t, "mgmt", cont.Args[0]) + + assert.Equal(t, "100", cont.Resources.Limits.Cpu().String()) + assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) +} + +func TestServiceSpec(t *testing.T) { + volSize := resource.NewQuantity(100000.0, resource.BinarySI) + c := New(&clusterd.Context{}, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + s := c.makeMgrService("rook-edgefs-mgr") + assert.NotNil(t, s) + assert.Equal(t, "rook-edgefs-mgr", s.Name) + assert.Equal(t, 2, len(s.Spec.Ports)) +} + +func TestHostNetwork(t *testing.T) { + volSize := resource.NewQuantity(100000.0, resource.BinarySI) + c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{ServerIfName: "eth0"}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + d := c.makeDeployment("mgr-a", "a", "edgefs", 1) + assert.NotNil(t, d) + + assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork) + assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy) +} diff --git a/pkg/operator/edgefs/cluster/target/config/config.go b/pkg/operator/edgefs/cluster/target/config/config.go new file mode 100644 index 000000000..58012936e --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/config/config.go @@ -0,0 +1,132 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package config for Edgefs target config managed by the operator +package config + +import ( + "strconv" + + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/operator/k8sutil" +) + +const ( + configStoreNameFmt = "rook-edgefs-%s-config" + osdDirsKeyName = "target-dirs" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "target-config") + +func GetConfigStoreName(nodeName string) string { + return k8sutil.TruncateNodeName(configStoreNameFmt, nodeName) +} + +const ( + RtVerifyChidKey = "rtVerifyChid" + LmdbPageSizeKey = "lmdbPageSize" + UseBcacheKey = "useBCache" + UseBcacheWBKey = "useBCacheWB" + UseMetadataMaskKey = "useMetadataMask" + UseMetadataOffloadKey = "useMetadataOffload" + UseAllSSDKey = "useAllSSD" + RtrdPlevelOverrideKey = "rtrdPLevelOverride" +) + +type StoreConfig struct { + // 0 (disabled), 1 (verify on write) or 2(verify on read/write) + RtVerifyChid int `json:"rtVerifyChid,omitempty"` + // 4096, 8192, 16384 or 32768 + LmdbPageSize int `json:"lmdbPageSize,omitempty"` + // enable use of back cache + UseBCache bool `json:"useBCache,omitempty"` + // enable write back cache + UseBCacheWB bool `json:"useBCacheWB,omitempty"` + // what guts needs to go to SSD and what not + UseMetadataMask string `json:"useMetadataMask,omitempty"` + // when useAllSSD is false, enable metadata offload on SSD + UseMetadataOffload bool `json:"useMetadataOffload,omitempty"` + // only look for SSD/NVMe + UseAllSSD bool `json:"useAllSSD,omitempty"` + // if > 0, override automatic partitioning numbering logic + RtrdPLevelOverride int `json:"rtrdPLevelOverride,omitempty"` +} + +func DefaultStoreConfig() StoreConfig { + return StoreConfig{ + RtVerifyChid: 1, + LmdbPageSize: 16348, + UseBCacheWB: false, + UseMetadataMask: "0xff", + UseMetadataOffload: true, + UseAllSSD: false, + RtrdPLevelOverride: 0, + } +} + +var validLmbdPageSize = map[int]bool{ + 4096: true, + 8192: true, + 16384: true, + 32768: true, +} + +func ToStoreConfig(config map[string]string) StoreConfig { + storeConfig := DefaultStoreConfig() + for k, v := range config { + switch k { + case RtVerifyChidKey: + if storeConfig.RtVerifyChid < 0 || storeConfig.RtVerifyChid > 2 { + storeConfig.RtVerifyChid = convertToIntIgnoreErr(v) + } + case LmdbPageSizeKey: + + if validLmbdPageSize[storeConfig.LmdbPageSize] { + storeConfig.LmdbPageSize = convertToIntIgnoreErr(v) + } + case UseBcacheWBKey: + storeConfig.UseBCacheWB = convertToBoolIgnoreErr(v) + case UseMetadataMaskKey: + storeConfig.UseMetadataMask = v + case UseMetadataOffloadKey: + storeConfig.UseMetadataOffload = convertToBoolIgnoreErr(v) + case UseAllSSDKey: + storeConfig.UseAllSSD = convertToBoolIgnoreErr(v) + case RtrdPlevelOverrideKey: + storeConfig.RtrdPLevelOverride = convertToIntIgnoreErr(v) + } + } + + return storeConfig +} + +func convertToIntIgnoreErr(raw string) int { + val, err := strconv.Atoi(raw) + if err != nil { + val = 0 + } + + return val +} + +func convertToBoolIgnoreErr(raw string) bool { + val, err := strconv.ParseBool(raw) + if err != nil { + val = false + } + + return val +} diff --git a/pkg/operator/edgefs/cluster/target/configmap.go b/pkg/operator/edgefs/cluster/target/configmap.go new file mode 100644 index 000000000..9c592a72d --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/configmap.go @@ -0,0 +1,207 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + "encoding/json" + + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + defaultServerIfName = "eth0" + defaultBrokerIfName = "eth0" +) + +type RtlfsDevice struct { + Name string `json:"name"` + Path string `json:"path"` + CheckMountpoint int `json:"check_mountpoint"` +} + +type RtlfsDevices struct { + Devices []RtlfsDevice `json:"devices"` +} + +type CcowTenant struct { + FailureDomain int `json:"failure_domain"` +} + +type CcowNetwork struct { + BrokerInterfaces string `json:"broker_interfaces"` + ServerUnixSocket string `json:"server_unix_socket"` + BrokerIP4addr string `json:"broker_ip4addr,omitempty"` + ServerIP4addr string `json:"server_ip4addr,omitempty"` +} + +type CcowConf struct { + Tenant CcowTenant `json:"tenant"` + Network CcowNetwork `json:"network"` +} + +type CcowdNetwork struct { + ServerInterfaces string `json:"server_interfaces"` + ServerUnixSocket string `json:"server_unix_socket"` + ServerIP4addr string `json:"server_ip4addr,omitempty"` +} + +type CcowdConf struct { + Network CcowdNetwork `json:"network"` + Transport []string `json:"transport"` +} + +type AuditdConf struct { + IsAggregator int `json:"is_aggregator"` +} + +type SetupNode struct { + Ccow CcowConf `json:"ccow"` + Ccowd CcowdConf `json:"ccowd"` + Auditd AuditdConf `json:"auditd"` + Ipv4Autodetect int `json:"ipv4_autodetect,omitempty"` + RtlfsAutodetect string `json:"rtlfs_autodetect,omitempty"` + ClusterNodes []string `json:"cluster_nodes,omitempty"` + Rtrd RTDevices `json:"rtrd"` + Rtlfs RtlfsDevices `json:"rtlfs"` +} + +// As we relying on StatefulSet, we want to build global ConfigMap shared +// to all the nodes in the cluster. This way configuration is simplified and +// available to all subcomponents at any point it time. +func (c *Cluster) createSetupConfigs(resurrect bool) error { + + cm := make(map[string]SetupNode) + + dnsRecords := make([]string, len(c.Storage.Nodes)) + for i := 0; i < len(c.Storage.Nodes); i++ { + dnsRecords[i] = createQualifiedHeadlessServiceName(i, c.Namespace) + } + + serverIfName := defaultServerIfName + brokerIfName := defaultBrokerIfName + if isHostNetworkDefined(c.HostNetworkSpec) { + + if len(c.HostNetworkSpec.ServerIfName) > 0 && len(c.HostNetworkSpec.BrokerIfName) > 0 { + serverIfName = c.HostNetworkSpec.ServerIfName + brokerIfName = c.HostNetworkSpec.BrokerIfName + } else if len(c.HostNetworkSpec.ServerIfName) > 0 { + serverIfName = c.HostNetworkSpec.ServerIfName + brokerIfName = c.HostNetworkSpec.ServerIfName + } else if len(c.HostNetworkSpec.BrokerIfName) > 0 { + serverIfName = c.HostNetworkSpec.BrokerIfName + brokerIfName = c.HostNetworkSpec.BrokerIfName + } + } + + // fully resolve the storage config and resources for all nodes + for _, node := range c.Storage.Nodes { + devConfig := c.deploymentConfig.devConfig[node.Name] + rtDevices := devConfig.rtrd.Devices + rtlfsDevices := devConfig.rtlfs.Devices + + rtlfsAutoDetectPath := "" + if c.deploymentConfig.deploymentType == deploymentAutoRtlfs && + !devConfig.isGatewayNode { + rtlfsAutoDetectPath = "/data" + } + + if resurrect || devConfig.isGatewayNode { + // In resurrection case we only need to adjust networking selections + // in ccow.json, ccowd.json and corosync.conf. And keep device transport + // same as before. Resurrection is "best effort" feature, we cannot + // guarnatee that cluster can be reconfigured, but at least we do try. + + rtDevices = make([]RTDevice, 0) + rtlfsDevices = make([]RtlfsDevice, 0) + } + + nodeConfig := SetupNode{ + Ccow: CcowConf{ + Tenant: CcowTenant{ + FailureDomain: 1, + }, + Network: CcowNetwork{ + BrokerInterfaces: brokerIfName, + ServerUnixSocket: "/opt/nedge/var/run/sock/ccowd.sock", + }, + }, + Ccowd: CcowdConf{ + Network: CcowdNetwork{ + ServerInterfaces: serverIfName, + ServerUnixSocket: "/opt/nedge/var/run/sock/ccowd.sock", + }, + Transport: []string{c.deploymentConfig.transportKey}, + }, + Auditd: AuditdConf{ + IsAggregator: 0, + }, + Rtrd: RTDevices{ + Devices: rtDevices, + }, + Rtlfs: RtlfsDevices{ + Devices: rtlfsDevices, + }, + Ipv4Autodetect: 1, + RtlfsAutodetect: rtlfsAutoDetectPath, + ClusterNodes: dnsRecords, + } + + cm[node.Name] = nodeConfig + + logger.Debugf("Resolved Node %s = %+v", node.Name, cm[node.Name]) + } + + nesetupJson, err := json.Marshal(&cm) + if err != nil { + return err + } + + dataMap := make(map[string]string, 1) + dataMap["nesetup"] = string(nesetupJson) + + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configName, + Namespace: c.Namespace, + }, + Data: dataMap, + } + + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &configMap.ObjectMeta, &c.ownerRef) + if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil { + if errors.IsAlreadyExists(err) { + if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(configMap); err != nil { + return nil + } + } else { + return err + } + } + + // success. do the labeling so that StatefulSet scheduler will + // select the right nodes. + for _, node := range c.Storage.Nodes { + k := c.Namespace + err = c.AddLabelsToNode(c.context.Clientset, node.Name, map[string]string{k: "cluster"}) + logger.Debugf("added label %s from %s: %+v", k, node.Name, err) + } + + return nil +} diff --git a/pkg/operator/edgefs/cluster/target/layout.go b/pkg/operator/edgefs/cluster/target/layout.go new file mode 100644 index 000000000..3e682c41d --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/layout.go @@ -0,0 +1,189 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + "fmt" + "math" + "path/filepath" + "strings" + + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/operator/edgefs/cluster/target/config" + "github.com/rook/rook/pkg/util/sys" +) + +type RTDevices struct { + Devices []RTDevice `json:"devices"` +} + +type RTDevice struct { + Name string `json:"name,omitempty"` + Device string `json:"device,omitempty"` + Psize int `json:"psize,omitempty"` + VerifyChid int `json:"verify_chid,omitempty"` + Journal string `json:"journal,omitempty"` + Metadata string `json:"metadata,omitempty"` + BcacheWritearound int `json:"bcache_writearound,omitempty"` + PlevelOverride int `json:"plevel_override,omitempty"` +} + +func getIdDevLinkName(dls string) (dl string) { + dlsArr := strings.Split(dls, " ") + for i := range dlsArr { + s := strings.Replace(dlsArr[i], "/dev/disk/by-id/", "", 1) + if strings.Contains(s, "/") || strings.Contains(s, "wwn-") { + continue + } + dl = s + break + } + return dl +} + +func GetRTDevices(nodeDisks []sys.LocalDisk, storeConfig *config.StoreConfig) (rtDevices []RTDevice, err error) { + rtDevices = make([]RTDevice, 0) + if storeConfig == nil { + return rtDevices, fmt.Errorf("no pointer to StoreConfig provided") + } + + if len(nodeDisks) == 0 { + return rtDevices, nil + } + + var ssds []sys.LocalDisk + var hdds []sys.LocalDisk + var devices []sys.LocalDisk + + for i := range nodeDisks { + if !nodeDisks[i].Empty || len(nodeDisks[i].Partitions) > 0 { + continue + } + if nodeDisks[i].Rotational { + hdds = append(hdds, nodeDisks[i]) + } else { + ssds = append(ssds, nodeDisks[i]) + } + devices = append(devices, nodeDisks[i]) + } + + //var rtdevs []RTDevice + if storeConfig.UseAllSSD { + // + // All flush media case (High Performance) + // + if len(ssds) == 0 { + return rtDevices, fmt.Errorf("No SSD/NVMe media found") + } + if storeConfig.UseMetadataOffload { + fmt.Println("Warning: useMetadataOffload parameter is ignored due to use useAllSSD=true") + } + + for i := range devices { + if devices[i].Rotational { + continue + } + rtdev := RTDevice{ + Name: getIdDevLinkName(devices[i].DevLinks), + Device: "/dev/" + devices[i].Name, + Psize: storeConfig.LmdbPageSize, + VerifyChid: storeConfig.RtVerifyChid, + } + if storeConfig.RtrdPLevelOverride != 0 { + rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride + } + rtDevices = append(rtDevices, rtdev) + } + return rtDevices, nil + } + + if len(hdds) == 0 { + return rtDevices, fmt.Errorf("No HDD media found") + } + + if !storeConfig.UseMetadataOffload { + // + // All HDD media case (capacity, cold archive) + // + for i := range devices { + if !devices[i].Rotational { + continue + } + rtdev := RTDevice{ + Name: getIdDevLinkName(devices[i].DevLinks), + Device: "/dev/" + devices[i].Name, + Psize: storeConfig.LmdbPageSize, + VerifyChid: storeConfig.RtVerifyChid, + } + if storeConfig.RtrdPLevelOverride != 0 { + rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride + } + rtDevices = append(rtDevices, rtdev) + } + return rtDevices, nil + } + + // + // Hybrid SSD/HDD media case (optimal) + // + if len(hdds) < len(ssds) || len(ssds) == 0 { + return rtDevices, fmt.Errorf("Confusing use of useMetadataOffload parameter HDDs(%d) < SSDs(%d)\n", len(hdds), len(ssds)) + } + + chunkSize := int(math.Ceil(float64(len(hdds) / len(ssds)))) + var hdds_divided [][]sys.LocalDisk + for i := 0; i < len(hdds); i += chunkSize { + end := i + chunkSize + + if end > len(hdds) { + end = len(hdds) + } + hdds_divided = append(hdds_divided, hdds[i:end]) + } + + for i := range hdds_divided { + for j := range hdds_divided[i] { + rtdev := RTDevice{ + Name: getIdDevLinkName(hdds_divided[i][j].DevLinks), + Device: "/dev/" + hdds_divided[i][j].Name, + Psize: storeConfig.LmdbPageSize, + VerifyChid: storeConfig.RtVerifyChid, + BcacheWritearound: (map[bool]int{true: 0, false: 1})[storeConfig.UseBCacheWB], + Journal: getIdDevLinkName(ssds[i].DevLinks), + Metadata: getIdDevLinkName(ssds[i].DevLinks) + "," + storeConfig.UseMetadataMask, + } + if storeConfig.RtrdPLevelOverride != 0 { + rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride + } + rtDevices = append(rtDevices, rtdev) + } + } + return rtDevices, nil +} + +func getRtlfsDevices(directories []rookalpha.Directory) []RtlfsDevice { + rtlfsDevices := make([]RtlfsDevice, 0) + for _, dir := range directories { + rtlfsDevice := RtlfsDevice{ + Name: filepath.Base(dir.Path), + Path: dir.Path, + CheckMountpoint: 1, + } + rtlfsDevices = append(rtlfsDevices, rtlfsDevice) + } + return rtlfsDevices +} diff --git a/pkg/operator/edgefs/cluster/target/pod.go b/pkg/operator/edgefs/cluster/target/pod.go new file mode 100644 index 000000000..0fffeabea --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/pod.go @@ -0,0 +1,451 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/operator/k8sutil" + appsv1beta1 "k8s.io/api/apps/v1beta1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/kubernetes/pkg/kubelet/apis" +) + +const ( + dataDirsEnvVarName = "ROOK_DATA_DIRECTORIES" + + udpTotemPortDefault = int32(5405) + udpTotemPortName = "totem" + volumeNameDataDir = "datadir" + + /* Volumes definitions */ + configVolumeName = "edgefs-configdir" + configName = "edgefs-config" + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" +) + +func (c *Cluster) createAppLabels() map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + } +} + +func (c *Cluster) makeCorosyncContainer(containerImage string) v1.Container { + + privileged := c.deploymentConfig.needPriviliges + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "IPC_LOCK"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + { + Name: configVolumeName, + MountPath: "/opt/nedge/etc/config", + }, + { + Name: dataVolumeName, + MountPath: "/data", + }, + } + + return v1.Container{ + Name: "corosync", + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"corosync"}, + SecurityContext: securityContext, + //Resources: c.resources, + VolumeMounts: volumeMounts, + Env: []v1.EnvVar{ + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + } +} + +func (c *Cluster) makeAuditdContainer(containerImage string) v1.Container { + + privileged := c.deploymentConfig.needPriviliges + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + return v1.Container{ + Name: "auditd", + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"auditd"}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + SecurityContext: securityContext, + //Resources: c.resources, + VolumeMounts: volumeMounts, + } +} + +func (c *Cluster) makeDaemonContainer(containerImage string, dro DevicesResurrectOptions, isInitContainer bool) v1.Container { + + privileged := c.deploymentConfig.needPriviliges + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + Privileged: &privileged, + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: "devices", + MountPath: "/dev", + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + if c.deploymentConfig.deploymentType == deploymentAutoRtlfs { + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: dataVolumeName, MountPath: "/data"}) + } else if c.deploymentConfig.deploymentType == deploymentRtlfs { + rtlfsDevices := getRtlfsDevices(c.Storage.Directories) + for _, device := range rtlfsDevices { + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: device.Name, MountPath: device.Path}) + } + } + + name := "daemon" + args := []string{"daemon"} + if isInitContainer { + if dro.needToZap { + name = "daemon-zap" + args = []string{"toolbox", "nezap --do-as-i-say"} + + // zap mode is InitContainer, it needs to mount config + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: configVolumeName, + MountPath: "/opt/nedge/etc/config", + }) + } + } else { + if dro.needToWait { + args = []string{"wait"} + } + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: args, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + SecurityContext: securityContext, + Resources: c.resources, + VolumeMounts: volumeMounts, + } +} + +func (c *Cluster) configOverrideVolume() v1.Volume { + cmSource := &v1.ConfigMapVolumeSource{Items: []v1.KeyToPath{{Key: "nesetup", Path: "nesetup.json"}}} + cmSource.Name = configName + return v1.Volume{Name: configVolumeName, VolumeSource: v1.VolumeSource{ConfigMap: cmSource}} +} + +func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool { + if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 { + return true + } + return false +} + +func (c *Cluster) createPodSpec(rookImage string, dro DevicesResurrectOptions) v1.PodSpec { + terminationGracePeriodSeconds := int64(60) + + DNSPolicy := v1.DNSClusterFirst + if isHostNetworkDefined(c.HostNetworkSpec) { + DNSPolicy = v1.DNSClusterFirstWithHostNet + } + + volumes := []v1.Volume{ + { + Name: "devices", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev", + }, + }, + }, + } + + hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + Type: &hostPathDirectoryOrCreate, + }, + }, + }) + } + + if c.deploymentConfig.deploymentType == deploymentRtlfs { + // RTLFS with specified folders + for _, folder := range c.deploymentConfig.directories { + volumes = append(volumes, v1.Volume{ + Name: folder.Name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: folder.Path, + Type: &hostPathDirectoryOrCreate, + }, + }, + }) + + } + + } + + var containers []v1.Container + var initContainers []v1.Container + if dro.needToZap { + // To execute "zap" functions (devices or directories) we + // create InitContainer to ensure it completes fully. + initContainers = []v1.Container{ + c.makeDaemonContainer(rookImage, dro, true), + } + } + + volumes = append(volumes, c.configOverrideVolume()) + + containers = []v1.Container{ + c.makeCorosyncContainer(rookImage), + c.makeAuditdContainer(rookImage), + c.makeDaemonContainer(rookImage, dro, false), + } + + return v1.PodSpec{ + ServiceAccountName: c.serviceAccount, + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: int32(100), + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: k8sutil.AppAttr, + Operator: metav1.LabelSelectorOpIn, + Values: []string{appName}, + }, + }, + }, + TopologyKey: apis.LabelHostname, + }, + }, + }, + }, + }, + NodeSelector: map[string]string{c.Namespace: "cluster"}, + InitContainers: initContainers, + Containers: containers, + // No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node. + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + DNSPolicy: DNSPolicy, + HostIPC: true, + HostNetwork: isHostNetworkDefined(c.HostNetworkSpec), + Volumes: volumes, + } +} + +func (c *Cluster) makeStatefulSet(replicas int32, rookImage string, dro DevicesResurrectOptions) (*appsv1beta1.StatefulSet, error) { + + statefulSet := &appsv1beta1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: c.Namespace, + Labels: c.createAppLabels(), + }, + Spec: appsv1beta1.StatefulSetSpec{ + ServiceName: appName, + Replicas: &replicas, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: c.Namespace, + Labels: c.createAppLabels(), + }, + Spec: c.createPodSpec(rookImage, dro), + }, + PodManagementPolicy: appsv1beta1.ParallelPodManagement, + UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{ + Type: appsv1beta1.RollingUpdateStatefulSetStrategyType, + }, + }, + } + + if c.dataVolumeSize.Value() > 0 { + statefulSet.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: dataVolumeName, + Namespace: c.Namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: c.dataVolumeSize, + }, + }, + }, + }, + } + } + + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &statefulSet.ObjectMeta, &c.ownerRef) + c.placement.ApplyToPodSpec(&statefulSet.Spec.Template.Spec) + + return statefulSet, nil +} + +func (c *Cluster) makeHeadlessServicePorts(totemPort int32) []v1.ServicePort { + return []v1.ServicePort{ + { + // The secondary port serves the UI as well as health and debug endpoints. + Name: udpTotemPortName, + Port: int32(totemPort), + Protocol: v1.ProtocolUDP, + TargetPort: intstr.FromInt(int(totemPort)), + }, + } +} + +// This service only exists to create DNS entries for each pod in the stateful +// set such that they can resolve each other's IP addresses. It does not +// create a load-balanced ClusterIP and should not be used directly by clients +// in most circumstances. +func (c *Cluster) makeHeadlessService() (*v1.Service, error) { + + headlessService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: c.Namespace, + Labels: c.createAppLabels(), + Annotations: map[string]string{ + "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", + }, + }, + Spec: v1.ServiceSpec{ + Selector: c.createAppLabels(), + PublishNotReadyAddresses: true, + ClusterIP: "None", + Ports: c.makeHeadlessServicePorts(udpTotemPortDefault), + }, + } + k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &headlessService.ObjectMeta, &c.ownerRef) + + return headlessService, nil +} diff --git a/pkg/operator/edgefs/cluster/target/pod_test.go b/pkg/operator/edgefs/cluster/target/pod_test.go new file mode 100644 index 000000000..248c04754 --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/pod_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + "testing" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/edgefs/cluster/target/config" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestStorageSpecDefaultPV(t *testing.T) { + storageSpec := rookalpha.StorageScopeSpec{ + Nodes: []rookalpha.Node{ + { + Name: "node1", + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", + storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI), + rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + c.createSetupConfigs(false) +} + +func TestStorageSpecDevices(t *testing.T) { + storageSpec := rookalpha.StorageScopeSpec{ + Selection: rookalpha.Selection{ + Directories: []rookalpha.Directory{{Path: "/rook/dir2"}}, + }, + Nodes: []rookalpha.Node{ + { + Name: "node1", + Selection: rookalpha.Selection{ + Devices: []rookalpha.Device{{Name: "sda"}, {Name: "sdb"}}, + }, + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", + storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI), + rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + c.createSetupConfigs(false) +} + +func TestStorageSpecDirectories(t *testing.T) { + storageSpec := rookalpha.StorageScopeSpec{ + Selection: rookalpha.Selection{ + Directories: []rookalpha.Directory{{Path: "/rook/dir2"}}, + }, + Nodes: []rookalpha.Node{ + { + Name: "node1", + Selection: rookalpha.Selection{ + Directories: []rookalpha.Directory{{Path: "/rook/dir1"}, {Path: "/rook/dir2"}}, + }, + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", + storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI), + rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + c.createSetupConfigs(false) +} + +func TestStorageSpecConfig(t *testing.T) { + storageSpec := rookalpha.StorageScopeSpec{ + Config: map[string]string{ + "useAllSSD": "false", + "useBCacheWB": "true", + "useBCache": "true", + }, + Nodes: []rookalpha.Node{ + { + Name: "node1", + Location: "zone1", + Config: map[string]string{ + "rtTransport": "rtrd", + "useAllSSD": "true", + "useMetadataOffload": "false", + }, + Selection: rookalpha.Selection{ + Devices: []rookalpha.Device{{Name: "sda"}, {Name: "sdb"}}, + }, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI), + }, + }, + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "", + storageSpec, "", *resource.NewQuantity(100000.0, resource.BinarySI), + rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + c.createSetupConfigs(false) + + n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name) + storeConfig := config.ToStoreConfig(storageSpec.Nodes[0].Config) + + // First Node's useAllSSD should override global useAllSSD value, -> should be different + assert.NotEqual(t, storageSpec.Config["useAllSSD"], storeConfig.UseAllSSD) + // useBCacheWB global in StorageSpec and does override config' UseBCacheWB prop, -> should be the same + assert.NotEqual(t, storageSpec.Config["useBCacheWB"], storeConfig.UseBCacheWB) + // useBCache global in StorageSpec and does override config' UseBCache prop, -> should be the same + assert.NotEqual(t, storageSpec.Config["useBCache"], storeConfig.UseBCache) + // UseMetadataOffload will override default value of true + assert.Equal(t, storeConfig.UseMetadataOffload, false) + + logger.Infof("Node Config is %+v", n.Config) + logger.Infof("storeConfig is %+v", storeConfig) + +} + +func TestHostNetwork(t *testing.T) { + storageSpec := rookalpha.StorageScopeSpec{ + Nodes: []rookalpha.Node{ + { + Name: "node1", + Location: "zone1", + Config: map[string]string{ + "rtTransport": "rtrd", + "useAllSSD": "true", + }, + }, + }, + } + + clientset := fake.NewSimpleClientset() + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", + storageSpec, "", *resource.NewQuantity(100000.0, resource.BinarySI), + rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{ServerIfName: "eth0"}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + c.createSetupConfigs(false) +} diff --git a/pkg/operator/edgefs/cluster/target/target.go b/pkg/operator/edgefs/cluster/target/target.go new file mode 100644 index 000000000..47e323f47 --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/target.go @@ -0,0 +1,469 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/coreos/pkg/capnslog" + "github.com/golang/glog" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/discover" + "github.com/rook/rook/pkg/operator/edgefs/cluster/target/config" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/sys" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-target") + +const ( + appName = "rook-edgefs-target" + targetAppNameFmt = "rook-edgefs-target-id-%d" + appNameFmt = "rook-edgefs-target-%s" + targetLabelKey = "edgefs-target-id" + defaultServiceAccountName = "rook-edgefs-cluster" + unknownID = -1 + labelingRetries = 5 + + //deployment types + deploymentRtlfs = "rtlfs" + deploymentRtrd = "rtrd" + deploymentAutoRtlfs = "autoRtlfs" + nodeTypeLabelFmt = "%s-nodetype" +) + +// Cluster keeps track of the Targets +type Cluster struct { + context *clusterd.Context + Namespace string + placement rookalpha.Placement + Version string + Storage rookalpha.StorageScopeSpec + dataDirHostPath string + dataVolumeSize resource.Quantity + HostNetworkSpec edgefsv1alpha1.NetworkSpec + Privileged bool + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference + serviceAccount string + deploymentConfig ClusterDeploymentConfig +} + +type ClusterDeploymentConfig struct { + deploymentType string //rtlfs, rtrd, autortlfs + transportKey string //rtlfs or rtrd + directories []RtlfsDevice //cluster wide directories + devConfig map[string]DevicesConfig + needPriviliges bool +} + +type DevicesConfig struct { + rtrd RTDevices + rtlfs RtlfsDevices + isGatewayNode bool +} + +// New creates an instance of the Target manager +func New( + context *clusterd.Context, + namespace, + version, + serviceAccount string, + storageSpec rookalpha.StorageScopeSpec, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, + hostNetworkSpec edgefsv1alpha1.NetworkSpec, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *Cluster { + + if serviceAccount == "" { + // if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient. + serviceAccount = defaultServiceAccountName + logger.Infof("setting the target pods to use the service account name: %s", serviceAccount) + } + return &Cluster{ + context: context, + Namespace: namespace, + serviceAccount: serviceAccount, + placement: placement, + Version: version, + Storage: storageSpec, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + HostNetworkSpec: hostNetworkSpec, + Privileged: (isHostNetworkDefined(hostNetworkSpec) || os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true"), + resources: resources, + ownerRef: ownerRef, + deploymentConfig: ClusterDeploymentConfig{devConfig: make(map[string]DevicesConfig, 0)}, + } +} + +type DevicesResurrectOptions struct { + needToResurrect bool + needToZap bool + needToWait bool +} + +func ParseDevicesResurrectMode(resurrectMode string) DevicesResurrectOptions { + drm := DevicesResurrectOptions{} + if len(resurrectMode) == 0 { + return drm + } + resurrectModeToLower := strings.ToLower(strings.TrimSpace(resurrectMode)) + switch resurrectModeToLower { + case "restore": + drm.needToResurrect = true + break + case "restorezap": + drm.needToResurrect = true + drm.needToZap = true + break + case "restorezapwait": + drm.needToResurrect = true + drm.needToZap = true + drm.needToWait = true + break + } + + return drm +} + +// Start the target management +func (c *Cluster) Start(rookImage, devicesResurrectMode string) (err error) { + logger.Infof("start running targets in namespace %s", c.Namespace) + + logger.Infof("Target Image is %s", rookImage) + if c.Storage.UseAllNodes == false && len(c.Storage.Nodes) == 0 { + logger.Warningf("useAllNodes is set to false and no nodes are specified, no Edgefs pods are going to be created") + } + + //logger.Infof("Cluster storage config %+v", c.Storage.Config) + if c.Storage.UseAllNodes { + // resolve all storage nodes + c.Storage.Nodes = nil + rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar) + allNodeDevices, err := discover.ListDevices(c.context, rookSystemNS, "" /* all nodes */) + //logger.Infof("allNodeDevices: %+v", allNodeDevices) + if err != nil { + logger.Warningf("failed to get storage nodes from namespace %s: %v", rookSystemNS, err) + return err + } + for nodeName := range allNodeDevices { + storageNode := rookalpha.Node{ + Name: nodeName, + } + c.Storage.Nodes = append(c.Storage.Nodes, storageNode) + } + } + validNodes := k8sutil.GetValidNodes(c.Storage.Nodes, c.context.Clientset, c.placement) + //logger.Infof("validNOdes: %+v", validNodes) + logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes)) + c.Storage.Nodes = validNodes + + rmo := ParseDevicesResurrectMode(devicesResurrectMode) + logger.Infof("DevicesResurrect mode: %s options %+v", devicesResurrectMode, rmo) + + err = c.createDeploymentConfig(rmo.needToResurrect) + if err != nil { + logger.Errorf("Failed to create deploymentConfig %+v", err) + return err + } + + logger.Infof("Deployment Config : %+v", c.deploymentConfig) + + if err := c.createSetupConfigs(rmo.needToResurrect); err != nil { + logger.Errorf("Failed to create/update Edgefs cluster configuration: %+v", err) + return err + } + + headlessService, _ := c.makeHeadlessService() + if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(headlessService); err != nil { + if !errors.IsAlreadyExists(err) { + return err + } + logger.Infof("headless service %s already exists in namespace %s", headlessService.Name, headlessService.Namespace) + } else { + logger.Infof("headless service %s started in namespace %s", headlessService.Name, headlessService.Namespace) + } + + statefulSet, _ := c.makeStatefulSet(int32(len(validNodes)), rookImage, rmo) + if _, err := c.context.Clientset.AppsV1beta1().StatefulSets(c.Namespace).Create(statefulSet); err != nil { + if !errors.IsAlreadyExists(err) { + return err + } + logger.Infof("stateful set %s already exists in namespace %s", statefulSet.Name, statefulSet.Namespace) + } else { + logger.Infof("stateful set %s created in namespace %s", statefulSet.Name, statefulSet.Namespace) + } + return nil +} + +func (c *Cluster) HasDevicesSpecification() bool { + + if len(c.Storage.DeviceFilter) > 0 || len(c.Storage.Devices) > 0 { + return true + } + + for _, node := range c.Storage.Nodes { + useAllDevices := node.UseAllDevices + if useAllDevices != nil && *useAllDevices { + return true + } + + if len(node.DeviceFilter) > 0 || len(node.Devices) > 0 { + return true + } + } + + return false +} + +func (c *Cluster) createDeploymentConfig(resurrect bool) error { + + //Fill deploymentConfig devices struct + for _, node := range c.Storage.Nodes { + n := c.resolveNode(node.Name) + + if n == nil { + return fmt.Errorf("node %s did not resolve to start target", node.Name) + } + + devicesConfig := DevicesConfig{} + devicesConfig.rtrd.Devices = make([]RTDevice, 0) + devicesConfig.rtlfs.Devices = make([]RtlfsDevice, 0) + + // if node labeled as gateway then return empty devises and skip RTDevices detection + if c.isGatewayLabeledNode(c.context.Clientset, node.Name) { + devicesConfig.isGatewayNode = true + c.deploymentConfig.devConfig[node.Name] = devicesConfig + continue + } + + rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar) + nodeDevices, _ := discover.ListDevices(c.context, rookSystemNS, n.Name) + + availDevs, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, + n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices()) + + if deviceErr != nil { + // Devices were specified but we couldn't find any. + // User needs to fix CRD. + return fmt.Errorf("failed to get devices for node %s cluster %s: %v", + n.Name, c.Namespace, deviceErr) + } + + // selects Disks from availDevs and translate to RTDevices + availDisks := []sys.LocalDisk{} + for _, dev := range availDevs { + for _, disk := range nodeDevices[n.Name] { + if disk.Name == dev.Name { + availDisks = append(availDisks, disk) + } + } + } + + storeConfig := config.ToStoreConfig(n.Config) + rtDevices, err := GetRTDevices(availDisks, &storeConfig) + if err != nil { + logger.Warningf("Can't get rtDevices for node %s due %v", n.Name, err) + rtDevices = make([]RTDevice, 0) + } + + devicesConfig.rtrd.Devices = rtDevices + devicesConfig.rtlfs.Devices = getRtlfsDevices(c.Storage.Directories) + c.deploymentConfig.devConfig[node.Name] = devicesConfig + } + + // Add Directories to deploymentConfig + c.deploymentConfig.directories = getRtlfsDevices(c.Storage.Directories) + + if len(c.Storage.Directories) > 0 && (len(c.dataDirHostPath) > 0 || c.dataVolumeSize.Value() != 0) { + c.deploymentConfig.deploymentType = deploymentRtlfs + c.deploymentConfig.transportKey = "rtlfs" + + // check directories devices count on all nodes + if len(c.Storage.Directories)*len(c.Storage.Nodes) < 3 { + return fmt.Errorf("Rtlfs devices should be more then 3 on all nodes summary") + } + + } else if c.HasDevicesSpecification() && (len(c.dataDirHostPath) > 0 || c.dataVolumeSize.Value() != 0) { + + // Check all deployment nodes has available disk devices + devicesCount := 0 + for nodeName, devCfg := range c.deploymentConfig.devConfig { + + if devCfg.isGatewayNode { + continue + } + + if len(devCfg.rtrd.Devices) == 0 && !resurrect { + return fmt.Errorf("Node %s has no available devices", nodeName) + } + devicesCount += len(devCfg.rtrd.Devices) + } + + // check new deployment devices count + if !resurrect && devicesCount < 3 { + return fmt.Errorf("Disk devices should be more then 3 on all nodes summary") + } + + c.deploymentConfig.deploymentType = deploymentRtrd + c.deploymentConfig.transportKey = "rtrd" + c.deploymentConfig.needPriviliges = true + } else if len(c.dataDirHostPath) == 0 || c.dataVolumeSize.Value() == 0 { + c.deploymentConfig.deploymentType = deploymentAutoRtlfs + c.deploymentConfig.transportKey = "rtlfs" + } else { + return fmt.Errorf("Unknown deployment type! Cluster spec:\n %+v", c) + } + + //set priviliges==true in case of HostNetwork + if len(c.HostNetworkSpec.ServerIfName) > 0 || len(c.HostNetworkSpec.BrokerIfName) > 0 { + c.deploymentConfig.needPriviliges = true + } + + return nil +} + +// creates a qualified name of the headless service for a given replica id and namespace, +// e.g., edgefs-0.edgefs.rook-edgefs +func createQualifiedHeadlessServiceName(replicaNum int, namespace string) string { + return fmt.Sprintf("%s-%d.%s.%s", appName, replicaNum, appName, namespace) +} + +func (c *Cluster) resolveNode(nodeName string) *rookalpha.Node { + // fully resolve the storage config and resources for this node + rookNode := c.Storage.ResolveNode(nodeName) + if rookNode == nil { + return nil + } + + // Apply directories from ClusterStorageSpec only + rookNode.Directories = c.Storage.Directories + + rookNode.Resources = k8sutil.MergeResourceRequirements(rookNode.Resources, c.resources) + + // ensure no invalid dirs are specified + var validDirs []rookalpha.Directory + for _, dir := range rookNode.Directories { + if dir.Path == k8sutil.DataDir || dir.Path == c.dataDirHostPath { + logger.Warningf("skipping directory %s that would conflict with the dataDirHostPath", dir.Path) + continue + } + validDirs = append(validDirs, dir) + } + rookNode.Directories = validDirs + + return rookNode +} + +func (c *Cluster) AddLabelsToNode(cs clientset.Interface, nodeName string, labels map[string]string) error { + tokens := make([]string, 0, len(labels)) + for k, v := range labels { + tokens = append(tokens, "\""+k+"\":\""+v+"\"") + } + labelString := "{" + strings.Join(tokens, ",") + "}" + patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString) + var err error + for attempt := 0; attempt < labelingRetries; attempt++ { + _, err = cs.CoreV1().Nodes().Patch(nodeName, types.MergePatchType, []byte(patch)) + if err != nil { + if !apierrs.IsConflict(err) { + return err + } + } else { + break + } + time.Sleep(100 * time.Millisecond) + } + return err +} + +// RemoveLabelOffNode is for cleaning up labels temporarily added to node, +// won't fail if target label doesn't exist or has been removed. +func (c *Cluster) RemoveLabelOffNode(cs clientset.Interface, nodeName string, labelKeys []string) error { + var node *v1.Node + var err error + for attempt := 0; attempt < labelingRetries; attempt++ { + node, err = cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + if node.Labels == nil { + return nil + } + for _, labelKey := range labelKeys { + if node.Labels == nil || len(node.Labels[labelKey]) == 0 { + break + } + delete(node.Labels, labelKey) + } + _, err = cs.CoreV1().Nodes().Update(node) + if err != nil { + if !apierrs.IsConflict(err) { + return err + } else { + glog.V(2).Infof("Conflict when trying to remove a labels %v from %v", labelKeys, nodeName) + } + } else { + break + } + time.Sleep(100 * time.Millisecond) + } + return err +} + +func (c *Cluster) isGatewayLabeledNode(cs clientset.Interface, nodeName string) bool { + labelMap, err := c.getNodeLabels(cs, nodeName) + if err != nil || labelMap == nil { + return false + } + + if nodeType, ok := labelMap[fmt.Sprintf(nodeTypeLabelFmt, c.Namespace)]; ok { + if nodeType == "gateway" { + return true + } + } + + return false +} + +func (c *Cluster) getNodeLabels(cs clientset.Interface, nodeName string) (map[string]string, error) { + node, err := cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + if node.Labels == nil { + return node.Labels, nil + } + return node.Labels, nil +} diff --git a/pkg/operator/edgefs/cluster/target/target_test.go b/pkg/operator/edgefs/cluster/target/target_test.go new file mode 100644 index 000000000..2428f8117 --- /dev/null +++ b/pkg/operator/edgefs/cluster/target/target_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package target + +import ( + "testing" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + discoverDaemon "github.com/rook/rook/pkg/daemon/discover" + "github.com/rook/rook/pkg/operator/k8sutil" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestStart(t *testing.T) { + rookImage := "edgefs/edgefs-operator" + clientset := fake.NewSimpleClientset() + volSize := resource.NewQuantity(100000.0, resource.BinarySI) + c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "", + rookalpha.StorageScopeSpec{}, "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{}) + + // Start the first time + err := c.Start(rookImage, "") + assert.Nil(t, err) + + // Should not fail if it already exists + err = c.Start(rookImage, "") + assert.Nil(t, err) +} + +func createDiscoverConfigmap(nodeName, ns string, clientset *fake.Clientset) error { + data := make(map[string]string, 1) + data[discoverDaemon.LocalDiskCMData] = `[{"name":"sdx","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405f826bd553d8c4dbf9f41c18be /dev/disk/by-id/wwn-0x6001405f826bd553d8c4dbf9f41c18be /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-1","size":10737418240,"uuid":"","serial":"36001405f826bd553d8c4dbf9f41c18be","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk02","wwn":"0x6001405f826bd553","wwnVendorExtension":"0x6001405f826bd553d8c4dbf9f41c18be","empty":true}]` + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-device-" + nodeName, + Namespace: ns, + Labels: map[string]string{ + k8sutil.AppAttr: discoverDaemon.AppName, + discoverDaemon.NodeAttr: nodeName, + }, + }, + Data: data, + } + _, err := clientset.CoreV1().ConfigMaps(ns).Create(cm) + return err +} diff --git a/pkg/operator/edgefs/iscsi/controller.go b/pkg/operator/edgefs/iscsi/controller.go new file mode 100644 index 000000000..2ed2f52ae --- /dev/null +++ b/pkg/operator/edgefs/iscsi/controller.go @@ -0,0 +1,171 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// scale-out, multi-cloud Edge-X ISCSI services controller +package iscsi + +import ( + "fmt" + "reflect" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/core/v1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +const ( + customResourceName = "iscsi" + customResourceNamePlural = "iscsis" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-iscsi") + +// ISCSIResource represents the iscsi custom resource +var ISCSIResource = opkit.CustomResource{ + Name: customResourceName, + Plural: customResourceNamePlural, + Group: edgefsv1alpha1.CustomResourceGroup, + Version: edgefsv1alpha1.Version, + Scope: apiextensionsv1beta1.NamespaceScoped, + Kind: reflect.TypeOf(edgefsv1alpha1.ISCSI{}).Name(), +} + +// ISCSIController represents a controller object for iscsi custom resources +type ISCSIController struct { + context *clusterd.Context + rookImage string + hostNetwork bool + dataDirHostPath string + dataVolumeSize resource.Quantity + placement rookalpha.Placement + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference +} + +// NewISCSIController create controller for watching ISCSI custom resources created +func NewISCSIController( + context *clusterd.Context, rookImage string, + hostNetwork bool, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *ISCSIController { + return &ISCSIController{ + context: context, + rookImage: rookImage, + hostNetwork: hostNetwork, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + placement: placement, + resources: resources, + ownerRef: ownerRef, + } +} + +// StartWatch watches for instances of ISCSI custom resources and acts on them +func (c *ISCSIController) StartWatch(namespace string, stopCh chan struct{}) error { + + resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ + AddFunc: c.onAdd, + UpdateFunc: c.onUpdate, + DeleteFunc: c.onDelete, + } + + logger.Infof("start watching iscsi resources in namespace %s", namespace) + watcher := opkit.NewWatcher(ISCSIResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient()) + go watcher.Watch(&edgefsv1alpha1.ISCSI{}, stopCh) + + return nil +} + +func (c *ISCSIController) onAdd(obj interface{}) { + iscsi, err := getISCSIObject(obj) + if err != nil { + logger.Errorf("failed to get iscsi object: %+v", err) + return + } + + if err = c.CreateService(*iscsi, c.serviceOwners(iscsi)); err != nil { + logger.Errorf("failed to create iscsi %s. %+v", iscsi.Name, err) + } +} + +func (c *ISCSIController) onUpdate(oldObj, newObj interface{}) { + oldService, err := getISCSIObject(oldObj) + if err != nil { + logger.Errorf("failed to get old iscsi object: %+v", err) + return + } + newService, err := getISCSIObject(newObj) + if err != nil { + logger.Errorf("failed to get new iscsi object: %+v", err) + return + } + + if !serviceChanged(oldService.Spec, newService.Spec) { + logger.Debugf("iscsi %s did not change", newService.Name) + return + } + + logger.Infof("applying iscsi %s changes", newService.Name) + if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil { + logger.Errorf("failed to create (modify) iscsi %s. %+v", newService.Name, err) + } +} + +func (c *ISCSIController) onDelete(obj interface{}) { + iscsi, err := getISCSIObject(obj) + if err != nil { + logger.Errorf("failed to get iscsi object: %+v", err) + return + } + + if err = c.DeleteService(*iscsi); err != nil { + logger.Errorf("failed to delete iscsi %s. %+v", iscsi.Name, err) + } +} + +func (c *ISCSIController) serviceOwners(service *edgefsv1alpha1.ISCSI) []metav1.OwnerReference { + // Only set the cluster crd as the owner of the ISCSI resources. + // If the ISCSI crd is deleted, the operator will explicitly remove the ISCSI resources. + // If the ISCSI crd still exists when the cluster crd is deleted, this will make sure the ISCSI + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + +func serviceChanged(oldService, newService edgefsv1alpha1.ISCSISpec) bool { + return false +} + +func getISCSIObject(obj interface{}) (iscsi *edgefsv1alpha1.ISCSI, err error) { + var ok bool + iscsi, ok = obj.(*edgefsv1alpha1.ISCSI) + if ok { + // the iscsi object is of the latest type, simply return it + return iscsi.DeepCopy(), nil + } + + return nil, fmt.Errorf("not a known iscsi object: %+v", obj) +} diff --git a/pkg/operator/edgefs/iscsi/iscsi.go b/pkg/operator/edgefs/iscsi/iscsi.go new file mode 100644 index 000000000..765363a02 --- /dev/null +++ b/pkg/operator/edgefs/iscsi/iscsi.go @@ -0,0 +1,362 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package ISCSI for the Edgefs manager. +package iscsi + +import ( + "encoding/json" + "fmt" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + appName = "rook-edgefs-iscsi" + + /* Volumes definitions */ + defaultTargetName = "iqn.2018-11.edgefs.io" + defaultTargetParams = "{}" + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" + defaultPort = 3260 +) + +// Start the ISCSI manager +func (c *ISCSIController) CreateService(s edgefsv1alpha1.ISCSI, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, false, ownerRefs) +} + +func (c *ISCSIController) UpdateService(s edgefsv1alpha1.ISCSI, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, true, ownerRefs) +} + +// Start the iscsi instance +func (c *ISCSIController) CreateOrUpdate(s edgefsv1alpha1.ISCSI, update bool, ownerRefs []metav1.OwnerReference) error { + logger.Infof("starting update=%v service=%s", update, s.Name) + + logger.Infof("ISCSI Base image is %s", c.rookImage) + // validate ISCSI service settings + if err := validateService(c.context, s); err != nil { + return fmt.Errorf("invalid ISCSI service %s arguments. %+v", s.Name, err) + } + + if len(s.Spec.TargetName) == 0 { + s.Spec.TargetName = defaultTargetName + } + + // check if ISCSI service already exists + exists, err := serviceExists(c.context, s) + if err == nil && exists { + if !update { + logger.Infof("ISCSI service %s exists in namespace %s", s.Name, s.Namespace) + return nil + } + logger.Infof("ISCSI service %s exists in namespace %s. checking for updates", s.Name, s.Namespace) + } + + // start the deployment + deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec) + if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s deployment. %+v", appName, err) + } + logger.Infof("%s deployment already exists", appName) + } else { + logger.Infof("%s deployment started", appName) + } + + // create the iscsi service + service := c.makeISCSIService(instanceName(s.Name), s.Name, s.Namespace, s.Spec) + if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create ISCSI service. %+v", err) + } + logger.Infof("ISCSI service %s already exists", service) + } else { + logger.Infof("ISCSI service %s started", service) + } + + return nil +} + +func (c *ISCSIController) makeISCSIService(name, svcname, namespace string, iscsiSpec edgefsv1alpha1.ISCSISpec) *v1.Service { + labels := getLabels(name, svcname, namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + {Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", Port: defaultPort, Protocol: v1.ProtocolTCP}, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *ISCSIController) makeDeployment(svcname, namespace, rookImage string, iscsiSpec edgefsv1alpha1.ISCSISpec) *extensions.Deployment { + + name := instanceName(svcname) + volumes := []v1.Volume{} + + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + // dataDir case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + }, + }, + }) + } + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: getLabels(name, svcname, namespace), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.iscsiContainer(svcname, name, rookImage, iscsiSpec)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + HostIPC: true, + HostNetwork: c.hostNetwork, + NodeSelector: map[string]string{namespace: "cluster"}, + }, + } + if c.hostNetwork { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + + // apply current ISCSI CRD options to pod's specification + iscsiSpec.Placement.ApplyToPodSpec(&podSpec.Spec) + + instancesCount := int32(1) + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &instancesCount}, + } + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *ISCSIController) iscsiContainer(svcname, name, containerImage string, iscsiSpec edgefsv1alpha1.ISCSISpec) v1.Container { + + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc.target", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"iscsi"}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "CCOW_SVCNAME", + Value: svcname, + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "KUBERNETES_NAMESPACE", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "EFSISCSI_TARGET_NAME", + Value: iscsiSpec.TargetName, + }, + { + Name: "EFSISCSI_TARGET_PARAMS", + Value: getTargetParamsJSON(iscsiSpec.TargetParams), + }, + }, + SecurityContext: securityContext, + Resources: iscsiSpec.Resources, + Ports: []v1.ContainerPort{ + {Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", ContainerPort: defaultPort, Protocol: v1.ProtocolTCP}, + }, + VolumeMounts: volumeMounts, + } +} + +// Delete ISCSI service and possibly some artifacts. +func (c *ISCSIController) DeleteService(s edgefsv1alpha1.ISCSI) error { + // check if service exists + exists, err := serviceExists(c.context, s) + if err != nil { + return fmt.Errorf("failed to detect if there is a ISCSI service to delete. %+v", err) + } + if !exists { + logger.Infof("ISCSI service %s does not exist in namespace %s", s.Name, s.Namespace) + return nil + } + + logger.Infof("Deleting ISCSI service %s from namespace %s", s.Name, s.Namespace) + + var gracePeriod int64 + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + + // Delete the iscsi service + err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options) + if err != nil && !errors.IsNotFound(err) { + logger.Warningf("failed to delete ISCSI service. %+v", err) + } + + // Make a best effort to delete the ISCSI pods + err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name)) + if err != nil { + logger.Warningf(err.Error()) + } + + logger.Infof("Completed deleting ISCSI service %s", s.Name) + return nil +} + +func getLabels(name, svcname, namespace string) map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: namespace, + "edgefs_svcname": svcname, + "edgefs_svctype": "iscsi", + } +} + +// Validate the ISCSI arguments +func validateService(context *clusterd.Context, s edgefsv1alpha1.ISCSI) error { + if s.Name == "" { + return fmt.Errorf("missing name") + } + if s.Namespace == "" { + return fmt.Errorf("missing namespace") + } + + return nil +} + +func instanceName(svcname string) string { + return fmt.Sprintf("%s-%s", appName, svcname) +} + +// Check if the ISCSI service exists +func serviceExists(context *clusterd.Context, s edgefsv1alpha1.ISCSI) (bool, error) { + _, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{}) + if err == nil { + // the deployment was found + return true, nil + } + if err != nil && !errors.IsNotFound(err) { + return false, err + } + + // not found + return false, nil +} + +func getTargetParamsJSON(params edgefsv1alpha1.TargetParametersSpec) string { + result := make(map[string]uint) + + if params.MaxRecvDataSegmentLength > 0 { + result["MaxRecvDataSegmentLength"] = params.MaxRecvDataSegmentLength + } + + if params.DefaultTime2Retain > 0 { + result["DefaultTime2Retain"] = params.DefaultTime2Retain + } + + if params.DefaultTime2Wait > 0 { + result["DefaultTime2Wait"] = params.DefaultTime2Wait + } + + if params.FirstBurstLength > 0 { + result["FirstBurstLength"] = params.FirstBurstLength + } + + if params.MaxBurstLength > 0 { + result["MaxBurstLength"] = params.MaxBurstLength + } + + if params.MaxQueueCmd > 0 { + result["MaxQueueCmd"] = params.MaxQueueCmd + } + + jsonString, _ := json.Marshal(result) + return string(jsonString) +} diff --git a/pkg/operator/edgefs/nfs/controller.go b/pkg/operator/edgefs/nfs/controller.go new file mode 100644 index 000000000..0870d1736 --- /dev/null +++ b/pkg/operator/edgefs/nfs/controller.go @@ -0,0 +1,171 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// scale-out, multi-cloud NFS +package nfs + +import ( + "fmt" + "reflect" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/core/v1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +const ( + customResourceName = "nfs" + customResourceNamePlural = "nfss" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-nfs") + +// NFSResource represents the nfs custom resource +var NFSResource = opkit.CustomResource{ + Name: customResourceName, + Plural: customResourceNamePlural, + Group: edgefsv1alpha1.CustomResourceGroup, + Version: edgefsv1alpha1.Version, + Scope: apiextensionsv1beta1.NamespaceScoped, + Kind: reflect.TypeOf(edgefsv1alpha1.NFS{}).Name(), +} + +// NFSController represents a controller object for nfs custom resources +type NFSController struct { + context *clusterd.Context + rookImage string + hostNetwork bool + dataDirHostPath string + dataVolumeSize resource.Quantity + placement rookalpha.Placement + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference +} + +// NewNFSController create controller for watching nfs custom resources created +func NewNFSController( + context *clusterd.Context, rookImage string, + hostNetwork bool, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *NFSController { + return &NFSController{ + context: context, + rookImage: rookImage, + hostNetwork: hostNetwork, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + placement: placement, + resources: resources, + ownerRef: ownerRef, + } +} + +// StartWatch watches for instances of NFS custom resources and acts on them +func (c *NFSController) StartWatch(namespace string, stopCh chan struct{}) error { + + resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ + AddFunc: c.onAdd, + UpdateFunc: c.onUpdate, + DeleteFunc: c.onDelete, + } + + logger.Infof("start watching nfs resources in namespace %s", namespace) + watcher := opkit.NewWatcher(NFSResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient()) + go watcher.Watch(&edgefsv1alpha1.NFS{}, stopCh) + + return nil +} + +func (c *NFSController) onAdd(obj interface{}) { + nfs, err := getNFSObject(obj) + if err != nil { + logger.Errorf("failed to get nfs object: %+v", err) + return + } + + if err = c.CreateService(*nfs, c.serviceOwners(nfs)); err != nil { + logger.Errorf("failed to create nfs %s. %+v", nfs.Name, err) + } +} + +func (c *NFSController) onUpdate(oldObj, newObj interface{}) { + oldService, err := getNFSObject(oldObj) + if err != nil { + logger.Errorf("failed to get old nfs object: %+v", err) + return + } + newService, err := getNFSObject(newObj) + if err != nil { + logger.Errorf("failed to get new nfs object: %+v", err) + return + } + + if !serviceChanged(oldService.Spec, newService.Spec) { + logger.Debugf("nfs %s did not change", newService.Name) + return + } + + logger.Infof("applying nfs %s changes", newService.Name) + if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil { + logger.Errorf("failed to create (modify) nfs %s. %+v", newService.Name, err) + } +} + +func (c *NFSController) onDelete(obj interface{}) { + nfs, err := getNFSObject(obj) + if err != nil { + logger.Errorf("failed to get nfs object: %+v", err) + return + } + + if err = c.DeleteService(*nfs); err != nil { + logger.Errorf("failed to delete nfs %s. %+v", nfs.Name, err) + } +} + +func (c *NFSController) serviceOwners(service *edgefsv1alpha1.NFS) []metav1.OwnerReference { + // Only set the cluster crd as the owner of the NFS resources. + // If the NFS crd is deleted, the operator will explicitly remove the NFS resources. + // If the NFS crd still exists when the cluster crd is deleted, this will make sure the NFS + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + +func serviceChanged(oldService, newService edgefsv1alpha1.NFSSpec) bool { + return false +} + +func getNFSObject(obj interface{}) (nfs *edgefsv1alpha1.NFS, err error) { + var ok bool + nfs, ok = obj.(*edgefsv1alpha1.NFS) + if ok { + // the nfs object is of the latest type, simply return it + return nfs.DeepCopy(), nil + } + + return nil, fmt.Errorf("not a known nfs object: %+v", obj) +} diff --git a/pkg/operator/edgefs/nfs/nfs.go b/pkg/operator/edgefs/nfs/nfs.go new file mode 100644 index 000000000..6f5cf532e --- /dev/null +++ b/pkg/operator/edgefs/nfs/nfs.go @@ -0,0 +1,326 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package nfs for the Edgefs manager. +package nfs + +import ( + "fmt" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + appName = "rook-edgefs-nfs" + + /* Volumes definitions */ + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" +) + +// Start the rgw manager +func (c *NFSController) CreateService(s edgefsv1alpha1.NFS, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, false, ownerRefs) +} + +func (c *NFSController) UpdateService(s edgefsv1alpha1.NFS, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, true, ownerRefs) +} + +// Start the nfs instance +func (c *NFSController) CreateOrUpdate(s edgefsv1alpha1.NFS, update bool, ownerRefs []metav1.OwnerReference) error { + logger.Infof("starting update=%v service=%s", update, s.Name) + + logger.Infof("NFS Base image is %s", c.rookImage) + // validate NFS service settings + if err := validateService(c.context, s); err != nil { + return fmt.Errorf("invalid NFS service %s arguments. %+v", s.Name, err) + } + + // check if NFS service already exists + exists, err := serviceExists(c.context, s) + if err == nil && exists { + if !update { + logger.Infof("NFS service %s exists in namespace %s", s.Name, s.Namespace) + return nil + } + logger.Infof("NFS service %s exists in namespace %s. checking for updates", s.Name, s.Namespace) + } + + // start the deployment + deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec) + if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s deployment. %+v", appName, err) + } + logger.Infof("%s deployment already exists", appName) + } else { + logger.Infof("%s deployment started", appName) + } + + // create the nfs service + service := c.makeNFSService(instanceName(s.Name), s.Name, s.Namespace) + if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create nfs service. %+v", err) + } + logger.Infof("nfs service %s already exists", service) + } else { + logger.Infof("nfs service %s started", service) + } + + return nil +} + +func (c *NFSController) makeNFSService(name, svcname, namespace string) *v1.Service { + labels := getLabels(name, svcname, namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + {Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP}, + {Name: "nfs-tcp", Port: 2049, Protocol: v1.ProtocolTCP}, + {Name: "nfs-udp", Port: 2049, Protocol: v1.ProtocolUDP}, + {Name: "nlockmgr-tcp", Port: 32803, Protocol: v1.ProtocolTCP}, + {Name: "nlockmgr-udp", Port: 32803, Protocol: v1.ProtocolUDP}, + {Name: "mountd-tcp", Port: 20048, Protocol: v1.ProtocolTCP}, + {Name: "mountd-udp", Port: 20048, Protocol: v1.ProtocolUDP}, + {Name: "portmapper-tcp", Port: 111, Protocol: v1.ProtocolTCP}, + {Name: "portmapper-udp", Port: 111, Protocol: v1.ProtocolUDP}, + {Name: "statd-tcp", Port: 662, Protocol: v1.ProtocolTCP}, + {Name: "statd-udp", Port: 662, Protocol: v1.ProtocolUDP}, + {Name: "rquotad-tcp", Port: 875, Protocol: v1.ProtocolTCP}, + {Name: "rquotad-udp", Port: 875, Protocol: v1.ProtocolUDP}, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *NFSController) makeDeployment(svcname, namespace, rookImage string, nfsSpec edgefsv1alpha1.NFSSpec) *extensions.Deployment { + + name := instanceName(svcname) + volumes := []v1.Volume{} + + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + // dataDir case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + }, + }, + }) + } + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: getLabels(name, svcname, namespace), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.nfsContainer(svcname, name, rookImage, nfsSpec)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + HostIPC: true, + HostNetwork: c.hostNetwork, + NodeSelector: map[string]string{namespace: "cluster"}, + }, + } + if c.hostNetwork { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + + // apply current NFS CRD options to pod's specification + nfsSpec.Placement.ApplyToPodSpec(&podSpec.Spec) + + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &nfsSpec.Instances}, + } + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *NFSController) nfsContainer(svcname, name, containerImage string, nfsSpec edgefsv1alpha1.NFSSpec) v1.Container { + + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"}, + }, + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"nfs"}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "CCOW_SVCNAME", + Value: svcname, + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + SecurityContext: securityContext, + Resources: nfsSpec.Resources, + Ports: []v1.ContainerPort{ + {Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP}, + {Name: "nfs-tcp", ContainerPort: 2049, Protocol: v1.ProtocolTCP}, + {Name: "nfs-udp", ContainerPort: 2049, Protocol: v1.ProtocolUDP}, + {Name: "nlockmgr-tcp", ContainerPort: 32803, Protocol: v1.ProtocolTCP}, + {Name: "nlockmgr-udp", ContainerPort: 32803, Protocol: v1.ProtocolUDP}, + {Name: "mountd-tcp", ContainerPort: 20048, Protocol: v1.ProtocolTCP}, + {Name: "mountd-udp", ContainerPort: 20048, Protocol: v1.ProtocolUDP}, + {Name: "portmapper-tcp", ContainerPort: 111, Protocol: v1.ProtocolTCP}, + {Name: "portmapper-udp", ContainerPort: 111, Protocol: v1.ProtocolUDP}, + {Name: "statd-tcp", ContainerPort: 662, Protocol: v1.ProtocolTCP}, + {Name: "statd-udp", ContainerPort: 662, Protocol: v1.ProtocolUDP}, + {Name: "rquotad-tcp", ContainerPort: 875, Protocol: v1.ProtocolTCP}, + {Name: "rquotad-udp", ContainerPort: 875, Protocol: v1.ProtocolUDP}, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc.target", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + }, + } +} + +// Delete NFS service and possibly some artifacts. +func (c *NFSController) DeleteService(s edgefsv1alpha1.NFS) error { + // check if service exists + exists, err := serviceExists(c.context, s) + if err != nil { + return fmt.Errorf("failed to detect if there is a NFS service to delete. %+v", err) + } + if !exists { + logger.Infof("NFS service %s does not exist in namespace %s", s.Name, s.Namespace) + return nil + } + + logger.Infof("Deleting NFS service %s from namespace %s", s.Name, s.Namespace) + + var gracePeriod int64 + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + + // Delete the nfs service + err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options) + if err != nil && !errors.IsNotFound(err) { + logger.Warningf("failed to delete NFS service. %+v", err) + } + + // Make a best effort to delete the NFS pods + err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name)) + if err != nil { + logger.Warningf(err.Error()) + } + + logger.Infof("Completed deleting NFS service %s", s.Name) + return nil +} + +func getLabels(name, svcname, namespace string) map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: namespace, + "edgefs_svcname": svcname, + "edgefs_svctype": "nfs", + } +} + +// Validate the NFS arguments +func validateService(context *clusterd.Context, s edgefsv1alpha1.NFS) error { + if s.Name == "" { + return fmt.Errorf("missing name") + } + if s.Namespace == "" { + return fmt.Errorf("missing namespace") + } + + return nil +} + +func instanceName(svcname string) string { + return fmt.Sprintf("%s-%s", appName, svcname) +} + +// Check if the NFS service exists +func serviceExists(context *clusterd.Context, s edgefsv1alpha1.NFS) (bool, error) { + _, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{}) + if err == nil { + // the deployment was found + return true, nil + } + if err != nil && !errors.IsNotFound(err) { + return false, err + } + + // not found + return false, nil +} diff --git a/pkg/operator/edgefs/operator.go b/pkg/operator/edgefs/operator.go new file mode 100644 index 000000000..2a3675d93 --- /dev/null +++ b/pkg/operator/edgefs/operator.go @@ -0,0 +1,87 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package operator to manage Kubernetes storage. +package edgefs + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/discover" + "github.com/rook/rook/pkg/operator/edgefs/cluster" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-operator") + +type Operator struct { + context *clusterd.Context + resources []opkit.CustomResource + rookImage string + securityAccount string + clusterController *cluster.ClusterController +} + +// New creates an operator instance +func New(context *clusterd.Context, rookImage string, securityAccount string) *Operator { + clusterController := cluster.NewClusterController(context, rookImage) + + schemes := []opkit.CustomResource{cluster.ClusterResource} + return &Operator{ + context: context, + clusterController: clusterController, + resources: schemes, + rookImage: rookImage, + securityAccount: securityAccount, + } +} + +// Run the operator instance +func (o *Operator) Run() error { + + namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) + if namespace == "" { + return fmt.Errorf("Rook operator namespace is not provided. Expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar) + } + + rookDiscover := discover.New(o.context.Clientset) + if err := rookDiscover.Start(namespace, o.rookImage, o.securityAccount); err != nil { + return fmt.Errorf("Error starting device discovery daemonset: %v", err) + } + + signalChan := make(chan os.Signal, 1) + stopChan := make(chan struct{}) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + // watch for changes to the edgefs clusters + o.clusterController.StartWatch(v1.NamespaceAll, stopChan) + + for { + select { + case <-signalChan: + logger.Infof("shutdown signal received, exiting...") + close(stopChan) + return nil + } + } +} diff --git a/pkg/operator/edgefs/operator_test.go b/pkg/operator/edgefs/operator_test.go new file mode 100644 index 000000000..c8ee321f2 --- /dev/null +++ b/pkg/operator/edgefs/operator_test.go @@ -0,0 +1,43 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package edgefs + +import ( + "fmt" + "testing" + + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/edgefs/cluster" + "github.com/rook/rook/pkg/operator/test" + "github.com/stretchr/testify/assert" +) + +func TestOperator(t *testing.T) { + clientset := test.New(3) + context := &clusterd.Context{Clientset: clientset} + o := New(context, "", "") + + assert.NotNil(t, o) + assert.NotNil(t, o.clusterController) + assert.NotNil(t, o.resources) + assert.Equal(t, context, o.context) + assert.Equal(t, len(o.resources), 1) + for _, r := range o.resources { + if r.Name != cluster.ClusterResource.Name { + assert.Fail(t, fmt.Sprintf("Resource %s is not valid", r.Name)) + } + } +} diff --git a/pkg/operator/edgefs/s3/controller.go b/pkg/operator/edgefs/s3/controller.go new file mode 100644 index 000000000..cae78f11f --- /dev/null +++ b/pkg/operator/edgefs/s3/controller.go @@ -0,0 +1,171 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// scale-out, multi-cloud Edge-X S3 services controller +package s3 + +import ( + "fmt" + "reflect" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/core/v1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +const ( + customResourceName = "s3" + customResourceNamePlural = "s3s" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-s3") + +// S3Resource represents the s3 custom resource +var S3Resource = opkit.CustomResource{ + Name: customResourceName, + Plural: customResourceNamePlural, + Group: edgefsv1alpha1.CustomResourceGroup, + Version: edgefsv1alpha1.Version, + Scope: apiextensionsv1beta1.NamespaceScoped, + Kind: reflect.TypeOf(edgefsv1alpha1.S3{}).Name(), +} + +// S3Controller represents a controller object for s3 custom resources +type S3Controller struct { + context *clusterd.Context + rookImage string + hostNetwork bool + dataDirHostPath string + dataVolumeSize resource.Quantity + placement rookalpha.Placement + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference +} + +// NewS3Controller create controller for watching S3 custom resources created +func NewS3Controller( + context *clusterd.Context, rookImage string, + hostNetwork bool, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *S3Controller { + return &S3Controller{ + context: context, + rookImage: rookImage, + hostNetwork: hostNetwork, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + placement: placement, + resources: resources, + ownerRef: ownerRef, + } +} + +// StartWatch watches for instances of S3 custom resources and acts on them +func (c *S3Controller) StartWatch(namespace string, stopCh chan struct{}) error { + + resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ + AddFunc: c.onAdd, + UpdateFunc: c.onUpdate, + DeleteFunc: c.onDelete, + } + + logger.Infof("start watching s3 resources in namespace %s", namespace) + watcher := opkit.NewWatcher(S3Resource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient()) + go watcher.Watch(&edgefsv1alpha1.S3{}, stopCh) + + return nil +} + +func (c *S3Controller) onAdd(obj interface{}) { + s3, err := getS3Object(obj) + if err != nil { + logger.Errorf("failed to get s3 object: %+v", err) + return + } + + if err = c.CreateService(*s3, c.serviceOwners(s3)); err != nil { + logger.Errorf("failed to create s3 %s. %+v", s3.Name, err) + } +} + +func (c *S3Controller) onUpdate(oldObj, newObj interface{}) { + oldService, err := getS3Object(oldObj) + if err != nil { + logger.Errorf("failed to get old s3 object: %+v", err) + return + } + newService, err := getS3Object(newObj) + if err != nil { + logger.Errorf("failed to get new s3 object: %+v", err) + return + } + + if !serviceChanged(oldService.Spec, newService.Spec) { + logger.Debugf("s3 %s did not change", newService.Name) + return + } + + logger.Infof("applying s3 %s changes", newService.Name) + if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil { + logger.Errorf("failed to create (modify) s3 %s. %+v", newService.Name, err) + } +} + +func (c *S3Controller) onDelete(obj interface{}) { + s3, err := getS3Object(obj) + if err != nil { + logger.Errorf("failed to get s3 object: %+v", err) + return + } + + if err = c.DeleteService(*s3); err != nil { + logger.Errorf("failed to delete s3 %s. %+v", s3.Name, err) + } +} + +func (c *S3Controller) serviceOwners(service *edgefsv1alpha1.S3) []metav1.OwnerReference { + // Only set the cluster crd as the owner of the S3 resources. + // If the S3 crd is deleted, the operator will explicitly remove the S3 resources. + // If the S3 crd still exists when the cluster crd is deleted, this will make sure the S3 + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + +func serviceChanged(oldService, newService edgefsv1alpha1.S3Spec) bool { + return false +} + +func getS3Object(obj interface{}) (s3 *edgefsv1alpha1.S3, err error) { + var ok bool + s3, ok = obj.(*edgefsv1alpha1.S3) + if ok { + // the s3 object is of the latest type, simply return it + return s3.DeepCopy(), nil + } + + return nil, fmt.Errorf("not a known s3 object: %+v", obj) +} diff --git a/pkg/operator/edgefs/s3/s3.go b/pkg/operator/edgefs/s3/s3.go new file mode 100644 index 000000000..481adf574 --- /dev/null +++ b/pkg/operator/edgefs/s3/s3.go @@ -0,0 +1,355 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package S3 for the Edgefs manager. +package s3 + +import ( + "fmt" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + appName = "rook-edgefs-s3" + + /* Volumes definitions */ + sslCertVolumeName = "ssl-cert-volume" + sslMountPath = "/opt/nedge/etc/ssl/" + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" + defaultPort = 9982 + defaultSecurePort = 8443 +) + +// Start the S3 manager +func (c *S3Controller) CreateService(s edgefsv1alpha1.S3, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, false, ownerRefs) +} + +func (c *S3Controller) UpdateService(s edgefsv1alpha1.S3, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, true, ownerRefs) +} + +// Start the s3 instance +func (c *S3Controller) CreateOrUpdate(s edgefsv1alpha1.S3, update bool, ownerRefs []metav1.OwnerReference) error { + logger.Infof("starting update=%v service=%s", update, s.Name) + + logger.Infof("S3 Base image is %s", c.rookImage) + // validate S3 service settings + if err := validateService(c.context, s); err != nil { + return fmt.Errorf("invalid S3 service %s arguments. %+v", s.Name, err) + } + + //check http settings + if s.Spec.Port == 0 { + s.Spec.Port = defaultPort + } + + if s.Spec.SecurePort == 0 { + s.Spec.SecurePort = defaultSecurePort + } + + // check if S3 service already exists + exists, err := serviceExists(c.context, s) + if err == nil && exists { + if !update { + logger.Infof("S3 service %s exists in namespace %s", s.Name, s.Namespace) + return nil + } + logger.Infof("S3 service %s exists in namespace %s. checking for updates", s.Name, s.Namespace) + } + + // start the deployment + deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec) + if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s deployment. %+v", appName, err) + } + logger.Infof("%s deployment already exists", appName) + } else { + logger.Infof("%s deployment started", appName) + } + + // create the s3 service + service := c.makeS3Service(instanceName(s.Name), s.Name, s.Namespace, s.Spec) + if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create s3 service. %+v", err) + } + logger.Infof("s3 service %s already exists", service) + } else { + logger.Infof("s3 service %s started", service) + } + + return nil +} + +func (c *S3Controller) makeS3Service(name, svcname, namespace string, s3Spec edgefsv1alpha1.S3Spec) *v1.Service { + labels := getLabels(name, svcname, namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + {Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", Port: int32(s3Spec.Port), Protocol: v1.ProtocolTCP}, + {Name: "secure-port", Port: int32(s3Spec.SecurePort), Protocol: v1.ProtocolTCP}, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *S3Controller) makeDeployment(svcname, namespace, rookImage string, s3Spec edgefsv1alpha1.S3Spec) *extensions.Deployment { + + name := instanceName(svcname) + volumes := []v1.Volume{} + + // add ssl certificate volume if defined + if len(s3Spec.SSLCertificateRef) > 0 { + volumes = append(volumes, v1.Volume{ + Name: sslCertVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: s3Spec.SSLCertificateRef, + Items: []v1.KeyToPath{ + { + Key: "sslkey", + Path: "ssl.key", + }, + { + Key: "sslcert", + Path: "ssl.crt", + }, + }, + }, + }, + }) + } + + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + // dataDir case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + }, + }, + }) + } + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: getLabels(name, svcname, namespace), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.s3Container(svcname, name, rookImage, s3Spec)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + HostIPC: true, + HostNetwork: c.hostNetwork, + NodeSelector: map[string]string{namespace: "cluster"}, + }, + } + if c.hostNetwork { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + + // apply current S3 CRD options to pod's specification + s3Spec.Placement.ApplyToPodSpec(&podSpec.Spec) + + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &s3Spec.Instances}, + } + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *S3Controller) s3Container(svcname, name, containerImage string, s3Spec edgefsv1alpha1.S3Spec) v1.Container { + + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc.target", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + if len(s3Spec.SSLCertificateRef) > 0 { + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath}) + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"s3"}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "CCOW_SVCNAME", + Value: svcname, + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "EFSS3_HTTP_PORT", + Value: fmt.Sprint(s3Spec.Port), + }, + { + Name: "EFSS3_HTTPS_PORT", + Value: fmt.Sprint(s3Spec.SecurePort), + }, + }, + SecurityContext: securityContext, + Resources: s3Spec.Resources, + Ports: []v1.ContainerPort{ + {Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", ContainerPort: int32(s3Spec.Port), Protocol: v1.ProtocolTCP}, + {Name: "secure-port", ContainerPort: int32(s3Spec.SecurePort), Protocol: v1.ProtocolTCP}, + }, + VolumeMounts: volumeMounts, + } +} + +// Delete S3 service and possibly some artifacts. +func (c *S3Controller) DeleteService(s edgefsv1alpha1.S3) error { + // check if service exists + exists, err := serviceExists(c.context, s) + if err != nil { + return fmt.Errorf("failed to detect if there is a S3 service to delete. %+v", err) + } + if !exists { + logger.Infof("S3 service %s does not exist in namespace %s", s.Name, s.Namespace) + return nil + } + + logger.Infof("Deleting S3 service %s from namespace %s", s.Name, s.Namespace) + + var gracePeriod int64 + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + + // Delete the s3 service + err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options) + if err != nil && !errors.IsNotFound(err) { + logger.Warningf("failed to delete S3 service. %+v", err) + } + + // Make a best effort to delete the S3 pods + err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name)) + if err != nil { + logger.Warningf(err.Error()) + } + + logger.Infof("Completed deleting S3 service %s", s.Name) + return nil +} + +func getLabels(name, svcname, namespace string) map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: namespace, + "edgefs_svcname": svcname, + "edgefs_svctype": "s3", + } +} + +// Validate the S3 arguments +func validateService(context *clusterd.Context, s edgefsv1alpha1.S3) error { + if s.Name == "" { + return fmt.Errorf("missing name") + } + if s.Namespace == "" { + return fmt.Errorf("missing namespace") + } + + return nil +} + +func instanceName(svcname string) string { + return fmt.Sprintf("%s-%s", appName, svcname) +} + +// Check if the S3 service exists +func serviceExists(context *clusterd.Context, s edgefsv1alpha1.S3) (bool, error) { + _, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{}) + if err == nil { + // the deployment was found + return true, nil + } + if err != nil && !errors.IsNotFound(err) { + return false, err + } + + // not found + return false, nil +} diff --git a/pkg/operator/edgefs/s3x/controller.go b/pkg/operator/edgefs/s3x/controller.go new file mode 100644 index 000000000..02a7d478d --- /dev/null +++ b/pkg/operator/edgefs/s3x/controller.go @@ -0,0 +1,171 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// scale-out, multi-cloud Edge-X S3 services controller +package s3x + +import ( + "fmt" + "reflect" + + "github.com/coreos/pkg/capnslog" + opkit "github.com/rook/operator-kit" + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" + "github.com/rook/rook/pkg/clusterd" + "k8s.io/api/core/v1" + apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +const ( + customResourceName = "s3x" + customResourceNamePlural = "s3xs" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-s3x") + +// S3XResource represents the s3x custom resource +var S3XResource = opkit.CustomResource{ + Name: customResourceName, + Plural: customResourceNamePlural, + Group: edgefsv1alpha1.CustomResourceGroup, + Version: edgefsv1alpha1.Version, + Scope: apiextensionsv1beta1.NamespaceScoped, + Kind: reflect.TypeOf(edgefsv1alpha1.S3X{}).Name(), +} + +// S3XController represents a controller object for s3x custom resources +type S3XController struct { + context *clusterd.Context + rookImage string + hostNetwork bool + dataDirHostPath string + dataVolumeSize resource.Quantity + placement rookalpha.Placement + resources v1.ResourceRequirements + ownerRef metav1.OwnerReference +} + +// NewS3XController create controller for watching S3X custom resources created +func NewS3XController( + context *clusterd.Context, rookImage string, + hostNetwork bool, + dataDirHostPath string, + dataVolumeSize resource.Quantity, + placement rookalpha.Placement, + resources v1.ResourceRequirements, + ownerRef metav1.OwnerReference, +) *S3XController { + return &S3XController{ + context: context, + rookImage: rookImage, + hostNetwork: hostNetwork, + dataDirHostPath: dataDirHostPath, + dataVolumeSize: dataVolumeSize, + placement: placement, + resources: resources, + ownerRef: ownerRef, + } +} + +// StartWatch watches for instances of S3X custom resources and acts on them +func (c *S3XController) StartWatch(namespace string, stopCh chan struct{}) error { + + resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{ + AddFunc: c.onAdd, + UpdateFunc: c.onUpdate, + DeleteFunc: c.onDelete, + } + + logger.Infof("start watching s3x resources in namespace %s", namespace) + watcher := opkit.NewWatcher(S3XResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient()) + go watcher.Watch(&edgefsv1alpha1.S3X{}, stopCh) + + return nil +} + +func (c *S3XController) onAdd(obj interface{}) { + s3x, err := getS3XObject(obj) + if err != nil { + logger.Errorf("failed to get s3x object: %+v", err) + return + } + + if err = c.CreateService(*s3x, c.serviceOwners(s3x)); err != nil { + logger.Errorf("failed to create s3x %s. %+v", s3x.Name, err) + } +} + +func (c *S3XController) onUpdate(oldObj, newObj interface{}) { + oldService, err := getS3XObject(oldObj) + if err != nil { + logger.Errorf("failed to get old s3x object: %+v", err) + return + } + newService, err := getS3XObject(newObj) + if err != nil { + logger.Errorf("failed to get new s3x object: %+v", err) + return + } + + if !serviceChanged(oldService.Spec, newService.Spec) { + logger.Debugf("s3x %s did not change", newService.Name) + return + } + + logger.Infof("applying s3x %s changes", newService.Name) + if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil { + logger.Errorf("failed to create (modify) s3x %s. %+v", newService.Name, err) + } +} + +func (c *S3XController) onDelete(obj interface{}) { + s3x, err := getS3XObject(obj) + if err != nil { + logger.Errorf("failed to get s3x object: %+v", err) + return + } + + if err = c.DeleteService(*s3x); err != nil { + logger.Errorf("failed to delete s3x %s. %+v", s3x.Name, err) + } +} + +func (c *S3XController) serviceOwners(service *edgefsv1alpha1.S3X) []metav1.OwnerReference { + // Only set the cluster crd as the owner of the S3X resources. + // If the S3X crd is deleted, the operator will explicitly remove the S3X resources. + // If the S3X crd still exists when the cluster crd is deleted, this will make sure the S3X + // resources are cleaned up. + return []metav1.OwnerReference{c.ownerRef} +} + +func serviceChanged(oldService, newService edgefsv1alpha1.S3XSpec) bool { + return false +} + +func getS3XObject(obj interface{}) (s3x *edgefsv1alpha1.S3X, err error) { + var ok bool + s3x, ok = obj.(*edgefsv1alpha1.S3X) + if ok { + // the s3x object is of the latest type, simply return it + return s3x.DeepCopy(), nil + } + + return nil, fmt.Errorf("not a known s3x object: %+v", obj) +} diff --git a/pkg/operator/edgefs/s3x/s3x.go b/pkg/operator/edgefs/s3x/s3x.go new file mode 100644 index 000000000..01d5cd8d8 --- /dev/null +++ b/pkg/operator/edgefs/s3x/s3x.go @@ -0,0 +1,355 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package s3x for the Edgefs manager. +package s3x + +import ( + "fmt" + + edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + appName = "rook-edgefs-s3x" + + /* Volumes definitions */ + sslCertVolumeName = "ssl-cert-volume" + sslMountPath = "/opt/nedge/etc/ssl/" + dataVolumeName = "edgefs-datadir" + stateVolumeFolder = ".state" + etcVolumeFolder = ".etc" + defaultPort = 3000 + defaultSecurePort = 3001 +) + +// Start the rgw manager +func (c *S3XController) CreateService(s edgefsv1alpha1.S3X, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, false, ownerRefs) +} + +func (c *S3XController) UpdateService(s edgefsv1alpha1.S3X, ownerRefs []metav1.OwnerReference) error { + return c.CreateOrUpdate(s, true, ownerRefs) +} + +// Start the s3x instance +func (c *S3XController) CreateOrUpdate(s edgefsv1alpha1.S3X, update bool, ownerRefs []metav1.OwnerReference) error { + logger.Infof("starting update=%v service=%s", update, s.Name) + + logger.Infof("S3X Base image is %s", c.rookImage) + // validate S3X service settings + if err := validateService(c.context, s); err != nil { + return fmt.Errorf("invalid S3X service %s arguments. %+v", s.Name, err) + } + + //check http settings + if s.Spec.Port == 0 { + s.Spec.Port = defaultPort + } + + if s.Spec.SecurePort == 0 { + s.Spec.SecurePort = defaultSecurePort + } + + // check if S3X service already exists + exists, err := serviceExists(c.context, s) + if err == nil && exists { + if !update { + logger.Infof("S3X service %s exists in namespace %s", s.Name, s.Namespace) + return nil + } + logger.Infof("S3X service %s exists in namespace %s. checking for updates", s.Name, s.Namespace) + } + + // start the deployment + deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec) + if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create %s deployment. %+v", appName, err) + } + logger.Infof("%s deployment already exists", appName) + } else { + logger.Infof("%s deployment started", appName) + } + + // create the s3x service + service := c.makeS3XService(instanceName(s.Name), s.Name, s.Namespace, s.Spec) + if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil { + if !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create s3x service. %+v", err) + } + logger.Infof("s3x service %s already exists", service) + } else { + logger.Infof("s3x service %s started", service) + } + + return nil +} + +func (c *S3XController) makeS3XService(name, svcname, namespace string, s3xSpec edgefsv1alpha1.S3XSpec) *v1.Service { + labels := getLabels(name, svcname, namespace) + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: v1.ServiceSpec{ + Selector: labels, + Type: v1.ServiceTypeNodePort, + Ports: []v1.ServicePort{ + {Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", Port: int32(s3xSpec.Port), Protocol: v1.ProtocolTCP}, + {Name: "secure-port", Port: int32(s3xSpec.SecurePort), Protocol: v1.ProtocolTCP}, + }, + }, + } + + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef) + return svc +} + +func (c *S3XController) makeDeployment(svcname, namespace, rookImage string, s3xSpec edgefsv1alpha1.S3XSpec) *extensions.Deployment { + + name := instanceName(svcname) + volumes := []v1.Volume{} + + // add ssl certificate volume if defined + if len(s3xSpec.SSLCertificateRef) > 0 { + volumes = append(volumes, v1.Volume{ + Name: sslCertVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: s3xSpec.SSLCertificateRef, + Items: []v1.KeyToPath{ + { + Key: "sslkey", + Path: "ssl.key", + }, + { + Key: "sslcert", + Path: "ssl.crt", + }, + }, + }, + }, + }) + } + + if c.dataVolumeSize.Value() > 0 { + // dataVolume case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: dataVolumeName, + }, + }, + }) + } else { + // dataDir case + volumes = append(volumes, v1.Volume{ + Name: dataVolumeName, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.dataDirHostPath, + }, + }, + }) + } + + podSpec := v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: getLabels(name, svcname, namespace), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{c.s3xContainer(svcname, name, rookImage, s3xSpec)}, + RestartPolicy: v1.RestartPolicyAlways, + Volumes: volumes, + HostIPC: true, + HostNetwork: c.hostNetwork, + NodeSelector: map[string]string{namespace: "cluster"}, + }, + } + if c.hostNetwork { + podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet + } + + // apply current S3X CRD options to pod's specification + s3xSpec.Placement.ApplyToPodSpec(&podSpec.Spec) + + d := &extensions.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &s3xSpec.Instances}, + } + k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef) + return d +} + +func (c *S3XController) s3xContainer(svcname, name, containerImage string, s3xSpec edgefsv1alpha1.S3XSpec) v1.Container { + + runAsUser := int64(0) + readOnlyRootFilesystem := false + securityContext := &v1.SecurityContext{ + RunAsUser: &runAsUser, + ReadOnlyRootFilesystem: &readOnlyRootFilesystem, + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"}, + }, + } + + volumeMounts := []v1.VolumeMount{ + { + Name: dataVolumeName, + MountPath: "/opt/nedge/etc.target", + SubPath: etcVolumeFolder, + }, + { + Name: dataVolumeName, + MountPath: "/opt/nedge/var/run", + SubPath: stateVolumeFolder, + }, + } + + if len(s3xSpec.SSLCertificateRef) > 0 { + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath}) + } + + return v1.Container{ + Name: name, + Image: containerImage, + ImagePullPolicy: v1.PullAlways, + Args: []string{"s3x"}, + Env: []v1.EnvVar{ + { + Name: "CCOW_LOG_LEVEL", + Value: "5", + }, + { + Name: "CCOW_SVCNAME", + Value: svcname, + }, + { + Name: "HOST_HOSTNAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "EFSS3X_HTTP_PORT", + Value: fmt.Sprint(s3xSpec.Port), + }, + { + Name: "EFSS3X_HTTPS_PORT", + Value: fmt.Sprint(s3xSpec.SecurePort), + }, + }, + SecurityContext: securityContext, + Resources: s3xSpec.Resources, + Ports: []v1.ContainerPort{ + {Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP}, + {Name: "port", ContainerPort: int32(s3xSpec.Port), Protocol: v1.ProtocolTCP}, + {Name: "secure-port", ContainerPort: int32(s3xSpec.SecurePort), Protocol: v1.ProtocolTCP}, + }, + VolumeMounts: volumeMounts, + } +} + +// Delete S3X service and possibly some artifacts. +func (c *S3XController) DeleteService(s edgefsv1alpha1.S3X) error { + // check if service exists + exists, err := serviceExists(c.context, s) + if err != nil { + return fmt.Errorf("failed to detect if there is a S3X service to delete. %+v", err) + } + if !exists { + logger.Infof("S3X service %s does not exist in namespace %s", s.Name, s.Namespace) + return nil + } + + logger.Infof("Deleting S3X service %s from namespace %s", s.Name, s.Namespace) + + var gracePeriod int64 + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + + // Delete the s3x service + err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options) + if err != nil && !errors.IsNotFound(err) { + logger.Warningf("failed to delete S3X service. %+v", err) + } + + // Make a best effort to delete the S3X pods + err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name)) + if err != nil { + logger.Warningf(err.Error()) + } + + logger.Infof("Completed deleting S3X service %s", s.Name) + return nil +} + +func getLabels(name, svcname, namespace string) map[string]string { + return map[string]string{ + k8sutil.AppAttr: appName, + k8sutil.ClusterAttr: namespace, + "edgefs_svcname": svcname, + "edgefs_svctype": "s3x", + } +} + +// Validate the S3X arguments +func validateService(context *clusterd.Context, s edgefsv1alpha1.S3X) error { + if s.Name == "" { + return fmt.Errorf("missing name") + } + if s.Namespace == "" { + return fmt.Errorf("missing namespace") + } + + return nil +} + +func instanceName(svcname string) string { + return fmt.Sprintf("%s-%s", appName, svcname) +} + +// Check if the S3X service exists +func serviceExists(context *clusterd.Context, s edgefsv1alpha1.S3X) (bool, error) { + _, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{}) + if err == nil { + // the deployment was found + return true, nil + } + if err != nil && !errors.IsNotFound(err) { + return false, err + } + + // not found + return false, nil +}