Edgefs operator and CRDs

Signed-off-by: Dmitry Yusupov <dmitry.yusupov@nexenta.com>
This commit is contained in:
Dmitry Yusupov committed 2018-12-05 23:05:42 -08:00
1 parent 4f958162e1
commit a29477f122
22 files changed
+5040

No files matched your search

+158
View File
@@ -0,0 +1,158 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cluster
import (
"fmt"
"reflect"
"sort"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/edgefs/cluster/mgr"
"github.com/rook/rook/pkg/operator/edgefs/cluster/target"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
httpPortDefault = int32(8080)
httpPortName = "http"
grpcPortDefault = int32(6789)
grpcPortName = "grpc"
udpTotemPortDefault = int32(5405)
udpTotemPortName = "totem"
volumeNameDataDir = "datadir"
defaultServiceAccountName = "rook-edgefs-cluster"
/* Volumes definitions */
configName = "edgefs-config"
)
type cluster struct {
context *clusterd.Context
Namespace string
Spec edgefsv1alpha1.ClusterSpec
ownerRef metav1.OwnerReference
targets *target.Cluster
mgrs *mgr.Cluster
stopCh chan struct{}
}
func newCluster(c *edgefsv1alpha1.Cluster, context *clusterd.Context) *cluster {
return &cluster{
context: context,
Namespace: c.Namespace,
Spec: c.Spec,
stopCh: make(chan struct{}),
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
}
}
func (c *cluster) createInstance(rookImage string) error {
logger.Infof("Cluster spec\n %+v", c.Spec)
//
// Validate Cluster CRD
//
if err := c.validateClusterSpec(); err != nil {
logger.Errorf("invalid cluster spec: %+v", err)
return err
}
// Create a configmap for overriding edgefs config settings
// These settings should only be modified by a user after they are initialized
placeholderConfig := map[string]string{
k8sutil.ConfigOverrideVal: "",
}
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: k8sutil.ConfigOverrideName,
},
Data: placeholderConfig,
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef)
_, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm)
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err)
}
//
// Create and start EdgeFS Targets StatefulSet
//
c.targets = target.New(c.context, c.Namespace, "latest", c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.DataVolumeSize,
edgefsv1alpha1.GetTargetPlacement(c.Spec.Placement), c.Spec.Network, c.Spec.Resources, c.ownerRef)
err = c.targets.Start(rookImage, c.Spec.DevicesResurrectMode)
if err != nil {
return fmt.Errorf("failed to start the targets. %+v", err)
}
//
// Create and start EdgeFS manager Deployment (gRPC proxy, Prometheus metrics)
//
c.mgrs = mgr.New(c.context, c.Namespace, "latest", c.Spec.ServiceAccount, c.Spec.DataDirHostPath, c.Spec.DataVolumeSize,
edgefsv1alpha1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network,
v1.ResourceRequirements{}, c.ownerRef)
err = c.mgrs.Start(rookImage)
if err != nil {
return fmt.Errorf("failed to start the edgefs mgr. %+v", err)
}
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
return nil
}
func (c *cluster) validateClusterSpec() error {
if len(c.Spec.DataDirHostPath) == 0 && c.Spec.DataVolumeSize.Value() == 0 {
return fmt.Errorf("DataDirHostPath or DataVolumeSize EdgeFS cluster's options not specified.")
}
if len(c.Spec.DataDirHostPath) > 0 && c.Spec.DataVolumeSize.Value() != 0 {
return fmt.Errorf("Both deployment options DataDirHostPath and DataVolumeSize are specified. Should be only one deployment option in cluster specification.")
}
if len(c.Spec.Storage.Directories) > 0 &&
((c.Spec.Storage.UseAllDevices != nil && *c.Spec.Storage.UseAllDevices) ||
len(c.Spec.Storage.Devices) > 0 || len(c.Spec.Storage.DeviceFilter) > 0) {
return fmt.Errorf("Directories option specified as well as Devices. Remove Directories or Devices option from cluster specification")
}
logger.Info("Validate cluster spec")
return nil
}
func clusterChanged(oldCluster, newCluster edgefsv1alpha1.ClusterSpec) bool {
changeFound := false
oldStorage := oldCluster.Storage
newStorage := newCluster.Storage
// sort the nodes by name then compare to see if there are changes
sort.Sort(rookv1alpha2.NodesByName(oldStorage.Nodes))
sort.Sort(rookv1alpha2.NodesByName(newStorage.Nodes))
if !reflect.DeepEqual(oldStorage.Nodes, newStorage.Nodes) {
logger.Infof("The list of nodes has changed")
changeFound = true
}
return changeFound
}
+432
View File
@@ -0,0 +1,432 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Portions of this file came from https://github.com/cockroachdb/cockroach, which uses the same license.
*/
// Package cluster to manage a edgefs cluster.
package cluster
import (
"fmt"
"reflect"
"time"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/discover"
"github.com/rook/rook/pkg/operator/edgefs/iscsi"
"github.com/rook/rook/pkg/operator/edgefs/nfs"
"github.com/rook/rook/pkg/operator/edgefs/s3"
"github.com/rook/rook/pkg/operator/edgefs/s3x"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
)
var (
logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-cluster")
finalizerName = fmt.Sprintf("%s.%s", ClusterResource.Name, ClusterResource.Group)
)
const (
CustomResourceName = "cluster"
CustomResourceNamePlural = "clusters"
appName = "rook-edgefs"
clusterCreateInterval = 6 * time.Second
clusterCreateTimeout = 5 * time.Minute
updateClusterInterval = 30 * time.Second
updateClusterTimeout = 1 * time.Hour
clusterDeleteRetryInterval = 2 //seconds
clusterDeleteMaxRetries = 15
defaultEdgefsImageName = "edgefs/edgefs:latest"
)
var ClusterResource = opkit.CustomResource{
Name: CustomResourceName,
Plural: CustomResourceNamePlural,
Group: edgefsv1alpha1.CustomResourceGroup,
Version: edgefsv1alpha1.Version,
Scope: apiextensionsv1beta1.NamespaceScoped,
Kind: reflect.TypeOf(edgefsv1alpha1.Cluster{}).Name(),
}
type ClusterController struct {
context *clusterd.Context
containerImage string
devicesInUse bool
clusterMap map[string]*cluster
}
func NewClusterController(context *clusterd.Context, containerImage string) *ClusterController {
return &ClusterController{
context: context,
containerImage: containerImage,
//createInitRetryInterval: createInitRetryIntervalDefault,
clusterMap: make(map[string]*cluster),
}
}
func ClusterOwnerRef(namespace, clusterID string) metav1.OwnerReference {
blockOwner := true
return metav1.OwnerReference{
APIVersion: ClusterResource.Version,
Kind: ClusterResource.Kind,
Name: namespace,
UID: types.UID(clusterID),
BlockOwnerDeletion: &blockOwner,
}
}
func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) error {
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
AddFunc: c.onAdd,
UpdateFunc: c.onUpdate,
DeleteFunc: c.onDelete,
}
logger.Infof("start watching edgefs clusters in all namespaces")
watcher := opkit.NewWatcher(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
go watcher.Watch(&edgefsv1alpha1.Cluster{}, stopCh)
return nil
}
func (c *ClusterController) StopWatch() {
for _, cluster := range c.clusterMap {
close(cluster.stopCh)
}
c.clusterMap = make(map[string]*cluster)
}
func (c *ClusterController) onAdd(obj interface{}) {
clusterObj := obj.(*edgefsv1alpha1.Cluster).DeepCopy()
logger.Infof("new cluster %s added to namespace %s", clusterObj.Name, clusterObj.Namespace)
cluster := newCluster(clusterObj, c.context)
c.clusterMap[cluster.Namespace] = cluster
//Override rook containerImage value
c.containerImage = defaultEdgefsImageName
if cluster.Spec.EdgefsImageName != "" {
c.containerImage = cluster.Spec.EdgefsImageName
}
logger.Infof("starting cluster in namespace %s", cluster.Namespace)
if c.devicesInUse && cluster.Spec.Storage.AnyUseAllDevices() {
message := "using all devices in more than one namespace not supported"
logger.Error(message)
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateError, message); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
}
return
}
if cluster.Spec.Storage.AnyUseAllDevices() {
c.devicesInUse = true
}
// Start the Rook cluster components. Retry several times in case of failure.
err := wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) {
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateCreating, ""); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
return false, nil
}
err := cluster.createInstance(c.containerImage)
if err != nil {
logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Namespace, err)
return false, nil
}
// cluster is created, update the cluster CRD status now
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateCreated, ""); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
return false, nil
}
return true, nil
})
if err != nil {
message := fmt.Sprintf("giving up creating cluster in namespace %s after %s", cluster.Namespace, clusterCreateTimeout)
logger.Error(message)
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateError, message); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
}
return
}
logger.Infof("succeeded creating and initializing EdgeFS cluster in namespace %s", cluster.Namespace)
// Start NFS service CRD watcher
NFSController := nfs.NewNFSController(c.context, c.containerImage,
isHostNetworkDefined(cluster.Spec.Network),
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
cluster.Spec.Resources,
cluster.ownerRef)
NFSController.StartWatch(cluster.Namespace, cluster.stopCh)
// Start S3 service CRD watcher
S3Controller := s3.NewS3Controller(c.context, c.containerImage,
isHostNetworkDefined(cluster.Spec.Network),
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
cluster.Spec.Resources,
cluster.ownerRef)
S3Controller.StartWatch(cluster.Namespace, cluster.stopCh)
// Start S3X service CRD watcher
S3XController := s3x.NewS3XController(c.context, c.containerImage,
isHostNetworkDefined(cluster.Spec.Network),
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
cluster.Spec.Resources,
cluster.ownerRef)
S3XController.StartWatch(cluster.Namespace, cluster.stopCh)
// Start ISCSI service CRD watcher
ISCSIController := iscsi.NewISCSIController(c.context, c.containerImage,
isHostNetworkDefined(cluster.Spec.Network),
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
cluster.Spec.Resources,
cluster.ownerRef)
ISCSIController.StartWatch(cluster.Namespace, cluster.stopCh)
// add the finalizer to the crd
err = c.addFinalizer(clusterObj)
if err != nil {
logger.Errorf("failed to add finalizer to cluster crd. %+v", err)
}
}
func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
oldCluster := oldObj.(*edgefsv1alpha1.Cluster).DeepCopy()
newCluster := newObj.(*edgefsv1alpha1.Cluster).DeepCopy()
logger.Infof("update event for cluster %s", newCluster.Namespace)
// Check if the cluster is being deleted. This code path is called when a finalizer is specified in the crd.
// When a cluster is requested for deletion, K8s will only set the deletion timestamp if there are any finalizers in the list.
// K8s will only delete the crd and child resources when the finalizers have been removed from the crd.
if newCluster.DeletionTimestamp != nil {
logger.Infof("cluster %s has a deletion timestamp", newCluster.Namespace)
err := c.handleDelete(newCluster, time.Duration(clusterDeleteRetryInterval)*time.Second)
if err != nil {
logger.Errorf("failed finalizer for cluster. %+v", err)
return
}
// remove the finalizer from the crd, which indicates to k8s that the resource can safely be deleted
c.removeFinalizer(newCluster)
return
}
if !clusterChanged(oldCluster.Spec, newCluster.Spec) {
logger.Infof("update event for cluster %s is not supported", newCluster.Namespace)
return
}
logger.Infof("update event for cluster %s is supported, orchestrating update now", newCluster.Namespace)
logger.Debugf("old cluster: %+v", oldCluster.Spec)
logger.Debugf("new cluster: %+v", newCluster.Spec)
cluster, ok := c.clusterMap[newCluster.Namespace]
if !ok {
logger.Errorf("Cannot update cluster %s that does not exist", newCluster.Namespace)
return
}
cluster.Spec = newCluster.Spec
// attempt to update the cluster. note this is done outside of wait.Poll because that function
// will wait for the retry interval before trying for the first time.
done, _ := c.handleUpdate(newCluster, cluster)
if done {
return
}
err := wait.Poll(updateClusterInterval, updateClusterTimeout, func() (bool, error) {
return c.handleUpdate(newCluster, cluster)
})
if err != nil {
message := fmt.Sprintf("giving up trying to update cluster in namespace %s after %s", cluster.Namespace, updateClusterTimeout)
logger.Error(message)
if err := c.updateClusterStatus(newCluster.Namespace, newCluster.Name, edgefsv1alpha1.ClusterStateError, message); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", newCluster.Namespace, err)
}
return
}
logger.Infof("cluster %s updated in namespace %s", newCluster.Name, newCluster.Namespace)
}
func (c *ClusterController) handleUpdate(newClust *edgefsv1alpha1.Cluster, cluster *cluster) (bool, error) {
if err := c.updateClusterStatus(newClust.Namespace, newClust.Name, edgefsv1alpha1.ClusterStateUpdating, ""); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", newClust.Namespace, err)
return false, nil
}
if err := cluster.createInstance(c.containerImage); err != nil {
logger.Errorf("failed to update cluster in namespace %s. %+v", newClust.Namespace, err)
return false, nil
}
if err := c.updateClusterStatus(newClust.Namespace, newClust.Name, edgefsv1alpha1.ClusterStateCreated, ""); err != nil {
logger.Errorf("failed to update cluster status in namespace %s: %+v", newClust.Namespace, err)
return false, nil
}
logger.Infof("succeeded updating cluster in namespace %s", newClust.Namespace)
return true, nil
}
func (c *ClusterController) onDelete(obj interface{}) {
clust := obj.(*edgefsv1alpha1.Cluster).DeepCopy()
logger.Infof("delete event for cluster %s in namespace %s", clust.Name, clust.Namespace)
err := c.handleDelete(clust, time.Duration(clusterDeleteRetryInterval)*time.Second)
if err != nil {
logger.Errorf("failed to delete cluster. %+v", err)
}
if cluster, ok := c.clusterMap[clust.Namespace]; ok {
close(cluster.stopCh)
delete(c.clusterMap, clust.Namespace)
}
if clust.Spec.Storage.AnyUseAllDevices() {
c.devicesInUse = false
}
discover.FreeDevicesByCluster(c.context, clust.Namespace)
}
func (c *ClusterController) handleDelete(clust *edgefsv1alpha1.Cluster, retryInterval time.Duration) error {
cluster, ok := c.clusterMap[clust.Namespace]
if !ok {
return fmt.Errorf("Cannot delete cluster %s that does not exist", clust.Namespace)
}
for _, node := range cluster.targets.Storage.Nodes {
k := cluster.Namespace
err := cluster.targets.RemoveLabelOffNode(cluster.context.Clientset, node.Name, []string{k})
logger.Infof("removed label %s from %s: %+v", k, node.Name, err)
}
// delete associated node labels
return nil
}
func (c *ClusterController) updateClusterStatus(namespace, name string, state edgefsv1alpha1.ClusterState, message string) error {
// get the most recent cluster CRD object
cluster, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get cluster from namespace %s prior to updating its status: %+v", namespace, err)
}
// update the status on the retrieved cluster object
cluster.Status = edgefsv1alpha1.ClusterStatus{State: state, Message: message}
if _, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Update(cluster); err != nil {
return fmt.Errorf("failed to update cluster %s status: %+v", cluster.Namespace, err)
}
return nil
}
func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool {
if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 {
return true
}
return false
}
func (c *ClusterController) addFinalizer(clust *edgefsv1alpha1.Cluster) error {
// get the latest cluster object since we probably updated it before we got to this point (e.g. by updating its status)
clust, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(clust.Namespace).Get(clust.Name, metav1.GetOptions{})
if err != nil {
return err
}
// add the finalizer (cluster.edgefs.rook.io) if it is not yet defined on the cluster CRD
for _, finalizer := range clust.Finalizers {
if finalizer == finalizerName {
logger.Infof("finalizer already set on cluster %s", clust.Namespace)
return nil
}
}
// adding finalizer to the cluster crd
clust.Finalizers = append(clust.Finalizers, finalizerName)
// update the crd
_, err = c.context.RookClientset.EdgefsV1alpha1().Clusters(clust.Namespace).Update(clust)
if err != nil {
return fmt.Errorf("failed to add finalizer to cluster. %+v", err)
}
logger.Infof("added finalizer to cluster %s", clust.Name)
return nil
}
func (c *ClusterController) removeFinalizer(obj interface{}) {
var fname string
var objectMeta *metav1.ObjectMeta
// first determine what type/version of cluster we are dealing with
if cl, ok := obj.(*edgefsv1alpha1.Cluster); ok {
fname = finalizerName
objectMeta = &cl.ObjectMeta
} else {
logger.Warningf("cannot remove finalizer from object that is not a cluster: %+v", obj)
return
}
// remove the finalizer from the slice if it exists
found := false
for i, finalizer := range objectMeta.Finalizers {
if finalizer == fname {
objectMeta.Finalizers = append(objectMeta.Finalizers[:i], objectMeta.Finalizers[i+1:]...)
found = true
break
}
}
if !found {
logger.Infof("finalizer %s not found in the cluster crd '%s'", fname, objectMeta.Name)
return
}
// update the crd to remove the finalizer for good. retry several times in case of intermittent failures.
maxRetries := 5
retrySeconds := 5 * time.Second
for i := 0; i < maxRetries; i++ {
var err error
if cluster, ok := obj.(*edgefsv1alpha1.Cluster); ok {
_, err = c.context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Update(cluster)
}
if err != nil {
logger.Errorf("failed to remove finalizer %s from cluster %s. %+v", fname, objectMeta.Name, err)
time.Sleep(retrySeconds)
continue
}
logger.Infof("removed finalizer %s from cluster %s", fname, objectMeta.Name)
return
}
logger.Warningf("giving up from removing the %s cluster finalizer", fname)
}
@@ -0,0 +1,126 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cluster
import (
"os"
"testing"
"time"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
testop "github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestClusterDelete(t *testing.T) {
//nodeName := "node841"
clusterName := "cluster684"
//pvName := "pvc-540"
rookSystemNamespace := "rook-system-6413"
os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace)
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
clientset := testop.New(3)
context := &clusterd.Context{
Clientset: clientset,
}
// create the cluster controller and tell it that the cluster has been deleted
controller := NewClusterController(context, "")
clusterToDelete := &edgefsv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
controller.handleDelete(clusterToDelete, time.Microsecond)
}
func TestClusterChanged(t *testing.T) {
// a new node added, should be a change
old := edgefsv1alpha1.ClusterSpec{
Storage: rookalpha.StorageScopeSpec{
Nodes: []rookalpha.Node{
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
},
},
}
new := edgefsv1alpha1.ClusterSpec{
Storage: rookalpha.StorageScopeSpec{
Nodes: []rookalpha.Node{
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
},
},
}
assert.True(t, clusterChanged(old, new))
// a node was removed, should be a change
old.Storage.Nodes = []rookalpha.Node{
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
}
new.Storage.Nodes = []rookalpha.Node{
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
}
assert.True(t, clusterChanged(old, new))
// the nodes being in a different order should not be a change
old.Storage.Nodes = []rookalpha.Node{
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
}
new.Storage.Nodes = []rookalpha.Node{
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
}
assert.False(t, clusterChanged(old, new))
}
func TestRemoveFinalizer(t *testing.T) {
clientset := testop.New(3)
context := &clusterd.Context{
Clientset: clientset,
RookClientset: rookfake.NewSimpleClientset(),
}
controller := NewClusterController(context, "")
// *****************************************
// start with a current version edgefs cluster
// *****************************************
cluster := &edgefsv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-1893",
Namespace: "namespace-6551",
Finalizers: []string{finalizerName},
},
}
// create the cluster initially so it exists in the k8s api
cluster, err := context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Create(cluster)
assert.NoError(t, err)
assert.Len(t, cluster.Finalizers, 1)
// remove the finalizer from the cluster object
controller.removeFinalizer(cluster)
// verify the finalier was removed
cluster, err = context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Get(cluster.Name, metav1.GetOptions{})
assert.NoError(t, err)
assert.NotNil(t, cluster)
assert.Len(t, cluster.Finalizers, 0)
}
+295
View File
@@ -0,0 +1,295 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package mgr for the Edgefs manager.
package mgr
import (
"fmt"
"strconv"
"github.com/coreos/pkg/capnslog"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-mgr")
const (
appName = "rook-edgefs-mgr"
defaultServiceAccountName = "rook-edgefs-cluster"
defaultGrpcPort = 6789
defaultMetricsPort = 8881
/* Volumes definitions */
dataVolumeName = "edgefs-datadir"
stateVolumeFolder = ".state"
etcVolumeFolder = ".etc"
)
// Cluster is the edgefs mgr manager
type Cluster struct {
Namespace string
Version string
serviceAccount string
Replicas int
dataDirHostPath string
dataVolumeSize resource.Quantity
placement rookalpha.Placement
context *clusterd.Context
HostNetworkSpec edgefsv1alpha1.NetworkSpec
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
}
// New creates an instance of the mgr
func New(
context *clusterd.Context, namespace, version string,
serviceAccount string,
dataDirHostPath string,
dataVolumeSize resource.Quantity,
placement rookalpha.Placement, HostNetworkSpec edgefsv1alpha1.NetworkSpec,
resources v1.ResourceRequirements,
ownerRef metav1.OwnerReference,
) *Cluster {
if serviceAccount == "" {
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
serviceAccount = defaultServiceAccountName
logger.Infof("setting the mgr pod to use the service account name: %s", serviceAccount)
}
return &Cluster{
context: context,
Namespace: namespace,
serviceAccount: serviceAccount,
placement: placement,
Version: version,
Replicas: 1,
dataDirHostPath: dataDirHostPath,
dataVolumeSize: dataVolumeSize,
HostNetworkSpec: HostNetworkSpec,
resources: resources,
ownerRef: ownerRef,
}
}
func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool {
if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 {
return true
}
return false
}
// Start the mgr instance
func (c *Cluster) Start(rookImage string) error {
logger.Infof("start running mgr")
logger.Infof("Mgr Image is %s", rookImage)
// start the deployment
deployment := c.makeDeployment(appName, c.Namespace, rookImage, 1)
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
}
logger.Infof("%s deployment already exists", appName)
} else {
logger.Infof("%s deployment started", appName)
}
// create the mgr service
service := c.makeMgrService(appName)
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(service); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create mgr service. %+v", err)
}
logger.Infof("mgr mgr service already exists")
} else {
logger.Infof("mgr mgr service started")
}
return nil
}
func (c *Cluster) makeMgrService(name string) *v1.Service {
labels := c.getLabels()
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: c.Namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Selector: labels,
Type: v1.ServiceTypeClusterIP,
Ports: []v1.ServicePort{
{
Name: "mgr",
Port: int32(defaultGrpcPort),
Protocol: v1.ProtocolTCP,
},
{
Name: "http-metrics",
Port: int32(defaultMetricsPort),
Protocol: v1.ProtocolTCP,
},
},
},
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef)
return svc
}
func (c *Cluster) makeDeployment(name, clusterName, rookImage string, replicas int32) *extensions.Deployment {
volumes := []v1.Volume{}
if c.dataVolumeSize.Value() > 0 {
// dataVolume case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: dataVolumeName,
},
},
})
} else {
// dataDir case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: c.dataDirHostPath,
},
},
})
}
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: c.getDaemonLabels(clusterName),
Annotations: map[string]string{"prometheus.io/scrape": "true",
"prometheus.io/port": strconv.Itoa(defaultMetricsPort)},
},
Spec: v1.PodSpec{
ServiceAccountName: c.serviceAccount,
Containers: []v1.Container{c.mgrContainer(name, rookImage)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
HostIPC: true,
HostNetwork: isHostNetworkDefined(c.HostNetworkSpec),
NodeSelector: map[string]string{c.Namespace: "cluster"},
},
}
if isHostNetworkDefined(c.HostNetworkSpec) {
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
c.placement.ApplyToPodSpec(&podSpec.Spec)
d := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: c.Namespace,
},
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas},
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &d.ObjectMeta, &c.ownerRef)
return d
}
func (c *Cluster) mgrContainer(name string, containerImage string) v1.Container {
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
},
}
volumeMounts := []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc.target",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
}
return v1.Container{
Name: name,
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"mgmt"},
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
},
SecurityContext: securityContext,
Resources: c.resources,
Ports: []v1.ContainerPort{
{
Name: "mgr",
ContainerPort: int32(defaultGrpcPort),
Protocol: v1.ProtocolTCP,
},
{
Name: "http-metrics",
ContainerPort: int32(defaultMetricsPort),
Protocol: v1.ProtocolTCP,
},
},
VolumeMounts: volumeMounts,
}
}
func (c *Cluster) getLabels() map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: c.Namespace,
}
}
func (c *Cluster) getDaemonLabels(clusterName string) map[string]string {
labels := c.getLabels()
labels["instance"] = clusterName
return labels
}
+131
View File
@@ -0,0 +1,131 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mgr
import (
"io/ioutil"
"os"
"strconv"
"testing"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestStartMGR(t *testing.T) {
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
return "{\"key\":\"mysecurekey\"}", nil
},
}
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
context := &clusterd.Context{
Executor: executor,
ConfigDir: configDir,
Clientset: testop.New(3)}
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
c := New(context, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
// start a basic service
err := c.Start("edgefs")
assert.Nil(t, err)
validateStart(t, c)
// starting again with more replicas
c.Replicas = 3
err = c.Start("edgefs")
assert.Nil(t, err)
validateStart(t, c)
}
func validateStart(t *testing.T, c *Cluster) {
_, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("rook-edgefs-mgr", metav1.GetOptions{})
assert.Nil(t, err)
_, err = c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-edgefs-mgr", metav1.GetOptions{})
assert.Nil(t, err)
}
func TestPodSpec(t *testing.T) {
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "rook/rook:myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
},
}, metav1.OwnerReference{})
d := c.makeDeployment("mgr-a", "rook-edgefs", "edgefs", 1)
assert.NotNil(t, d)
assert.Equal(t, "mgr-a", d.Name)
assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy)
assert.Equal(t, 1, len(d.Spec.Template.Spec.Volumes))
//check ports
assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers[0].Ports))
assert.Equal(t, int32(6789), d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort)
assert.Equal(t, int32(8881), d.Spec.Template.Spec.Containers[0].Ports[1].ContainerPort)
assert.Equal(t, "edgefs-datadir", d.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, "mgr-a", d.ObjectMeta.Name)
assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"])
assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"])
assert.Equal(t, 0, len(d.ObjectMeta.Annotations))
assert.Equal(t, 2, len(d.Spec.Template.ObjectMeta.Annotations))
assert.Equal(t, "true", d.Spec.Template.ObjectMeta.Annotations["prometheus.io/scrape"])
assert.Equal(t, strconv.Itoa(defaultMetricsPort), d.Spec.Template.ObjectMeta.Annotations["prometheus.io/port"])
cont := d.Spec.Template.Spec.Containers[0]
assert.Equal(t, 2, len(cont.VolumeMounts))
assert.Equal(t, "mgmt", cont.Args[0])
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
}
func TestServiceSpec(t *testing.T) {
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
c := New(&clusterd.Context{}, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
s := c.makeMgrService("rook-edgefs-mgr")
assert.NotNil(t, s)
assert.Equal(t, "rook-edgefs-mgr", s.Name)
assert.Equal(t, 2, len(s.Spec.Ports))
}
func TestHostNetwork(t *testing.T) {
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{ServerIfName: "eth0"}, v1.ResourceRequirements{}, metav1.OwnerReference{})
d := c.makeDeployment("mgr-a", "a", "edgefs", 1)
assert.NotNil(t, d)
assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork)
assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy)
}
@@ -0,0 +1,132 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package config for Edgefs target config managed by the operator
package config
import (
"strconv"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/operator/k8sutil"
)
const (
configStoreNameFmt = "rook-edgefs-%s-config"
osdDirsKeyName = "target-dirs"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "target-config")
func GetConfigStoreName(nodeName string) string {
return k8sutil.TruncateNodeName(configStoreNameFmt, nodeName)
}
const (
RtVerifyChidKey = "rtVerifyChid"
LmdbPageSizeKey = "lmdbPageSize"
UseBcacheKey = "useBCache"
UseBcacheWBKey = "useBCacheWB"
UseMetadataMaskKey = "useMetadataMask"
UseMetadataOffloadKey = "useMetadataOffload"
UseAllSSDKey = "useAllSSD"
RtrdPlevelOverrideKey = "rtrdPLevelOverride"
)
type StoreConfig struct {
// 0 (disabled), 1 (verify on write) or 2(verify on read/write)
RtVerifyChid int `json:"rtVerifyChid,omitempty"`
// 4096, 8192, 16384 or 32768
LmdbPageSize int `json:"lmdbPageSize,omitempty"`
// enable use of back cache
UseBCache bool `json:"useBCache,omitempty"`
// enable write back cache
UseBCacheWB bool `json:"useBCacheWB,omitempty"`
// what guts needs to go to SSD and what not
UseMetadataMask string `json:"useMetadataMask,omitempty"`
// when useAllSSD is false, enable metadata offload on SSD
UseMetadataOffload bool `json:"useMetadataOffload,omitempty"`
// only look for SSD/NVMe
UseAllSSD bool `json:"useAllSSD,omitempty"`
// if > 0, override automatic partitioning numbering logic
RtrdPLevelOverride int `json:"rtrdPLevelOverride,omitempty"`
}
func DefaultStoreConfig() StoreConfig {
return StoreConfig{
RtVerifyChid: 1,
LmdbPageSize: 16348,
UseBCacheWB: false,
UseMetadataMask: "0xff",
UseMetadataOffload: true,
UseAllSSD: false,
RtrdPLevelOverride: 0,
}
}
var validLmbdPageSize = map[int]bool{
4096: true,
8192: true,
16384: true,
32768: true,
}
func ToStoreConfig(config map[string]string) StoreConfig {
storeConfig := DefaultStoreConfig()
for k, v := range config {
switch k {
case RtVerifyChidKey:
if storeConfig.RtVerifyChid < 0 || storeConfig.RtVerifyChid > 2 {
storeConfig.RtVerifyChid = convertToIntIgnoreErr(v)
}
case LmdbPageSizeKey:
if validLmbdPageSize[storeConfig.LmdbPageSize] {
storeConfig.LmdbPageSize = convertToIntIgnoreErr(v)
}
case UseBcacheWBKey:
storeConfig.UseBCacheWB = convertToBoolIgnoreErr(v)
case UseMetadataMaskKey:
storeConfig.UseMetadataMask = v
case UseMetadataOffloadKey:
storeConfig.UseMetadataOffload = convertToBoolIgnoreErr(v)
case UseAllSSDKey:
storeConfig.UseAllSSD = convertToBoolIgnoreErr(v)
case RtrdPlevelOverrideKey:
storeConfig.RtrdPLevelOverride = convertToIntIgnoreErr(v)
}
}
return storeConfig
}
func convertToIntIgnoreErr(raw string) int {
val, err := strconv.Atoi(raw)
if err != nil {
val = 0
}
return val
}
func convertToBoolIgnoreErr(raw string) bool {
val, err := strconv.ParseBool(raw)
if err != nil {
val = false
}
return val
}
@@ -0,0 +1,207 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package target
import (
"encoding/json"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
defaultServerIfName = "eth0"
defaultBrokerIfName = "eth0"
)
type RtlfsDevice struct {
Name string `json:"name"`
Path string `json:"path"`
CheckMountpoint int `json:"check_mountpoint"`
}
type RtlfsDevices struct {
Devices []RtlfsDevice `json:"devices"`
}
type CcowTenant struct {
FailureDomain int `json:"failure_domain"`
}
type CcowNetwork struct {
BrokerInterfaces string `json:"broker_interfaces"`
ServerUnixSocket string `json:"server_unix_socket"`
BrokerIP4addr string `json:"broker_ip4addr,omitempty"`
ServerIP4addr string `json:"server_ip4addr,omitempty"`
}
type CcowConf struct {
Tenant CcowTenant `json:"tenant"`
Network CcowNetwork `json:"network"`
}
type CcowdNetwork struct {
ServerInterfaces string `json:"server_interfaces"`
ServerUnixSocket string `json:"server_unix_socket"`
ServerIP4addr string `json:"server_ip4addr,omitempty"`
}
type CcowdConf struct {
Network CcowdNetwork `json:"network"`
Transport []string `json:"transport"`
}
type AuditdConf struct {
IsAggregator int `json:"is_aggregator"`
}
type SetupNode struct {
Ccow CcowConf `json:"ccow"`
Ccowd CcowdConf `json:"ccowd"`
Auditd AuditdConf `json:"auditd"`
Ipv4Autodetect int `json:"ipv4_autodetect,omitempty"`
RtlfsAutodetect string `json:"rtlfs_autodetect,omitempty"`
ClusterNodes []string `json:"cluster_nodes,omitempty"`
Rtrd RTDevices `json:"rtrd"`
Rtlfs RtlfsDevices `json:"rtlfs"`
}
// As we relying on StatefulSet, we want to build global ConfigMap shared
// to all the nodes in the cluster. This way configuration is simplified and
// available to all subcomponents at any point it time.
func (c *Cluster) createSetupConfigs(resurrect bool) error {
cm := make(map[string]SetupNode)
dnsRecords := make([]string, len(c.Storage.Nodes))
for i := 0; i < len(c.Storage.Nodes); i++ {
dnsRecords[i] = createQualifiedHeadlessServiceName(i, c.Namespace)
}
serverIfName := defaultServerIfName
brokerIfName := defaultBrokerIfName
if isHostNetworkDefined(c.HostNetworkSpec) {
if len(c.HostNetworkSpec.ServerIfName) > 0 && len(c.HostNetworkSpec.BrokerIfName) > 0 {
serverIfName = c.HostNetworkSpec.ServerIfName
brokerIfName = c.HostNetworkSpec.BrokerIfName
} else if len(c.HostNetworkSpec.ServerIfName) > 0 {
serverIfName = c.HostNetworkSpec.ServerIfName
brokerIfName = c.HostNetworkSpec.ServerIfName
} else if len(c.HostNetworkSpec.BrokerIfName) > 0 {
serverIfName = c.HostNetworkSpec.BrokerIfName
brokerIfName = c.HostNetworkSpec.BrokerIfName
}
}
// fully resolve the storage config and resources for all nodes
for _, node := range c.Storage.Nodes {
devConfig := c.deploymentConfig.devConfig[node.Name]
rtDevices := devConfig.rtrd.Devices
rtlfsDevices := devConfig.rtlfs.Devices
rtlfsAutoDetectPath := ""
if c.deploymentConfig.deploymentType == deploymentAutoRtlfs &&
!devConfig.isGatewayNode {
rtlfsAutoDetectPath = "/data"
}
if resurrect || devConfig.isGatewayNode {
// In resurrection case we only need to adjust networking selections
// in ccow.json, ccowd.json and corosync.conf. And keep device transport
// same as before. Resurrection is "best effort" feature, we cannot
// guarnatee that cluster can be reconfigured, but at least we do try.
rtDevices = make([]RTDevice, 0)
rtlfsDevices = make([]RtlfsDevice, 0)
}
nodeConfig := SetupNode{
Ccow: CcowConf{
Tenant: CcowTenant{
FailureDomain: 1,
},
Network: CcowNetwork{
BrokerInterfaces: brokerIfName,
ServerUnixSocket: "/opt/nedge/var/run/sock/ccowd.sock",
},
},
Ccowd: CcowdConf{
Network: CcowdNetwork{
ServerInterfaces: serverIfName,
ServerUnixSocket: "/opt/nedge/var/run/sock/ccowd.sock",
},
Transport: []string{c.deploymentConfig.transportKey},
},
Auditd: AuditdConf{
IsAggregator: 0,
},
Rtrd: RTDevices{
Devices: rtDevices,
},
Rtlfs: RtlfsDevices{
Devices: rtlfsDevices,
},
Ipv4Autodetect: 1,
RtlfsAutodetect: rtlfsAutoDetectPath,
ClusterNodes: dnsRecords,
}
cm[node.Name] = nodeConfig
logger.Debugf("Resolved Node %s = %+v", node.Name, cm[node.Name])
}
nesetupJson, err := json.Marshal(&cm)
if err != nil {
return err
}
dataMap := make(map[string]string, 1)
dataMap["nesetup"] = string(nesetupJson)
configMap := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: configName,
Namespace: c.Namespace,
},
Data: dataMap,
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &configMap.ObjectMeta, &c.ownerRef)
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil {
if errors.IsAlreadyExists(err) {
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(configMap); err != nil {
return nil
}
} else {
return err
}
}
// success. do the labeling so that StatefulSet scheduler will
// select the right nodes.
for _, node := range c.Storage.Nodes {
k := c.Namespace
err = c.AddLabelsToNode(c.context.Clientset, node.Name, map[string]string{k: "cluster"})
logger.Debugf("added label %s from %s: %+v", k, node.Name, err)
}
return nil
}
@@ -0,0 +1,189 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package target
import (
"fmt"
"math"
"path/filepath"
"strings"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/operator/edgefs/cluster/target/config"
"github.com/rook/rook/pkg/util/sys"
)
type RTDevices struct {
Devices []RTDevice `json:"devices"`
}
type RTDevice struct {
Name string `json:"name,omitempty"`
Device string `json:"device,omitempty"`
Psize int `json:"psize,omitempty"`
VerifyChid int `json:"verify_chid,omitempty"`
Journal string `json:"journal,omitempty"`
Metadata string `json:"metadata,omitempty"`
BcacheWritearound int `json:"bcache_writearound,omitempty"`
PlevelOverride int `json:"plevel_override,omitempty"`
}
func getIdDevLinkName(dls string) (dl string) {
dlsArr := strings.Split(dls, " ")
for i := range dlsArr {
s := strings.Replace(dlsArr[i], "/dev/disk/by-id/", "", 1)
if strings.Contains(s, "/") || strings.Contains(s, "wwn-") {
continue
}
dl = s
break
}
return dl
}
func GetRTDevices(nodeDisks []sys.LocalDisk, storeConfig *config.StoreConfig) (rtDevices []RTDevice, err error) {
rtDevices = make([]RTDevice, 0)
if storeConfig == nil {
return rtDevices, fmt.Errorf("no pointer to StoreConfig provided")
}
if len(nodeDisks) == 0 {
return rtDevices, nil
}
var ssds []sys.LocalDisk
var hdds []sys.LocalDisk
var devices []sys.LocalDisk
for i := range nodeDisks {
if !nodeDisks[i].Empty || len(nodeDisks[i].Partitions) > 0 {
continue
}
if nodeDisks[i].Rotational {
hdds = append(hdds, nodeDisks[i])
} else {
ssds = append(ssds, nodeDisks[i])
}
devices = append(devices, nodeDisks[i])
}
//var rtdevs []RTDevice
if storeConfig.UseAllSSD {
//
// All flush media case (High Performance)
//
if len(ssds) == 0 {
return rtDevices, fmt.Errorf("No SSD/NVMe media found")
}
if storeConfig.UseMetadataOffload {
fmt.Println("Warning: useMetadataOffload parameter is ignored due to use useAllSSD=true")
}
for i := range devices {
if devices[i].Rotational {
continue
}
rtdev := RTDevice{
Name: getIdDevLinkName(devices[i].DevLinks),
Device: "/dev/" + devices[i].Name,
Psize: storeConfig.LmdbPageSize,
VerifyChid: storeConfig.RtVerifyChid,
}
if storeConfig.RtrdPLevelOverride != 0 {
rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride
}
rtDevices = append(rtDevices, rtdev)
}
return rtDevices, nil
}
if len(hdds) == 0 {
return rtDevices, fmt.Errorf("No HDD media found")
}
if !storeConfig.UseMetadataOffload {
//
// All HDD media case (capacity, cold archive)
//
for i := range devices {
if !devices[i].Rotational {
continue
}
rtdev := RTDevice{
Name: getIdDevLinkName(devices[i].DevLinks),
Device: "/dev/" + devices[i].Name,
Psize: storeConfig.LmdbPageSize,
VerifyChid: storeConfig.RtVerifyChid,
}
if storeConfig.RtrdPLevelOverride != 0 {
rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride
}
rtDevices = append(rtDevices, rtdev)
}
return rtDevices, nil
}
//
// Hybrid SSD/HDD media case (optimal)
//
if len(hdds) < len(ssds) || len(ssds) == 0 {
return rtDevices, fmt.Errorf("Confusing use of useMetadataOffload parameter HDDs(%d) < SSDs(%d)\n", len(hdds), len(ssds))
}
chunkSize := int(math.Ceil(float64(len(hdds) / len(ssds))))
var hdds_divided [][]sys.LocalDisk
for i := 0; i < len(hdds); i += chunkSize {
end := i + chunkSize
if end > len(hdds) {
end = len(hdds)
}
hdds_divided = append(hdds_divided, hdds[i:end])
}
for i := range hdds_divided {
for j := range hdds_divided[i] {
rtdev := RTDevice{
Name: getIdDevLinkName(hdds_divided[i][j].DevLinks),
Device: "/dev/" + hdds_divided[i][j].Name,
Psize: storeConfig.LmdbPageSize,
VerifyChid: storeConfig.RtVerifyChid,
BcacheWritearound: (map[bool]int{true: 0, false: 1})[storeConfig.UseBCacheWB],
Journal: getIdDevLinkName(ssds[i].DevLinks),
Metadata: getIdDevLinkName(ssds[i].DevLinks) + "," + storeConfig.UseMetadataMask,
}
if storeConfig.RtrdPLevelOverride != 0 {
rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride
}
rtDevices = append(rtDevices, rtdev)
}
}
return rtDevices, nil
}
func getRtlfsDevices(directories []rookalpha.Directory) []RtlfsDevice {
rtlfsDevices := make([]RtlfsDevice, 0)
for _, dir := range directories {
rtlfsDevice := RtlfsDevice{
Name: filepath.Base(dir.Path),
Path: dir.Path,
CheckMountpoint: 1,
}
rtlfsDevices = append(rtlfsDevices, rtlfsDevice)
}
return rtlfsDevices
}
+451
View File
@@ -0,0 +1,451 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package target
import (
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
"github.com/rook/rook/pkg/operator/k8sutil"
appsv1beta1 "k8s.io/api/apps/v1beta1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/kubernetes/pkg/kubelet/apis"
)
const (
dataDirsEnvVarName = "ROOK_DATA_DIRECTORIES"
udpTotemPortDefault = int32(5405)
udpTotemPortName = "totem"
volumeNameDataDir = "datadir"
/* Volumes definitions */
configVolumeName = "edgefs-configdir"
configName = "edgefs-config"
dataVolumeName = "edgefs-datadir"
stateVolumeFolder = ".state"
etcVolumeFolder = ".etc"
)
func (c *Cluster) createAppLabels() map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
}
}
func (c *Cluster) makeCorosyncContainer(containerImage string) v1.Container {
privileged := c.deploymentConfig.needPriviliges
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
Privileged: &privileged,
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "IPC_LOCK"},
},
}
volumeMounts := []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
{
Name: configVolumeName,
MountPath: "/opt/nedge/etc/config",
},
{
Name: dataVolumeName,
MountPath: "/data",
},
}
return v1.Container{
Name: "corosync",
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"corosync"},
SecurityContext: securityContext,
//Resources: c.resources,
VolumeMounts: volumeMounts,
Env: []v1.EnvVar{
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
},
}
}
func (c *Cluster) makeAuditdContainer(containerImage string) v1.Container {
privileged := c.deploymentConfig.needPriviliges
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
Privileged: &privileged,
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
}
volumeMounts := []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
}
return v1.Container{
Name: "auditd",
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"auditd"},
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
},
SecurityContext: securityContext,
//Resources: c.resources,
VolumeMounts: volumeMounts,
}
}
func (c *Cluster) makeDaemonContainer(containerImage string, dro DevicesResurrectOptions, isInitContainer bool) v1.Container {
privileged := c.deploymentConfig.needPriviliges
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
Privileged: &privileged,
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE"},
},
}
volumeMounts := []v1.VolumeMount{
{
Name: "devices",
MountPath: "/dev",
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
}
if c.deploymentConfig.deploymentType == deploymentAutoRtlfs {
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: dataVolumeName, MountPath: "/data"})
} else if c.deploymentConfig.deploymentType == deploymentRtlfs {
rtlfsDevices := getRtlfsDevices(c.Storage.Directories)
for _, device := range rtlfsDevices {
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: device.Name, MountPath: device.Path})
}
}
name := "daemon"
args := []string{"daemon"}
if isInitContainer {
if dro.needToZap {
name = "daemon-zap"
args = []string{"toolbox", "nezap --do-as-i-say"}
// zap mode is InitContainer, it needs to mount config
volumeMounts = append(volumeMounts, v1.VolumeMount{
Name: configVolumeName,
MountPath: "/opt/nedge/etc/config",
})
}
} else {
if dro.needToWait {
args = []string{"wait"}
}
}
return v1.Container{
Name: name,
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: args,
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
},
SecurityContext: securityContext,
Resources: c.resources,
VolumeMounts: volumeMounts,
}
}
func (c *Cluster) configOverrideVolume() v1.Volume {
cmSource := &v1.ConfigMapVolumeSource{Items: []v1.KeyToPath{{Key: "nesetup", Path: "nesetup.json"}}}
cmSource.Name = configName
return v1.Volume{Name: configVolumeName, VolumeSource: v1.VolumeSource{ConfigMap: cmSource}}
}
func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool {
if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 {
return true
}
return false
}
func (c *Cluster) createPodSpec(rookImage string, dro DevicesResurrectOptions) v1.PodSpec {
terminationGracePeriodSeconds := int64(60)
DNSPolicy := v1.DNSClusterFirst
if isHostNetworkDefined(c.HostNetworkSpec) {
DNSPolicy = v1.DNSClusterFirstWithHostNet
}
volumes := []v1.Volume{
{
Name: "devices",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/dev",
},
},
},
}
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
if c.dataVolumeSize.Value() > 0 {
// dataVolume case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: dataVolumeName,
},
},
})
} else {
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: c.dataDirHostPath,
Type: &hostPathDirectoryOrCreate,
},
},
})
}
if c.deploymentConfig.deploymentType == deploymentRtlfs {
// RTLFS with specified folders
for _, folder := range c.deploymentConfig.directories {
volumes = append(volumes, v1.Volume{
Name: folder.Name,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: folder.Path,
Type: &hostPathDirectoryOrCreate,
},
},
})
}
}
var containers []v1.Container
var initContainers []v1.Container
if dro.needToZap {
// To execute "zap" functions (devices or directories) we
// create InitContainer to ensure it completes fully.
initContainers = []v1.Container{
c.makeDaemonContainer(rookImage, dro, true),
}
}
volumes = append(volumes, c.configOverrideVolume())
containers = []v1.Container{
c.makeCorosyncContainer(rookImage),
c.makeAuditdContainer(rookImage),
c.makeDaemonContainer(rookImage, dro, false),
}
return v1.PodSpec{
ServiceAccountName: c.serviceAccount,
Affinity: &v1.Affinity{
PodAntiAffinity: &v1.PodAntiAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
{
Weight: int32(100),
PodAffinityTerm: v1.PodAffinityTerm{
LabelSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: k8sutil.AppAttr,
Operator: metav1.LabelSelectorOpIn,
Values: []string{appName},
},
},
},
TopologyKey: apis.LabelHostname,
},
},
},
},
},
NodeSelector: map[string]string{c.Namespace: "cluster"},
InitContainers: initContainers,
Containers: containers,
// No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node.
TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,
DNSPolicy: DNSPolicy,
HostIPC: true,
HostNetwork: isHostNetworkDefined(c.HostNetworkSpec),
Volumes: volumes,
}
}
func (c *Cluster) makeStatefulSet(replicas int32, rookImage string, dro DevicesResurrectOptions) (*appsv1beta1.StatefulSet, error) {
statefulSet := &appsv1beta1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: appName,
Namespace: c.Namespace,
Labels: c.createAppLabels(),
},
Spec: appsv1beta1.StatefulSetSpec{
ServiceName: appName,
Replicas: &replicas,
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Namespace: c.Namespace,
Labels: c.createAppLabels(),
},
Spec: c.createPodSpec(rookImage, dro),
},
PodManagementPolicy: appsv1beta1.ParallelPodManagement,
UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{
Type: appsv1beta1.RollingUpdateStatefulSetStrategyType,
},
},
}
if c.dataVolumeSize.Value() > 0 {
statefulSet.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{
{
ObjectMeta: metav1.ObjectMeta{
Name: dataVolumeName,
Namespace: c.Namespace,
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceStorage: c.dataVolumeSize,
},
},
},
},
}
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &statefulSet.ObjectMeta, &c.ownerRef)
c.placement.ApplyToPodSpec(&statefulSet.Spec.Template.Spec)
return statefulSet, nil
}
func (c *Cluster) makeHeadlessServicePorts(totemPort int32) []v1.ServicePort {
return []v1.ServicePort{
{
// The secondary port serves the UI as well as health and debug endpoints.
Name: udpTotemPortName,
Port: int32(totemPort),
Protocol: v1.ProtocolUDP,
TargetPort: intstr.FromInt(int(totemPort)),
},
}
}
// This service only exists to create DNS entries for each pod in the stateful
// set such that they can resolve each other's IP addresses. It does not
// create a load-balanced ClusterIP and should not be used directly by clients
// in most circumstances.
func (c *Cluster) makeHeadlessService() (*v1.Service, error) {
headlessService := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: appName,
Namespace: c.Namespace,
Labels: c.createAppLabels(),
Annotations: map[string]string{
"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
},
},
Spec: v1.ServiceSpec{
Selector: c.createAppLabels(),
PublishNotReadyAddresses: true,
ClusterIP: "None",
Ports: c.makeHeadlessServicePorts(udpTotemPortDefault),
},
}
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &headlessService.ObjectMeta, &c.ownerRef)
return headlessService, nil
}
@@ -0,0 +1,172 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package target
import (
"testing"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/edgefs/cluster/target/config"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestStorageSpecDefaultPV(t *testing.T) {
storageSpec := rookalpha.StorageScopeSpec{
Nodes: []rookalpha.Node{
{
Name: "node1",
},
},
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI),
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
c.createSetupConfigs(false)
}
func TestStorageSpecDevices(t *testing.T) {
storageSpec := rookalpha.StorageScopeSpec{
Selection: rookalpha.Selection{
Directories: []rookalpha.Directory{{Path: "/rook/dir2"}},
},
Nodes: []rookalpha.Node{
{
Name: "node1",
Selection: rookalpha.Selection{
Devices: []rookalpha.Device{{Name: "sda"}, {Name: "sdb"}},
},
},
},
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI),
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
c.createSetupConfigs(false)
}
func TestStorageSpecDirectories(t *testing.T) {
storageSpec := rookalpha.StorageScopeSpec{
Selection: rookalpha.Selection{
Directories: []rookalpha.Directory{{Path: "/rook/dir2"}},
},
Nodes: []rookalpha.Node{
{
Name: "node1",
Selection: rookalpha.Selection{
Directories: []rookalpha.Directory{{Path: "/rook/dir1"}, {Path: "/rook/dir2"}},
},
},
},
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI),
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
c.createSetupConfigs(false)
}
func TestStorageSpecConfig(t *testing.T) {
storageSpec := rookalpha.StorageScopeSpec{
Config: map[string]string{
"useAllSSD": "false",
"useBCacheWB": "true",
"useBCache": "true",
},
Nodes: []rookalpha.Node{
{
Name: "node1",
Location: "zone1",
Config: map[string]string{
"rtTransport": "rtrd",
"useAllSSD": "true",
"useMetadataOffload": "false",
},
Selection: rookalpha.Selection{
Devices: []rookalpha.Device{{Name: "sda"}, {Name: "sdb"}},
},
Resources: v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
},
Requests: v1.ResourceList{
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
},
},
},
},
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
storageSpec, "", *resource.NewQuantity(100000.0, resource.BinarySI),
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
c.createSetupConfigs(false)
n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name)
storeConfig := config.ToStoreConfig(storageSpec.Nodes[0].Config)
// First Node's useAllSSD should override global useAllSSD value, -> should be different
assert.NotEqual(t, storageSpec.Config["useAllSSD"], storeConfig.UseAllSSD)
// useBCacheWB global in StorageSpec and does override config' UseBCacheWB prop, -> should be the same
assert.NotEqual(t, storageSpec.Config["useBCacheWB"], storeConfig.UseBCacheWB)
// useBCache global in StorageSpec and does override config' UseBCache prop, -> should be the same
assert.NotEqual(t, storageSpec.Config["useBCache"], storeConfig.UseBCache)
// UseMetadataOffload will override default value of true
assert.Equal(t, storeConfig.UseMetadataOffload, false)
logger.Infof("Node Config is %+v", n.Config)
logger.Infof("storeConfig is %+v", storeConfig)
}
func TestHostNetwork(t *testing.T) {
storageSpec := rookalpha.StorageScopeSpec{
Nodes: []rookalpha.Node{
{
Name: "node1",
Location: "zone1",
Config: map[string]string{
"rtTransport": "rtrd",
"useAllSSD": "true",
},
},
},
}
clientset := fake.NewSimpleClientset()
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
storageSpec, "", *resource.NewQuantity(100000.0, resource.BinarySI),
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{ServerIfName: "eth0"}, v1.ResourceRequirements{}, metav1.OwnerReference{})
c.createSetupConfigs(false)
}
@@ -0,0 +1,469 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package target
import (
"fmt"
"os"
"strings"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/golang/glog"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/discover"
"github.com/rook/rook/pkg/operator/edgefs/cluster/target/config"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/sys"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientset "k8s.io/client-go/kubernetes"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-target")
const (
appName = "rook-edgefs-target"
targetAppNameFmt = "rook-edgefs-target-id-%d"
appNameFmt = "rook-edgefs-target-%s"
targetLabelKey = "edgefs-target-id"
defaultServiceAccountName = "rook-edgefs-cluster"
unknownID = -1
labelingRetries = 5
//deployment types
deploymentRtlfs = "rtlfs"
deploymentRtrd = "rtrd"
deploymentAutoRtlfs = "autoRtlfs"
nodeTypeLabelFmt = "%s-nodetype"
)
// Cluster keeps track of the Targets
type Cluster struct {
context *clusterd.Context
Namespace string
placement rookalpha.Placement
Version string
Storage rookalpha.StorageScopeSpec
dataDirHostPath string
dataVolumeSize resource.Quantity
HostNetworkSpec edgefsv1alpha1.NetworkSpec
Privileged bool
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
serviceAccount string
deploymentConfig ClusterDeploymentConfig
}
type ClusterDeploymentConfig struct {
deploymentType string //rtlfs, rtrd, autortlfs
transportKey string //rtlfs or rtrd
directories []RtlfsDevice //cluster wide directories
devConfig map[string]DevicesConfig
needPriviliges bool
}
type DevicesConfig struct {
rtrd RTDevices
rtlfs RtlfsDevices
isGatewayNode bool
}
// New creates an instance of the Target manager
func New(
context *clusterd.Context,
namespace,
version,
serviceAccount string,
storageSpec rookalpha.StorageScopeSpec,
dataDirHostPath string,
dataVolumeSize resource.Quantity,
placement rookalpha.Placement,
hostNetworkSpec edgefsv1alpha1.NetworkSpec,
resources v1.ResourceRequirements,
ownerRef metav1.OwnerReference,
) *Cluster {
if serviceAccount == "" {
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
serviceAccount = defaultServiceAccountName
logger.Infof("setting the target pods to use the service account name: %s", serviceAccount)
}
return &Cluster{
context: context,
Namespace: namespace,
serviceAccount: serviceAccount,
placement: placement,
Version: version,
Storage: storageSpec,
dataDirHostPath: dataDirHostPath,
dataVolumeSize: dataVolumeSize,
HostNetworkSpec: hostNetworkSpec,
Privileged: (isHostNetworkDefined(hostNetworkSpec) || os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true"),
resources: resources,
ownerRef: ownerRef,
deploymentConfig: ClusterDeploymentConfig{devConfig: make(map[string]DevicesConfig, 0)},
}
}
type DevicesResurrectOptions struct {
needToResurrect bool
needToZap bool
needToWait bool
}
func ParseDevicesResurrectMode(resurrectMode string) DevicesResurrectOptions {
drm := DevicesResurrectOptions{}
if len(resurrectMode) == 0 {
return drm
}
resurrectModeToLower := strings.ToLower(strings.TrimSpace(resurrectMode))
switch resurrectModeToLower {
case "restore":
drm.needToResurrect = true
break
case "restorezap":
drm.needToResurrect = true
drm.needToZap = true
break
case "restorezapwait":
drm.needToResurrect = true
drm.needToZap = true
drm.needToWait = true
break
}
return drm
}
// Start the target management
func (c *Cluster) Start(rookImage, devicesResurrectMode string) (err error) {
logger.Infof("start running targets in namespace %s", c.Namespace)
logger.Infof("Target Image is %s", rookImage)
if c.Storage.UseAllNodes == false && len(c.Storage.Nodes) == 0 {
logger.Warningf("useAllNodes is set to false and no nodes are specified, no Edgefs pods are going to be created")
}
//logger.Infof("Cluster storage config %+v", c.Storage.Config)
if c.Storage.UseAllNodes {
// resolve all storage nodes
c.Storage.Nodes = nil
rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar)
allNodeDevices, err := discover.ListDevices(c.context, rookSystemNS, "" /* all nodes */)
//logger.Infof("allNodeDevices: %+v", allNodeDevices)
if err != nil {
logger.Warningf("failed to get storage nodes from namespace %s: %v", rookSystemNS, err)
return err
}
for nodeName := range allNodeDevices {
storageNode := rookalpha.Node{
Name: nodeName,
}
c.Storage.Nodes = append(c.Storage.Nodes, storageNode)
}
}
validNodes := k8sutil.GetValidNodes(c.Storage.Nodes, c.context.Clientset, c.placement)
//logger.Infof("validNOdes: %+v", validNodes)
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes))
c.Storage.Nodes = validNodes
rmo := ParseDevicesResurrectMode(devicesResurrectMode)
logger.Infof("DevicesResurrect mode: %s options %+v", devicesResurrectMode, rmo)
err = c.createDeploymentConfig(rmo.needToResurrect)
if err != nil {
logger.Errorf("Failed to create deploymentConfig %+v", err)
return err
}
logger.Infof("Deployment Config : %+v", c.deploymentConfig)
if err := c.createSetupConfigs(rmo.needToResurrect); err != nil {
logger.Errorf("Failed to create/update Edgefs cluster configuration: %+v", err)
return err
}
headlessService, _ := c.makeHeadlessService()
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(headlessService); err != nil {
if !errors.IsAlreadyExists(err) {
return err
}
logger.Infof("headless service %s already exists in namespace %s", headlessService.Name, headlessService.Namespace)
} else {
logger.Infof("headless service %s started in namespace %s", headlessService.Name, headlessService.Namespace)
}
statefulSet, _ := c.makeStatefulSet(int32(len(validNodes)), rookImage, rmo)
if _, err := c.context.Clientset.AppsV1beta1().StatefulSets(c.Namespace).Create(statefulSet); err != nil {
if !errors.IsAlreadyExists(err) {
return err
}
logger.Infof("stateful set %s already exists in namespace %s", statefulSet.Name, statefulSet.Namespace)
} else {
logger.Infof("stateful set %s created in namespace %s", statefulSet.Name, statefulSet.Namespace)
}
return nil
}
func (c *Cluster) HasDevicesSpecification() bool {
if len(c.Storage.DeviceFilter) > 0 || len(c.Storage.Devices) > 0 {
return true
}
for _, node := range c.Storage.Nodes {
useAllDevices := node.UseAllDevices
if useAllDevices != nil && *useAllDevices {
return true
}
if len(node.DeviceFilter) > 0 || len(node.Devices) > 0 {
return true
}
}
return false
}
func (c *Cluster) createDeploymentConfig(resurrect bool) error {
//Fill deploymentConfig devices struct
for _, node := range c.Storage.Nodes {
n := c.resolveNode(node.Name)
if n == nil {
return fmt.Errorf("node %s did not resolve to start target", node.Name)
}
devicesConfig := DevicesConfig{}
devicesConfig.rtrd.Devices = make([]RTDevice, 0)
devicesConfig.rtlfs.Devices = make([]RtlfsDevice, 0)
// if node labeled as gateway then return empty devises and skip RTDevices detection
if c.isGatewayLabeledNode(c.context.Clientset, node.Name) {
devicesConfig.isGatewayNode = true
c.deploymentConfig.devConfig[node.Name] = devicesConfig
continue
}
rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar)
nodeDevices, _ := discover.ListDevices(c.context, rookSystemNS, n.Name)
availDevs, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace,
n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices())
if deviceErr != nil {
// Devices were specified but we couldn't find any.
// User needs to fix CRD.
return fmt.Errorf("failed to get devices for node %s cluster %s: %v",
n.Name, c.Namespace, deviceErr)
}
// selects Disks from availDevs and translate to RTDevices
availDisks := []sys.LocalDisk{}
for _, dev := range availDevs {
for _, disk := range nodeDevices[n.Name] {
if disk.Name == dev.Name {
availDisks = append(availDisks, disk)
}
}
}
storeConfig := config.ToStoreConfig(n.Config)
rtDevices, err := GetRTDevices(availDisks, &storeConfig)
if err != nil {
logger.Warningf("Can't get rtDevices for node %s due %v", n.Name, err)
rtDevices = make([]RTDevice, 0)
}
devicesConfig.rtrd.Devices = rtDevices
devicesConfig.rtlfs.Devices = getRtlfsDevices(c.Storage.Directories)
c.deploymentConfig.devConfig[node.Name] = devicesConfig
}
// Add Directories to deploymentConfig
c.deploymentConfig.directories = getRtlfsDevices(c.Storage.Directories)
if len(c.Storage.Directories) > 0 && (len(c.dataDirHostPath) > 0 || c.dataVolumeSize.Value() != 0) {
c.deploymentConfig.deploymentType = deploymentRtlfs
c.deploymentConfig.transportKey = "rtlfs"
// check directories devices count on all nodes
if len(c.Storage.Directories)*len(c.Storage.Nodes) < 3 {
return fmt.Errorf("Rtlfs devices should be more then 3 on all nodes summary")
}
} else if c.HasDevicesSpecification() && (len(c.dataDirHostPath) > 0 || c.dataVolumeSize.Value() != 0) {
// Check all deployment nodes has available disk devices
devicesCount := 0
for nodeName, devCfg := range c.deploymentConfig.devConfig {
if devCfg.isGatewayNode {
continue
}
if len(devCfg.rtrd.Devices) == 0 && !resurrect {
return fmt.Errorf("Node %s has no available devices", nodeName)
}
devicesCount += len(devCfg.rtrd.Devices)
}
// check new deployment devices count
if !resurrect && devicesCount < 3 {
return fmt.Errorf("Disk devices should be more then 3 on all nodes summary")
}
c.deploymentConfig.deploymentType = deploymentRtrd
c.deploymentConfig.transportKey = "rtrd"
c.deploymentConfig.needPriviliges = true
} else if len(c.dataDirHostPath) == 0 || c.dataVolumeSize.Value() == 0 {
c.deploymentConfig.deploymentType = deploymentAutoRtlfs
c.deploymentConfig.transportKey = "rtlfs"
} else {
return fmt.Errorf("Unknown deployment type! Cluster spec:\n %+v", c)
}
//set priviliges==true in case of HostNetwork
if len(c.HostNetworkSpec.ServerIfName) > 0 || len(c.HostNetworkSpec.BrokerIfName) > 0 {
c.deploymentConfig.needPriviliges = true
}
return nil
}
// creates a qualified name of the headless service for a given replica id and namespace,
// e.g., edgefs-0.edgefs.rook-edgefs
func createQualifiedHeadlessServiceName(replicaNum int, namespace string) string {
return fmt.Sprintf("%s-%d.%s.%s", appName, replicaNum, appName, namespace)
}
func (c *Cluster) resolveNode(nodeName string) *rookalpha.Node {
// fully resolve the storage config and resources for this node
rookNode := c.Storage.ResolveNode(nodeName)
if rookNode == nil {
return nil
}
// Apply directories from ClusterStorageSpec only
rookNode.Directories = c.Storage.Directories
rookNode.Resources = k8sutil.MergeResourceRequirements(rookNode.Resources, c.resources)
// ensure no invalid dirs are specified
var validDirs []rookalpha.Directory
for _, dir := range rookNode.Directories {
if dir.Path == k8sutil.DataDir || dir.Path == c.dataDirHostPath {
logger.Warningf("skipping directory %s that would conflict with the dataDirHostPath", dir.Path)
continue
}
validDirs = append(validDirs, dir)
}
rookNode.Directories = validDirs
return rookNode
}
func (c *Cluster) AddLabelsToNode(cs clientset.Interface, nodeName string, labels map[string]string) error {
tokens := make([]string, 0, len(labels))
for k, v := range labels {
tokens = append(tokens, "\""+k+"\":\""+v+"\"")
}
labelString := "{" + strings.Join(tokens, ",") + "}"
patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString)
var err error
for attempt := 0; attempt < labelingRetries; attempt++ {
_, err = cs.CoreV1().Nodes().Patch(nodeName, types.MergePatchType, []byte(patch))
if err != nil {
if !apierrs.IsConflict(err) {
return err
}
} else {
break
}
time.Sleep(100 * time.Millisecond)
}
return err
}
// RemoveLabelOffNode is for cleaning up labels temporarily added to node,
// won't fail if target label doesn't exist or has been removed.
func (c *Cluster) RemoveLabelOffNode(cs clientset.Interface, nodeName string, labelKeys []string) error {
var node *v1.Node
var err error
for attempt := 0; attempt < labelingRetries; attempt++ {
node, err = cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil {
return err
}
if node.Labels == nil {
return nil
}
for _, labelKey := range labelKeys {
if node.Labels == nil || len(node.Labels[labelKey]) == 0 {
break
}
delete(node.Labels, labelKey)
}
_, err = cs.CoreV1().Nodes().Update(node)
if err != nil {
if !apierrs.IsConflict(err) {
return err
} else {
glog.V(2).Infof("Conflict when trying to remove a labels %v from %v", labelKeys, nodeName)
}
} else {
break
}
time.Sleep(100 * time.Millisecond)
}
return err
}
func (c *Cluster) isGatewayLabeledNode(cs clientset.Interface, nodeName string) bool {
labelMap, err := c.getNodeLabels(cs, nodeName)
if err != nil || labelMap == nil {
return false
}
if nodeType, ok := labelMap[fmt.Sprintf(nodeTypeLabelFmt, c.Namespace)]; ok {
if nodeType == "gateway" {
return true
}
}
return false
}
func (c *Cluster) getNodeLabels(cs clientset.Interface, nodeName string) (map[string]string, error) {
node, err := cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
if err != nil {
return nil, err
}
if node.Labels == nil {
return node.Labels, nil
}
return node.Labels, nil
}
@@ -0,0 +1,66 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package target
import (
"testing"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
"github.com/rook/rook/pkg/operator/k8sutil"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestStart(t *testing.T) {
rookImage := "edgefs/edgefs-operator"
clientset := fake.NewSimpleClientset()
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
rookalpha.StorageScopeSpec{}, "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
// Start the first time
err := c.Start(rookImage, "")
assert.Nil(t, err)
// Should not fail if it already exists
err = c.Start(rookImage, "")
assert.Nil(t, err)
}
func createDiscoverConfigmap(nodeName, ns string, clientset *fake.Clientset) error {
data := make(map[string]string, 1)
data[discoverDaemon.LocalDiskCMData] = `[{"name":"sdx","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405f826bd553d8c4dbf9f41c18be /dev/disk/by-id/wwn-0x6001405f826bd553d8c4dbf9f41c18be /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-1","size":10737418240,"uuid":"","serial":"36001405f826bd553d8c4dbf9f41c18be","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk02","wwn":"0x6001405f826bd553","wwnVendorExtension":"0x6001405f826bd553d8c4dbf9f41c18be","empty":true}]`
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "local-device-" + nodeName,
Namespace: ns,
Labels: map[string]string{
k8sutil.AppAttr: discoverDaemon.AppName,
discoverDaemon.NodeAttr: nodeName,
},
},
Data: data,
}
_, err := clientset.CoreV1().ConfigMaps(ns).Create(cm)
return err
}
+171
View File
@@ -0,0 +1,171 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// scale-out, multi-cloud Edge-X ISCSI services controller
package iscsi
import (
"fmt"
"reflect"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/cache"
)
const (
customResourceName = "iscsi"
customResourceNamePlural = "iscsis"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-iscsi")
// ISCSIResource represents the iscsi custom resource
var ISCSIResource = opkit.CustomResource{
Name: customResourceName,
Plural: customResourceNamePlural,
Group: edgefsv1alpha1.CustomResourceGroup,
Version: edgefsv1alpha1.Version,
Scope: apiextensionsv1beta1.NamespaceScoped,
Kind: reflect.TypeOf(edgefsv1alpha1.ISCSI{}).Name(),
}
// ISCSIController represents a controller object for iscsi custom resources
type ISCSIController struct {
context *clusterd.Context
rookImage string
hostNetwork bool
dataDirHostPath string
dataVolumeSize resource.Quantity
placement rookalpha.Placement
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
}
// NewISCSIController create controller for watching ISCSI custom resources created
func NewISCSIController(
context *clusterd.Context, rookImage string,
hostNetwork bool,
dataDirHostPath string,
dataVolumeSize resource.Quantity,
placement rookalpha.Placement,
resources v1.ResourceRequirements,
ownerRef metav1.OwnerReference,
) *ISCSIController {
return &ISCSIController{
context: context,
rookImage: rookImage,
hostNetwork: hostNetwork,
dataDirHostPath: dataDirHostPath,
dataVolumeSize: dataVolumeSize,
placement: placement,
resources: resources,
ownerRef: ownerRef,
}
}
// StartWatch watches for instances of ISCSI custom resources and acts on them
func (c *ISCSIController) StartWatch(namespace string, stopCh chan struct{}) error {
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
AddFunc: c.onAdd,
UpdateFunc: c.onUpdate,
DeleteFunc: c.onDelete,
}
logger.Infof("start watching iscsi resources in namespace %s", namespace)
watcher := opkit.NewWatcher(ISCSIResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
go watcher.Watch(&edgefsv1alpha1.ISCSI{}, stopCh)
return nil
}
func (c *ISCSIController) onAdd(obj interface{}) {
iscsi, err := getISCSIObject(obj)
if err != nil {
logger.Errorf("failed to get iscsi object: %+v", err)
return
}
if err = c.CreateService(*iscsi, c.serviceOwners(iscsi)); err != nil {
logger.Errorf("failed to create iscsi %s. %+v", iscsi.Name, err)
}
}
func (c *ISCSIController) onUpdate(oldObj, newObj interface{}) {
oldService, err := getISCSIObject(oldObj)
if err != nil {
logger.Errorf("failed to get old iscsi object: %+v", err)
return
}
newService, err := getISCSIObject(newObj)
if err != nil {
logger.Errorf("failed to get new iscsi object: %+v", err)
return
}
if !serviceChanged(oldService.Spec, newService.Spec) {
logger.Debugf("iscsi %s did not change", newService.Name)
return
}
logger.Infof("applying iscsi %s changes", newService.Name)
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
logger.Errorf("failed to create (modify) iscsi %s. %+v", newService.Name, err)
}
}
func (c *ISCSIController) onDelete(obj interface{}) {
iscsi, err := getISCSIObject(obj)
if err != nil {
logger.Errorf("failed to get iscsi object: %+v", err)
return
}
if err = c.DeleteService(*iscsi); err != nil {
logger.Errorf("failed to delete iscsi %s. %+v", iscsi.Name, err)
}
}
func (c *ISCSIController) serviceOwners(service *edgefsv1alpha1.ISCSI) []metav1.OwnerReference {
// Only set the cluster crd as the owner of the ISCSI resources.
// If the ISCSI crd is deleted, the operator will explicitly remove the ISCSI resources.
// If the ISCSI crd still exists when the cluster crd is deleted, this will make sure the ISCSI
// resources are cleaned up.
return []metav1.OwnerReference{c.ownerRef}
}
func serviceChanged(oldService, newService edgefsv1alpha1.ISCSISpec) bool {
return false
}
func getISCSIObject(obj interface{}) (iscsi *edgefsv1alpha1.ISCSI, err error) {
var ok bool
iscsi, ok = obj.(*edgefsv1alpha1.ISCSI)
if ok {
// the iscsi object is of the latest type, simply return it
return iscsi.DeepCopy(), nil
}
return nil, fmt.Errorf("not a known iscsi object: %+v", obj)
}
+362
View File
@@ -0,0 +1,362 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package ISCSI for the Edgefs manager.
package iscsi
import (
"encoding/json"
"fmt"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
appName = "rook-edgefs-iscsi"
/* Volumes definitions */
defaultTargetName = "iqn.2018-11.edgefs.io"
defaultTargetParams = "{}"
dataVolumeName = "edgefs-datadir"
stateVolumeFolder = ".state"
etcVolumeFolder = ".etc"
defaultPort = 3260
)
// Start the ISCSI manager
func (c *ISCSIController) CreateService(s edgefsv1alpha1.ISCSI, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, false, ownerRefs)
}
func (c *ISCSIController) UpdateService(s edgefsv1alpha1.ISCSI, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, true, ownerRefs)
}
// Start the iscsi instance
func (c *ISCSIController) CreateOrUpdate(s edgefsv1alpha1.ISCSI, update bool, ownerRefs []metav1.OwnerReference) error {
logger.Infof("starting update=%v service=%s", update, s.Name)
logger.Infof("ISCSI Base image is %s", c.rookImage)
// validate ISCSI service settings
if err := validateService(c.context, s); err != nil {
return fmt.Errorf("invalid ISCSI service %s arguments. %+v", s.Name, err)
}
if len(s.Spec.TargetName) == 0 {
s.Spec.TargetName = defaultTargetName
}
// check if ISCSI service already exists
exists, err := serviceExists(c.context, s)
if err == nil && exists {
if !update {
logger.Infof("ISCSI service %s exists in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("ISCSI service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
}
// start the deployment
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
}
logger.Infof("%s deployment already exists", appName)
} else {
logger.Infof("%s deployment started", appName)
}
// create the iscsi service
service := c.makeISCSIService(instanceName(s.Name), s.Name, s.Namespace, s.Spec)
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create ISCSI service. %+v", err)
}
logger.Infof("ISCSI service %s already exists", service)
} else {
logger.Infof("ISCSI service %s started", service)
}
return nil
}
func (c *ISCSIController) makeISCSIService(name, svcname, namespace string, iscsiSpec edgefsv1alpha1.ISCSISpec) *v1.Service {
labels := getLabels(name, svcname, namespace)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Selector: labels,
Type: v1.ServiceTypeNodePort,
Ports: []v1.ServicePort{
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
{Name: "port", Port: defaultPort, Protocol: v1.ProtocolTCP},
},
},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
return svc
}
func (c *ISCSIController) makeDeployment(svcname, namespace, rookImage string, iscsiSpec edgefsv1alpha1.ISCSISpec) *extensions.Deployment {
name := instanceName(svcname)
volumes := []v1.Volume{}
if c.dataVolumeSize.Value() > 0 {
// dataVolume case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: dataVolumeName,
},
},
})
} else {
// dataDir case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: c.dataDirHostPath,
},
},
})
}
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: getLabels(name, svcname, namespace),
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.iscsiContainer(svcname, name, rookImage, iscsiSpec)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
HostIPC: true,
HostNetwork: c.hostNetwork,
NodeSelector: map[string]string{namespace: "cluster"},
},
}
if c.hostNetwork {
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// apply current ISCSI CRD options to pod's specification
iscsiSpec.Placement.ApplyToPodSpec(&podSpec.Spec)
instancesCount := int32(1)
d := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &instancesCount},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
return d
}
func (c *ISCSIController) iscsiContainer(svcname, name, containerImage string, iscsiSpec edgefsv1alpha1.ISCSISpec) v1.Container {
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
},
}
volumeMounts := []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc.target",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
}
return v1.Container{
Name: name,
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"iscsi"},
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "CCOW_SVCNAME",
Value: svcname,
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
{
Name: "KUBERNETES_NAMESPACE",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
{
Name: "EFSISCSI_TARGET_NAME",
Value: iscsiSpec.TargetName,
},
{
Name: "EFSISCSI_TARGET_PARAMS",
Value: getTargetParamsJSON(iscsiSpec.TargetParams),
},
},
SecurityContext: securityContext,
Resources: iscsiSpec.Resources,
Ports: []v1.ContainerPort{
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
{Name: "port", ContainerPort: defaultPort, Protocol: v1.ProtocolTCP},
},
VolumeMounts: volumeMounts,
}
}
// Delete ISCSI service and possibly some artifacts.
func (c *ISCSIController) DeleteService(s edgefsv1alpha1.ISCSI) error {
// check if service exists
exists, err := serviceExists(c.context, s)
if err != nil {
return fmt.Errorf("failed to detect if there is a ISCSI service to delete. %+v", err)
}
if !exists {
logger.Infof("ISCSI service %s does not exist in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("Deleting ISCSI service %s from namespace %s", s.Name, s.Namespace)
var gracePeriod int64
propagation := metav1.DeletePropagationForeground
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
// Delete the iscsi service
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
if err != nil && !errors.IsNotFound(err) {
logger.Warningf("failed to delete ISCSI service. %+v", err)
}
// Make a best effort to delete the ISCSI pods
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
if err != nil {
logger.Warningf(err.Error())
}
logger.Infof("Completed deleting ISCSI service %s", s.Name)
return nil
}
func getLabels(name, svcname, namespace string) map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: namespace,
"edgefs_svcname": svcname,
"edgefs_svctype": "iscsi",
}
}
// Validate the ISCSI arguments
func validateService(context *clusterd.Context, s edgefsv1alpha1.ISCSI) error {
if s.Name == "" {
return fmt.Errorf("missing name")
}
if s.Namespace == "" {
return fmt.Errorf("missing namespace")
}
return nil
}
func instanceName(svcname string) string {
return fmt.Sprintf("%s-%s", appName, svcname)
}
// Check if the ISCSI service exists
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.ISCSI) (bool, error) {
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
if err == nil {
// the deployment was found
return true, nil
}
if err != nil && !errors.IsNotFound(err) {
return false, err
}
// not found
return false, nil
}
func getTargetParamsJSON(params edgefsv1alpha1.TargetParametersSpec) string {
result := make(map[string]uint)
if params.MaxRecvDataSegmentLength > 0 {
result["MaxRecvDataSegmentLength"] = params.MaxRecvDataSegmentLength
}
if params.DefaultTime2Retain > 0 {
result["DefaultTime2Retain"] = params.DefaultTime2Retain
}
if params.DefaultTime2Wait > 0 {
result["DefaultTime2Wait"] = params.DefaultTime2Wait
}
if params.FirstBurstLength > 0 {
result["FirstBurstLength"] = params.FirstBurstLength
}
if params.MaxBurstLength > 0 {
result["MaxBurstLength"] = params.MaxBurstLength
}
if params.MaxQueueCmd > 0 {
result["MaxQueueCmd"] = params.MaxQueueCmd
}
jsonString, _ := json.Marshal(result)
return string(jsonString)
}
+171
View File
@@ -0,0 +1,171 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// scale-out, multi-cloud NFS
package nfs
import (
"fmt"
"reflect"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/cache"
)
const (
customResourceName = "nfs"
customResourceNamePlural = "nfss"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-nfs")
// NFSResource represents the nfs custom resource
var NFSResource = opkit.CustomResource{
Name: customResourceName,
Plural: customResourceNamePlural,
Group: edgefsv1alpha1.CustomResourceGroup,
Version: edgefsv1alpha1.Version,
Scope: apiextensionsv1beta1.NamespaceScoped,
Kind: reflect.TypeOf(edgefsv1alpha1.NFS{}).Name(),
}
// NFSController represents a controller object for nfs custom resources
type NFSController struct {
context *clusterd.Context
rookImage string
hostNetwork bool
dataDirHostPath string
dataVolumeSize resource.Quantity
placement rookalpha.Placement
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
}
// NewNFSController create controller for watching nfs custom resources created
func NewNFSController(
context *clusterd.Context, rookImage string,
hostNetwork bool,
dataDirHostPath string,
dataVolumeSize resource.Quantity,
placement rookalpha.Placement,
resources v1.ResourceRequirements,
ownerRef metav1.OwnerReference,
) *NFSController {
return &NFSController{
context: context,
rookImage: rookImage,
hostNetwork: hostNetwork,
dataDirHostPath: dataDirHostPath,
dataVolumeSize: dataVolumeSize,
placement: placement,
resources: resources,
ownerRef: ownerRef,
}
}
// StartWatch watches for instances of NFS custom resources and acts on them
func (c *NFSController) StartWatch(namespace string, stopCh chan struct{}) error {
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
AddFunc: c.onAdd,
UpdateFunc: c.onUpdate,
DeleteFunc: c.onDelete,
}
logger.Infof("start watching nfs resources in namespace %s", namespace)
watcher := opkit.NewWatcher(NFSResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
go watcher.Watch(&edgefsv1alpha1.NFS{}, stopCh)
return nil
}
func (c *NFSController) onAdd(obj interface{}) {
nfs, err := getNFSObject(obj)
if err != nil {
logger.Errorf("failed to get nfs object: %+v", err)
return
}
if err = c.CreateService(*nfs, c.serviceOwners(nfs)); err != nil {
logger.Errorf("failed to create nfs %s. %+v", nfs.Name, err)
}
}
func (c *NFSController) onUpdate(oldObj, newObj interface{}) {
oldService, err := getNFSObject(oldObj)
if err != nil {
logger.Errorf("failed to get old nfs object: %+v", err)
return
}
newService, err := getNFSObject(newObj)
if err != nil {
logger.Errorf("failed to get new nfs object: %+v", err)
return
}
if !serviceChanged(oldService.Spec, newService.Spec) {
logger.Debugf("nfs %s did not change", newService.Name)
return
}
logger.Infof("applying nfs %s changes", newService.Name)
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
logger.Errorf("failed to create (modify) nfs %s. %+v", newService.Name, err)
}
}
func (c *NFSController) onDelete(obj interface{}) {
nfs, err := getNFSObject(obj)
if err != nil {
logger.Errorf("failed to get nfs object: %+v", err)
return
}
if err = c.DeleteService(*nfs); err != nil {
logger.Errorf("failed to delete nfs %s. %+v", nfs.Name, err)
}
}
func (c *NFSController) serviceOwners(service *edgefsv1alpha1.NFS) []metav1.OwnerReference {
// Only set the cluster crd as the owner of the NFS resources.
// If the NFS crd is deleted, the operator will explicitly remove the NFS resources.
// If the NFS crd still exists when the cluster crd is deleted, this will make sure the NFS
// resources are cleaned up.
return []metav1.OwnerReference{c.ownerRef}
}
func serviceChanged(oldService, newService edgefsv1alpha1.NFSSpec) bool {
return false
}
func getNFSObject(obj interface{}) (nfs *edgefsv1alpha1.NFS, err error) {
var ok bool
nfs, ok = obj.(*edgefsv1alpha1.NFS)
if ok {
// the nfs object is of the latest type, simply return it
return nfs.DeepCopy(), nil
}
return nil, fmt.Errorf("not a known nfs object: %+v", obj)
}
+326
View File
@@ -0,0 +1,326 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package nfs for the Edgefs manager.
package nfs
import (
"fmt"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
appName = "rook-edgefs-nfs"
/* Volumes definitions */
dataVolumeName = "edgefs-datadir"
stateVolumeFolder = ".state"
etcVolumeFolder = ".etc"
)
// Start the rgw manager
func (c *NFSController) CreateService(s edgefsv1alpha1.NFS, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, false, ownerRefs)
}
func (c *NFSController) UpdateService(s edgefsv1alpha1.NFS, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, true, ownerRefs)
}
// Start the nfs instance
func (c *NFSController) CreateOrUpdate(s edgefsv1alpha1.NFS, update bool, ownerRefs []metav1.OwnerReference) error {
logger.Infof("starting update=%v service=%s", update, s.Name)
logger.Infof("NFS Base image is %s", c.rookImage)
// validate NFS service settings
if err := validateService(c.context, s); err != nil {
return fmt.Errorf("invalid NFS service %s arguments. %+v", s.Name, err)
}
// check if NFS service already exists
exists, err := serviceExists(c.context, s)
if err == nil && exists {
if !update {
logger.Infof("NFS service %s exists in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("NFS service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
}
// start the deployment
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
}
logger.Infof("%s deployment already exists", appName)
} else {
logger.Infof("%s deployment started", appName)
}
// create the nfs service
service := c.makeNFSService(instanceName(s.Name), s.Name, s.Namespace)
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create nfs service. %+v", err)
}
logger.Infof("nfs service %s already exists", service)
} else {
logger.Infof("nfs service %s started", service)
}
return nil
}
func (c *NFSController) makeNFSService(name, svcname, namespace string) *v1.Service {
labels := getLabels(name, svcname, namespace)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Selector: labels,
Type: v1.ServiceTypeNodePort,
Ports: []v1.ServicePort{
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
{Name: "nfs-tcp", Port: 2049, Protocol: v1.ProtocolTCP},
{Name: "nfs-udp", Port: 2049, Protocol: v1.ProtocolUDP},
{Name: "nlockmgr-tcp", Port: 32803, Protocol: v1.ProtocolTCP},
{Name: "nlockmgr-udp", Port: 32803, Protocol: v1.ProtocolUDP},
{Name: "mountd-tcp", Port: 20048, Protocol: v1.ProtocolTCP},
{Name: "mountd-udp", Port: 20048, Protocol: v1.ProtocolUDP},
{Name: "portmapper-tcp", Port: 111, Protocol: v1.ProtocolTCP},
{Name: "portmapper-udp", Port: 111, Protocol: v1.ProtocolUDP},
{Name: "statd-tcp", Port: 662, Protocol: v1.ProtocolTCP},
{Name: "statd-udp", Port: 662, Protocol: v1.ProtocolUDP},
{Name: "rquotad-tcp", Port: 875, Protocol: v1.ProtocolTCP},
{Name: "rquotad-udp", Port: 875, Protocol: v1.ProtocolUDP},
},
},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
return svc
}
func (c *NFSController) makeDeployment(svcname, namespace, rookImage string, nfsSpec edgefsv1alpha1.NFSSpec) *extensions.Deployment {
name := instanceName(svcname)
volumes := []v1.Volume{}
if c.dataVolumeSize.Value() > 0 {
// dataVolume case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: dataVolumeName,
},
},
})
} else {
// dataDir case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: c.dataDirHostPath,
},
},
})
}
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: getLabels(name, svcname, namespace),
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.nfsContainer(svcname, name, rookImage, nfsSpec)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
HostIPC: true,
HostNetwork: c.hostNetwork,
NodeSelector: map[string]string{namespace: "cluster"},
},
}
if c.hostNetwork {
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// apply current NFS CRD options to pod's specification
nfsSpec.Placement.ApplyToPodSpec(&podSpec.Spec)
d := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &nfsSpec.Instances},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
return d
}
func (c *NFSController) nfsContainer(svcname, name, containerImage string, nfsSpec edgefsv1alpha1.NFSSpec) v1.Container {
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
},
}
return v1.Container{
Name: name,
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"nfs"},
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "CCOW_SVCNAME",
Value: svcname,
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
},
SecurityContext: securityContext,
Resources: nfsSpec.Resources,
Ports: []v1.ContainerPort{
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
{Name: "nfs-tcp", ContainerPort: 2049, Protocol: v1.ProtocolTCP},
{Name: "nfs-udp", ContainerPort: 2049, Protocol: v1.ProtocolUDP},
{Name: "nlockmgr-tcp", ContainerPort: 32803, Protocol: v1.ProtocolTCP},
{Name: "nlockmgr-udp", ContainerPort: 32803, Protocol: v1.ProtocolUDP},
{Name: "mountd-tcp", ContainerPort: 20048, Protocol: v1.ProtocolTCP},
{Name: "mountd-udp", ContainerPort: 20048, Protocol: v1.ProtocolUDP},
{Name: "portmapper-tcp", ContainerPort: 111, Protocol: v1.ProtocolTCP},
{Name: "portmapper-udp", ContainerPort: 111, Protocol: v1.ProtocolUDP},
{Name: "statd-tcp", ContainerPort: 662, Protocol: v1.ProtocolTCP},
{Name: "statd-udp", ContainerPort: 662, Protocol: v1.ProtocolUDP},
{Name: "rquotad-tcp", ContainerPort: 875, Protocol: v1.ProtocolTCP},
{Name: "rquotad-udp", ContainerPort: 875, Protocol: v1.ProtocolUDP},
},
VolumeMounts: []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc.target",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
},
}
}
// Delete NFS service and possibly some artifacts.
func (c *NFSController) DeleteService(s edgefsv1alpha1.NFS) error {
// check if service exists
exists, err := serviceExists(c.context, s)
if err != nil {
return fmt.Errorf("failed to detect if there is a NFS service to delete. %+v", err)
}
if !exists {
logger.Infof("NFS service %s does not exist in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("Deleting NFS service %s from namespace %s", s.Name, s.Namespace)
var gracePeriod int64
propagation := metav1.DeletePropagationForeground
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
// Delete the nfs service
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
if err != nil && !errors.IsNotFound(err) {
logger.Warningf("failed to delete NFS service. %+v", err)
}
// Make a best effort to delete the NFS pods
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
if err != nil {
logger.Warningf(err.Error())
}
logger.Infof("Completed deleting NFS service %s", s.Name)
return nil
}
func getLabels(name, svcname, namespace string) map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: namespace,
"edgefs_svcname": svcname,
"edgefs_svctype": "nfs",
}
}
// Validate the NFS arguments
func validateService(context *clusterd.Context, s edgefsv1alpha1.NFS) error {
if s.Name == "" {
return fmt.Errorf("missing name")
}
if s.Namespace == "" {
return fmt.Errorf("missing namespace")
}
return nil
}
func instanceName(svcname string) string {
return fmt.Sprintf("%s-%s", appName, svcname)
}
// Check if the NFS service exists
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.NFS) (bool, error) {
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
if err == nil {
// the deployment was found
return true, nil
}
if err != nil && !errors.IsNotFound(err) {
return false, err
}
// not found
return false, nil
}
+87
View File
@@ -0,0 +1,87 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package operator to manage Kubernetes storage.
package edgefs
import (
"fmt"
"os"
"os/signal"
"syscall"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/discover"
"github.com/rook/rook/pkg/operator/edgefs/cluster"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-operator")
type Operator struct {
context *clusterd.Context
resources []opkit.CustomResource
rookImage string
securityAccount string
clusterController *cluster.ClusterController
}
// New creates an operator instance
func New(context *clusterd.Context, rookImage string, securityAccount string) *Operator {
clusterController := cluster.NewClusterController(context, rookImage)
schemes := []opkit.CustomResource{cluster.ClusterResource}
return &Operator{
context: context,
clusterController: clusterController,
resources: schemes,
rookImage: rookImage,
securityAccount: securityAccount,
}
}
// Run the operator instance
func (o *Operator) Run() error {
namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
if namespace == "" {
return fmt.Errorf("Rook operator namespace is not provided. Expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
}
rookDiscover := discover.New(o.context.Clientset)
if err := rookDiscover.Start(namespace, o.rookImage, o.securityAccount); err != nil {
return fmt.Errorf("Error starting device discovery daemonset: %v", err)
}
signalChan := make(chan os.Signal, 1)
stopChan := make(chan struct{})
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// watch for changes to the edgefs clusters
o.clusterController.StartWatch(v1.NamespaceAll, stopChan)
for {
select {
case <-signalChan:
logger.Infof("shutdown signal received, exiting...")
close(stopChan)
return nil
}
}
}
+43
View File
@@ -0,0 +1,43 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package edgefs
import (
"fmt"
"testing"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/edgefs/cluster"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
)
func TestOperator(t *testing.T) {
clientset := test.New(3)
context := &clusterd.Context{Clientset: clientset}
o := New(context, "", "")
assert.NotNil(t, o)
assert.NotNil(t, o.clusterController)
assert.NotNil(t, o.resources)
assert.Equal(t, context, o.context)
assert.Equal(t, len(o.resources), 1)
for _, r := range o.resources {
if r.Name != cluster.ClusterResource.Name {
assert.Fail(t, fmt.Sprintf("Resource %s is not valid", r.Name))
}
}
}
+171
View File
@@ -0,0 +1,171 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// scale-out, multi-cloud Edge-X S3 services controller
package s3
import (
"fmt"
"reflect"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/cache"
)
const (
customResourceName = "s3"
customResourceNamePlural = "s3s"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-s3")
// S3Resource represents the s3 custom resource
var S3Resource = opkit.CustomResource{
Name: customResourceName,
Plural: customResourceNamePlural,
Group: edgefsv1alpha1.CustomResourceGroup,
Version: edgefsv1alpha1.Version,
Scope: apiextensionsv1beta1.NamespaceScoped,
Kind: reflect.TypeOf(edgefsv1alpha1.S3{}).Name(),
}
// S3Controller represents a controller object for s3 custom resources
type S3Controller struct {
context *clusterd.Context
rookImage string
hostNetwork bool
dataDirHostPath string
dataVolumeSize resource.Quantity
placement rookalpha.Placement
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
}
// NewS3Controller create controller for watching S3 custom resources created
func NewS3Controller(
context *clusterd.Context, rookImage string,
hostNetwork bool,
dataDirHostPath string,
dataVolumeSize resource.Quantity,
placement rookalpha.Placement,
resources v1.ResourceRequirements,
ownerRef metav1.OwnerReference,
) *S3Controller {
return &S3Controller{
context: context,
rookImage: rookImage,
hostNetwork: hostNetwork,
dataDirHostPath: dataDirHostPath,
dataVolumeSize: dataVolumeSize,
placement: placement,
resources: resources,
ownerRef: ownerRef,
}
}
// StartWatch watches for instances of S3 custom resources and acts on them
func (c *S3Controller) StartWatch(namespace string, stopCh chan struct{}) error {
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
AddFunc: c.onAdd,
UpdateFunc: c.onUpdate,
DeleteFunc: c.onDelete,
}
logger.Infof("start watching s3 resources in namespace %s", namespace)
watcher := opkit.NewWatcher(S3Resource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
go watcher.Watch(&edgefsv1alpha1.S3{}, stopCh)
return nil
}
func (c *S3Controller) onAdd(obj interface{}) {
s3, err := getS3Object(obj)
if err != nil {
logger.Errorf("failed to get s3 object: %+v", err)
return
}
if err = c.CreateService(*s3, c.serviceOwners(s3)); err != nil {
logger.Errorf("failed to create s3 %s. %+v", s3.Name, err)
}
}
func (c *S3Controller) onUpdate(oldObj, newObj interface{}) {
oldService, err := getS3Object(oldObj)
if err != nil {
logger.Errorf("failed to get old s3 object: %+v", err)
return
}
newService, err := getS3Object(newObj)
if err != nil {
logger.Errorf("failed to get new s3 object: %+v", err)
return
}
if !serviceChanged(oldService.Spec, newService.Spec) {
logger.Debugf("s3 %s did not change", newService.Name)
return
}
logger.Infof("applying s3 %s changes", newService.Name)
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
logger.Errorf("failed to create (modify) s3 %s. %+v", newService.Name, err)
}
}
func (c *S3Controller) onDelete(obj interface{}) {
s3, err := getS3Object(obj)
if err != nil {
logger.Errorf("failed to get s3 object: %+v", err)
return
}
if err = c.DeleteService(*s3); err != nil {
logger.Errorf("failed to delete s3 %s. %+v", s3.Name, err)
}
}
func (c *S3Controller) serviceOwners(service *edgefsv1alpha1.S3) []metav1.OwnerReference {
// Only set the cluster crd as the owner of the S3 resources.
// If the S3 crd is deleted, the operator will explicitly remove the S3 resources.
// If the S3 crd still exists when the cluster crd is deleted, this will make sure the S3
// resources are cleaned up.
return []metav1.OwnerReference{c.ownerRef}
}
func serviceChanged(oldService, newService edgefsv1alpha1.S3Spec) bool {
return false
}
func getS3Object(obj interface{}) (s3 *edgefsv1alpha1.S3, err error) {
var ok bool
s3, ok = obj.(*edgefsv1alpha1.S3)
if ok {
// the s3 object is of the latest type, simply return it
return s3.DeepCopy(), nil
}
return nil, fmt.Errorf("not a known s3 object: %+v", obj)
}
+355
View File
@@ -0,0 +1,355 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package S3 for the Edgefs manager.
package s3
import (
"fmt"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
appName = "rook-edgefs-s3"
/* Volumes definitions */
sslCertVolumeName = "ssl-cert-volume"
sslMountPath = "/opt/nedge/etc/ssl/"
dataVolumeName = "edgefs-datadir"
stateVolumeFolder = ".state"
etcVolumeFolder = ".etc"
defaultPort = 9982
defaultSecurePort = 8443
)
// Start the S3 manager
func (c *S3Controller) CreateService(s edgefsv1alpha1.S3, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, false, ownerRefs)
}
func (c *S3Controller) UpdateService(s edgefsv1alpha1.S3, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, true, ownerRefs)
}
// Start the s3 instance
func (c *S3Controller) CreateOrUpdate(s edgefsv1alpha1.S3, update bool, ownerRefs []metav1.OwnerReference) error {
logger.Infof("starting update=%v service=%s", update, s.Name)
logger.Infof("S3 Base image is %s", c.rookImage)
// validate S3 service settings
if err := validateService(c.context, s); err != nil {
return fmt.Errorf("invalid S3 service %s arguments. %+v", s.Name, err)
}
//check http settings
if s.Spec.Port == 0 {
s.Spec.Port = defaultPort
}
if s.Spec.SecurePort == 0 {
s.Spec.SecurePort = defaultSecurePort
}
// check if S3 service already exists
exists, err := serviceExists(c.context, s)
if err == nil && exists {
if !update {
logger.Infof("S3 service %s exists in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("S3 service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
}
// start the deployment
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
}
logger.Infof("%s deployment already exists", appName)
} else {
logger.Infof("%s deployment started", appName)
}
// create the s3 service
service := c.makeS3Service(instanceName(s.Name), s.Name, s.Namespace, s.Spec)
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create s3 service. %+v", err)
}
logger.Infof("s3 service %s already exists", service)
} else {
logger.Infof("s3 service %s started", service)
}
return nil
}
func (c *S3Controller) makeS3Service(name, svcname, namespace string, s3Spec edgefsv1alpha1.S3Spec) *v1.Service {
labels := getLabels(name, svcname, namespace)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Selector: labels,
Type: v1.ServiceTypeNodePort,
Ports: []v1.ServicePort{
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
{Name: "port", Port: int32(s3Spec.Port), Protocol: v1.ProtocolTCP},
{Name: "secure-port", Port: int32(s3Spec.SecurePort), Protocol: v1.ProtocolTCP},
},
},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
return svc
}
func (c *S3Controller) makeDeployment(svcname, namespace, rookImage string, s3Spec edgefsv1alpha1.S3Spec) *extensions.Deployment {
name := instanceName(svcname)
volumes := []v1.Volume{}
// add ssl certificate volume if defined
if len(s3Spec.SSLCertificateRef) > 0 {
volumes = append(volumes, v1.Volume{
Name: sslCertVolumeName,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: s3Spec.SSLCertificateRef,
Items: []v1.KeyToPath{
{
Key: "sslkey",
Path: "ssl.key",
},
{
Key: "sslcert",
Path: "ssl.crt",
},
},
},
},
})
}
if c.dataVolumeSize.Value() > 0 {
// dataVolume case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: dataVolumeName,
},
},
})
} else {
// dataDir case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: c.dataDirHostPath,
},
},
})
}
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: getLabels(name, svcname, namespace),
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.s3Container(svcname, name, rookImage, s3Spec)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
HostIPC: true,
HostNetwork: c.hostNetwork,
NodeSelector: map[string]string{namespace: "cluster"},
},
}
if c.hostNetwork {
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// apply current S3 CRD options to pod's specification
s3Spec.Placement.ApplyToPodSpec(&podSpec.Spec)
d := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &s3Spec.Instances},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
return d
}
func (c *S3Controller) s3Container(svcname, name, containerImage string, s3Spec edgefsv1alpha1.S3Spec) v1.Container {
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
},
}
volumeMounts := []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc.target",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
}
if len(s3Spec.SSLCertificateRef) > 0 {
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath})
}
return v1.Container{
Name: name,
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"s3"},
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "CCOW_SVCNAME",
Value: svcname,
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
{
Name: "EFSS3_HTTP_PORT",
Value: fmt.Sprint(s3Spec.Port),
},
{
Name: "EFSS3_HTTPS_PORT",
Value: fmt.Sprint(s3Spec.SecurePort),
},
},
SecurityContext: securityContext,
Resources: s3Spec.Resources,
Ports: []v1.ContainerPort{
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
{Name: "port", ContainerPort: int32(s3Spec.Port), Protocol: v1.ProtocolTCP},
{Name: "secure-port", ContainerPort: int32(s3Spec.SecurePort), Protocol: v1.ProtocolTCP},
},
VolumeMounts: volumeMounts,
}
}
// Delete S3 service and possibly some artifacts.
func (c *S3Controller) DeleteService(s edgefsv1alpha1.S3) error {
// check if service exists
exists, err := serviceExists(c.context, s)
if err != nil {
return fmt.Errorf("failed to detect if there is a S3 service to delete. %+v", err)
}
if !exists {
logger.Infof("S3 service %s does not exist in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("Deleting S3 service %s from namespace %s", s.Name, s.Namespace)
var gracePeriod int64
propagation := metav1.DeletePropagationForeground
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
// Delete the s3 service
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
if err != nil && !errors.IsNotFound(err) {
logger.Warningf("failed to delete S3 service. %+v", err)
}
// Make a best effort to delete the S3 pods
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
if err != nil {
logger.Warningf(err.Error())
}
logger.Infof("Completed deleting S3 service %s", s.Name)
return nil
}
func getLabels(name, svcname, namespace string) map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: namespace,
"edgefs_svcname": svcname,
"edgefs_svctype": "s3",
}
}
// Validate the S3 arguments
func validateService(context *clusterd.Context, s edgefsv1alpha1.S3) error {
if s.Name == "" {
return fmt.Errorf("missing name")
}
if s.Namespace == "" {
return fmt.Errorf("missing namespace")
}
return nil
}
func instanceName(svcname string) string {
return fmt.Sprintf("%s-%s", appName, svcname)
}
// Check if the S3 service exists
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.S3) (bool, error) {
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
if err == nil {
// the deployment was found
return true, nil
}
if err != nil && !errors.IsNotFound(err) {
return false, err
}
// not found
return false, nil
}
+171
View File
@@ -0,0 +1,171 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// scale-out, multi-cloud Edge-X S3 services controller
package s3x
import (
"fmt"
"reflect"
"github.com/coreos/pkg/capnslog"
opkit "github.com/rook/operator-kit"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"github.com/rook/rook/pkg/clusterd"
"k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/cache"
)
const (
customResourceName = "s3x"
customResourceNamePlural = "s3xs"
)
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-s3x")
// S3XResource represents the s3x custom resource
var S3XResource = opkit.CustomResource{
Name: customResourceName,
Plural: customResourceNamePlural,
Group: edgefsv1alpha1.CustomResourceGroup,
Version: edgefsv1alpha1.Version,
Scope: apiextensionsv1beta1.NamespaceScoped,
Kind: reflect.TypeOf(edgefsv1alpha1.S3X{}).Name(),
}
// S3XController represents a controller object for s3x custom resources
type S3XController struct {
context *clusterd.Context
rookImage string
hostNetwork bool
dataDirHostPath string
dataVolumeSize resource.Quantity
placement rookalpha.Placement
resources v1.ResourceRequirements
ownerRef metav1.OwnerReference
}
// NewS3XController create controller for watching S3X custom resources created
func NewS3XController(
context *clusterd.Context, rookImage string,
hostNetwork bool,
dataDirHostPath string,
dataVolumeSize resource.Quantity,
placement rookalpha.Placement,
resources v1.ResourceRequirements,
ownerRef metav1.OwnerReference,
) *S3XController {
return &S3XController{
context: context,
rookImage: rookImage,
hostNetwork: hostNetwork,
dataDirHostPath: dataDirHostPath,
dataVolumeSize: dataVolumeSize,
placement: placement,
resources: resources,
ownerRef: ownerRef,
}
}
// StartWatch watches for instances of S3X custom resources and acts on them
func (c *S3XController) StartWatch(namespace string, stopCh chan struct{}) error {
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
AddFunc: c.onAdd,
UpdateFunc: c.onUpdate,
DeleteFunc: c.onDelete,
}
logger.Infof("start watching s3x resources in namespace %s", namespace)
watcher := opkit.NewWatcher(S3XResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
go watcher.Watch(&edgefsv1alpha1.S3X{}, stopCh)
return nil
}
func (c *S3XController) onAdd(obj interface{}) {
s3x, err := getS3XObject(obj)
if err != nil {
logger.Errorf("failed to get s3x object: %+v", err)
return
}
if err = c.CreateService(*s3x, c.serviceOwners(s3x)); err != nil {
logger.Errorf("failed to create s3x %s. %+v", s3x.Name, err)
}
}
func (c *S3XController) onUpdate(oldObj, newObj interface{}) {
oldService, err := getS3XObject(oldObj)
if err != nil {
logger.Errorf("failed to get old s3x object: %+v", err)
return
}
newService, err := getS3XObject(newObj)
if err != nil {
logger.Errorf("failed to get new s3x object: %+v", err)
return
}
if !serviceChanged(oldService.Spec, newService.Spec) {
logger.Debugf("s3x %s did not change", newService.Name)
return
}
logger.Infof("applying s3x %s changes", newService.Name)
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
logger.Errorf("failed to create (modify) s3x %s. %+v", newService.Name, err)
}
}
func (c *S3XController) onDelete(obj interface{}) {
s3x, err := getS3XObject(obj)
if err != nil {
logger.Errorf("failed to get s3x object: %+v", err)
return
}
if err = c.DeleteService(*s3x); err != nil {
logger.Errorf("failed to delete s3x %s. %+v", s3x.Name, err)
}
}
func (c *S3XController) serviceOwners(service *edgefsv1alpha1.S3X) []metav1.OwnerReference {
// Only set the cluster crd as the owner of the S3X resources.
// If the S3X crd is deleted, the operator will explicitly remove the S3X resources.
// If the S3X crd still exists when the cluster crd is deleted, this will make sure the S3X
// resources are cleaned up.
return []metav1.OwnerReference{c.ownerRef}
}
func serviceChanged(oldService, newService edgefsv1alpha1.S3XSpec) bool {
return false
}
func getS3XObject(obj interface{}) (s3x *edgefsv1alpha1.S3X, err error) {
var ok bool
s3x, ok = obj.(*edgefsv1alpha1.S3X)
if ok {
// the s3x object is of the latest type, simply return it
return s3x.DeepCopy(), nil
}
return nil, fmt.Errorf("not a known s3x object: %+v", obj)
}
+355
View File
@@ -0,0 +1,355 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package s3x for the Edgefs manager.
package s3x
import (
"fmt"
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
appName = "rook-edgefs-s3x"
/* Volumes definitions */
sslCertVolumeName = "ssl-cert-volume"
sslMountPath = "/opt/nedge/etc/ssl/"
dataVolumeName = "edgefs-datadir"
stateVolumeFolder = ".state"
etcVolumeFolder = ".etc"
defaultPort = 3000
defaultSecurePort = 3001
)
// Start the rgw manager
func (c *S3XController) CreateService(s edgefsv1alpha1.S3X, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, false, ownerRefs)
}
func (c *S3XController) UpdateService(s edgefsv1alpha1.S3X, ownerRefs []metav1.OwnerReference) error {
return c.CreateOrUpdate(s, true, ownerRefs)
}
// Start the s3x instance
func (c *S3XController) CreateOrUpdate(s edgefsv1alpha1.S3X, update bool, ownerRefs []metav1.OwnerReference) error {
logger.Infof("starting update=%v service=%s", update, s.Name)
logger.Infof("S3X Base image is %s", c.rookImage)
// validate S3X service settings
if err := validateService(c.context, s); err != nil {
return fmt.Errorf("invalid S3X service %s arguments. %+v", s.Name, err)
}
//check http settings
if s.Spec.Port == 0 {
s.Spec.Port = defaultPort
}
if s.Spec.SecurePort == 0 {
s.Spec.SecurePort = defaultSecurePort
}
// check if S3X service already exists
exists, err := serviceExists(c.context, s)
if err == nil && exists {
if !update {
logger.Infof("S3X service %s exists in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("S3X service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
}
// start the deployment
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
}
logger.Infof("%s deployment already exists", appName)
} else {
logger.Infof("%s deployment started", appName)
}
// create the s3x service
service := c.makeS3XService(instanceName(s.Name), s.Name, s.Namespace, s.Spec)
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create s3x service. %+v", err)
}
logger.Infof("s3x service %s already exists", service)
} else {
logger.Infof("s3x service %s started", service)
}
return nil
}
func (c *S3XController) makeS3XService(name, svcname, namespace string, s3xSpec edgefsv1alpha1.S3XSpec) *v1.Service {
labels := getLabels(name, svcname, namespace)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Selector: labels,
Type: v1.ServiceTypeNodePort,
Ports: []v1.ServicePort{
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
{Name: "port", Port: int32(s3xSpec.Port), Protocol: v1.ProtocolTCP},
{Name: "secure-port", Port: int32(s3xSpec.SecurePort), Protocol: v1.ProtocolTCP},
},
},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
return svc
}
func (c *S3XController) makeDeployment(svcname, namespace, rookImage string, s3xSpec edgefsv1alpha1.S3XSpec) *extensions.Deployment {
name := instanceName(svcname)
volumes := []v1.Volume{}
// add ssl certificate volume if defined
if len(s3xSpec.SSLCertificateRef) > 0 {
volumes = append(volumes, v1.Volume{
Name: sslCertVolumeName,
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: s3xSpec.SSLCertificateRef,
Items: []v1.KeyToPath{
{
Key: "sslkey",
Path: "ssl.key",
},
{
Key: "sslcert",
Path: "ssl.crt",
},
},
},
},
})
}
if c.dataVolumeSize.Value() > 0 {
// dataVolume case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: dataVolumeName,
},
},
})
} else {
// dataDir case
volumes = append(volumes, v1.Volume{
Name: dataVolumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: c.dataDirHostPath,
},
},
})
}
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: getLabels(name, svcname, namespace),
},
Spec: v1.PodSpec{
Containers: []v1.Container{c.s3xContainer(svcname, name, rookImage, s3xSpec)},
RestartPolicy: v1.RestartPolicyAlways,
Volumes: volumes,
HostIPC: true,
HostNetwork: c.hostNetwork,
NodeSelector: map[string]string{namespace: "cluster"},
},
}
if c.hostNetwork {
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
}
// apply current S3X CRD options to pod's specification
s3xSpec.Placement.ApplyToPodSpec(&podSpec.Spec)
d := &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &s3xSpec.Instances},
}
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
return d
}
func (c *S3XController) s3xContainer(svcname, name, containerImage string, s3xSpec edgefsv1alpha1.S3XSpec) v1.Container {
runAsUser := int64(0)
readOnlyRootFilesystem := false
securityContext := &v1.SecurityContext{
RunAsUser: &runAsUser,
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
Capabilities: &v1.Capabilities{
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
},
}
volumeMounts := []v1.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/opt/nedge/etc.target",
SubPath: etcVolumeFolder,
},
{
Name: dataVolumeName,
MountPath: "/opt/nedge/var/run",
SubPath: stateVolumeFolder,
},
}
if len(s3xSpec.SSLCertificateRef) > 0 {
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath})
}
return v1.Container{
Name: name,
Image: containerImage,
ImagePullPolicy: v1.PullAlways,
Args: []string{"s3x"},
Env: []v1.EnvVar{
{
Name: "CCOW_LOG_LEVEL",
Value: "5",
},
{
Name: "CCOW_SVCNAME",
Value: svcname,
},
{
Name: "HOST_HOSTNAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
FieldPath: "spec.nodeName",
},
},
},
{
Name: "EFSS3X_HTTP_PORT",
Value: fmt.Sprint(s3xSpec.Port),
},
{
Name: "EFSS3X_HTTPS_PORT",
Value: fmt.Sprint(s3xSpec.SecurePort),
},
},
SecurityContext: securityContext,
Resources: s3xSpec.Resources,
Ports: []v1.ContainerPort{
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
{Name: "port", ContainerPort: int32(s3xSpec.Port), Protocol: v1.ProtocolTCP},
{Name: "secure-port", ContainerPort: int32(s3xSpec.SecurePort), Protocol: v1.ProtocolTCP},
},
VolumeMounts: volumeMounts,
}
}
// Delete S3X service and possibly some artifacts.
func (c *S3XController) DeleteService(s edgefsv1alpha1.S3X) error {
// check if service exists
exists, err := serviceExists(c.context, s)
if err != nil {
return fmt.Errorf("failed to detect if there is a S3X service to delete. %+v", err)
}
if !exists {
logger.Infof("S3X service %s does not exist in namespace %s", s.Name, s.Namespace)
return nil
}
logger.Infof("Deleting S3X service %s from namespace %s", s.Name, s.Namespace)
var gracePeriod int64
propagation := metav1.DeletePropagationForeground
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
// Delete the s3x service
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
if err != nil && !errors.IsNotFound(err) {
logger.Warningf("failed to delete S3X service. %+v", err)
}
// Make a best effort to delete the S3X pods
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
if err != nil {
logger.Warningf(err.Error())
}
logger.Infof("Completed deleting S3X service %s", s.Name)
return nil
}
func getLabels(name, svcname, namespace string) map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: namespace,
"edgefs_svcname": svcname,
"edgefs_svctype": "s3x",
}
}
// Validate the S3X arguments
func validateService(context *clusterd.Context, s edgefsv1alpha1.S3X) error {
if s.Name == "" {
return fmt.Errorf("missing name")
}
if s.Namespace == "" {
return fmt.Errorf("missing namespace")
}
return nil
}
func instanceName(svcname string) string {
return fmt.Sprintf("%s-%s", appName, svcname)
}
// Check if the S3X service exists
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.S3X) (bool, error) {
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
if err == nil {
// the deployment was found
return true, nil
}
if err != nil && !errors.IsNotFound(err) {
return false, err
}
// not found
return false, nil
}