Edgefs operator and CRDs
Signed-off-by: Dmitry Yusupov <dmitry.yusupov@nexenta.com>
This commit is contained in:
1 parent
4f958162e1
commit
a29477f122
22 files changed
+5040
No files matched your search
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster/mgr"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster/target"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
httpPortDefault = int32(8080)
|
||||
httpPortName = "http"
|
||||
grpcPortDefault = int32(6789)
|
||||
grpcPortName = "grpc"
|
||||
udpTotemPortDefault = int32(5405)
|
||||
udpTotemPortName = "totem"
|
||||
volumeNameDataDir = "datadir"
|
||||
defaultServiceAccountName = "rook-edgefs-cluster"
|
||||
|
||||
/* Volumes definitions */
|
||||
configName = "edgefs-config"
|
||||
)
|
||||
|
||||
type cluster struct {
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
Spec edgefsv1alpha1.ClusterSpec
|
||||
ownerRef metav1.OwnerReference
|
||||
targets *target.Cluster
|
||||
mgrs *mgr.Cluster
|
||||
stopCh chan struct{}
|
||||
}
|
||||
|
||||
func newCluster(c *edgefsv1alpha1.Cluster, context *clusterd.Context) *cluster {
|
||||
|
||||
return &cluster{
|
||||
context: context,
|
||||
Namespace: c.Namespace,
|
||||
Spec: c.Spec,
|
||||
stopCh: make(chan struct{}),
|
||||
ownerRef: ClusterOwnerRef(c.Namespace, string(c.UID)),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cluster) createInstance(rookImage string) error {
|
||||
|
||||
logger.Infof("Cluster spec\n %+v", c.Spec)
|
||||
|
||||
//
|
||||
// Validate Cluster CRD
|
||||
//
|
||||
if err := c.validateClusterSpec(); err != nil {
|
||||
logger.Errorf("invalid cluster spec: %+v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a configmap for overriding edgefs config settings
|
||||
// These settings should only be modified by a user after they are initialized
|
||||
placeholderConfig := map[string]string{
|
||||
k8sutil.ConfigOverrideVal: "",
|
||||
}
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: k8sutil.ConfigOverrideName,
|
||||
},
|
||||
Data: placeholderConfig,
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &cm.ObjectMeta, &c.ownerRef)
|
||||
_, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(cm)
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create override configmap %s. %+v", c.Namespace, err)
|
||||
}
|
||||
|
||||
//
|
||||
// Create and start EdgeFS Targets StatefulSet
|
||||
//
|
||||
c.targets = target.New(c.context, c.Namespace, "latest", c.Spec.ServiceAccount, c.Spec.Storage, c.Spec.DataDirHostPath, c.Spec.DataVolumeSize,
|
||||
edgefsv1alpha1.GetTargetPlacement(c.Spec.Placement), c.Spec.Network, c.Spec.Resources, c.ownerRef)
|
||||
|
||||
err = c.targets.Start(rookImage, c.Spec.DevicesResurrectMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the targets. %+v", err)
|
||||
}
|
||||
|
||||
//
|
||||
// Create and start EdgeFS manager Deployment (gRPC proxy, Prometheus metrics)
|
||||
//
|
||||
c.mgrs = mgr.New(c.context, c.Namespace, "latest", c.Spec.ServiceAccount, c.Spec.DataDirHostPath, c.Spec.DataVolumeSize,
|
||||
edgefsv1alpha1.GetMgrPlacement(c.Spec.Placement), c.Spec.Network,
|
||||
v1.ResourceRequirements{}, c.ownerRef)
|
||||
err = c.mgrs.Start(rookImage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start the edgefs mgr. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("Done creating rook instance in namespace %s", c.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *cluster) validateClusterSpec() error {
|
||||
if len(c.Spec.DataDirHostPath) == 0 && c.Spec.DataVolumeSize.Value() == 0 {
|
||||
return fmt.Errorf("DataDirHostPath or DataVolumeSize EdgeFS cluster's options not specified.")
|
||||
}
|
||||
|
||||
if len(c.Spec.DataDirHostPath) > 0 && c.Spec.DataVolumeSize.Value() != 0 {
|
||||
return fmt.Errorf("Both deployment options DataDirHostPath and DataVolumeSize are specified. Should be only one deployment option in cluster specification.")
|
||||
}
|
||||
|
||||
if len(c.Spec.Storage.Directories) > 0 &&
|
||||
((c.Spec.Storage.UseAllDevices != nil && *c.Spec.Storage.UseAllDevices) ||
|
||||
len(c.Spec.Storage.Devices) > 0 || len(c.Spec.Storage.DeviceFilter) > 0) {
|
||||
return fmt.Errorf("Directories option specified as well as Devices. Remove Directories or Devices option from cluster specification")
|
||||
}
|
||||
|
||||
logger.Info("Validate cluster spec")
|
||||
return nil
|
||||
}
|
||||
|
||||
func clusterChanged(oldCluster, newCluster edgefsv1alpha1.ClusterSpec) bool {
|
||||
changeFound := false
|
||||
oldStorage := oldCluster.Storage
|
||||
newStorage := newCluster.Storage
|
||||
|
||||
// sort the nodes by name then compare to see if there are changes
|
||||
sort.Sort(rookv1alpha2.NodesByName(oldStorage.Nodes))
|
||||
sort.Sort(rookv1alpha2.NodesByName(newStorage.Nodes))
|
||||
if !reflect.DeepEqual(oldStorage.Nodes, newStorage.Nodes) {
|
||||
logger.Infof("The list of nodes has changed")
|
||||
changeFound = true
|
||||
}
|
||||
|
||||
return changeFound
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
Portions of this file came from https://github.com/cockroachdb/cockroach, which uses the same license.
|
||||
*/
|
||||
|
||||
// Package cluster to manage a edgefs cluster.
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/discover"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/iscsi"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/nfs"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/s3"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/s3x"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
var (
|
||||
logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-cluster")
|
||||
finalizerName = fmt.Sprintf("%s.%s", ClusterResource.Name, ClusterResource.Group)
|
||||
)
|
||||
|
||||
const (
|
||||
CustomResourceName = "cluster"
|
||||
CustomResourceNamePlural = "clusters"
|
||||
appName = "rook-edgefs"
|
||||
clusterCreateInterval = 6 * time.Second
|
||||
clusterCreateTimeout = 5 * time.Minute
|
||||
updateClusterInterval = 30 * time.Second
|
||||
updateClusterTimeout = 1 * time.Hour
|
||||
clusterDeleteRetryInterval = 2 //seconds
|
||||
clusterDeleteMaxRetries = 15
|
||||
defaultEdgefsImageName = "edgefs/edgefs:latest"
|
||||
)
|
||||
|
||||
var ClusterResource = opkit.CustomResource{
|
||||
Name: CustomResourceName,
|
||||
Plural: CustomResourceNamePlural,
|
||||
Group: edgefsv1alpha1.CustomResourceGroup,
|
||||
Version: edgefsv1alpha1.Version,
|
||||
Scope: apiextensionsv1beta1.NamespaceScoped,
|
||||
Kind: reflect.TypeOf(edgefsv1alpha1.Cluster{}).Name(),
|
||||
}
|
||||
|
||||
type ClusterController struct {
|
||||
context *clusterd.Context
|
||||
containerImage string
|
||||
devicesInUse bool
|
||||
clusterMap map[string]*cluster
|
||||
}
|
||||
|
||||
func NewClusterController(context *clusterd.Context, containerImage string) *ClusterController {
|
||||
return &ClusterController{
|
||||
context: context,
|
||||
containerImage: containerImage,
|
||||
//createInitRetryInterval: createInitRetryIntervalDefault,
|
||||
clusterMap: make(map[string]*cluster),
|
||||
}
|
||||
}
|
||||
|
||||
func ClusterOwnerRef(namespace, clusterID string) metav1.OwnerReference {
|
||||
blockOwner := true
|
||||
return metav1.OwnerReference{
|
||||
APIVersion: ClusterResource.Version,
|
||||
Kind: ClusterResource.Kind,
|
||||
Name: namespace,
|
||||
UID: types.UID(clusterID),
|
||||
BlockOwnerDeletion: &blockOwner,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ClusterController) StartWatch(namespace string, stopCh chan struct{}) error {
|
||||
|
||||
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onAdd,
|
||||
UpdateFunc: c.onUpdate,
|
||||
DeleteFunc: c.onDelete,
|
||||
}
|
||||
|
||||
logger.Infof("start watching edgefs clusters in all namespaces")
|
||||
watcher := opkit.NewWatcher(ClusterResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
|
||||
go watcher.Watch(&edgefsv1alpha1.Cluster{}, stopCh)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) StopWatch() {
|
||||
for _, cluster := range c.clusterMap {
|
||||
close(cluster.stopCh)
|
||||
}
|
||||
c.clusterMap = make(map[string]*cluster)
|
||||
}
|
||||
|
||||
func (c *ClusterController) onAdd(obj interface{}) {
|
||||
clusterObj := obj.(*edgefsv1alpha1.Cluster).DeepCopy()
|
||||
logger.Infof("new cluster %s added to namespace %s", clusterObj.Name, clusterObj.Namespace)
|
||||
|
||||
cluster := newCluster(clusterObj, c.context)
|
||||
c.clusterMap[cluster.Namespace] = cluster
|
||||
|
||||
//Override rook containerImage value
|
||||
c.containerImage = defaultEdgefsImageName
|
||||
if cluster.Spec.EdgefsImageName != "" {
|
||||
c.containerImage = cluster.Spec.EdgefsImageName
|
||||
}
|
||||
|
||||
logger.Infof("starting cluster in namespace %s", cluster.Namespace)
|
||||
|
||||
if c.devicesInUse && cluster.Spec.Storage.AnyUseAllDevices() {
|
||||
message := "using all devices in more than one namespace not supported"
|
||||
logger.Error(message)
|
||||
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateError, message); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if cluster.Spec.Storage.AnyUseAllDevices() {
|
||||
c.devicesInUse = true
|
||||
}
|
||||
|
||||
// Start the Rook cluster components. Retry several times in case of failure.
|
||||
err := wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) {
|
||||
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateCreating, ""); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
err := cluster.createInstance(c.containerImage)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create cluster in namespace %s. %+v", cluster.Namespace, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// cluster is created, update the cluster CRD status now
|
||||
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateCreated, ""); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
message := fmt.Sprintf("giving up creating cluster in namespace %s after %s", cluster.Namespace, clusterCreateTimeout)
|
||||
logger.Error(message)
|
||||
if err := c.updateClusterStatus(clusterObj.Namespace, clusterObj.Name, edgefsv1alpha1.ClusterStateError, message); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", cluster.Namespace, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("succeeded creating and initializing EdgeFS cluster in namespace %s", cluster.Namespace)
|
||||
|
||||
// Start NFS service CRD watcher
|
||||
NFSController := nfs.NewNFSController(c.context, c.containerImage,
|
||||
isHostNetworkDefined(cluster.Spec.Network),
|
||||
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
|
||||
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
|
||||
cluster.Spec.Resources,
|
||||
cluster.ownerRef)
|
||||
NFSController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start S3 service CRD watcher
|
||||
S3Controller := s3.NewS3Controller(c.context, c.containerImage,
|
||||
isHostNetworkDefined(cluster.Spec.Network),
|
||||
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
|
||||
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
|
||||
cluster.Spec.Resources,
|
||||
cluster.ownerRef)
|
||||
S3Controller.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start S3X service CRD watcher
|
||||
S3XController := s3x.NewS3XController(c.context, c.containerImage,
|
||||
isHostNetworkDefined(cluster.Spec.Network),
|
||||
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
|
||||
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
|
||||
cluster.Spec.Resources,
|
||||
cluster.ownerRef)
|
||||
S3XController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// Start ISCSI service CRD watcher
|
||||
ISCSIController := iscsi.NewISCSIController(c.context, c.containerImage,
|
||||
isHostNetworkDefined(cluster.Spec.Network),
|
||||
cluster.Spec.DataDirHostPath, cluster.Spec.DataVolumeSize,
|
||||
edgefsv1alpha1.GetTargetPlacement(cluster.Spec.Placement),
|
||||
cluster.Spec.Resources,
|
||||
cluster.ownerRef)
|
||||
ISCSIController.StartWatch(cluster.Namespace, cluster.stopCh)
|
||||
|
||||
// add the finalizer to the crd
|
||||
err = c.addFinalizer(clusterObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to add finalizer to cluster crd. %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
|
||||
oldCluster := oldObj.(*edgefsv1alpha1.Cluster).DeepCopy()
|
||||
newCluster := newObj.(*edgefsv1alpha1.Cluster).DeepCopy()
|
||||
logger.Infof("update event for cluster %s", newCluster.Namespace)
|
||||
|
||||
// Check if the cluster is being deleted. This code path is called when a finalizer is specified in the crd.
|
||||
// When a cluster is requested for deletion, K8s will only set the deletion timestamp if there are any finalizers in the list.
|
||||
// K8s will only delete the crd and child resources when the finalizers have been removed from the crd.
|
||||
if newCluster.DeletionTimestamp != nil {
|
||||
logger.Infof("cluster %s has a deletion timestamp", newCluster.Namespace)
|
||||
err := c.handleDelete(newCluster, time.Duration(clusterDeleteRetryInterval)*time.Second)
|
||||
if err != nil {
|
||||
logger.Errorf("failed finalizer for cluster. %+v", err)
|
||||
return
|
||||
}
|
||||
// remove the finalizer from the crd, which indicates to k8s that the resource can safely be deleted
|
||||
c.removeFinalizer(newCluster)
|
||||
return
|
||||
}
|
||||
|
||||
if !clusterChanged(oldCluster.Spec, newCluster.Spec) {
|
||||
logger.Infof("update event for cluster %s is not supported", newCluster.Namespace)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("update event for cluster %s is supported, orchestrating update now", newCluster.Namespace)
|
||||
logger.Debugf("old cluster: %+v", oldCluster.Spec)
|
||||
logger.Debugf("new cluster: %+v", newCluster.Spec)
|
||||
|
||||
cluster, ok := c.clusterMap[newCluster.Namespace]
|
||||
if !ok {
|
||||
logger.Errorf("Cannot update cluster %s that does not exist", newCluster.Namespace)
|
||||
return
|
||||
}
|
||||
cluster.Spec = newCluster.Spec
|
||||
|
||||
// attempt to update the cluster. note this is done outside of wait.Poll because that function
|
||||
// will wait for the retry interval before trying for the first time.
|
||||
done, _ := c.handleUpdate(newCluster, cluster)
|
||||
if done {
|
||||
return
|
||||
}
|
||||
|
||||
err := wait.Poll(updateClusterInterval, updateClusterTimeout, func() (bool, error) {
|
||||
return c.handleUpdate(newCluster, cluster)
|
||||
})
|
||||
if err != nil {
|
||||
message := fmt.Sprintf("giving up trying to update cluster in namespace %s after %s", cluster.Namespace, updateClusterTimeout)
|
||||
logger.Error(message)
|
||||
if err := c.updateClusterStatus(newCluster.Namespace, newCluster.Name, edgefsv1alpha1.ClusterStateError, message); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", newCluster.Namespace, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
logger.Infof("cluster %s updated in namespace %s", newCluster.Name, newCluster.Namespace)
|
||||
}
|
||||
|
||||
func (c *ClusterController) handleUpdate(newClust *edgefsv1alpha1.Cluster, cluster *cluster) (bool, error) {
|
||||
if err := c.updateClusterStatus(newClust.Namespace, newClust.Name, edgefsv1alpha1.ClusterStateUpdating, ""); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", newClust.Namespace, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err := cluster.createInstance(c.containerImage); err != nil {
|
||||
logger.Errorf("failed to update cluster in namespace %s. %+v", newClust.Namespace, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if err := c.updateClusterStatus(newClust.Namespace, newClust.Name, edgefsv1alpha1.ClusterStateCreated, ""); err != nil {
|
||||
logger.Errorf("failed to update cluster status in namespace %s: %+v", newClust.Namespace, err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
logger.Infof("succeeded updating cluster in namespace %s", newClust.Namespace)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) onDelete(obj interface{}) {
|
||||
clust := obj.(*edgefsv1alpha1.Cluster).DeepCopy()
|
||||
|
||||
logger.Infof("delete event for cluster %s in namespace %s", clust.Name, clust.Namespace)
|
||||
|
||||
err := c.handleDelete(clust, time.Duration(clusterDeleteRetryInterval)*time.Second)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to delete cluster. %+v", err)
|
||||
}
|
||||
if cluster, ok := c.clusterMap[clust.Namespace]; ok {
|
||||
close(cluster.stopCh)
|
||||
delete(c.clusterMap, clust.Namespace)
|
||||
}
|
||||
if clust.Spec.Storage.AnyUseAllDevices() {
|
||||
c.devicesInUse = false
|
||||
}
|
||||
discover.FreeDevicesByCluster(c.context, clust.Namespace)
|
||||
}
|
||||
|
||||
func (c *ClusterController) handleDelete(clust *edgefsv1alpha1.Cluster, retryInterval time.Duration) error {
|
||||
cluster, ok := c.clusterMap[clust.Namespace]
|
||||
if !ok {
|
||||
return fmt.Errorf("Cannot delete cluster %s that does not exist", clust.Namespace)
|
||||
}
|
||||
|
||||
for _, node := range cluster.targets.Storage.Nodes {
|
||||
k := cluster.Namespace
|
||||
err := cluster.targets.RemoveLabelOffNode(cluster.context.Clientset, node.Name, []string{k})
|
||||
logger.Infof("removed label %s from %s: %+v", k, node.Name, err)
|
||||
}
|
||||
|
||||
// delete associated node labels
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) updateClusterStatus(namespace, name string, state edgefsv1alpha1.ClusterState, message string) error {
|
||||
// get the most recent cluster CRD object
|
||||
cluster, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(namespace).Get(name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get cluster from namespace %s prior to updating its status: %+v", namespace, err)
|
||||
}
|
||||
|
||||
// update the status on the retrieved cluster object
|
||||
cluster.Status = edgefsv1alpha1.ClusterStatus{State: state, Message: message}
|
||||
if _, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Update(cluster); err != nil {
|
||||
return fmt.Errorf("failed to update cluster %s status: %+v", cluster.Namespace, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool {
|
||||
if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *ClusterController) addFinalizer(clust *edgefsv1alpha1.Cluster) error {
|
||||
|
||||
// get the latest cluster object since we probably updated it before we got to this point (e.g. by updating its status)
|
||||
clust, err := c.context.RookClientset.EdgefsV1alpha1().Clusters(clust.Namespace).Get(clust.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// add the finalizer (cluster.edgefs.rook.io) if it is not yet defined on the cluster CRD
|
||||
for _, finalizer := range clust.Finalizers {
|
||||
if finalizer == finalizerName {
|
||||
logger.Infof("finalizer already set on cluster %s", clust.Namespace)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// adding finalizer to the cluster crd
|
||||
clust.Finalizers = append(clust.Finalizers, finalizerName)
|
||||
|
||||
// update the crd
|
||||
_, err = c.context.RookClientset.EdgefsV1alpha1().Clusters(clust.Namespace).Update(clust)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add finalizer to cluster. %+v", err)
|
||||
}
|
||||
|
||||
logger.Infof("added finalizer to cluster %s", clust.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ClusterController) removeFinalizer(obj interface{}) {
|
||||
var fname string
|
||||
var objectMeta *metav1.ObjectMeta
|
||||
|
||||
// first determine what type/version of cluster we are dealing with
|
||||
if cl, ok := obj.(*edgefsv1alpha1.Cluster); ok {
|
||||
fname = finalizerName
|
||||
objectMeta = &cl.ObjectMeta
|
||||
} else {
|
||||
logger.Warningf("cannot remove finalizer from object that is not a cluster: %+v", obj)
|
||||
return
|
||||
}
|
||||
|
||||
// remove the finalizer from the slice if it exists
|
||||
found := false
|
||||
for i, finalizer := range objectMeta.Finalizers {
|
||||
if finalizer == fname {
|
||||
objectMeta.Finalizers = append(objectMeta.Finalizers[:i], objectMeta.Finalizers[i+1:]...)
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
logger.Infof("finalizer %s not found in the cluster crd '%s'", fname, objectMeta.Name)
|
||||
return
|
||||
}
|
||||
|
||||
// update the crd to remove the finalizer for good. retry several times in case of intermittent failures.
|
||||
maxRetries := 5
|
||||
retrySeconds := 5 * time.Second
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
var err error
|
||||
if cluster, ok := obj.(*edgefsv1alpha1.Cluster); ok {
|
||||
_, err = c.context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Update(cluster)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
logger.Errorf("failed to remove finalizer %s from cluster %s. %+v", fname, objectMeta.Name, err)
|
||||
time.Sleep(retrySeconds)
|
||||
continue
|
||||
}
|
||||
logger.Infof("removed finalizer %s from cluster %s", fname, objectMeta.Name)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Warningf("giving up from removing the %s cluster finalizer", fname)
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
rookfake "github.com/rook/rook/pkg/client/clientset/versioned/fake"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestClusterDelete(t *testing.T) {
|
||||
//nodeName := "node841"
|
||||
clusterName := "cluster684"
|
||||
//pvName := "pvc-540"
|
||||
rookSystemNamespace := "rook-system-6413"
|
||||
|
||||
os.Setenv(k8sutil.PodNamespaceEnvVar, rookSystemNamespace)
|
||||
defer os.Unsetenv(k8sutil.PodNamespaceEnvVar)
|
||||
|
||||
clientset := testop.New(3)
|
||||
context := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
}
|
||||
|
||||
// create the cluster controller and tell it that the cluster has been deleted
|
||||
controller := NewClusterController(context, "")
|
||||
clusterToDelete := &edgefsv1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: clusterName}}
|
||||
controller.handleDelete(clusterToDelete, time.Microsecond)
|
||||
}
|
||||
|
||||
func TestClusterChanged(t *testing.T) {
|
||||
// a new node added, should be a change
|
||||
old := edgefsv1alpha1.ClusterSpec{
|
||||
Storage: rookalpha.StorageScopeSpec{
|
||||
Nodes: []rookalpha.Node{
|
||||
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
new := edgefsv1alpha1.ClusterSpec{
|
||||
Storage: rookalpha.StorageScopeSpec{
|
||||
Nodes: []rookalpha.Node{
|
||||
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.True(t, clusterChanged(old, new))
|
||||
|
||||
// a node was removed, should be a change
|
||||
old.Storage.Nodes = []rookalpha.Node{
|
||||
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
}
|
||||
new.Storage.Nodes = []rookalpha.Node{
|
||||
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
}
|
||||
assert.True(t, clusterChanged(old, new))
|
||||
|
||||
// the nodes being in a different order should not be a change
|
||||
old.Storage.Nodes = []rookalpha.Node{
|
||||
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
}
|
||||
new.Storage.Nodes = []rookalpha.Node{
|
||||
{Name: "node2", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
{Name: "node1", Selection: rookalpha.Selection{Devices: []rookalpha.Device{{Name: "sda"}}}},
|
||||
}
|
||||
assert.False(t, clusterChanged(old, new))
|
||||
}
|
||||
|
||||
func TestRemoveFinalizer(t *testing.T) {
|
||||
clientset := testop.New(3)
|
||||
context := &clusterd.Context{
|
||||
Clientset: clientset,
|
||||
RookClientset: rookfake.NewSimpleClientset(),
|
||||
}
|
||||
controller := NewClusterController(context, "")
|
||||
|
||||
// *****************************************
|
||||
// start with a current version edgefs cluster
|
||||
// *****************************************
|
||||
cluster := &edgefsv1alpha1.Cluster{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "cluster-1893",
|
||||
Namespace: "namespace-6551",
|
||||
Finalizers: []string{finalizerName},
|
||||
},
|
||||
}
|
||||
|
||||
// create the cluster initially so it exists in the k8s api
|
||||
cluster, err := context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Create(cluster)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, cluster.Finalizers, 1)
|
||||
|
||||
// remove the finalizer from the cluster object
|
||||
controller.removeFinalizer(cluster)
|
||||
|
||||
// verify the finalier was removed
|
||||
cluster, err = context.RookClientset.EdgefsV1alpha1().Clusters(cluster.Namespace).Get(cluster.Name, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, cluster)
|
||||
assert.Len(t, cluster.Finalizers, 0)
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package mgr for the Edgefs manager.
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-mgr")
|
||||
|
||||
const (
|
||||
appName = "rook-edgefs-mgr"
|
||||
defaultServiceAccountName = "rook-edgefs-cluster"
|
||||
defaultGrpcPort = 6789
|
||||
defaultMetricsPort = 8881
|
||||
|
||||
/* Volumes definitions */
|
||||
dataVolumeName = "edgefs-datadir"
|
||||
stateVolumeFolder = ".state"
|
||||
etcVolumeFolder = ".etc"
|
||||
)
|
||||
|
||||
// Cluster is the edgefs mgr manager
|
||||
type Cluster struct {
|
||||
Namespace string
|
||||
Version string
|
||||
serviceAccount string
|
||||
Replicas int
|
||||
dataDirHostPath string
|
||||
dataVolumeSize resource.Quantity
|
||||
placement rookalpha.Placement
|
||||
context *clusterd.Context
|
||||
HostNetworkSpec edgefsv1alpha1.NetworkSpec
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// New creates an instance of the mgr
|
||||
func New(
|
||||
context *clusterd.Context, namespace, version string,
|
||||
serviceAccount string,
|
||||
dataDirHostPath string,
|
||||
dataVolumeSize resource.Quantity,
|
||||
placement rookalpha.Placement, HostNetworkSpec edgefsv1alpha1.NetworkSpec,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *Cluster {
|
||||
|
||||
if serviceAccount == "" {
|
||||
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
|
||||
serviceAccount = defaultServiceAccountName
|
||||
logger.Infof("setting the mgr pod to use the service account name: %s", serviceAccount)
|
||||
}
|
||||
|
||||
return &Cluster{
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
serviceAccount: serviceAccount,
|
||||
placement: placement,
|
||||
Version: version,
|
||||
Replicas: 1,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
dataVolumeSize: dataVolumeSize,
|
||||
HostNetworkSpec: HostNetworkSpec,
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
}
|
||||
|
||||
func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool {
|
||||
if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Start the mgr instance
|
||||
func (c *Cluster) Start(rookImage string) error {
|
||||
logger.Infof("start running mgr")
|
||||
|
||||
logger.Infof("Mgr Image is %s", rookImage)
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(appName, c.Namespace, rookImage, 1)
|
||||
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
|
||||
}
|
||||
logger.Infof("%s deployment already exists", appName)
|
||||
} else {
|
||||
logger.Infof("%s deployment started", appName)
|
||||
}
|
||||
|
||||
// create the mgr service
|
||||
service := c.makeMgrService(appName)
|
||||
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(service); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create mgr service. %+v", err)
|
||||
}
|
||||
logger.Infof("mgr mgr service already exists")
|
||||
} else {
|
||||
logger.Infof("mgr mgr service started")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeMgrService(name string) *v1.Service {
|
||||
labels := c.getLabels()
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: c.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeClusterIP,
|
||||
Ports: []v1.ServicePort{
|
||||
{
|
||||
Name: "mgr",
|
||||
Port: int32(defaultGrpcPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "http-metrics",
|
||||
Port: int32(defaultMetricsPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDeployment(name, clusterName, rookImage string, replicas int32) *extensions.Deployment {
|
||||
|
||||
volumes := []v1.Volume{}
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
// dataVolume case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: dataVolumeName,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// dataDir case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: c.dataDirHostPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Labels: c.getDaemonLabels(clusterName),
|
||||
Annotations: map[string]string{"prometheus.io/scrape": "true",
|
||||
"prometheus.io/port": strconv.Itoa(defaultMetricsPort)},
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
ServiceAccountName: c.serviceAccount,
|
||||
Containers: []v1.Container{c.mgrContainer(name, rookImage)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: volumes,
|
||||
HostIPC: true,
|
||||
HostNetwork: isHostNetworkDefined(c.HostNetworkSpec),
|
||||
NodeSelector: map[string]string{c.Namespace: "cluster"},
|
||||
},
|
||||
}
|
||||
if isHostNetworkDefined(c.HostNetworkSpec) {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
c.placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
d := &extensions.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: c.Namespace,
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &replicas},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &d.ObjectMeta, &c.ownerRef)
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *Cluster) mgrContainer(name string, containerImage string) v1.Container {
|
||||
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
||||
},
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc.target",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: name,
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"mgmt"},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
Resources: c.resources,
|
||||
Ports: []v1.ContainerPort{
|
||||
{
|
||||
Name: "mgr",
|
||||
ContainerPort: int32(defaultGrpcPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "http-metrics",
|
||||
ContainerPort: int32(defaultMetricsPort),
|
||||
Protocol: v1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) getLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: c.Namespace,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) getDaemonLabels(clusterName string) map[string]string {
|
||||
labels := c.getLabels()
|
||||
labels["instance"] = clusterName
|
||||
return labels
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package mgr
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestStartMGR(t *testing.T) {
|
||||
executor := &exectest.MockExecutor{
|
||||
MockExecuteCommandWithOutputFile: func(debug bool, actionName string, command string, outFileArg string, args ...string) (string, error) {
|
||||
return "{\"key\":\"mysecurekey\"}", nil
|
||||
},
|
||||
}
|
||||
|
||||
configDir, _ := ioutil.TempDir("", "")
|
||||
defer os.RemoveAll(configDir)
|
||||
context := &clusterd.Context{
|
||||
Executor: executor,
|
||||
ConfigDir: configDir,
|
||||
Clientset: testop.New(3)}
|
||||
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
|
||||
c := New(context, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// start a basic service
|
||||
err := c.Start("edgefs")
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, c)
|
||||
|
||||
// starting again with more replicas
|
||||
c.Replicas = 3
|
||||
err = c.Start("edgefs")
|
||||
assert.Nil(t, err)
|
||||
validateStart(t, c)
|
||||
}
|
||||
|
||||
func validateStart(t *testing.T, c *Cluster) {
|
||||
|
||||
_, err := c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get("rook-edgefs-mgr", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
|
||||
_, err = c.context.Clientset.CoreV1().Services(c.Namespace).Get("rook-edgefs-mgr", metav1.GetOptions{})
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestPodSpec(t *testing.T) {
|
||||
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
|
||||
c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "rook/rook:myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
|
||||
},
|
||||
}, metav1.OwnerReference{})
|
||||
|
||||
d := c.makeDeployment("mgr-a", "rook-edgefs", "edgefs", 1)
|
||||
assert.NotNil(t, d)
|
||||
assert.Equal(t, "mgr-a", d.Name)
|
||||
assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy)
|
||||
assert.Equal(t, 1, len(d.Spec.Template.Spec.Volumes))
|
||||
|
||||
//check ports
|
||||
assert.Equal(t, 2, len(d.Spec.Template.Spec.Containers[0].Ports))
|
||||
assert.Equal(t, int32(6789), d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort)
|
||||
assert.Equal(t, int32(8881), d.Spec.Template.Spec.Containers[0].Ports[1].ContainerPort)
|
||||
|
||||
assert.Equal(t, "edgefs-datadir", d.Spec.Template.Spec.Volumes[0].Name)
|
||||
assert.Equal(t, "mgr-a", d.ObjectMeta.Name)
|
||||
assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"])
|
||||
assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"])
|
||||
assert.Equal(t, 0, len(d.ObjectMeta.Annotations))
|
||||
|
||||
assert.Equal(t, 2, len(d.Spec.Template.ObjectMeta.Annotations))
|
||||
assert.Equal(t, "true", d.Spec.Template.ObjectMeta.Annotations["prometheus.io/scrape"])
|
||||
assert.Equal(t, strconv.Itoa(defaultMetricsPort), d.Spec.Template.ObjectMeta.Annotations["prometheus.io/port"])
|
||||
|
||||
cont := d.Spec.Template.Spec.Containers[0]
|
||||
assert.Equal(t, 2, len(cont.VolumeMounts))
|
||||
assert.Equal(t, "mgmt", cont.Args[0])
|
||||
|
||||
assert.Equal(t, "100", cont.Resources.Limits.Cpu().String())
|
||||
assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
|
||||
}
|
||||
|
||||
func TestServiceSpec(t *testing.T) {
|
||||
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
|
||||
c := New(&clusterd.Context{}, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
s := c.makeMgrService("rook-edgefs-mgr")
|
||||
assert.NotNil(t, s)
|
||||
assert.Equal(t, "rook-edgefs-mgr", s.Name)
|
||||
assert.Equal(t, 2, len(s.Spec.Ports))
|
||||
}
|
||||
|
||||
func TestHostNetwork(t *testing.T) {
|
||||
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
|
||||
c := New(&clusterd.Context{Clientset: testop.New(1)}, "ns", "myversion", "", "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{ServerIfName: "eth0"}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
d := c.makeDeployment("mgr-a", "a", "edgefs", 1)
|
||||
assert.NotNil(t, d)
|
||||
|
||||
assert.Equal(t, true, d.Spec.Template.Spec.HostNetwork)
|
||||
assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy)
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package config for Edgefs target config managed by the operator
|
||||
package config
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
)
|
||||
|
||||
const (
|
||||
configStoreNameFmt = "rook-edgefs-%s-config"
|
||||
osdDirsKeyName = "target-dirs"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "target-config")
|
||||
|
||||
func GetConfigStoreName(nodeName string) string {
|
||||
return k8sutil.TruncateNodeName(configStoreNameFmt, nodeName)
|
||||
}
|
||||
|
||||
const (
|
||||
RtVerifyChidKey = "rtVerifyChid"
|
||||
LmdbPageSizeKey = "lmdbPageSize"
|
||||
UseBcacheKey = "useBCache"
|
||||
UseBcacheWBKey = "useBCacheWB"
|
||||
UseMetadataMaskKey = "useMetadataMask"
|
||||
UseMetadataOffloadKey = "useMetadataOffload"
|
||||
UseAllSSDKey = "useAllSSD"
|
||||
RtrdPlevelOverrideKey = "rtrdPLevelOverride"
|
||||
)
|
||||
|
||||
type StoreConfig struct {
|
||||
// 0 (disabled), 1 (verify on write) or 2(verify on read/write)
|
||||
RtVerifyChid int `json:"rtVerifyChid,omitempty"`
|
||||
// 4096, 8192, 16384 or 32768
|
||||
LmdbPageSize int `json:"lmdbPageSize,omitempty"`
|
||||
// enable use of back cache
|
||||
UseBCache bool `json:"useBCache,omitempty"`
|
||||
// enable write back cache
|
||||
UseBCacheWB bool `json:"useBCacheWB,omitempty"`
|
||||
// what guts needs to go to SSD and what not
|
||||
UseMetadataMask string `json:"useMetadataMask,omitempty"`
|
||||
// when useAllSSD is false, enable metadata offload on SSD
|
||||
UseMetadataOffload bool `json:"useMetadataOffload,omitempty"`
|
||||
// only look for SSD/NVMe
|
||||
UseAllSSD bool `json:"useAllSSD,omitempty"`
|
||||
// if > 0, override automatic partitioning numbering logic
|
||||
RtrdPLevelOverride int `json:"rtrdPLevelOverride,omitempty"`
|
||||
}
|
||||
|
||||
func DefaultStoreConfig() StoreConfig {
|
||||
return StoreConfig{
|
||||
RtVerifyChid: 1,
|
||||
LmdbPageSize: 16348,
|
||||
UseBCacheWB: false,
|
||||
UseMetadataMask: "0xff",
|
||||
UseMetadataOffload: true,
|
||||
UseAllSSD: false,
|
||||
RtrdPLevelOverride: 0,
|
||||
}
|
||||
}
|
||||
|
||||
var validLmbdPageSize = map[int]bool{
|
||||
4096: true,
|
||||
8192: true,
|
||||
16384: true,
|
||||
32768: true,
|
||||
}
|
||||
|
||||
func ToStoreConfig(config map[string]string) StoreConfig {
|
||||
storeConfig := DefaultStoreConfig()
|
||||
for k, v := range config {
|
||||
switch k {
|
||||
case RtVerifyChidKey:
|
||||
if storeConfig.RtVerifyChid < 0 || storeConfig.RtVerifyChid > 2 {
|
||||
storeConfig.RtVerifyChid = convertToIntIgnoreErr(v)
|
||||
}
|
||||
case LmdbPageSizeKey:
|
||||
|
||||
if validLmbdPageSize[storeConfig.LmdbPageSize] {
|
||||
storeConfig.LmdbPageSize = convertToIntIgnoreErr(v)
|
||||
}
|
||||
case UseBcacheWBKey:
|
||||
storeConfig.UseBCacheWB = convertToBoolIgnoreErr(v)
|
||||
case UseMetadataMaskKey:
|
||||
storeConfig.UseMetadataMask = v
|
||||
case UseMetadataOffloadKey:
|
||||
storeConfig.UseMetadataOffload = convertToBoolIgnoreErr(v)
|
||||
case UseAllSSDKey:
|
||||
storeConfig.UseAllSSD = convertToBoolIgnoreErr(v)
|
||||
case RtrdPlevelOverrideKey:
|
||||
storeConfig.RtrdPLevelOverride = convertToIntIgnoreErr(v)
|
||||
}
|
||||
}
|
||||
|
||||
return storeConfig
|
||||
}
|
||||
|
||||
func convertToIntIgnoreErr(raw string) int {
|
||||
val, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
val = 0
|
||||
}
|
||||
|
||||
return val
|
||||
}
|
||||
|
||||
func convertToBoolIgnoreErr(raw string) bool {
|
||||
val, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
val = false
|
||||
}
|
||||
|
||||
return val
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultServerIfName = "eth0"
|
||||
defaultBrokerIfName = "eth0"
|
||||
)
|
||||
|
||||
type RtlfsDevice struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
CheckMountpoint int `json:"check_mountpoint"`
|
||||
}
|
||||
|
||||
type RtlfsDevices struct {
|
||||
Devices []RtlfsDevice `json:"devices"`
|
||||
}
|
||||
|
||||
type CcowTenant struct {
|
||||
FailureDomain int `json:"failure_domain"`
|
||||
}
|
||||
|
||||
type CcowNetwork struct {
|
||||
BrokerInterfaces string `json:"broker_interfaces"`
|
||||
ServerUnixSocket string `json:"server_unix_socket"`
|
||||
BrokerIP4addr string `json:"broker_ip4addr,omitempty"`
|
||||
ServerIP4addr string `json:"server_ip4addr,omitempty"`
|
||||
}
|
||||
|
||||
type CcowConf struct {
|
||||
Tenant CcowTenant `json:"tenant"`
|
||||
Network CcowNetwork `json:"network"`
|
||||
}
|
||||
|
||||
type CcowdNetwork struct {
|
||||
ServerInterfaces string `json:"server_interfaces"`
|
||||
ServerUnixSocket string `json:"server_unix_socket"`
|
||||
ServerIP4addr string `json:"server_ip4addr,omitempty"`
|
||||
}
|
||||
|
||||
type CcowdConf struct {
|
||||
Network CcowdNetwork `json:"network"`
|
||||
Transport []string `json:"transport"`
|
||||
}
|
||||
|
||||
type AuditdConf struct {
|
||||
IsAggregator int `json:"is_aggregator"`
|
||||
}
|
||||
|
||||
type SetupNode struct {
|
||||
Ccow CcowConf `json:"ccow"`
|
||||
Ccowd CcowdConf `json:"ccowd"`
|
||||
Auditd AuditdConf `json:"auditd"`
|
||||
Ipv4Autodetect int `json:"ipv4_autodetect,omitempty"`
|
||||
RtlfsAutodetect string `json:"rtlfs_autodetect,omitempty"`
|
||||
ClusterNodes []string `json:"cluster_nodes,omitempty"`
|
||||
Rtrd RTDevices `json:"rtrd"`
|
||||
Rtlfs RtlfsDevices `json:"rtlfs"`
|
||||
}
|
||||
|
||||
// As we relying on StatefulSet, we want to build global ConfigMap shared
|
||||
// to all the nodes in the cluster. This way configuration is simplified and
|
||||
// available to all subcomponents at any point it time.
|
||||
func (c *Cluster) createSetupConfigs(resurrect bool) error {
|
||||
|
||||
cm := make(map[string]SetupNode)
|
||||
|
||||
dnsRecords := make([]string, len(c.Storage.Nodes))
|
||||
for i := 0; i < len(c.Storage.Nodes); i++ {
|
||||
dnsRecords[i] = createQualifiedHeadlessServiceName(i, c.Namespace)
|
||||
}
|
||||
|
||||
serverIfName := defaultServerIfName
|
||||
brokerIfName := defaultBrokerIfName
|
||||
if isHostNetworkDefined(c.HostNetworkSpec) {
|
||||
|
||||
if len(c.HostNetworkSpec.ServerIfName) > 0 && len(c.HostNetworkSpec.BrokerIfName) > 0 {
|
||||
serverIfName = c.HostNetworkSpec.ServerIfName
|
||||
brokerIfName = c.HostNetworkSpec.BrokerIfName
|
||||
} else if len(c.HostNetworkSpec.ServerIfName) > 0 {
|
||||
serverIfName = c.HostNetworkSpec.ServerIfName
|
||||
brokerIfName = c.HostNetworkSpec.ServerIfName
|
||||
} else if len(c.HostNetworkSpec.BrokerIfName) > 0 {
|
||||
serverIfName = c.HostNetworkSpec.BrokerIfName
|
||||
brokerIfName = c.HostNetworkSpec.BrokerIfName
|
||||
}
|
||||
}
|
||||
|
||||
// fully resolve the storage config and resources for all nodes
|
||||
for _, node := range c.Storage.Nodes {
|
||||
devConfig := c.deploymentConfig.devConfig[node.Name]
|
||||
rtDevices := devConfig.rtrd.Devices
|
||||
rtlfsDevices := devConfig.rtlfs.Devices
|
||||
|
||||
rtlfsAutoDetectPath := ""
|
||||
if c.deploymentConfig.deploymentType == deploymentAutoRtlfs &&
|
||||
!devConfig.isGatewayNode {
|
||||
rtlfsAutoDetectPath = "/data"
|
||||
}
|
||||
|
||||
if resurrect || devConfig.isGatewayNode {
|
||||
// In resurrection case we only need to adjust networking selections
|
||||
// in ccow.json, ccowd.json and corosync.conf. And keep device transport
|
||||
// same as before. Resurrection is "best effort" feature, we cannot
|
||||
// guarnatee that cluster can be reconfigured, but at least we do try.
|
||||
|
||||
rtDevices = make([]RTDevice, 0)
|
||||
rtlfsDevices = make([]RtlfsDevice, 0)
|
||||
}
|
||||
|
||||
nodeConfig := SetupNode{
|
||||
Ccow: CcowConf{
|
||||
Tenant: CcowTenant{
|
||||
FailureDomain: 1,
|
||||
},
|
||||
Network: CcowNetwork{
|
||||
BrokerInterfaces: brokerIfName,
|
||||
ServerUnixSocket: "/opt/nedge/var/run/sock/ccowd.sock",
|
||||
},
|
||||
},
|
||||
Ccowd: CcowdConf{
|
||||
Network: CcowdNetwork{
|
||||
ServerInterfaces: serverIfName,
|
||||
ServerUnixSocket: "/opt/nedge/var/run/sock/ccowd.sock",
|
||||
},
|
||||
Transport: []string{c.deploymentConfig.transportKey},
|
||||
},
|
||||
Auditd: AuditdConf{
|
||||
IsAggregator: 0,
|
||||
},
|
||||
Rtrd: RTDevices{
|
||||
Devices: rtDevices,
|
||||
},
|
||||
Rtlfs: RtlfsDevices{
|
||||
Devices: rtlfsDevices,
|
||||
},
|
||||
Ipv4Autodetect: 1,
|
||||
RtlfsAutodetect: rtlfsAutoDetectPath,
|
||||
ClusterNodes: dnsRecords,
|
||||
}
|
||||
|
||||
cm[node.Name] = nodeConfig
|
||||
|
||||
logger.Debugf("Resolved Node %s = %+v", node.Name, cm[node.Name])
|
||||
}
|
||||
|
||||
nesetupJson, err := json.Marshal(&cm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataMap := make(map[string]string, 1)
|
||||
dataMap["nesetup"] = string(nesetupJson)
|
||||
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: configName,
|
||||
Namespace: c.Namespace,
|
||||
},
|
||||
Data: dataMap,
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &configMap.ObjectMeta, &c.ownerRef)
|
||||
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap); err != nil {
|
||||
if errors.IsAlreadyExists(err) {
|
||||
if _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Update(configMap); err != nil {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// success. do the labeling so that StatefulSet scheduler will
|
||||
// select the right nodes.
|
||||
for _, node := range c.Storage.Nodes {
|
||||
k := c.Namespace
|
||||
err = c.AddLabelsToNode(c.context.Clientset, node.Name, map[string]string{k: "cluster"})
|
||||
logger.Debugf("added label %s from %s: %+v", k, node.Name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster/target/config"
|
||||
"github.com/rook/rook/pkg/util/sys"
|
||||
)
|
||||
|
||||
type RTDevices struct {
|
||||
Devices []RTDevice `json:"devices"`
|
||||
}
|
||||
|
||||
type RTDevice struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Device string `json:"device,omitempty"`
|
||||
Psize int `json:"psize,omitempty"`
|
||||
VerifyChid int `json:"verify_chid,omitempty"`
|
||||
Journal string `json:"journal,omitempty"`
|
||||
Metadata string `json:"metadata,omitempty"`
|
||||
BcacheWritearound int `json:"bcache_writearound,omitempty"`
|
||||
PlevelOverride int `json:"plevel_override,omitempty"`
|
||||
}
|
||||
|
||||
func getIdDevLinkName(dls string) (dl string) {
|
||||
dlsArr := strings.Split(dls, " ")
|
||||
for i := range dlsArr {
|
||||
s := strings.Replace(dlsArr[i], "/dev/disk/by-id/", "", 1)
|
||||
if strings.Contains(s, "/") || strings.Contains(s, "wwn-") {
|
||||
continue
|
||||
}
|
||||
dl = s
|
||||
break
|
||||
}
|
||||
return dl
|
||||
}
|
||||
|
||||
func GetRTDevices(nodeDisks []sys.LocalDisk, storeConfig *config.StoreConfig) (rtDevices []RTDevice, err error) {
|
||||
rtDevices = make([]RTDevice, 0)
|
||||
if storeConfig == nil {
|
||||
return rtDevices, fmt.Errorf("no pointer to StoreConfig provided")
|
||||
}
|
||||
|
||||
if len(nodeDisks) == 0 {
|
||||
return rtDevices, nil
|
||||
}
|
||||
|
||||
var ssds []sys.LocalDisk
|
||||
var hdds []sys.LocalDisk
|
||||
var devices []sys.LocalDisk
|
||||
|
||||
for i := range nodeDisks {
|
||||
if !nodeDisks[i].Empty || len(nodeDisks[i].Partitions) > 0 {
|
||||
continue
|
||||
}
|
||||
if nodeDisks[i].Rotational {
|
||||
hdds = append(hdds, nodeDisks[i])
|
||||
} else {
|
||||
ssds = append(ssds, nodeDisks[i])
|
||||
}
|
||||
devices = append(devices, nodeDisks[i])
|
||||
}
|
||||
|
||||
//var rtdevs []RTDevice
|
||||
if storeConfig.UseAllSSD {
|
||||
//
|
||||
// All flush media case (High Performance)
|
||||
//
|
||||
if len(ssds) == 0 {
|
||||
return rtDevices, fmt.Errorf("No SSD/NVMe media found")
|
||||
}
|
||||
if storeConfig.UseMetadataOffload {
|
||||
fmt.Println("Warning: useMetadataOffload parameter is ignored due to use useAllSSD=true")
|
||||
}
|
||||
|
||||
for i := range devices {
|
||||
if devices[i].Rotational {
|
||||
continue
|
||||
}
|
||||
rtdev := RTDevice{
|
||||
Name: getIdDevLinkName(devices[i].DevLinks),
|
||||
Device: "/dev/" + devices[i].Name,
|
||||
Psize: storeConfig.LmdbPageSize,
|
||||
VerifyChid: storeConfig.RtVerifyChid,
|
||||
}
|
||||
if storeConfig.RtrdPLevelOverride != 0 {
|
||||
rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride
|
||||
}
|
||||
rtDevices = append(rtDevices, rtdev)
|
||||
}
|
||||
return rtDevices, nil
|
||||
}
|
||||
|
||||
if len(hdds) == 0 {
|
||||
return rtDevices, fmt.Errorf("No HDD media found")
|
||||
}
|
||||
|
||||
if !storeConfig.UseMetadataOffload {
|
||||
//
|
||||
// All HDD media case (capacity, cold archive)
|
||||
//
|
||||
for i := range devices {
|
||||
if !devices[i].Rotational {
|
||||
continue
|
||||
}
|
||||
rtdev := RTDevice{
|
||||
Name: getIdDevLinkName(devices[i].DevLinks),
|
||||
Device: "/dev/" + devices[i].Name,
|
||||
Psize: storeConfig.LmdbPageSize,
|
||||
VerifyChid: storeConfig.RtVerifyChid,
|
||||
}
|
||||
if storeConfig.RtrdPLevelOverride != 0 {
|
||||
rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride
|
||||
}
|
||||
rtDevices = append(rtDevices, rtdev)
|
||||
}
|
||||
return rtDevices, nil
|
||||
}
|
||||
|
||||
//
|
||||
// Hybrid SSD/HDD media case (optimal)
|
||||
//
|
||||
if len(hdds) < len(ssds) || len(ssds) == 0 {
|
||||
return rtDevices, fmt.Errorf("Confusing use of useMetadataOffload parameter HDDs(%d) < SSDs(%d)\n", len(hdds), len(ssds))
|
||||
}
|
||||
|
||||
chunkSize := int(math.Ceil(float64(len(hdds) / len(ssds))))
|
||||
var hdds_divided [][]sys.LocalDisk
|
||||
for i := 0; i < len(hdds); i += chunkSize {
|
||||
end := i + chunkSize
|
||||
|
||||
if end > len(hdds) {
|
||||
end = len(hdds)
|
||||
}
|
||||
hdds_divided = append(hdds_divided, hdds[i:end])
|
||||
}
|
||||
|
||||
for i := range hdds_divided {
|
||||
for j := range hdds_divided[i] {
|
||||
rtdev := RTDevice{
|
||||
Name: getIdDevLinkName(hdds_divided[i][j].DevLinks),
|
||||
Device: "/dev/" + hdds_divided[i][j].Name,
|
||||
Psize: storeConfig.LmdbPageSize,
|
||||
VerifyChid: storeConfig.RtVerifyChid,
|
||||
BcacheWritearound: (map[bool]int{true: 0, false: 1})[storeConfig.UseBCacheWB],
|
||||
Journal: getIdDevLinkName(ssds[i].DevLinks),
|
||||
Metadata: getIdDevLinkName(ssds[i].DevLinks) + "," + storeConfig.UseMetadataMask,
|
||||
}
|
||||
if storeConfig.RtrdPLevelOverride != 0 {
|
||||
rtdev.PlevelOverride = storeConfig.RtrdPLevelOverride
|
||||
}
|
||||
rtDevices = append(rtDevices, rtdev)
|
||||
}
|
||||
}
|
||||
return rtDevices, nil
|
||||
}
|
||||
|
||||
func getRtlfsDevices(directories []rookalpha.Directory) []RtlfsDevice {
|
||||
rtlfsDevices := make([]RtlfsDevice, 0)
|
||||
for _, dir := range directories {
|
||||
rtlfsDevice := RtlfsDevice{
|
||||
Name: filepath.Base(dir.Path),
|
||||
Path: dir.Path,
|
||||
CheckMountpoint: 1,
|
||||
}
|
||||
rtlfsDevices = append(rtlfsDevices, rtlfsDevice)
|
||||
}
|
||||
return rtlfsDevices
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
appsv1beta1 "k8s.io/api/apps/v1beta1"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
)
|
||||
|
||||
const (
|
||||
dataDirsEnvVarName = "ROOK_DATA_DIRECTORIES"
|
||||
|
||||
udpTotemPortDefault = int32(5405)
|
||||
udpTotemPortName = "totem"
|
||||
volumeNameDataDir = "datadir"
|
||||
|
||||
/* Volumes definitions */
|
||||
configVolumeName = "edgefs-configdir"
|
||||
configName = "edgefs-config"
|
||||
dataVolumeName = "edgefs-datadir"
|
||||
stateVolumeFolder = ".state"
|
||||
etcVolumeFolder = ".etc"
|
||||
)
|
||||
|
||||
func (c *Cluster) createAppLabels() map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeCorosyncContainer(containerImage string) v1.Container {
|
||||
|
||||
privileged := c.deploymentConfig.needPriviliges
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "IPC_LOCK"},
|
||||
},
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: configVolumeName,
|
||||
MountPath: "/opt/nedge/etc/config",
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/data",
|
||||
},
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: "corosync",
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"corosync"},
|
||||
SecurityContext: securityContext,
|
||||
//Resources: c.resources,
|
||||
VolumeMounts: volumeMounts,
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeAuditdContainer(containerImage string) v1.Container {
|
||||
|
||||
privileged := c.deploymentConfig.needPriviliges
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: "auditd",
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"auditd"},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
//Resources: c.resources,
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeDaemonContainer(containerImage string, dro DevicesResurrectOptions, isInitContainer bool) v1.Container {
|
||||
|
||||
privileged := c.deploymentConfig.needPriviliges
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE"},
|
||||
},
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: "devices",
|
||||
MountPath: "/dev",
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
}
|
||||
|
||||
if c.deploymentConfig.deploymentType == deploymentAutoRtlfs {
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: dataVolumeName, MountPath: "/data"})
|
||||
} else if c.deploymentConfig.deploymentType == deploymentRtlfs {
|
||||
rtlfsDevices := getRtlfsDevices(c.Storage.Directories)
|
||||
for _, device := range rtlfsDevices {
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: device.Name, MountPath: device.Path})
|
||||
}
|
||||
}
|
||||
|
||||
name := "daemon"
|
||||
args := []string{"daemon"}
|
||||
if isInitContainer {
|
||||
if dro.needToZap {
|
||||
name = "daemon-zap"
|
||||
args = []string{"toolbox", "nezap --do-as-i-say"}
|
||||
|
||||
// zap mode is InitContainer, it needs to mount config
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{
|
||||
Name: configVolumeName,
|
||||
MountPath: "/opt/nedge/etc/config",
|
||||
})
|
||||
}
|
||||
} else {
|
||||
if dro.needToWait {
|
||||
args = []string{"wait"}
|
||||
}
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: name,
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: args,
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
Resources: c.resources,
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) configOverrideVolume() v1.Volume {
|
||||
cmSource := &v1.ConfigMapVolumeSource{Items: []v1.KeyToPath{{Key: "nesetup", Path: "nesetup.json"}}}
|
||||
cmSource.Name = configName
|
||||
return v1.Volume{Name: configVolumeName, VolumeSource: v1.VolumeSource{ConfigMap: cmSource}}
|
||||
}
|
||||
|
||||
func isHostNetworkDefined(hostNetworkSpec edgefsv1alpha1.NetworkSpec) bool {
|
||||
if len(hostNetworkSpec.ServerIfName) > 0 || len(hostNetworkSpec.ServerIfName) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Cluster) createPodSpec(rookImage string, dro DevicesResurrectOptions) v1.PodSpec {
|
||||
terminationGracePeriodSeconds := int64(60)
|
||||
|
||||
DNSPolicy := v1.DNSClusterFirst
|
||||
if isHostNetworkDefined(c.HostNetworkSpec) {
|
||||
DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
volumes := []v1.Volume{
|
||||
{
|
||||
Name: "devices",
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: "/dev",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
// dataVolume case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: dataVolumeName,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: c.dataDirHostPath,
|
||||
Type: &hostPathDirectoryOrCreate,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if c.deploymentConfig.deploymentType == deploymentRtlfs {
|
||||
// RTLFS with specified folders
|
||||
for _, folder := range c.deploymentConfig.directories {
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: folder.Name,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: folder.Path,
|
||||
Type: &hostPathDirectoryOrCreate,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var containers []v1.Container
|
||||
var initContainers []v1.Container
|
||||
if dro.needToZap {
|
||||
// To execute "zap" functions (devices or directories) we
|
||||
// create InitContainer to ensure it completes fully.
|
||||
initContainers = []v1.Container{
|
||||
c.makeDaemonContainer(rookImage, dro, true),
|
||||
}
|
||||
}
|
||||
|
||||
volumes = append(volumes, c.configOverrideVolume())
|
||||
|
||||
containers = []v1.Container{
|
||||
c.makeCorosyncContainer(rookImage),
|
||||
c.makeAuditdContainer(rookImage),
|
||||
c.makeDaemonContainer(rookImage, dro, false),
|
||||
}
|
||||
|
||||
return v1.PodSpec{
|
||||
ServiceAccountName: c.serviceAccount,
|
||||
Affinity: &v1.Affinity{
|
||||
PodAntiAffinity: &v1.PodAntiAffinity{
|
||||
PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
|
||||
{
|
||||
Weight: int32(100),
|
||||
PodAffinityTerm: v1.PodAffinityTerm{
|
||||
LabelSelector: &metav1.LabelSelector{
|
||||
MatchExpressions: []metav1.LabelSelectorRequirement{
|
||||
{
|
||||
Key: k8sutil.AppAttr,
|
||||
Operator: metav1.LabelSelectorOpIn,
|
||||
Values: []string{appName},
|
||||
},
|
||||
},
|
||||
},
|
||||
TopologyKey: apis.LabelHostname,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NodeSelector: map[string]string{c.Namespace: "cluster"},
|
||||
InitContainers: initContainers,
|
||||
Containers: containers,
|
||||
// No pre-stop hook is required, a SIGTERM plus some time is all that's needed for graceful shutdown of a node.
|
||||
TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,
|
||||
DNSPolicy: DNSPolicy,
|
||||
HostIPC: true,
|
||||
HostNetwork: isHostNetworkDefined(c.HostNetworkSpec),
|
||||
Volumes: volumes,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cluster) makeStatefulSet(replicas int32, rookImage string, dro DevicesResurrectOptions) (*appsv1beta1.StatefulSet, error) {
|
||||
|
||||
statefulSet := &appsv1beta1.StatefulSet{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: appName,
|
||||
Namespace: c.Namespace,
|
||||
Labels: c.createAppLabels(),
|
||||
},
|
||||
Spec: appsv1beta1.StatefulSetSpec{
|
||||
ServiceName: appName,
|
||||
Replicas: &replicas,
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: c.Namespace,
|
||||
Labels: c.createAppLabels(),
|
||||
},
|
||||
Spec: c.createPodSpec(rookImage, dro),
|
||||
},
|
||||
PodManagementPolicy: appsv1beta1.ParallelPodManagement,
|
||||
UpdateStrategy: appsv1beta1.StatefulSetUpdateStrategy{
|
||||
Type: appsv1beta1.RollingUpdateStatefulSetStrategyType,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
statefulSet.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: dataVolumeName,
|
||||
Namespace: c.Namespace,
|
||||
},
|
||||
Spec: v1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceStorage: c.dataVolumeSize,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &statefulSet.ObjectMeta, &c.ownerRef)
|
||||
c.placement.ApplyToPodSpec(&statefulSet.Spec.Template.Spec)
|
||||
|
||||
return statefulSet, nil
|
||||
}
|
||||
|
||||
func (c *Cluster) makeHeadlessServicePorts(totemPort int32) []v1.ServicePort {
|
||||
return []v1.ServicePort{
|
||||
{
|
||||
// The secondary port serves the UI as well as health and debug endpoints.
|
||||
Name: udpTotemPortName,
|
||||
Port: int32(totemPort),
|
||||
Protocol: v1.ProtocolUDP,
|
||||
TargetPort: intstr.FromInt(int(totemPort)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// This service only exists to create DNS entries for each pod in the stateful
|
||||
// set such that they can resolve each other's IP addresses. It does not
|
||||
// create a load-balanced ClusterIP and should not be used directly by clients
|
||||
// in most circumstances.
|
||||
func (c *Cluster) makeHeadlessService() (*v1.Service, error) {
|
||||
|
||||
headlessService := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: appName,
|
||||
Namespace: c.Namespace,
|
||||
Labels: c.createAppLabels(),
|
||||
Annotations: map[string]string{
|
||||
"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
|
||||
},
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: c.createAppLabels(),
|
||||
PublishNotReadyAddresses: true,
|
||||
ClusterIP: "None",
|
||||
Ports: c.makeHeadlessServicePorts(udpTotemPortDefault),
|
||||
},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, c.Namespace, &headlessService.ObjectMeta, &c.ownerRef)
|
||||
|
||||
return headlessService, nil
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster/target/config"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestStorageSpecDefaultPV(t *testing.T) {
|
||||
storageSpec := rookalpha.StorageScopeSpec{
|
||||
Nodes: []rookalpha.Node{
|
||||
{
|
||||
Name: "node1",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI),
|
||||
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
c.createSetupConfigs(false)
|
||||
}
|
||||
|
||||
func TestStorageSpecDevices(t *testing.T) {
|
||||
storageSpec := rookalpha.StorageScopeSpec{
|
||||
Selection: rookalpha.Selection{
|
||||
Directories: []rookalpha.Directory{{Path: "/rook/dir2"}},
|
||||
},
|
||||
Nodes: []rookalpha.Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Selection: rookalpha.Selection{
|
||||
Devices: []rookalpha.Device{{Name: "sda"}, {Name: "sdb"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI),
|
||||
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
c.createSetupConfigs(false)
|
||||
}
|
||||
|
||||
func TestStorageSpecDirectories(t *testing.T) {
|
||||
storageSpec := rookalpha.StorageScopeSpec{
|
||||
Selection: rookalpha.Selection{
|
||||
Directories: []rookalpha.Directory{{Path: "/rook/dir2"}},
|
||||
},
|
||||
Nodes: []rookalpha.Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Selection: rookalpha.Selection{
|
||||
Directories: []rookalpha.Directory{{Path: "/rook/dir1"}, {Path: "/rook/dir2"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
storageSpec, "/var/lib/rook", *resource.NewQuantity(100000.0, resource.BinarySI),
|
||||
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
c.createSetupConfigs(false)
|
||||
}
|
||||
|
||||
func TestStorageSpecConfig(t *testing.T) {
|
||||
storageSpec := rookalpha.StorageScopeSpec{
|
||||
Config: map[string]string{
|
||||
"useAllSSD": "false",
|
||||
"useBCacheWB": "true",
|
||||
"useBCache": "true",
|
||||
},
|
||||
Nodes: []rookalpha.Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Location: "zone1",
|
||||
Config: map[string]string{
|
||||
"rtTransport": "rtrd",
|
||||
"useAllSSD": "true",
|
||||
"useMetadataOffload": "false",
|
||||
},
|
||||
Selection: rookalpha.Selection{
|
||||
Devices: []rookalpha.Device{{Name: "sda"}, {Name: "sdb"}},
|
||||
},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "rook/rook:myversion", "",
|
||||
storageSpec, "", *resource.NewQuantity(100000.0, resource.BinarySI),
|
||||
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
c.createSetupConfigs(false)
|
||||
|
||||
n := c.Storage.ResolveNode(storageSpec.Nodes[0].Name)
|
||||
storeConfig := config.ToStoreConfig(storageSpec.Nodes[0].Config)
|
||||
|
||||
// First Node's useAllSSD should override global useAllSSD value, -> should be different
|
||||
assert.NotEqual(t, storageSpec.Config["useAllSSD"], storeConfig.UseAllSSD)
|
||||
// useBCacheWB global in StorageSpec and does override config' UseBCacheWB prop, -> should be the same
|
||||
assert.NotEqual(t, storageSpec.Config["useBCacheWB"], storeConfig.UseBCacheWB)
|
||||
// useBCache global in StorageSpec and does override config' UseBCache prop, -> should be the same
|
||||
assert.NotEqual(t, storageSpec.Config["useBCache"], storeConfig.UseBCache)
|
||||
// UseMetadataOffload will override default value of true
|
||||
assert.Equal(t, storeConfig.UseMetadataOffload, false)
|
||||
|
||||
logger.Infof("Node Config is %+v", n.Config)
|
||||
logger.Infof("storeConfig is %+v", storeConfig)
|
||||
|
||||
}
|
||||
|
||||
func TestHostNetwork(t *testing.T) {
|
||||
storageSpec := rookalpha.StorageScopeSpec{
|
||||
Nodes: []rookalpha.Node{
|
||||
{
|
||||
Name: "node1",
|
||||
Location: "zone1",
|
||||
Config: map[string]string{
|
||||
"rtTransport": "rtrd",
|
||||
"useAllSSD": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
clientset := fake.NewSimpleClientset()
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
storageSpec, "", *resource.NewQuantity(100000.0, resource.BinarySI),
|
||||
rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{ServerIfName: "eth0"}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
c.createSetupConfigs(false)
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package target
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/golang/glog"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/discover"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster/target/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/sys"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-target")
|
||||
|
||||
const (
|
||||
appName = "rook-edgefs-target"
|
||||
targetAppNameFmt = "rook-edgefs-target-id-%d"
|
||||
appNameFmt = "rook-edgefs-target-%s"
|
||||
targetLabelKey = "edgefs-target-id"
|
||||
defaultServiceAccountName = "rook-edgefs-cluster"
|
||||
unknownID = -1
|
||||
labelingRetries = 5
|
||||
|
||||
//deployment types
|
||||
deploymentRtlfs = "rtlfs"
|
||||
deploymentRtrd = "rtrd"
|
||||
deploymentAutoRtlfs = "autoRtlfs"
|
||||
nodeTypeLabelFmt = "%s-nodetype"
|
||||
)
|
||||
|
||||
// Cluster keeps track of the Targets
|
||||
type Cluster struct {
|
||||
context *clusterd.Context
|
||||
Namespace string
|
||||
placement rookalpha.Placement
|
||||
Version string
|
||||
Storage rookalpha.StorageScopeSpec
|
||||
dataDirHostPath string
|
||||
dataVolumeSize resource.Quantity
|
||||
HostNetworkSpec edgefsv1alpha1.NetworkSpec
|
||||
Privileged bool
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
serviceAccount string
|
||||
deploymentConfig ClusterDeploymentConfig
|
||||
}
|
||||
|
||||
type ClusterDeploymentConfig struct {
|
||||
deploymentType string //rtlfs, rtrd, autortlfs
|
||||
transportKey string //rtlfs or rtrd
|
||||
directories []RtlfsDevice //cluster wide directories
|
||||
devConfig map[string]DevicesConfig
|
||||
needPriviliges bool
|
||||
}
|
||||
|
||||
type DevicesConfig struct {
|
||||
rtrd RTDevices
|
||||
rtlfs RtlfsDevices
|
||||
isGatewayNode bool
|
||||
}
|
||||
|
||||
// New creates an instance of the Target manager
|
||||
func New(
|
||||
context *clusterd.Context,
|
||||
namespace,
|
||||
version,
|
||||
serviceAccount string,
|
||||
storageSpec rookalpha.StorageScopeSpec,
|
||||
dataDirHostPath string,
|
||||
dataVolumeSize resource.Quantity,
|
||||
placement rookalpha.Placement,
|
||||
hostNetworkSpec edgefsv1alpha1.NetworkSpec,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *Cluster {
|
||||
|
||||
if serviceAccount == "" {
|
||||
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
|
||||
serviceAccount = defaultServiceAccountName
|
||||
logger.Infof("setting the target pods to use the service account name: %s", serviceAccount)
|
||||
}
|
||||
return &Cluster{
|
||||
context: context,
|
||||
Namespace: namespace,
|
||||
serviceAccount: serviceAccount,
|
||||
placement: placement,
|
||||
Version: version,
|
||||
Storage: storageSpec,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
dataVolumeSize: dataVolumeSize,
|
||||
HostNetworkSpec: hostNetworkSpec,
|
||||
Privileged: (isHostNetworkDefined(hostNetworkSpec) || os.Getenv("ROOK_HOSTPATH_REQUIRES_PRIVILEGED") == "true"),
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
deploymentConfig: ClusterDeploymentConfig{devConfig: make(map[string]DevicesConfig, 0)},
|
||||
}
|
||||
}
|
||||
|
||||
type DevicesResurrectOptions struct {
|
||||
needToResurrect bool
|
||||
needToZap bool
|
||||
needToWait bool
|
||||
}
|
||||
|
||||
func ParseDevicesResurrectMode(resurrectMode string) DevicesResurrectOptions {
|
||||
drm := DevicesResurrectOptions{}
|
||||
if len(resurrectMode) == 0 {
|
||||
return drm
|
||||
}
|
||||
resurrectModeToLower := strings.ToLower(strings.TrimSpace(resurrectMode))
|
||||
switch resurrectModeToLower {
|
||||
case "restore":
|
||||
drm.needToResurrect = true
|
||||
break
|
||||
case "restorezap":
|
||||
drm.needToResurrect = true
|
||||
drm.needToZap = true
|
||||
break
|
||||
case "restorezapwait":
|
||||
drm.needToResurrect = true
|
||||
drm.needToZap = true
|
||||
drm.needToWait = true
|
||||
break
|
||||
}
|
||||
|
||||
return drm
|
||||
}
|
||||
|
||||
// Start the target management
|
||||
func (c *Cluster) Start(rookImage, devicesResurrectMode string) (err error) {
|
||||
logger.Infof("start running targets in namespace %s", c.Namespace)
|
||||
|
||||
logger.Infof("Target Image is %s", rookImage)
|
||||
if c.Storage.UseAllNodes == false && len(c.Storage.Nodes) == 0 {
|
||||
logger.Warningf("useAllNodes is set to false and no nodes are specified, no Edgefs pods are going to be created")
|
||||
}
|
||||
|
||||
//logger.Infof("Cluster storage config %+v", c.Storage.Config)
|
||||
if c.Storage.UseAllNodes {
|
||||
// resolve all storage nodes
|
||||
c.Storage.Nodes = nil
|
||||
rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
allNodeDevices, err := discover.ListDevices(c.context, rookSystemNS, "" /* all nodes */)
|
||||
//logger.Infof("allNodeDevices: %+v", allNodeDevices)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get storage nodes from namespace %s: %v", rookSystemNS, err)
|
||||
return err
|
||||
}
|
||||
for nodeName := range allNodeDevices {
|
||||
storageNode := rookalpha.Node{
|
||||
Name: nodeName,
|
||||
}
|
||||
c.Storage.Nodes = append(c.Storage.Nodes, storageNode)
|
||||
}
|
||||
}
|
||||
validNodes := k8sutil.GetValidNodes(c.Storage.Nodes, c.context.Clientset, c.placement)
|
||||
//logger.Infof("validNOdes: %+v", validNodes)
|
||||
logger.Infof("%d of the %d storage nodes are valid", len(validNodes), len(c.Storage.Nodes))
|
||||
c.Storage.Nodes = validNodes
|
||||
|
||||
rmo := ParseDevicesResurrectMode(devicesResurrectMode)
|
||||
logger.Infof("DevicesResurrect mode: %s options %+v", devicesResurrectMode, rmo)
|
||||
|
||||
err = c.createDeploymentConfig(rmo.needToResurrect)
|
||||
if err != nil {
|
||||
logger.Errorf("Failed to create deploymentConfig %+v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof("Deployment Config : %+v", c.deploymentConfig)
|
||||
|
||||
if err := c.createSetupConfigs(rmo.needToResurrect); err != nil {
|
||||
logger.Errorf("Failed to create/update Edgefs cluster configuration: %+v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
headlessService, _ := c.makeHeadlessService()
|
||||
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(headlessService); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return err
|
||||
}
|
||||
logger.Infof("headless service %s already exists in namespace %s", headlessService.Name, headlessService.Namespace)
|
||||
} else {
|
||||
logger.Infof("headless service %s started in namespace %s", headlessService.Name, headlessService.Namespace)
|
||||
}
|
||||
|
||||
statefulSet, _ := c.makeStatefulSet(int32(len(validNodes)), rookImage, rmo)
|
||||
if _, err := c.context.Clientset.AppsV1beta1().StatefulSets(c.Namespace).Create(statefulSet); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return err
|
||||
}
|
||||
logger.Infof("stateful set %s already exists in namespace %s", statefulSet.Name, statefulSet.Namespace)
|
||||
} else {
|
||||
logger.Infof("stateful set %s created in namespace %s", statefulSet.Name, statefulSet.Namespace)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cluster) HasDevicesSpecification() bool {
|
||||
|
||||
if len(c.Storage.DeviceFilter) > 0 || len(c.Storage.Devices) > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
for _, node := range c.Storage.Nodes {
|
||||
useAllDevices := node.UseAllDevices
|
||||
if useAllDevices != nil && *useAllDevices {
|
||||
return true
|
||||
}
|
||||
|
||||
if len(node.DeviceFilter) > 0 || len(node.Devices) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Cluster) createDeploymentConfig(resurrect bool) error {
|
||||
|
||||
//Fill deploymentConfig devices struct
|
||||
for _, node := range c.Storage.Nodes {
|
||||
n := c.resolveNode(node.Name)
|
||||
|
||||
if n == nil {
|
||||
return fmt.Errorf("node %s did not resolve to start target", node.Name)
|
||||
}
|
||||
|
||||
devicesConfig := DevicesConfig{}
|
||||
devicesConfig.rtrd.Devices = make([]RTDevice, 0)
|
||||
devicesConfig.rtlfs.Devices = make([]RtlfsDevice, 0)
|
||||
|
||||
// if node labeled as gateway then return empty devises and skip RTDevices detection
|
||||
if c.isGatewayLabeledNode(c.context.Clientset, node.Name) {
|
||||
devicesConfig.isGatewayNode = true
|
||||
c.deploymentConfig.devConfig[node.Name] = devicesConfig
|
||||
continue
|
||||
}
|
||||
|
||||
rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
nodeDevices, _ := discover.ListDevices(c.context, rookSystemNS, n.Name)
|
||||
|
||||
availDevs, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace,
|
||||
n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices())
|
||||
|
||||
if deviceErr != nil {
|
||||
// Devices were specified but we couldn't find any.
|
||||
// User needs to fix CRD.
|
||||
return fmt.Errorf("failed to get devices for node %s cluster %s: %v",
|
||||
n.Name, c.Namespace, deviceErr)
|
||||
}
|
||||
|
||||
// selects Disks from availDevs and translate to RTDevices
|
||||
availDisks := []sys.LocalDisk{}
|
||||
for _, dev := range availDevs {
|
||||
for _, disk := range nodeDevices[n.Name] {
|
||||
if disk.Name == dev.Name {
|
||||
availDisks = append(availDisks, disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
storeConfig := config.ToStoreConfig(n.Config)
|
||||
rtDevices, err := GetRTDevices(availDisks, &storeConfig)
|
||||
if err != nil {
|
||||
logger.Warningf("Can't get rtDevices for node %s due %v", n.Name, err)
|
||||
rtDevices = make([]RTDevice, 0)
|
||||
}
|
||||
|
||||
devicesConfig.rtrd.Devices = rtDevices
|
||||
devicesConfig.rtlfs.Devices = getRtlfsDevices(c.Storage.Directories)
|
||||
c.deploymentConfig.devConfig[node.Name] = devicesConfig
|
||||
}
|
||||
|
||||
// Add Directories to deploymentConfig
|
||||
c.deploymentConfig.directories = getRtlfsDevices(c.Storage.Directories)
|
||||
|
||||
if len(c.Storage.Directories) > 0 && (len(c.dataDirHostPath) > 0 || c.dataVolumeSize.Value() != 0) {
|
||||
c.deploymentConfig.deploymentType = deploymentRtlfs
|
||||
c.deploymentConfig.transportKey = "rtlfs"
|
||||
|
||||
// check directories devices count on all nodes
|
||||
if len(c.Storage.Directories)*len(c.Storage.Nodes) < 3 {
|
||||
return fmt.Errorf("Rtlfs devices should be more then 3 on all nodes summary")
|
||||
}
|
||||
|
||||
} else if c.HasDevicesSpecification() && (len(c.dataDirHostPath) > 0 || c.dataVolumeSize.Value() != 0) {
|
||||
|
||||
// Check all deployment nodes has available disk devices
|
||||
devicesCount := 0
|
||||
for nodeName, devCfg := range c.deploymentConfig.devConfig {
|
||||
|
||||
if devCfg.isGatewayNode {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(devCfg.rtrd.Devices) == 0 && !resurrect {
|
||||
return fmt.Errorf("Node %s has no available devices", nodeName)
|
||||
}
|
||||
devicesCount += len(devCfg.rtrd.Devices)
|
||||
}
|
||||
|
||||
// check new deployment devices count
|
||||
if !resurrect && devicesCount < 3 {
|
||||
return fmt.Errorf("Disk devices should be more then 3 on all nodes summary")
|
||||
}
|
||||
|
||||
c.deploymentConfig.deploymentType = deploymentRtrd
|
||||
c.deploymentConfig.transportKey = "rtrd"
|
||||
c.deploymentConfig.needPriviliges = true
|
||||
} else if len(c.dataDirHostPath) == 0 || c.dataVolumeSize.Value() == 0 {
|
||||
c.deploymentConfig.deploymentType = deploymentAutoRtlfs
|
||||
c.deploymentConfig.transportKey = "rtlfs"
|
||||
} else {
|
||||
return fmt.Errorf("Unknown deployment type! Cluster spec:\n %+v", c)
|
||||
}
|
||||
|
||||
//set priviliges==true in case of HostNetwork
|
||||
if len(c.HostNetworkSpec.ServerIfName) > 0 || len(c.HostNetworkSpec.BrokerIfName) > 0 {
|
||||
c.deploymentConfig.needPriviliges = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// creates a qualified name of the headless service for a given replica id and namespace,
|
||||
// e.g., edgefs-0.edgefs.rook-edgefs
|
||||
func createQualifiedHeadlessServiceName(replicaNum int, namespace string) string {
|
||||
return fmt.Sprintf("%s-%d.%s.%s", appName, replicaNum, appName, namespace)
|
||||
}
|
||||
|
||||
func (c *Cluster) resolveNode(nodeName string) *rookalpha.Node {
|
||||
// fully resolve the storage config and resources for this node
|
||||
rookNode := c.Storage.ResolveNode(nodeName)
|
||||
if rookNode == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Apply directories from ClusterStorageSpec only
|
||||
rookNode.Directories = c.Storage.Directories
|
||||
|
||||
rookNode.Resources = k8sutil.MergeResourceRequirements(rookNode.Resources, c.resources)
|
||||
|
||||
// ensure no invalid dirs are specified
|
||||
var validDirs []rookalpha.Directory
|
||||
for _, dir := range rookNode.Directories {
|
||||
if dir.Path == k8sutil.DataDir || dir.Path == c.dataDirHostPath {
|
||||
logger.Warningf("skipping directory %s that would conflict with the dataDirHostPath", dir.Path)
|
||||
continue
|
||||
}
|
||||
validDirs = append(validDirs, dir)
|
||||
}
|
||||
rookNode.Directories = validDirs
|
||||
|
||||
return rookNode
|
||||
}
|
||||
|
||||
func (c *Cluster) AddLabelsToNode(cs clientset.Interface, nodeName string, labels map[string]string) error {
|
||||
tokens := make([]string, 0, len(labels))
|
||||
for k, v := range labels {
|
||||
tokens = append(tokens, "\""+k+"\":\""+v+"\"")
|
||||
}
|
||||
labelString := "{" + strings.Join(tokens, ",") + "}"
|
||||
patch := fmt.Sprintf(`{"metadata":{"labels":%v}}`, labelString)
|
||||
var err error
|
||||
for attempt := 0; attempt < labelingRetries; attempt++ {
|
||||
_, err = cs.CoreV1().Nodes().Patch(nodeName, types.MergePatchType, []byte(patch))
|
||||
if err != nil {
|
||||
if !apierrs.IsConflict(err) {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// RemoveLabelOffNode is for cleaning up labels temporarily added to node,
|
||||
// won't fail if target label doesn't exist or has been removed.
|
||||
func (c *Cluster) RemoveLabelOffNode(cs clientset.Interface, nodeName string, labelKeys []string) error {
|
||||
var node *v1.Node
|
||||
var err error
|
||||
for attempt := 0; attempt < labelingRetries; attempt++ {
|
||||
node, err = cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if node.Labels == nil {
|
||||
return nil
|
||||
}
|
||||
for _, labelKey := range labelKeys {
|
||||
if node.Labels == nil || len(node.Labels[labelKey]) == 0 {
|
||||
break
|
||||
}
|
||||
delete(node.Labels, labelKey)
|
||||
}
|
||||
_, err = cs.CoreV1().Nodes().Update(node)
|
||||
if err != nil {
|
||||
if !apierrs.IsConflict(err) {
|
||||
return err
|
||||
} else {
|
||||
glog.V(2).Infof("Conflict when trying to remove a labels %v from %v", labelKeys, nodeName)
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Cluster) isGatewayLabeledNode(cs clientset.Interface, nodeName string) bool {
|
||||
labelMap, err := c.getNodeLabels(cs, nodeName)
|
||||
if err != nil || labelMap == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if nodeType, ok := labelMap[fmt.Sprintf(nodeTypeLabelFmt, c.Namespace)]; ok {
|
||||
if nodeType == "gateway" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Cluster) getNodeLabels(cs clientset.Interface, nodeName string) (map[string]string, error) {
|
||||
node, err := cs.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if node.Labels == nil {
|
||||
return node.Labels, nil
|
||||
}
|
||||
return node.Labels, nil
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package target
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestStart(t *testing.T) {
|
||||
rookImage := "edgefs/edgefs-operator"
|
||||
clientset := fake.NewSimpleClientset()
|
||||
volSize := resource.NewQuantity(100000.0, resource.BinarySI)
|
||||
c := New(&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook", Executor: &exectest.MockExecutor{}}, "ns", "myversion", "",
|
||||
rookalpha.StorageScopeSpec{}, "", *volSize, rookalpha.Placement{}, edgefsv1alpha1.NetworkSpec{}, v1.ResourceRequirements{}, metav1.OwnerReference{})
|
||||
|
||||
// Start the first time
|
||||
err := c.Start(rookImage, "")
|
||||
assert.Nil(t, err)
|
||||
|
||||
// Should not fail if it already exists
|
||||
err = c.Start(rookImage, "")
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
func createDiscoverConfigmap(nodeName, ns string, clientset *fake.Clientset) error {
|
||||
data := make(map[string]string, 1)
|
||||
data[discoverDaemon.LocalDiskCMData] = `[{"name":"sdx","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405f826bd553d8c4dbf9f41c18be /dev/disk/by-id/wwn-0x6001405f826bd553d8c4dbf9f41c18be /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-1","size":10737418240,"uuid":"","serial":"36001405f826bd553d8c4dbf9f41c18be","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk02","wwn":"0x6001405f826bd553","wwnVendorExtension":"0x6001405f826bd553d8c4dbf9f41c18be","empty":true}]`
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "local-device-" + nodeName,
|
||||
Namespace: ns,
|
||||
Labels: map[string]string{
|
||||
k8sutil.AppAttr: discoverDaemon.AppName,
|
||||
discoverDaemon.NodeAttr: nodeName,
|
||||
},
|
||||
},
|
||||
Data: data,
|
||||
}
|
||||
_, err := clientset.CoreV1().ConfigMaps(ns).Create(cm)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// scale-out, multi-cloud Edge-X ISCSI services controller
|
||||
package iscsi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"k8s.io/api/core/v1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
customResourceName = "iscsi"
|
||||
customResourceNamePlural = "iscsis"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-iscsi")
|
||||
|
||||
// ISCSIResource represents the iscsi custom resource
|
||||
var ISCSIResource = opkit.CustomResource{
|
||||
Name: customResourceName,
|
||||
Plural: customResourceNamePlural,
|
||||
Group: edgefsv1alpha1.CustomResourceGroup,
|
||||
Version: edgefsv1alpha1.Version,
|
||||
Scope: apiextensionsv1beta1.NamespaceScoped,
|
||||
Kind: reflect.TypeOf(edgefsv1alpha1.ISCSI{}).Name(),
|
||||
}
|
||||
|
||||
// ISCSIController represents a controller object for iscsi custom resources
|
||||
type ISCSIController struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
hostNetwork bool
|
||||
dataDirHostPath string
|
||||
dataVolumeSize resource.Quantity
|
||||
placement rookalpha.Placement
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// NewISCSIController create controller for watching ISCSI custom resources created
|
||||
func NewISCSIController(
|
||||
context *clusterd.Context, rookImage string,
|
||||
hostNetwork bool,
|
||||
dataDirHostPath string,
|
||||
dataVolumeSize resource.Quantity,
|
||||
placement rookalpha.Placement,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *ISCSIController {
|
||||
return &ISCSIController{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
hostNetwork: hostNetwork,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
dataVolumeSize: dataVolumeSize,
|
||||
placement: placement,
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
}
|
||||
|
||||
// StartWatch watches for instances of ISCSI custom resources and acts on them
|
||||
func (c *ISCSIController) StartWatch(namespace string, stopCh chan struct{}) error {
|
||||
|
||||
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onAdd,
|
||||
UpdateFunc: c.onUpdate,
|
||||
DeleteFunc: c.onDelete,
|
||||
}
|
||||
|
||||
logger.Infof("start watching iscsi resources in namespace %s", namespace)
|
||||
watcher := opkit.NewWatcher(ISCSIResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
|
||||
go watcher.Watch(&edgefsv1alpha1.ISCSI{}, stopCh)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ISCSIController) onAdd(obj interface{}) {
|
||||
iscsi, err := getISCSIObject(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get iscsi object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.CreateService(*iscsi, c.serviceOwners(iscsi)); err != nil {
|
||||
logger.Errorf("failed to create iscsi %s. %+v", iscsi.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ISCSIController) onUpdate(oldObj, newObj interface{}) {
|
||||
oldService, err := getISCSIObject(oldObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get old iscsi object: %+v", err)
|
||||
return
|
||||
}
|
||||
newService, err := getISCSIObject(newObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get new iscsi object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !serviceChanged(oldService.Spec, newService.Spec) {
|
||||
logger.Debugf("iscsi %s did not change", newService.Name)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("applying iscsi %s changes", newService.Name)
|
||||
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
|
||||
logger.Errorf("failed to create (modify) iscsi %s. %+v", newService.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ISCSIController) onDelete(obj interface{}) {
|
||||
iscsi, err := getISCSIObject(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get iscsi object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.DeleteService(*iscsi); err != nil {
|
||||
logger.Errorf("failed to delete iscsi %s. %+v", iscsi.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ISCSIController) serviceOwners(service *edgefsv1alpha1.ISCSI) []metav1.OwnerReference {
|
||||
// Only set the cluster crd as the owner of the ISCSI resources.
|
||||
// If the ISCSI crd is deleted, the operator will explicitly remove the ISCSI resources.
|
||||
// If the ISCSI crd still exists when the cluster crd is deleted, this will make sure the ISCSI
|
||||
// resources are cleaned up.
|
||||
return []metav1.OwnerReference{c.ownerRef}
|
||||
}
|
||||
|
||||
func serviceChanged(oldService, newService edgefsv1alpha1.ISCSISpec) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func getISCSIObject(obj interface{}) (iscsi *edgefsv1alpha1.ISCSI, err error) {
|
||||
var ok bool
|
||||
iscsi, ok = obj.(*edgefsv1alpha1.ISCSI)
|
||||
if ok {
|
||||
// the iscsi object is of the latest type, simply return it
|
||||
return iscsi.DeepCopy(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not a known iscsi object: %+v", obj)
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package ISCSI for the Edgefs manager.
|
||||
package iscsi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
appName = "rook-edgefs-iscsi"
|
||||
|
||||
/* Volumes definitions */
|
||||
defaultTargetName = "iqn.2018-11.edgefs.io"
|
||||
defaultTargetParams = "{}"
|
||||
dataVolumeName = "edgefs-datadir"
|
||||
stateVolumeFolder = ".state"
|
||||
etcVolumeFolder = ".etc"
|
||||
defaultPort = 3260
|
||||
)
|
||||
|
||||
// Start the ISCSI manager
|
||||
func (c *ISCSIController) CreateService(s edgefsv1alpha1.ISCSI, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, false, ownerRefs)
|
||||
}
|
||||
|
||||
func (c *ISCSIController) UpdateService(s edgefsv1alpha1.ISCSI, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, true, ownerRefs)
|
||||
}
|
||||
|
||||
// Start the iscsi instance
|
||||
func (c *ISCSIController) CreateOrUpdate(s edgefsv1alpha1.ISCSI, update bool, ownerRefs []metav1.OwnerReference) error {
|
||||
logger.Infof("starting update=%v service=%s", update, s.Name)
|
||||
|
||||
logger.Infof("ISCSI Base image is %s", c.rookImage)
|
||||
// validate ISCSI service settings
|
||||
if err := validateService(c.context, s); err != nil {
|
||||
return fmt.Errorf("invalid ISCSI service %s arguments. %+v", s.Name, err)
|
||||
}
|
||||
|
||||
if len(s.Spec.TargetName) == 0 {
|
||||
s.Spec.TargetName = defaultTargetName
|
||||
}
|
||||
|
||||
// check if ISCSI service already exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err == nil && exists {
|
||||
if !update {
|
||||
logger.Infof("ISCSI service %s exists in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("ISCSI service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
|
||||
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
|
||||
}
|
||||
logger.Infof("%s deployment already exists", appName)
|
||||
} else {
|
||||
logger.Infof("%s deployment started", appName)
|
||||
}
|
||||
|
||||
// create the iscsi service
|
||||
service := c.makeISCSIService(instanceName(s.Name), s.Name, s.Namespace, s.Spec)
|
||||
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create ISCSI service. %+v", err)
|
||||
}
|
||||
logger.Infof("ISCSI service %s already exists", service)
|
||||
} else {
|
||||
logger.Infof("ISCSI service %s started", service)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ISCSIController) makeISCSIService(name, svcname, namespace string, iscsiSpec edgefsv1alpha1.ISCSISpec) *v1.Service {
|
||||
labels := getLabels(name, svcname, namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeNodePort,
|
||||
Ports: []v1.ServicePort{
|
||||
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "port", Port: defaultPort, Protocol: v1.ProtocolTCP},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *ISCSIController) makeDeployment(svcname, namespace, rookImage string, iscsiSpec edgefsv1alpha1.ISCSISpec) *extensions.Deployment {
|
||||
|
||||
name := instanceName(svcname)
|
||||
volumes := []v1.Volume{}
|
||||
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
// dataVolume case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: dataVolumeName,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// dataDir case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: c.dataDirHostPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Labels: getLabels(name, svcname, namespace),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.iscsiContainer(svcname, name, rookImage, iscsiSpec)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: volumes,
|
||||
HostIPC: true,
|
||||
HostNetwork: c.hostNetwork,
|
||||
NodeSelector: map[string]string{namespace: "cluster"},
|
||||
},
|
||||
}
|
||||
if c.hostNetwork {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// apply current ISCSI CRD options to pod's specification
|
||||
iscsiSpec.Placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
instancesCount := int32(1)
|
||||
d := &extensions.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &instancesCount},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *ISCSIController) iscsiContainer(svcname, name, containerImage string, iscsiSpec edgefsv1alpha1.ISCSISpec) v1.Container {
|
||||
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
||||
},
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc.target",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: name,
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"iscsi"},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "CCOW_SVCNAME",
|
||||
Value: svcname,
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "KUBERNETES_NAMESPACE",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "metadata.namespace",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "EFSISCSI_TARGET_NAME",
|
||||
Value: iscsiSpec.TargetName,
|
||||
},
|
||||
{
|
||||
Name: "EFSISCSI_TARGET_PARAMS",
|
||||
Value: getTargetParamsJSON(iscsiSpec.TargetParams),
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
Resources: iscsiSpec.Resources,
|
||||
Ports: []v1.ContainerPort{
|
||||
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "port", ContainerPort: defaultPort, Protocol: v1.ProtocolTCP},
|
||||
},
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
|
||||
// Delete ISCSI service and possibly some artifacts.
|
||||
func (c *ISCSIController) DeleteService(s edgefsv1alpha1.ISCSI) error {
|
||||
// check if service exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect if there is a ISCSI service to delete. %+v", err)
|
||||
}
|
||||
if !exists {
|
||||
logger.Infof("ISCSI service %s does not exist in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("Deleting ISCSI service %s from namespace %s", s.Name, s.Namespace)
|
||||
|
||||
var gracePeriod int64
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
// Delete the iscsi service
|
||||
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete ISCSI service. %+v", err)
|
||||
}
|
||||
|
||||
// Make a best effort to delete the ISCSI pods
|
||||
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
|
||||
if err != nil {
|
||||
logger.Warningf(err.Error())
|
||||
}
|
||||
|
||||
logger.Infof("Completed deleting ISCSI service %s", s.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLabels(name, svcname, namespace string) map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: namespace,
|
||||
"edgefs_svcname": svcname,
|
||||
"edgefs_svctype": "iscsi",
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the ISCSI arguments
|
||||
func validateService(context *clusterd.Context, s edgefsv1alpha1.ISCSI) error {
|
||||
if s.Name == "" {
|
||||
return fmt.Errorf("missing name")
|
||||
}
|
||||
if s.Namespace == "" {
|
||||
return fmt.Errorf("missing namespace")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func instanceName(svcname string) string {
|
||||
return fmt.Sprintf("%s-%s", appName, svcname)
|
||||
}
|
||||
|
||||
// Check if the ISCSI service exists
|
||||
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.ISCSI) (bool, error) {
|
||||
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
// the deployment was found
|
||||
return true, nil
|
||||
}
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// not found
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func getTargetParamsJSON(params edgefsv1alpha1.TargetParametersSpec) string {
|
||||
result := make(map[string]uint)
|
||||
|
||||
if params.MaxRecvDataSegmentLength > 0 {
|
||||
result["MaxRecvDataSegmentLength"] = params.MaxRecvDataSegmentLength
|
||||
}
|
||||
|
||||
if params.DefaultTime2Retain > 0 {
|
||||
result["DefaultTime2Retain"] = params.DefaultTime2Retain
|
||||
}
|
||||
|
||||
if params.DefaultTime2Wait > 0 {
|
||||
result["DefaultTime2Wait"] = params.DefaultTime2Wait
|
||||
}
|
||||
|
||||
if params.FirstBurstLength > 0 {
|
||||
result["FirstBurstLength"] = params.FirstBurstLength
|
||||
}
|
||||
|
||||
if params.MaxBurstLength > 0 {
|
||||
result["MaxBurstLength"] = params.MaxBurstLength
|
||||
}
|
||||
|
||||
if params.MaxQueueCmd > 0 {
|
||||
result["MaxQueueCmd"] = params.MaxQueueCmd
|
||||
}
|
||||
|
||||
jsonString, _ := json.Marshal(result)
|
||||
return string(jsonString)
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// scale-out, multi-cloud NFS
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"k8s.io/api/core/v1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
customResourceName = "nfs"
|
||||
customResourceNamePlural = "nfss"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-nfs")
|
||||
|
||||
// NFSResource represents the nfs custom resource
|
||||
var NFSResource = opkit.CustomResource{
|
||||
Name: customResourceName,
|
||||
Plural: customResourceNamePlural,
|
||||
Group: edgefsv1alpha1.CustomResourceGroup,
|
||||
Version: edgefsv1alpha1.Version,
|
||||
Scope: apiextensionsv1beta1.NamespaceScoped,
|
||||
Kind: reflect.TypeOf(edgefsv1alpha1.NFS{}).Name(),
|
||||
}
|
||||
|
||||
// NFSController represents a controller object for nfs custom resources
|
||||
type NFSController struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
hostNetwork bool
|
||||
dataDirHostPath string
|
||||
dataVolumeSize resource.Quantity
|
||||
placement rookalpha.Placement
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// NewNFSController create controller for watching nfs custom resources created
|
||||
func NewNFSController(
|
||||
context *clusterd.Context, rookImage string,
|
||||
hostNetwork bool,
|
||||
dataDirHostPath string,
|
||||
dataVolumeSize resource.Quantity,
|
||||
placement rookalpha.Placement,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *NFSController {
|
||||
return &NFSController{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
hostNetwork: hostNetwork,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
dataVolumeSize: dataVolumeSize,
|
||||
placement: placement,
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
}
|
||||
|
||||
// StartWatch watches for instances of NFS custom resources and acts on them
|
||||
func (c *NFSController) StartWatch(namespace string, stopCh chan struct{}) error {
|
||||
|
||||
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onAdd,
|
||||
UpdateFunc: c.onUpdate,
|
||||
DeleteFunc: c.onDelete,
|
||||
}
|
||||
|
||||
logger.Infof("start watching nfs resources in namespace %s", namespace)
|
||||
watcher := opkit.NewWatcher(NFSResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
|
||||
go watcher.Watch(&edgefsv1alpha1.NFS{}, stopCh)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *NFSController) onAdd(obj interface{}) {
|
||||
nfs, err := getNFSObject(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get nfs object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.CreateService(*nfs, c.serviceOwners(nfs)); err != nil {
|
||||
logger.Errorf("failed to create nfs %s. %+v", nfs.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *NFSController) onUpdate(oldObj, newObj interface{}) {
|
||||
oldService, err := getNFSObject(oldObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get old nfs object: %+v", err)
|
||||
return
|
||||
}
|
||||
newService, err := getNFSObject(newObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get new nfs object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !serviceChanged(oldService.Spec, newService.Spec) {
|
||||
logger.Debugf("nfs %s did not change", newService.Name)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("applying nfs %s changes", newService.Name)
|
||||
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
|
||||
logger.Errorf("failed to create (modify) nfs %s. %+v", newService.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *NFSController) onDelete(obj interface{}) {
|
||||
nfs, err := getNFSObject(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get nfs object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.DeleteService(*nfs); err != nil {
|
||||
logger.Errorf("failed to delete nfs %s. %+v", nfs.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *NFSController) serviceOwners(service *edgefsv1alpha1.NFS) []metav1.OwnerReference {
|
||||
// Only set the cluster crd as the owner of the NFS resources.
|
||||
// If the NFS crd is deleted, the operator will explicitly remove the NFS resources.
|
||||
// If the NFS crd still exists when the cluster crd is deleted, this will make sure the NFS
|
||||
// resources are cleaned up.
|
||||
return []metav1.OwnerReference{c.ownerRef}
|
||||
}
|
||||
|
||||
func serviceChanged(oldService, newService edgefsv1alpha1.NFSSpec) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func getNFSObject(obj interface{}) (nfs *edgefsv1alpha1.NFS, err error) {
|
||||
var ok bool
|
||||
nfs, ok = obj.(*edgefsv1alpha1.NFS)
|
||||
if ok {
|
||||
// the nfs object is of the latest type, simply return it
|
||||
return nfs.DeepCopy(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not a known nfs object: %+v", obj)
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package nfs for the Edgefs manager.
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
appName = "rook-edgefs-nfs"
|
||||
|
||||
/* Volumes definitions */
|
||||
dataVolumeName = "edgefs-datadir"
|
||||
stateVolumeFolder = ".state"
|
||||
etcVolumeFolder = ".etc"
|
||||
)
|
||||
|
||||
// Start the rgw manager
|
||||
func (c *NFSController) CreateService(s edgefsv1alpha1.NFS, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, false, ownerRefs)
|
||||
}
|
||||
|
||||
func (c *NFSController) UpdateService(s edgefsv1alpha1.NFS, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, true, ownerRefs)
|
||||
}
|
||||
|
||||
// Start the nfs instance
|
||||
func (c *NFSController) CreateOrUpdate(s edgefsv1alpha1.NFS, update bool, ownerRefs []metav1.OwnerReference) error {
|
||||
logger.Infof("starting update=%v service=%s", update, s.Name)
|
||||
|
||||
logger.Infof("NFS Base image is %s", c.rookImage)
|
||||
// validate NFS service settings
|
||||
if err := validateService(c.context, s); err != nil {
|
||||
return fmt.Errorf("invalid NFS service %s arguments. %+v", s.Name, err)
|
||||
}
|
||||
|
||||
// check if NFS service already exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err == nil && exists {
|
||||
if !update {
|
||||
logger.Infof("NFS service %s exists in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("NFS service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
|
||||
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
|
||||
}
|
||||
logger.Infof("%s deployment already exists", appName)
|
||||
} else {
|
||||
logger.Infof("%s deployment started", appName)
|
||||
}
|
||||
|
||||
// create the nfs service
|
||||
service := c.makeNFSService(instanceName(s.Name), s.Name, s.Namespace)
|
||||
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create nfs service. %+v", err)
|
||||
}
|
||||
logger.Infof("nfs service %s already exists", service)
|
||||
} else {
|
||||
logger.Infof("nfs service %s started", service)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *NFSController) makeNFSService(name, svcname, namespace string) *v1.Service {
|
||||
labels := getLabels(name, svcname, namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeNodePort,
|
||||
Ports: []v1.ServicePort{
|
||||
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "nfs-tcp", Port: 2049, Protocol: v1.ProtocolTCP},
|
||||
{Name: "nfs-udp", Port: 2049, Protocol: v1.ProtocolUDP},
|
||||
{Name: "nlockmgr-tcp", Port: 32803, Protocol: v1.ProtocolTCP},
|
||||
{Name: "nlockmgr-udp", Port: 32803, Protocol: v1.ProtocolUDP},
|
||||
{Name: "mountd-tcp", Port: 20048, Protocol: v1.ProtocolTCP},
|
||||
{Name: "mountd-udp", Port: 20048, Protocol: v1.ProtocolUDP},
|
||||
{Name: "portmapper-tcp", Port: 111, Protocol: v1.ProtocolTCP},
|
||||
{Name: "portmapper-udp", Port: 111, Protocol: v1.ProtocolUDP},
|
||||
{Name: "statd-tcp", Port: 662, Protocol: v1.ProtocolTCP},
|
||||
{Name: "statd-udp", Port: 662, Protocol: v1.ProtocolUDP},
|
||||
{Name: "rquotad-tcp", Port: 875, Protocol: v1.ProtocolTCP},
|
||||
{Name: "rquotad-udp", Port: 875, Protocol: v1.ProtocolUDP},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *NFSController) makeDeployment(svcname, namespace, rookImage string, nfsSpec edgefsv1alpha1.NFSSpec) *extensions.Deployment {
|
||||
|
||||
name := instanceName(svcname)
|
||||
volumes := []v1.Volume{}
|
||||
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
// dataVolume case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: dataVolumeName,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// dataDir case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: c.dataDirHostPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Labels: getLabels(name, svcname, namespace),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.nfsContainer(svcname, name, rookImage, nfsSpec)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: volumes,
|
||||
HostIPC: true,
|
||||
HostNetwork: c.hostNetwork,
|
||||
NodeSelector: map[string]string{namespace: "cluster"},
|
||||
},
|
||||
}
|
||||
if c.hostNetwork {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// apply current NFS CRD options to pod's specification
|
||||
nfsSpec.Placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
d := &extensions.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &nfsSpec.Instances},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *NFSController) nfsContainer(svcname, name, containerImage string, nfsSpec edgefsv1alpha1.NFSSpec) v1.Container {
|
||||
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
||||
},
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: name,
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"nfs"},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "CCOW_SVCNAME",
|
||||
Value: svcname,
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
Resources: nfsSpec.Resources,
|
||||
Ports: []v1.ContainerPort{
|
||||
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "nfs-tcp", ContainerPort: 2049, Protocol: v1.ProtocolTCP},
|
||||
{Name: "nfs-udp", ContainerPort: 2049, Protocol: v1.ProtocolUDP},
|
||||
{Name: "nlockmgr-tcp", ContainerPort: 32803, Protocol: v1.ProtocolTCP},
|
||||
{Name: "nlockmgr-udp", ContainerPort: 32803, Protocol: v1.ProtocolUDP},
|
||||
{Name: "mountd-tcp", ContainerPort: 20048, Protocol: v1.ProtocolTCP},
|
||||
{Name: "mountd-udp", ContainerPort: 20048, Protocol: v1.ProtocolUDP},
|
||||
{Name: "portmapper-tcp", ContainerPort: 111, Protocol: v1.ProtocolTCP},
|
||||
{Name: "portmapper-udp", ContainerPort: 111, Protocol: v1.ProtocolUDP},
|
||||
{Name: "statd-tcp", ContainerPort: 662, Protocol: v1.ProtocolTCP},
|
||||
{Name: "statd-udp", ContainerPort: 662, Protocol: v1.ProtocolUDP},
|
||||
{Name: "rquotad-tcp", ContainerPort: 875, Protocol: v1.ProtocolTCP},
|
||||
{Name: "rquotad-udp", ContainerPort: 875, Protocol: v1.ProtocolUDP},
|
||||
},
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc.target",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Delete NFS service and possibly some artifacts.
|
||||
func (c *NFSController) DeleteService(s edgefsv1alpha1.NFS) error {
|
||||
// check if service exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect if there is a NFS service to delete. %+v", err)
|
||||
}
|
||||
if !exists {
|
||||
logger.Infof("NFS service %s does not exist in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("Deleting NFS service %s from namespace %s", s.Name, s.Namespace)
|
||||
|
||||
var gracePeriod int64
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
// Delete the nfs service
|
||||
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete NFS service. %+v", err)
|
||||
}
|
||||
|
||||
// Make a best effort to delete the NFS pods
|
||||
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
|
||||
if err != nil {
|
||||
logger.Warningf(err.Error())
|
||||
}
|
||||
|
||||
logger.Infof("Completed deleting NFS service %s", s.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLabels(name, svcname, namespace string) map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: namespace,
|
||||
"edgefs_svcname": svcname,
|
||||
"edgefs_svctype": "nfs",
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the NFS arguments
|
||||
func validateService(context *clusterd.Context, s edgefsv1alpha1.NFS) error {
|
||||
if s.Name == "" {
|
||||
return fmt.Errorf("missing name")
|
||||
}
|
||||
if s.Namespace == "" {
|
||||
return fmt.Errorf("missing namespace")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func instanceName(svcname string) string {
|
||||
return fmt.Sprintf("%s-%s", appName, svcname)
|
||||
}
|
||||
|
||||
// Check if the NFS service exists
|
||||
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.NFS) (bool, error) {
|
||||
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
// the deployment was found
|
||||
return true, nil
|
||||
}
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// not found
|
||||
return false, nil
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
Copyright 2018 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package operator to manage Kubernetes storage.
|
||||
package edgefs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/discover"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-operator")
|
||||
|
||||
type Operator struct {
|
||||
context *clusterd.Context
|
||||
resources []opkit.CustomResource
|
||||
rookImage string
|
||||
securityAccount string
|
||||
clusterController *cluster.ClusterController
|
||||
}
|
||||
|
||||
// New creates an operator instance
|
||||
func New(context *clusterd.Context, rookImage string, securityAccount string) *Operator {
|
||||
clusterController := cluster.NewClusterController(context, rookImage)
|
||||
|
||||
schemes := []opkit.CustomResource{cluster.ClusterResource}
|
||||
return &Operator{
|
||||
context: context,
|
||||
clusterController: clusterController,
|
||||
resources: schemes,
|
||||
rookImage: rookImage,
|
||||
securityAccount: securityAccount,
|
||||
}
|
||||
}
|
||||
|
||||
// Run the operator instance
|
||||
func (o *Operator) Run() error {
|
||||
|
||||
namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
if namespace == "" {
|
||||
return fmt.Errorf("Rook operator namespace is not provided. Expose it via downward API in the rook operator manifest file using environment variable %s", k8sutil.PodNamespaceEnvVar)
|
||||
}
|
||||
|
||||
rookDiscover := discover.New(o.context.Clientset)
|
||||
if err := rookDiscover.Start(namespace, o.rookImage, o.securityAccount); err != nil {
|
||||
return fmt.Errorf("Error starting device discovery daemonset: %v", err)
|
||||
}
|
||||
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
stopChan := make(chan struct{})
|
||||
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// watch for changes to the edgefs clusters
|
||||
o.clusterController.StartWatch(v1.NamespaceAll, stopChan)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-signalChan:
|
||||
logger.Infof("shutdown signal received, exiting...")
|
||||
close(stopChan)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package edgefs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/edgefs/cluster"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestOperator(t *testing.T) {
|
||||
clientset := test.New(3)
|
||||
context := &clusterd.Context{Clientset: clientset}
|
||||
o := New(context, "", "")
|
||||
|
||||
assert.NotNil(t, o)
|
||||
assert.NotNil(t, o.clusterController)
|
||||
assert.NotNil(t, o.resources)
|
||||
assert.Equal(t, context, o.context)
|
||||
assert.Equal(t, len(o.resources), 1)
|
||||
for _, r := range o.resources {
|
||||
if r.Name != cluster.ClusterResource.Name {
|
||||
assert.Fail(t, fmt.Sprintf("Resource %s is not valid", r.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// scale-out, multi-cloud Edge-X S3 services controller
|
||||
package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"k8s.io/api/core/v1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
customResourceName = "s3"
|
||||
customResourceNamePlural = "s3s"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-s3")
|
||||
|
||||
// S3Resource represents the s3 custom resource
|
||||
var S3Resource = opkit.CustomResource{
|
||||
Name: customResourceName,
|
||||
Plural: customResourceNamePlural,
|
||||
Group: edgefsv1alpha1.CustomResourceGroup,
|
||||
Version: edgefsv1alpha1.Version,
|
||||
Scope: apiextensionsv1beta1.NamespaceScoped,
|
||||
Kind: reflect.TypeOf(edgefsv1alpha1.S3{}).Name(),
|
||||
}
|
||||
|
||||
// S3Controller represents a controller object for s3 custom resources
|
||||
type S3Controller struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
hostNetwork bool
|
||||
dataDirHostPath string
|
||||
dataVolumeSize resource.Quantity
|
||||
placement rookalpha.Placement
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// NewS3Controller create controller for watching S3 custom resources created
|
||||
func NewS3Controller(
|
||||
context *clusterd.Context, rookImage string,
|
||||
hostNetwork bool,
|
||||
dataDirHostPath string,
|
||||
dataVolumeSize resource.Quantity,
|
||||
placement rookalpha.Placement,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *S3Controller {
|
||||
return &S3Controller{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
hostNetwork: hostNetwork,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
dataVolumeSize: dataVolumeSize,
|
||||
placement: placement,
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
}
|
||||
|
||||
// StartWatch watches for instances of S3 custom resources and acts on them
|
||||
func (c *S3Controller) StartWatch(namespace string, stopCh chan struct{}) error {
|
||||
|
||||
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onAdd,
|
||||
UpdateFunc: c.onUpdate,
|
||||
DeleteFunc: c.onDelete,
|
||||
}
|
||||
|
||||
logger.Infof("start watching s3 resources in namespace %s", namespace)
|
||||
watcher := opkit.NewWatcher(S3Resource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
|
||||
go watcher.Watch(&edgefsv1alpha1.S3{}, stopCh)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *S3Controller) onAdd(obj interface{}) {
|
||||
s3, err := getS3Object(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get s3 object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.CreateService(*s3, c.serviceOwners(s3)); err != nil {
|
||||
logger.Errorf("failed to create s3 %s. %+v", s3.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3Controller) onUpdate(oldObj, newObj interface{}) {
|
||||
oldService, err := getS3Object(oldObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get old s3 object: %+v", err)
|
||||
return
|
||||
}
|
||||
newService, err := getS3Object(newObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get new s3 object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !serviceChanged(oldService.Spec, newService.Spec) {
|
||||
logger.Debugf("s3 %s did not change", newService.Name)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("applying s3 %s changes", newService.Name)
|
||||
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
|
||||
logger.Errorf("failed to create (modify) s3 %s. %+v", newService.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3Controller) onDelete(obj interface{}) {
|
||||
s3, err := getS3Object(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get s3 object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.DeleteService(*s3); err != nil {
|
||||
logger.Errorf("failed to delete s3 %s. %+v", s3.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3Controller) serviceOwners(service *edgefsv1alpha1.S3) []metav1.OwnerReference {
|
||||
// Only set the cluster crd as the owner of the S3 resources.
|
||||
// If the S3 crd is deleted, the operator will explicitly remove the S3 resources.
|
||||
// If the S3 crd still exists when the cluster crd is deleted, this will make sure the S3
|
||||
// resources are cleaned up.
|
||||
return []metav1.OwnerReference{c.ownerRef}
|
||||
}
|
||||
|
||||
func serviceChanged(oldService, newService edgefsv1alpha1.S3Spec) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func getS3Object(obj interface{}) (s3 *edgefsv1alpha1.S3, err error) {
|
||||
var ok bool
|
||||
s3, ok = obj.(*edgefsv1alpha1.S3)
|
||||
if ok {
|
||||
// the s3 object is of the latest type, simply return it
|
||||
return s3.DeepCopy(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not a known s3 object: %+v", obj)
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package S3 for the Edgefs manager.
|
||||
package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
appName = "rook-edgefs-s3"
|
||||
|
||||
/* Volumes definitions */
|
||||
sslCertVolumeName = "ssl-cert-volume"
|
||||
sslMountPath = "/opt/nedge/etc/ssl/"
|
||||
dataVolumeName = "edgefs-datadir"
|
||||
stateVolumeFolder = ".state"
|
||||
etcVolumeFolder = ".etc"
|
||||
defaultPort = 9982
|
||||
defaultSecurePort = 8443
|
||||
)
|
||||
|
||||
// Start the S3 manager
|
||||
func (c *S3Controller) CreateService(s edgefsv1alpha1.S3, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, false, ownerRefs)
|
||||
}
|
||||
|
||||
func (c *S3Controller) UpdateService(s edgefsv1alpha1.S3, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, true, ownerRefs)
|
||||
}
|
||||
|
||||
// Start the s3 instance
|
||||
func (c *S3Controller) CreateOrUpdate(s edgefsv1alpha1.S3, update bool, ownerRefs []metav1.OwnerReference) error {
|
||||
logger.Infof("starting update=%v service=%s", update, s.Name)
|
||||
|
||||
logger.Infof("S3 Base image is %s", c.rookImage)
|
||||
// validate S3 service settings
|
||||
if err := validateService(c.context, s); err != nil {
|
||||
return fmt.Errorf("invalid S3 service %s arguments. %+v", s.Name, err)
|
||||
}
|
||||
|
||||
//check http settings
|
||||
if s.Spec.Port == 0 {
|
||||
s.Spec.Port = defaultPort
|
||||
}
|
||||
|
||||
if s.Spec.SecurePort == 0 {
|
||||
s.Spec.SecurePort = defaultSecurePort
|
||||
}
|
||||
|
||||
// check if S3 service already exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err == nil && exists {
|
||||
if !update {
|
||||
logger.Infof("S3 service %s exists in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("S3 service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
|
||||
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
|
||||
}
|
||||
logger.Infof("%s deployment already exists", appName)
|
||||
} else {
|
||||
logger.Infof("%s deployment started", appName)
|
||||
}
|
||||
|
||||
// create the s3 service
|
||||
service := c.makeS3Service(instanceName(s.Name), s.Name, s.Namespace, s.Spec)
|
||||
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create s3 service. %+v", err)
|
||||
}
|
||||
logger.Infof("s3 service %s already exists", service)
|
||||
} else {
|
||||
logger.Infof("s3 service %s started", service)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *S3Controller) makeS3Service(name, svcname, namespace string, s3Spec edgefsv1alpha1.S3Spec) *v1.Service {
|
||||
labels := getLabels(name, svcname, namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeNodePort,
|
||||
Ports: []v1.ServicePort{
|
||||
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "port", Port: int32(s3Spec.Port), Protocol: v1.ProtocolTCP},
|
||||
{Name: "secure-port", Port: int32(s3Spec.SecurePort), Protocol: v1.ProtocolTCP},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *S3Controller) makeDeployment(svcname, namespace, rookImage string, s3Spec edgefsv1alpha1.S3Spec) *extensions.Deployment {
|
||||
|
||||
name := instanceName(svcname)
|
||||
volumes := []v1.Volume{}
|
||||
|
||||
// add ssl certificate volume if defined
|
||||
if len(s3Spec.SSLCertificateRef) > 0 {
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: sslCertVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
Secret: &v1.SecretVolumeSource{
|
||||
SecretName: s3Spec.SSLCertificateRef,
|
||||
Items: []v1.KeyToPath{
|
||||
{
|
||||
Key: "sslkey",
|
||||
Path: "ssl.key",
|
||||
},
|
||||
{
|
||||
Key: "sslcert",
|
||||
Path: "ssl.crt",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
// dataVolume case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: dataVolumeName,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// dataDir case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: c.dataDirHostPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Labels: getLabels(name, svcname, namespace),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.s3Container(svcname, name, rookImage, s3Spec)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: volumes,
|
||||
HostIPC: true,
|
||||
HostNetwork: c.hostNetwork,
|
||||
NodeSelector: map[string]string{namespace: "cluster"},
|
||||
},
|
||||
}
|
||||
if c.hostNetwork {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// apply current S3 CRD options to pod's specification
|
||||
s3Spec.Placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
d := &extensions.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &s3Spec.Instances},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *S3Controller) s3Container(svcname, name, containerImage string, s3Spec edgefsv1alpha1.S3Spec) v1.Container {
|
||||
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
||||
},
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc.target",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
}
|
||||
|
||||
if len(s3Spec.SSLCertificateRef) > 0 {
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath})
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: name,
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"s3"},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "CCOW_SVCNAME",
|
||||
Value: svcname,
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "EFSS3_HTTP_PORT",
|
||||
Value: fmt.Sprint(s3Spec.Port),
|
||||
},
|
||||
{
|
||||
Name: "EFSS3_HTTPS_PORT",
|
||||
Value: fmt.Sprint(s3Spec.SecurePort),
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
Resources: s3Spec.Resources,
|
||||
Ports: []v1.ContainerPort{
|
||||
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "port", ContainerPort: int32(s3Spec.Port), Protocol: v1.ProtocolTCP},
|
||||
{Name: "secure-port", ContainerPort: int32(s3Spec.SecurePort), Protocol: v1.ProtocolTCP},
|
||||
},
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
|
||||
// Delete S3 service and possibly some artifacts.
|
||||
func (c *S3Controller) DeleteService(s edgefsv1alpha1.S3) error {
|
||||
// check if service exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect if there is a S3 service to delete. %+v", err)
|
||||
}
|
||||
if !exists {
|
||||
logger.Infof("S3 service %s does not exist in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("Deleting S3 service %s from namespace %s", s.Name, s.Namespace)
|
||||
|
||||
var gracePeriod int64
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
// Delete the s3 service
|
||||
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete S3 service. %+v", err)
|
||||
}
|
||||
|
||||
// Make a best effort to delete the S3 pods
|
||||
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
|
||||
if err != nil {
|
||||
logger.Warningf(err.Error())
|
||||
}
|
||||
|
||||
logger.Infof("Completed deleting S3 service %s", s.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLabels(name, svcname, namespace string) map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: namespace,
|
||||
"edgefs_svcname": svcname,
|
||||
"edgefs_svctype": "s3",
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the S3 arguments
|
||||
func validateService(context *clusterd.Context, s edgefsv1alpha1.S3) error {
|
||||
if s.Name == "" {
|
||||
return fmt.Errorf("missing name")
|
||||
}
|
||||
if s.Namespace == "" {
|
||||
return fmt.Errorf("missing namespace")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func instanceName(svcname string) string {
|
||||
return fmt.Sprintf("%s-%s", appName, svcname)
|
||||
}
|
||||
|
||||
// Check if the S3 service exists
|
||||
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.S3) (bool, error) {
|
||||
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
// the deployment was found
|
||||
return true, nil
|
||||
}
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// not found
|
||||
return false, nil
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// scale-out, multi-cloud Edge-X S3 services controller
|
||||
package s3x
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
opkit "github.com/rook/operator-kit"
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"k8s.io/api/core/v1"
|
||||
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
)
|
||||
|
||||
const (
|
||||
customResourceName = "s3x"
|
||||
customResourceNamePlural = "s3xs"
|
||||
)
|
||||
|
||||
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-s3x")
|
||||
|
||||
// S3XResource represents the s3x custom resource
|
||||
var S3XResource = opkit.CustomResource{
|
||||
Name: customResourceName,
|
||||
Plural: customResourceNamePlural,
|
||||
Group: edgefsv1alpha1.CustomResourceGroup,
|
||||
Version: edgefsv1alpha1.Version,
|
||||
Scope: apiextensionsv1beta1.NamespaceScoped,
|
||||
Kind: reflect.TypeOf(edgefsv1alpha1.S3X{}).Name(),
|
||||
}
|
||||
|
||||
// S3XController represents a controller object for s3x custom resources
|
||||
type S3XController struct {
|
||||
context *clusterd.Context
|
||||
rookImage string
|
||||
hostNetwork bool
|
||||
dataDirHostPath string
|
||||
dataVolumeSize resource.Quantity
|
||||
placement rookalpha.Placement
|
||||
resources v1.ResourceRequirements
|
||||
ownerRef metav1.OwnerReference
|
||||
}
|
||||
|
||||
// NewS3XController create controller for watching S3X custom resources created
|
||||
func NewS3XController(
|
||||
context *clusterd.Context, rookImage string,
|
||||
hostNetwork bool,
|
||||
dataDirHostPath string,
|
||||
dataVolumeSize resource.Quantity,
|
||||
placement rookalpha.Placement,
|
||||
resources v1.ResourceRequirements,
|
||||
ownerRef metav1.OwnerReference,
|
||||
) *S3XController {
|
||||
return &S3XController{
|
||||
context: context,
|
||||
rookImage: rookImage,
|
||||
hostNetwork: hostNetwork,
|
||||
dataDirHostPath: dataDirHostPath,
|
||||
dataVolumeSize: dataVolumeSize,
|
||||
placement: placement,
|
||||
resources: resources,
|
||||
ownerRef: ownerRef,
|
||||
}
|
||||
}
|
||||
|
||||
// StartWatch watches for instances of S3X custom resources and acts on them
|
||||
func (c *S3XController) StartWatch(namespace string, stopCh chan struct{}) error {
|
||||
|
||||
resourceHandlerFuncs := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.onAdd,
|
||||
UpdateFunc: c.onUpdate,
|
||||
DeleteFunc: c.onDelete,
|
||||
}
|
||||
|
||||
logger.Infof("start watching s3x resources in namespace %s", namespace)
|
||||
watcher := opkit.NewWatcher(S3XResource, namespace, resourceHandlerFuncs, c.context.RookClientset.EdgefsV1alpha1().RESTClient())
|
||||
go watcher.Watch(&edgefsv1alpha1.S3X{}, stopCh)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *S3XController) onAdd(obj interface{}) {
|
||||
s3x, err := getS3XObject(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get s3x object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.CreateService(*s3x, c.serviceOwners(s3x)); err != nil {
|
||||
logger.Errorf("failed to create s3x %s. %+v", s3x.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3XController) onUpdate(oldObj, newObj interface{}) {
|
||||
oldService, err := getS3XObject(oldObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get old s3x object: %+v", err)
|
||||
return
|
||||
}
|
||||
newService, err := getS3XObject(newObj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get new s3x object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !serviceChanged(oldService.Spec, newService.Spec) {
|
||||
logger.Debugf("s3x %s did not change", newService.Name)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof("applying s3x %s changes", newService.Name)
|
||||
if err = c.UpdateService(*newService, c.serviceOwners(newService)); err != nil {
|
||||
logger.Errorf("failed to create (modify) s3x %s. %+v", newService.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3XController) onDelete(obj interface{}) {
|
||||
s3x, err := getS3XObject(obj)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get s3x object: %+v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = c.DeleteService(*s3x); err != nil {
|
||||
logger.Errorf("failed to delete s3x %s. %+v", s3x.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *S3XController) serviceOwners(service *edgefsv1alpha1.S3X) []metav1.OwnerReference {
|
||||
// Only set the cluster crd as the owner of the S3X resources.
|
||||
// If the S3X crd is deleted, the operator will explicitly remove the S3X resources.
|
||||
// If the S3X crd still exists when the cluster crd is deleted, this will make sure the S3X
|
||||
// resources are cleaned up.
|
||||
return []metav1.OwnerReference{c.ownerRef}
|
||||
}
|
||||
|
||||
func serviceChanged(oldService, newService edgefsv1alpha1.S3XSpec) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func getS3XObject(obj interface{}) (s3x *edgefsv1alpha1.S3X, err error) {
|
||||
var ok bool
|
||||
s3x, ok = obj.(*edgefsv1alpha1.S3X)
|
||||
if ok {
|
||||
// the s3x object is of the latest type, simply return it
|
||||
return s3x.DeepCopy(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("not a known s3x object: %+v", obj)
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
/*
|
||||
Copyright 2016 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package s3x for the Edgefs manager.
|
||||
package s3x
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
edgefsv1alpha1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1alpha1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
extensions "k8s.io/api/extensions/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
appName = "rook-edgefs-s3x"
|
||||
|
||||
/* Volumes definitions */
|
||||
sslCertVolumeName = "ssl-cert-volume"
|
||||
sslMountPath = "/opt/nedge/etc/ssl/"
|
||||
dataVolumeName = "edgefs-datadir"
|
||||
stateVolumeFolder = ".state"
|
||||
etcVolumeFolder = ".etc"
|
||||
defaultPort = 3000
|
||||
defaultSecurePort = 3001
|
||||
)
|
||||
|
||||
// Start the rgw manager
|
||||
func (c *S3XController) CreateService(s edgefsv1alpha1.S3X, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, false, ownerRefs)
|
||||
}
|
||||
|
||||
func (c *S3XController) UpdateService(s edgefsv1alpha1.S3X, ownerRefs []metav1.OwnerReference) error {
|
||||
return c.CreateOrUpdate(s, true, ownerRefs)
|
||||
}
|
||||
|
||||
// Start the s3x instance
|
||||
func (c *S3XController) CreateOrUpdate(s edgefsv1alpha1.S3X, update bool, ownerRefs []metav1.OwnerReference) error {
|
||||
logger.Infof("starting update=%v service=%s", update, s.Name)
|
||||
|
||||
logger.Infof("S3X Base image is %s", c.rookImage)
|
||||
// validate S3X service settings
|
||||
if err := validateService(c.context, s); err != nil {
|
||||
return fmt.Errorf("invalid S3X service %s arguments. %+v", s.Name, err)
|
||||
}
|
||||
|
||||
//check http settings
|
||||
if s.Spec.Port == 0 {
|
||||
s.Spec.Port = defaultPort
|
||||
}
|
||||
|
||||
if s.Spec.SecurePort == 0 {
|
||||
s.Spec.SecurePort = defaultSecurePort
|
||||
}
|
||||
|
||||
// check if S3X service already exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err == nil && exists {
|
||||
if !update {
|
||||
logger.Infof("S3X service %s exists in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
logger.Infof("S3X service %s exists in namespace %s. checking for updates", s.Name, s.Namespace)
|
||||
}
|
||||
|
||||
// start the deployment
|
||||
deployment := c.makeDeployment(s.Name, s.Namespace, c.rookImage, s.Spec)
|
||||
if _, err := c.context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Create(deployment); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
|
||||
}
|
||||
logger.Infof("%s deployment already exists", appName)
|
||||
} else {
|
||||
logger.Infof("%s deployment started", appName)
|
||||
}
|
||||
|
||||
// create the s3x service
|
||||
service := c.makeS3XService(instanceName(s.Name), s.Name, s.Namespace, s.Spec)
|
||||
if _, err := c.context.Clientset.CoreV1().Services(s.Namespace).Create(service); err != nil {
|
||||
if !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("failed to create s3x service. %+v", err)
|
||||
}
|
||||
logger.Infof("s3x service %s already exists", service)
|
||||
} else {
|
||||
logger.Infof("s3x service %s started", service)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *S3XController) makeS3XService(name, svcname, namespace string, s3xSpec edgefsv1alpha1.S3XSpec) *v1.Service {
|
||||
labels := getLabels(name, svcname, namespace)
|
||||
svc := &v1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Spec: v1.ServiceSpec{
|
||||
Selector: labels,
|
||||
Type: v1.ServiceTypeNodePort,
|
||||
Ports: []v1.ServicePort{
|
||||
{Name: "grpc", Port: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "port", Port: int32(s3xSpec.Port), Protocol: v1.ProtocolTCP},
|
||||
{Name: "secure-port", Port: int32(s3xSpec.SecurePort), Protocol: v1.ProtocolTCP},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &svc.ObjectMeta, &c.ownerRef)
|
||||
return svc
|
||||
}
|
||||
|
||||
func (c *S3XController) makeDeployment(svcname, namespace, rookImage string, s3xSpec edgefsv1alpha1.S3XSpec) *extensions.Deployment {
|
||||
|
||||
name := instanceName(svcname)
|
||||
volumes := []v1.Volume{}
|
||||
|
||||
// add ssl certificate volume if defined
|
||||
if len(s3xSpec.SSLCertificateRef) > 0 {
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: sslCertVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
Secret: &v1.SecretVolumeSource{
|
||||
SecretName: s3xSpec.SSLCertificateRef,
|
||||
Items: []v1.KeyToPath{
|
||||
{
|
||||
Key: "sslkey",
|
||||
Path: "ssl.key",
|
||||
},
|
||||
{
|
||||
Key: "sslcert",
|
||||
Path: "ssl.crt",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if c.dataVolumeSize.Value() > 0 {
|
||||
// dataVolume case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: dataVolumeName,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// dataDir case
|
||||
volumes = append(volumes, v1.Volume{
|
||||
Name: dataVolumeName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
HostPath: &v1.HostPathVolumeSource{
|
||||
Path: c.dataDirHostPath,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
podSpec := v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Labels: getLabels(name, svcname, namespace),
|
||||
},
|
||||
Spec: v1.PodSpec{
|
||||
Containers: []v1.Container{c.s3xContainer(svcname, name, rookImage, s3xSpec)},
|
||||
RestartPolicy: v1.RestartPolicyAlways,
|
||||
Volumes: volumes,
|
||||
HostIPC: true,
|
||||
HostNetwork: c.hostNetwork,
|
||||
NodeSelector: map[string]string{namespace: "cluster"},
|
||||
},
|
||||
}
|
||||
if c.hostNetwork {
|
||||
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
||||
}
|
||||
|
||||
// apply current S3X CRD options to pod's specification
|
||||
s3xSpec.Placement.ApplyToPodSpec(&podSpec.Spec)
|
||||
|
||||
d := &extensions.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: extensions.DeploymentSpec{Template: podSpec, Replicas: &s3xSpec.Instances},
|
||||
}
|
||||
k8sutil.SetOwnerRef(c.context.Clientset, namespace, &d.ObjectMeta, &c.ownerRef)
|
||||
return d
|
||||
}
|
||||
|
||||
func (c *S3XController) s3xContainer(svcname, name, containerImage string, s3xSpec edgefsv1alpha1.S3XSpec) v1.Container {
|
||||
|
||||
runAsUser := int64(0)
|
||||
readOnlyRootFilesystem := false
|
||||
securityContext := &v1.SecurityContext{
|
||||
RunAsUser: &runAsUser,
|
||||
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
||||
Capabilities: &v1.Capabilities{
|
||||
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
||||
},
|
||||
}
|
||||
|
||||
volumeMounts := []v1.VolumeMount{
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/etc.target",
|
||||
SubPath: etcVolumeFolder,
|
||||
},
|
||||
{
|
||||
Name: dataVolumeName,
|
||||
MountPath: "/opt/nedge/var/run",
|
||||
SubPath: stateVolumeFolder,
|
||||
},
|
||||
}
|
||||
|
||||
if len(s3xSpec.SSLCertificateRef) > 0 {
|
||||
volumeMounts = append(volumeMounts, v1.VolumeMount{Name: sslCertVolumeName, MountPath: sslMountPath})
|
||||
}
|
||||
|
||||
return v1.Container{
|
||||
Name: name,
|
||||
Image: containerImage,
|
||||
ImagePullPolicy: v1.PullAlways,
|
||||
Args: []string{"s3x"},
|
||||
Env: []v1.EnvVar{
|
||||
{
|
||||
Name: "CCOW_LOG_LEVEL",
|
||||
Value: "5",
|
||||
},
|
||||
{
|
||||
Name: "CCOW_SVCNAME",
|
||||
Value: svcname,
|
||||
},
|
||||
{
|
||||
Name: "HOST_HOSTNAME",
|
||||
ValueFrom: &v1.EnvVarSource{
|
||||
FieldRef: &v1.ObjectFieldSelector{
|
||||
FieldPath: "spec.nodeName",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "EFSS3X_HTTP_PORT",
|
||||
Value: fmt.Sprint(s3xSpec.Port),
|
||||
},
|
||||
{
|
||||
Name: "EFSS3X_HTTPS_PORT",
|
||||
Value: fmt.Sprint(s3xSpec.SecurePort),
|
||||
},
|
||||
},
|
||||
SecurityContext: securityContext,
|
||||
Resources: s3xSpec.Resources,
|
||||
Ports: []v1.ContainerPort{
|
||||
{Name: "grpc", ContainerPort: 49000, Protocol: v1.ProtocolTCP},
|
||||
{Name: "port", ContainerPort: int32(s3xSpec.Port), Protocol: v1.ProtocolTCP},
|
||||
{Name: "secure-port", ContainerPort: int32(s3xSpec.SecurePort), Protocol: v1.ProtocolTCP},
|
||||
},
|
||||
VolumeMounts: volumeMounts,
|
||||
}
|
||||
}
|
||||
|
||||
// Delete S3X service and possibly some artifacts.
|
||||
func (c *S3XController) DeleteService(s edgefsv1alpha1.S3X) error {
|
||||
// check if service exists
|
||||
exists, err := serviceExists(c.context, s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect if there is a S3X service to delete. %+v", err)
|
||||
}
|
||||
if !exists {
|
||||
logger.Infof("S3X service %s does not exist in namespace %s", s.Name, s.Namespace)
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof("Deleting S3X service %s from namespace %s", s.Name, s.Namespace)
|
||||
|
||||
var gracePeriod int64
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
|
||||
// Delete the s3x service
|
||||
err = c.context.Clientset.CoreV1().Services(s.Namespace).Delete(instanceName(s.Name), options)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
logger.Warningf("failed to delete S3X service. %+v", err)
|
||||
}
|
||||
|
||||
// Make a best effort to delete the S3X pods
|
||||
err = k8sutil.DeleteDeployment(c.context.Clientset, s.Namespace, instanceName(s.Name))
|
||||
if err != nil {
|
||||
logger.Warningf(err.Error())
|
||||
}
|
||||
|
||||
logger.Infof("Completed deleting S3X service %s", s.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLabels(name, svcname, namespace string) map[string]string {
|
||||
return map[string]string{
|
||||
k8sutil.AppAttr: appName,
|
||||
k8sutil.ClusterAttr: namespace,
|
||||
"edgefs_svcname": svcname,
|
||||
"edgefs_svctype": "s3x",
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the S3X arguments
|
||||
func validateService(context *clusterd.Context, s edgefsv1alpha1.S3X) error {
|
||||
if s.Name == "" {
|
||||
return fmt.Errorf("missing name")
|
||||
}
|
||||
if s.Namespace == "" {
|
||||
return fmt.Errorf("missing namespace")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func instanceName(svcname string) string {
|
||||
return fmt.Sprintf("%s-%s", appName, svcname)
|
||||
}
|
||||
|
||||
// Check if the S3X service exists
|
||||
func serviceExists(context *clusterd.Context, s edgefsv1alpha1.S3X) (bool, error) {
|
||||
_, err := context.Clientset.ExtensionsV1beta1().Deployments(s.Namespace).Get(instanceName(s.Name), metav1.GetOptions{})
|
||||
if err == nil {
|
||||
// the deployment was found
|
||||
return true, nil
|
||||
}
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// not found
|
||||
return false, nil
|
||||
}
|
||||
Reference in new issue
Block a user