The rook types used across the storage providers moved from the v1alpha2 package to the v1 package. This commit points the packages at their new location. Implementation is expected to remain unchanged. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
630 lines
16 KiB
Go
630 lines
16 KiB
Go
/*
|
|
Copyright 2019 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
// Package mgr for the Edgefs manager.
|
|
package mgr
|
|
|
|
import (
|
|
"fmt"
|
|
"net"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
edgefsv1 "github.com/rook/rook/pkg/apis/edgefs.rook.io/v1"
|
|
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
apps "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
)
|
|
|
|
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "edgefs-op-mgr")
|
|
|
|
const (
|
|
appName = "rook-edgefs-mgr"
|
|
restapiSvcName = "rook-edgefs-restapi"
|
|
uiSvcName = "rook-edgefs-ui"
|
|
defaultServiceAccountName = "rook-edgefs-cluster"
|
|
defaultGrpcPort = 6789
|
|
defaultRestPort = 8080
|
|
defaultRestSPort = 4443
|
|
defaultMetricsPort = 8881
|
|
defaultUIPort = 3000
|
|
defaultUISPort = 3443
|
|
|
|
/* Volumes definitions */
|
|
dataVolumeName = "edgefs-datadir"
|
|
stateVolumeFolder = ".state"
|
|
etcVolumeFolder = ".etc"
|
|
)
|
|
|
|
// Cluster is the edgefs mgr manager
|
|
type Cluster struct {
|
|
Namespace string
|
|
Version string
|
|
serviceAccount string
|
|
Replicas int
|
|
dataDirHostPath string
|
|
dataVolumeSize resource.Quantity
|
|
annotations rookv1.Annotations
|
|
placement rookv1.Placement
|
|
context *clusterd.Context
|
|
NetworkSpec rookv1.NetworkSpec
|
|
dashboardSpec edgefsv1.DashboardSpec
|
|
resources v1.ResourceRequirements
|
|
resourceProfile string
|
|
ownerRef metav1.OwnerReference
|
|
useHostLocalTime bool
|
|
}
|
|
|
|
// New creates an instance of the mgr
|
|
func New(
|
|
context *clusterd.Context, namespace, version string,
|
|
serviceAccount string,
|
|
dataDirHostPath string,
|
|
dataVolumeSize resource.Quantity,
|
|
annotations rookv1.Annotations,
|
|
placement rookv1.Placement,
|
|
NetworkSpec rookv1.NetworkSpec,
|
|
dashboardSpec edgefsv1.DashboardSpec,
|
|
resources v1.ResourceRequirements,
|
|
resourceProfile string,
|
|
ownerRef metav1.OwnerReference,
|
|
useHostLocalTime bool,
|
|
) *Cluster {
|
|
|
|
if serviceAccount == "" {
|
|
// if the service account was not set, make a best effort with the example service account name since the default is unlikely to be sufficient.
|
|
serviceAccount = defaultServiceAccountName
|
|
logger.Infof("setting the mgr pod to use the service account name: %s", serviceAccount)
|
|
}
|
|
|
|
return &Cluster{
|
|
context: context,
|
|
Namespace: namespace,
|
|
serviceAccount: serviceAccount,
|
|
annotations: annotations,
|
|
placement: placement,
|
|
Version: version,
|
|
Replicas: 1,
|
|
dataDirHostPath: dataDirHostPath,
|
|
dataVolumeSize: dataVolumeSize,
|
|
NetworkSpec: NetworkSpec,
|
|
dashboardSpec: dashboardSpec,
|
|
resources: resources,
|
|
resourceProfile: resourceProfile,
|
|
ownerRef: ownerRef,
|
|
useHostLocalTime: useHostLocalTime,
|
|
}
|
|
}
|
|
|
|
// Start the mgr instance
|
|
func (c *Cluster) Start(rookImage string) error {
|
|
logger.Infof("start running mgr")
|
|
|
|
logger.Infof("Mgr Image is %s", rookImage)
|
|
// start the deployment
|
|
deployment := c.makeDeployment(appName, c.Namespace, rookImage, 1)
|
|
if _, err := c.context.Clientset.AppsV1().Deployments(c.Namespace).Create(deployment); err != nil {
|
|
if !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("failed to create %s deployment. %+v", appName, err)
|
|
}
|
|
logger.Infof("deployment for mgr %s already exists. updating if needed", appName)
|
|
|
|
// If mgr deployment already exists, then we need to force deployment update to prevent placement manager over the node with unexisting target pod on it
|
|
deployment.Spec.Template.Annotations["edgefs.io/update-timestamp"] = fmt.Sprintf("%d", time.Now().Unix())
|
|
|
|
// placeholder for a verify callback
|
|
// see comments on k8sutil.UpdateDeploymentAndWait's definition to understand its purpose
|
|
callback := func(action string) error {
|
|
return nil
|
|
}
|
|
if _, err := k8sutil.UpdateDeploymentAndWait(c.context, deployment, c.Namespace, callback); err != nil {
|
|
return fmt.Errorf("failed to update mgr deployment %s. %+v", appName, err)
|
|
}
|
|
} else {
|
|
logger.Infof("%s deployment started", appName)
|
|
}
|
|
|
|
// create the mgr service
|
|
mgrService := c.makeMgrService(appName)
|
|
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(mgrService); err != nil {
|
|
if !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("failed to create mgr service. %+v", err)
|
|
}
|
|
logger.Infof("mgr service already exists")
|
|
} else {
|
|
logger.Infof("mgr service started")
|
|
}
|
|
|
|
// create the restapi service
|
|
restapiService := c.makeRestapiService(restapiSvcName)
|
|
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(restapiService); err != nil {
|
|
if !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("failed to create restapi service. %+v", err)
|
|
}
|
|
logger.Infof("restapi service already exists")
|
|
} else {
|
|
logger.Infof("restapi service started")
|
|
}
|
|
|
|
// create the ui/dashboard service
|
|
uiService := c.makeUIService(uiSvcName)
|
|
if _, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(uiService); err != nil {
|
|
if !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("failed to create ui service. %+v", err)
|
|
}
|
|
logger.Infof("ui service already exists")
|
|
} else {
|
|
logger.Infof("ui service started")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *Cluster) makeMgrService(name string) *v1.Service {
|
|
labels := c.getLabels()
|
|
svc := &v1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: c.Namespace,
|
|
Labels: labels,
|
|
},
|
|
Spec: v1.ServiceSpec{
|
|
Selector: labels,
|
|
Type: v1.ServiceTypeClusterIP,
|
|
Ports: []v1.ServicePort{
|
|
{
|
|
Name: "mgr",
|
|
Port: int32(defaultGrpcPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.ownerRef)
|
|
return svc
|
|
}
|
|
|
|
func (c *Cluster) makeRestapiService(name string) *v1.Service {
|
|
labels := c.getLabels()
|
|
svc := &v1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: c.Namespace,
|
|
Labels: labels,
|
|
},
|
|
Spec: v1.ServiceSpec{
|
|
Selector: labels,
|
|
Type: v1.ServiceTypeClusterIP,
|
|
Ports: []v1.ServicePort{
|
|
{
|
|
Name: "http-metrics",
|
|
Port: int32(defaultMetricsPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: "http-restapi",
|
|
Port: int32(defaultRestPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: "https-restapi",
|
|
Port: int32(defaultRestSPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.ownerRef)
|
|
return svc
|
|
}
|
|
|
|
func (c *Cluster) makeUIService(name string) *v1.Service {
|
|
labels := c.getLabels()
|
|
svc := &v1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: c.Namespace,
|
|
Labels: labels,
|
|
},
|
|
Spec: v1.ServiceSpec{
|
|
Selector: labels,
|
|
Type: v1.ServiceTypeClusterIP,
|
|
Ports: []v1.ServicePort{
|
|
{
|
|
Name: "http-ui",
|
|
Port: int32(defaultUIPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: "https-ui",
|
|
Port: int32(defaultUISPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.ownerRef)
|
|
|
|
if c.dashboardSpec.LocalAddr != "" {
|
|
ip := net.ParseIP(c.dashboardSpec.LocalAddr)
|
|
if ip == nil {
|
|
logger.Errorf("wrong dashboard localAddr format")
|
|
return svc
|
|
}
|
|
|
|
if !ip.IsUnspecified() {
|
|
logger.Infof("Cluster dashboard assigned with externalIP=%s", c.dashboardSpec.LocalAddr)
|
|
svc.Spec.ExternalIPs = []string{c.dashboardSpec.LocalAddr}
|
|
} else {
|
|
logger.Errorf("Cluster dashboard externalIP cannot be assigned to %s", c.dashboardSpec.LocalAddr)
|
|
}
|
|
}
|
|
|
|
return svc
|
|
}
|
|
|
|
func (c *Cluster) makeDeployment(name, clusterName, rookImage string, replicas int32) *apps.Deployment {
|
|
|
|
volumes := []v1.Volume{}
|
|
|
|
if c.useHostLocalTime {
|
|
volumes = append(volumes, edgefsv1.GetHostLocalTimeVolume())
|
|
}
|
|
|
|
if c.dataVolumeSize.Value() > 0 {
|
|
// dataVolume case
|
|
volumes = append(volumes, v1.Volume{
|
|
Name: dataVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
|
|
ClaimName: dataVolumeName,
|
|
},
|
|
},
|
|
})
|
|
} else {
|
|
// dataDir case
|
|
volumes = append(volumes, v1.Volume{
|
|
Name: dataVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: c.dataDirHostPath,
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
podSpec := v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Labels: c.getDaemonLabels(clusterName),
|
|
},
|
|
Spec: v1.PodSpec{
|
|
ServiceAccountName: c.serviceAccount,
|
|
Containers: []v1.Container{
|
|
c.restApiContainer(name, edgefsv1.GetModifiedRookImagePath(rookImage, "restapi")),
|
|
c.grpcProxyContainer("grpc", rookImage),
|
|
c.uiContainer("ui", edgefsv1.GetModifiedRookImagePath(rookImage, "ui")),
|
|
},
|
|
RestartPolicy: v1.RestartPolicyAlways,
|
|
Volumes: volumes,
|
|
HostIPC: true,
|
|
HostNetwork: c.NetworkSpec.IsHost(),
|
|
NodeSelector: map[string]string{c.Namespace: "cluster"},
|
|
},
|
|
}
|
|
|
|
// Add the prometheus.io scrape annoations by default
|
|
podSpec.ObjectMeta.Annotations = map[string]string{
|
|
"prometheus.io/scrape": "true",
|
|
"prometheus.io/port": strconv.Itoa(defaultMetricsPort),
|
|
}
|
|
|
|
if c.NetworkSpec.IsHost() {
|
|
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
|
} else if c.NetworkSpec.IsMultus() {
|
|
if err := k8sutil.ApplyMultus(c.NetworkSpec, &podSpec.ObjectMeta); err != nil {
|
|
logger.Errorf("failed to apply multus spec to pod template metadata. %v", err)
|
|
}
|
|
}
|
|
|
|
c.annotations.ApplyToObjectMeta(&podSpec.ObjectMeta)
|
|
c.placement.ApplyToPodSpec(&podSpec.Spec)
|
|
|
|
d := &apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
Namespace: c.Namespace,
|
|
},
|
|
Spec: apps.DeploymentSpec{
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: podSpec.Labels,
|
|
},
|
|
Template: podSpec,
|
|
Replicas: &replicas,
|
|
},
|
|
}
|
|
k8sutil.SetOwnerRef(&d.ObjectMeta, &c.ownerRef)
|
|
c.annotations.ApplyToObjectMeta(&d.ObjectMeta)
|
|
return d
|
|
}
|
|
|
|
func (c *Cluster) uiContainer(name string, containerImage string) v1.Container {
|
|
|
|
runAsUser := int64(0)
|
|
readOnlyRootFilesystem := false
|
|
securityContext := &v1.SecurityContext{
|
|
RunAsUser: &runAsUser,
|
|
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
|
}
|
|
|
|
volumeMounts := []v1.VolumeMount{}
|
|
if c.useHostLocalTime {
|
|
volumeMounts = append(volumeMounts, edgefsv1.GetHostLocalTimeVolumeMount())
|
|
}
|
|
|
|
return v1.Container{
|
|
Name: name,
|
|
Image: containerImage,
|
|
ImagePullPolicy: v1.PullAlways,
|
|
Args: []string{},
|
|
Env: []v1.EnvVar{
|
|
{
|
|
Name: "MGR_POD_IP",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "status.podIP",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "API_ENDPOINT",
|
|
Value: "http://$(MGR_POD_IP):8080",
|
|
},
|
|
{
|
|
Name: "K8S_NAMESPACE",
|
|
Value: c.Namespace,
|
|
},
|
|
},
|
|
SecurityContext: securityContext,
|
|
Resources: c.resources,
|
|
Ports: []v1.ContainerPort{
|
|
{
|
|
Name: "http-ui",
|
|
ContainerPort: int32(defaultUIPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: "https-ui",
|
|
ContainerPort: int32(defaultUISPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
},
|
|
VolumeMounts: volumeMounts,
|
|
}
|
|
}
|
|
|
|
func (c *Cluster) restApiContainer(name string, containerImage string) v1.Container {
|
|
|
|
runAsUser := int64(0)
|
|
readOnlyRootFilesystem := false
|
|
securityContext := &v1.SecurityContext{
|
|
RunAsUser: &runAsUser,
|
|
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
|
Capabilities: &v1.Capabilities{
|
|
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
|
},
|
|
}
|
|
|
|
volumeMounts := []v1.VolumeMount{
|
|
{
|
|
Name: dataVolumeName,
|
|
MountPath: "/opt/nedge/etc.target",
|
|
SubPath: etcVolumeFolder,
|
|
},
|
|
{
|
|
Name: dataVolumeName,
|
|
MountPath: "/opt/nedge/var/run",
|
|
SubPath: stateVolumeFolder,
|
|
},
|
|
}
|
|
|
|
if c.useHostLocalTime {
|
|
volumeMounts = append(volumeMounts, edgefsv1.GetHostLocalTimeVolumeMount())
|
|
}
|
|
|
|
cont := v1.Container{
|
|
Name: name,
|
|
Image: containerImage,
|
|
ImagePullPolicy: v1.PullAlways,
|
|
Args: []string{"mgmt"},
|
|
Env: []v1.EnvVar{
|
|
{
|
|
Name: "CCOW_LOG_LEVEL",
|
|
Value: "5",
|
|
},
|
|
{
|
|
Name: "DEBUG",
|
|
Value: "alert,error,info",
|
|
},
|
|
{
|
|
Name: "K8S_NAMESPACE",
|
|
Value: c.Namespace,
|
|
},
|
|
{
|
|
Name: "HOST_HOSTNAME",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "spec.nodeName",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "EFSROOK_CRD_API",
|
|
Value: fmt.Sprintf("%s/%s", edgefsv1.CustomResourceGroup, edgefsv1.Version),
|
|
},
|
|
},
|
|
SecurityContext: securityContext,
|
|
Resources: c.resources,
|
|
Ports: []v1.ContainerPort{
|
|
{
|
|
Name: "http-metrics",
|
|
ContainerPort: int32(defaultMetricsPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: "http-mgmt",
|
|
ContainerPort: int32(defaultRestPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: "https-mgmt",
|
|
ContainerPort: int32(defaultRestSPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
},
|
|
VolumeMounts: volumeMounts,
|
|
}
|
|
|
|
if c.resourceProfile == "embedded" {
|
|
cont.Env = append(cont.Env, v1.EnvVar{
|
|
Name: "CCOW_EMBEDDED",
|
|
Value: "1",
|
|
})
|
|
cont.Env = append(cont.Env, v1.EnvVar{
|
|
Name: "JE_MALLOC_CONF",
|
|
Value: "tcache:false",
|
|
})
|
|
}
|
|
|
|
return cont
|
|
}
|
|
|
|
func (c *Cluster) grpcProxyContainer(name string, containerImage string) v1.Container {
|
|
|
|
runAsUser := int64(0)
|
|
readOnlyRootFilesystem := false
|
|
securityContext := &v1.SecurityContext{
|
|
RunAsUser: &runAsUser,
|
|
ReadOnlyRootFilesystem: &readOnlyRootFilesystem,
|
|
Capabilities: &v1.Capabilities{
|
|
Add: []v1.Capability{"SYS_NICE", "SYS_RESOURCE", "IPC_LOCK"},
|
|
},
|
|
}
|
|
|
|
volumeMounts := []v1.VolumeMount{
|
|
{
|
|
Name: dataVolumeName,
|
|
MountPath: "/opt/nedge/etc.target",
|
|
SubPath: etcVolumeFolder,
|
|
},
|
|
{
|
|
Name: dataVolumeName,
|
|
MountPath: "/opt/nedge/var/run",
|
|
SubPath: stateVolumeFolder,
|
|
},
|
|
}
|
|
|
|
if c.useHostLocalTime {
|
|
volumeMounts = append(volumeMounts, edgefsv1.GetHostLocalTimeVolumeMount())
|
|
}
|
|
|
|
cont := v1.Container{
|
|
Name: name,
|
|
Image: containerImage,
|
|
ImagePullPolicy: v1.PullAlways,
|
|
Args: []string{"mgmt"},
|
|
LivenessProbe: c.getLivenessProbe(),
|
|
Env: []v1.EnvVar{
|
|
{
|
|
Name: "CCOW_LOG_LEVEL",
|
|
Value: "5",
|
|
},
|
|
{
|
|
Name: "K8S_NAMESPACE",
|
|
Value: c.Namespace,
|
|
},
|
|
{
|
|
Name: "HOST_HOSTNAME",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "spec.nodeName",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
SecurityContext: securityContext,
|
|
Resources: c.resources,
|
|
Ports: []v1.ContainerPort{
|
|
{
|
|
Name: "mgr",
|
|
ContainerPort: int32(defaultGrpcPort),
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
},
|
|
VolumeMounts: volumeMounts,
|
|
}
|
|
|
|
if c.resourceProfile == "embedded" {
|
|
cont.Env = append(cont.Env, v1.EnvVar{
|
|
Name: "CCOW_EMBEDDED",
|
|
Value: "1",
|
|
})
|
|
cont.Env = append(cont.Env, v1.EnvVar{
|
|
Name: "JE_MALLOC_CONF",
|
|
Value: "tcache:false",
|
|
})
|
|
}
|
|
|
|
return cont
|
|
}
|
|
|
|
func (c *Cluster) getLivenessProbe() *v1.Probe {
|
|
return &v1.Probe{
|
|
Handler: v1.Handler{
|
|
Exec: &v1.ExecAction{
|
|
Command: []string{"/opt/nedge/sbin/grpc-proxy-liveness.sh"},
|
|
},
|
|
},
|
|
InitialDelaySeconds: 20,
|
|
PeriodSeconds: 20,
|
|
TimeoutSeconds: 10,
|
|
SuccessThreshold: 1,
|
|
FailureThreshold: 6,
|
|
}
|
|
}
|
|
|
|
func (c *Cluster) getLabels() map[string]string {
|
|
return map[string]string{
|
|
k8sutil.AppAttr: appName,
|
|
k8sutil.ClusterAttr: c.Namespace,
|
|
}
|
|
}
|
|
|
|
func (c *Cluster) getDaemonLabels(clusterName string) map[string]string {
|
|
labels := c.getLabels()
|
|
labels["instance"] = clusterName
|
|
return labels
|
|
}
|