forked from rook/rook
Peter Razumovsky
516eab4d8a
Resolves CIS benchmark rule 5.7.3, Pods part. SecurityContext should be explicitly defined in pod level of Pod spec section. It is sufficient to specify empty securityContext to satisfy CIS 5.7.3 rule. 5.7.3 Apply Security Context to Your Pods and Containers When designing your containers and pods, make sure that you configure the security context for your pods, containers, and volumes. Signed-off-by: Peter Razumovsky <prazumovsky@mirantis.com>
490 lines
16 KiB
Go
490 lines
16 KiB
Go
/*
|
|
Copyright 2018 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
// Package discover to discover devices on storage nodes.
|
|
package discover
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
|
|
"github.com/rook/rook/pkg/operator/ceph/controller"
|
|
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
|
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
|
|
"github.com/rook/rook/pkg/util/sys"
|
|
|
|
apps "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/kubernetes"
|
|
)
|
|
|
|
const (
|
|
discoverDaemonsetName = "rook-discover"
|
|
discoverDaemonsetPriorityClassNameEnv = "DISCOVER_PRIORITY_CLASS_NAME"
|
|
discoverDaemonsetTolerationEnv = "DISCOVER_TOLERATION"
|
|
discoverDaemonsetTolerationKeyEnv = "DISCOVER_TOLERATION_KEY"
|
|
discoverDaemonsetTolerationsEnv = "DISCOVER_TOLERATIONS"
|
|
discoverDaemonSetNodeAffinityEnv = "DISCOVER_AGENT_NODE_AFFINITY"
|
|
discoverDaemonSetPodLabelsEnv = "DISCOVER_AGENT_POD_LABELS"
|
|
deviceInUseCMName = "local-device-in-use-cluster-%s-node-%s"
|
|
deviceInUseAppName = "rook-claimed-devices"
|
|
deviceInUseClusterAttr = "rook.io/cluster"
|
|
discoverIntervalEnv = "ROOK_DISCOVER_DEVICES_INTERVAL"
|
|
defaultDiscoverInterval = "60m"
|
|
discoverDaemonResourcesEnv = "DISCOVER_DAEMON_RESOURCES"
|
|
)
|
|
|
|
var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-discover")
|
|
|
|
// Discover reference to be deployed
|
|
type Discover struct {
|
|
clientset kubernetes.Interface
|
|
}
|
|
|
|
// New creates an instance of Discover
|
|
func New(clientset kubernetes.Interface) *Discover {
|
|
return &Discover{
|
|
clientset: clientset,
|
|
}
|
|
}
|
|
|
|
// Start the discover
|
|
func (d *Discover) Start(ctx context.Context, namespace, discoverImage, securityAccount string, data map[string]string, useCephVolume bool) error {
|
|
err := d.createDiscoverDaemonSet(ctx, namespace, discoverImage, securityAccount, data, useCephVolume)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to start discover daemonset. %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, discoverImage, securityAccount string, data map[string]string, useCephVolume bool) error {
|
|
discoveryInterval := k8sutil.GetValue(data, discoverIntervalEnv, defaultDiscoverInterval)
|
|
|
|
discoveryParameters := []string{"discover",
|
|
"--discover-interval", discoveryInterval}
|
|
if useCephVolume {
|
|
discoveryParameters = append(discoveryParameters, "--use-ceph-volume")
|
|
}
|
|
|
|
discoverDaemonResourcesRaw := k8sutil.GetValue(data, discoverDaemonResourcesEnv, "")
|
|
discoverDaemonResources, err := k8sutil.YamlToContainerResource(discoverDaemonResourcesRaw)
|
|
if err != nil {
|
|
logger.Warningf("failed to parse.%s %v", discoverDaemonResourcesRaw, err)
|
|
}
|
|
|
|
ds := &apps.DaemonSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: discoverDaemonsetName,
|
|
Labels: getLabels(),
|
|
},
|
|
Spec: apps.DaemonSetSpec{
|
|
RevisionHistoryLimit: opcontroller.RevisionHistoryLimit(),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{
|
|
"app": discoverDaemonsetName,
|
|
},
|
|
},
|
|
UpdateStrategy: apps.DaemonSetUpdateStrategy{
|
|
Type: apps.RollingUpdateDaemonSetStrategyType,
|
|
},
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: getLabels(),
|
|
},
|
|
Spec: v1.PodSpec{
|
|
ServiceAccountName: securityAccount,
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: discoverDaemonsetName,
|
|
Image: discoverImage,
|
|
Args: discoveryParameters,
|
|
SecurityContext: controller.PrivilegedContext(true),
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "dev",
|
|
MountPath: "/dev",
|
|
// discovery pod could fail to start if /dev is mounted ro
|
|
ReadOnly: false,
|
|
},
|
|
{
|
|
Name: "sys",
|
|
MountPath: "/sys",
|
|
ReadOnly: true,
|
|
},
|
|
{
|
|
Name: "udev",
|
|
MountPath: "/run/udev",
|
|
ReadOnly: true,
|
|
},
|
|
},
|
|
Resources: discoverDaemonResources,
|
|
Env: []v1.EnvVar{
|
|
k8sutil.NamespaceEnvVar(),
|
|
k8sutil.NodeEnvVar(),
|
|
k8sutil.NameEnvVar(),
|
|
},
|
|
},
|
|
},
|
|
Volumes: []v1.Volume{
|
|
{
|
|
Name: "dev",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/dev",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "sys",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/sys",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "udev",
|
|
VolumeSource: v1.VolumeSource{
|
|
HostPath: &v1.HostPathVolumeSource{
|
|
Path: "/run/udev",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
HostNetwork: opcontroller.EnforceHostNetwork(),
|
|
PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""),
|
|
SecurityContext: &v1.PodSecurityContext{},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// Get the operator pod details to attach the owner reference to the discover daemon set
|
|
operatorPod, err := k8sutil.GetRunningPod(ctx, d.clientset)
|
|
if err != nil {
|
|
logger.Errorf("failed to get operator pod. %+v", err)
|
|
} else {
|
|
k8sutil.SetOwnerRefsWithoutBlockOwner(&ds.ObjectMeta, operatorPod.OwnerReferences)
|
|
}
|
|
|
|
// Add toleration if any
|
|
tolerationValue := k8sutil.GetValue(data, discoverDaemonsetTolerationEnv, "")
|
|
if tolerationValue != "" {
|
|
ds.Spec.Template.Spec.Tolerations = []v1.Toleration{
|
|
{
|
|
Effect: v1.TaintEffect(tolerationValue),
|
|
Operator: v1.TolerationOpExists,
|
|
Key: k8sutil.GetValue(data, discoverDaemonsetTolerationKeyEnv, ""),
|
|
},
|
|
}
|
|
}
|
|
|
|
tolerationsRaw := k8sutil.GetValue(data, discoverDaemonsetTolerationsEnv, "")
|
|
tolerations, err := k8sutil.YamlToTolerations(tolerationsRaw)
|
|
if err != nil {
|
|
logger.Warningf("failed to parse %s. %+v", tolerationsRaw, err)
|
|
}
|
|
logger.Infof("tolerations: %v", tolerations)
|
|
ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, tolerations...)
|
|
|
|
// Add NodeAffinity if any
|
|
nodeAffinity := k8sutil.GetValue(data, discoverDaemonSetNodeAffinityEnv, "")
|
|
if nodeAffinity != "" {
|
|
v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity)
|
|
if err != nil {
|
|
logger.Errorf("failed to create NodeAffinity. %+v", err)
|
|
} else {
|
|
ds.Spec.Template.Spec.Affinity = &v1.Affinity{
|
|
NodeAffinity: v1NodeAffinity,
|
|
}
|
|
}
|
|
logger.Infof("nodeAffinity: %s", v1NodeAffinity)
|
|
}
|
|
|
|
podLabels := k8sutil.GetValue(data, discoverDaemonSetPodLabelsEnv, "")
|
|
if podLabels != "" {
|
|
podLabels := k8sutil.ParseStringToLabels(podLabels)
|
|
// Override / Set the app label even if set by the user as
|
|
// otherwise the DaemonSet pod selector may be broken
|
|
podLabels["app"] = discoverDaemonsetName
|
|
ds.Spec.Template.ObjectMeta.Labels = podLabels
|
|
}
|
|
|
|
if controller.LoopDevicesAllowed() {
|
|
ds.Spec.Template.Spec.Containers[0].Env = append(ds.Spec.Template.Spec.Containers[0].Env,
|
|
v1.EnvVar{Name: "CEPH_VOLUME_ALLOW_LOOP_DEVICES", Value: "true"})
|
|
}
|
|
|
|
_, err = d.clientset.AppsV1().DaemonSets(namespace).Create(ctx, ds, metav1.CreateOptions{})
|
|
if err != nil {
|
|
if !k8serrors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("failed to create rook-discover daemon set. %+v", err)
|
|
}
|
|
logger.Infof("rook-discover daemonset already exists, updating ...")
|
|
_, err = d.clientset.AppsV1().DaemonSets(namespace).Update(ctx, ds, metav1.UpdateOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to update rook-discover daemon set. %+v", err)
|
|
}
|
|
} else {
|
|
logger.Infof("rook-discover daemonset started")
|
|
}
|
|
return nil
|
|
|
|
}
|
|
|
|
func getLabels() map[string]string {
|
|
labels := make(map[string]string)
|
|
k8sutil.AddRecommendedLabels(labels, "rook-discover", "rook-ceph-operator", "rook-discover", "rook-discover")
|
|
labels["app"] = discoverDaemonsetName
|
|
return labels
|
|
}
|
|
|
|
// ListDevices lists all devices discovered on all nodes or specific node if node name is provided.
|
|
func ListDevices(ctx context.Context, clusterdContext *clusterd.Context, namespace, nodeName string) (map[string][]sys.LocalDisk, error) {
|
|
// convert the host name label to the k8s node name to look up the configmap with the devices
|
|
if len(nodeName) > 0 {
|
|
var err error
|
|
nodeName, err = k8sutil.GetNodeNameFromHostname(ctx, clusterdContext.Clientset, nodeName)
|
|
if err != nil {
|
|
logger.Warningf("failed to get node name from hostname. %+v", err)
|
|
}
|
|
}
|
|
|
|
var devices map[string][]sys.LocalDisk
|
|
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, discoverDaemon.AppName)}
|
|
// wait for device discovery configmaps
|
|
retryCount := 0
|
|
retryMax := 30
|
|
sleepTime := 5
|
|
for {
|
|
retryCount++
|
|
if retryCount > retryMax {
|
|
return devices, fmt.Errorf("exceeded max retry count waiting for device configmap to appear")
|
|
}
|
|
|
|
if retryCount > 1 {
|
|
// only sleep after the first time
|
|
<-time.After(time.Duration(sleepTime) * time.Second)
|
|
}
|
|
|
|
cms, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).List(ctx, listOpts)
|
|
if err != nil {
|
|
logger.Warningf("failed to list device configmaps: %v", err)
|
|
return devices, fmt.Errorf("failed to list device configmaps: %+v", err)
|
|
}
|
|
if len(cms.Items) == 0 {
|
|
logger.Infof("no configmap match, retry #%d", retryCount)
|
|
continue
|
|
}
|
|
devices = make(map[string][]sys.LocalDisk, len(cms.Items))
|
|
for _, cm := range cms.Items {
|
|
node := cm.ObjectMeta.Labels[discoverDaemon.NodeAttr]
|
|
if len(nodeName) > 0 && node != nodeName {
|
|
continue
|
|
}
|
|
deviceJson := cm.Data[discoverDaemon.LocalDiskCMData]
|
|
logger.Debugf("node %s, device %s", node, deviceJson)
|
|
|
|
if len(node) == 0 || len(deviceJson) == 0 {
|
|
continue
|
|
}
|
|
var d []sys.LocalDisk
|
|
err = json.Unmarshal([]byte(deviceJson), &d)
|
|
if err != nil {
|
|
logger.Warningf("failed to unmarshal %s", deviceJson)
|
|
continue
|
|
}
|
|
devices[node] = d
|
|
}
|
|
break
|
|
}
|
|
logger.Debugf("discovery found the following devices %+v", devices)
|
|
return devices, nil
|
|
}
|
|
|
|
// ListDevicesInUse lists all devices on a node that are already used by existing clusters.
|
|
func ListDevicesInUse(ctx context.Context, clusterdContext *clusterd.Context, namespace, nodeName string) ([]sys.LocalDisk, error) {
|
|
var devices []sys.LocalDisk
|
|
|
|
if len(nodeName) == 0 {
|
|
return devices, fmt.Errorf("empty node name")
|
|
}
|
|
|
|
listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, deviceInUseAppName)}
|
|
cms, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).List(ctx, listOpts)
|
|
if err != nil {
|
|
return devices, fmt.Errorf("failed to list device in use configmaps: %+v", err)
|
|
}
|
|
|
|
for _, cm := range cms.Items {
|
|
node := cm.ObjectMeta.Labels[discoverDaemon.NodeAttr]
|
|
if node != nodeName {
|
|
continue
|
|
}
|
|
deviceJson := cm.Data[discoverDaemon.LocalDiskCMData]
|
|
logger.Debugf("node %s, device in use %s", node, deviceJson)
|
|
|
|
if len(node) == 0 || len(deviceJson) == 0 {
|
|
continue
|
|
}
|
|
var d []sys.LocalDisk
|
|
err = json.Unmarshal([]byte(deviceJson), &d)
|
|
if err != nil {
|
|
logger.Warningf("failed to unmarshal %s", deviceJson)
|
|
continue
|
|
}
|
|
devices = append(devices, d...)
|
|
}
|
|
logger.Debugf("devices in use %+v", devices)
|
|
return devices, nil
|
|
}
|
|
|
|
func matchDeviceFullPath(devLinks, fullpath string) bool {
|
|
dlsArr := strings.Split(devLinks, " ")
|
|
for i := range dlsArr {
|
|
if dlsArr[i] == fullpath {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// GetAvailableDevices conducts outer join using input filters with free devices that a node has. It marks the devices from join result as in-use.
|
|
func GetAvailableDevices(ctx context.Context, clusterdContext *clusterd.Context, nodeName, clusterName string, devices []cephv1.Device, filter string, useAllDevices bool) ([]cephv1.Device, error) {
|
|
results := []cephv1.Device{}
|
|
if len(devices) == 0 && len(filter) == 0 && !useAllDevices {
|
|
return results, nil
|
|
}
|
|
namespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
|
|
// find all devices
|
|
allDevices, err := ListDevices(ctx, clusterdContext, namespace, nodeName)
|
|
if err != nil {
|
|
return results, err
|
|
}
|
|
// find those on the node
|
|
nodeAllDevices, ok := allDevices[nodeName]
|
|
if !ok {
|
|
return results, fmt.Errorf("node %s has no devices", nodeName)
|
|
}
|
|
// find those in use on the node
|
|
devicesInUse, err := ListDevicesInUse(ctx, clusterdContext, namespace, nodeName)
|
|
if err != nil {
|
|
return results, err
|
|
}
|
|
|
|
nodeDevices := []sys.LocalDisk{}
|
|
for _, nodeDevice := range nodeAllDevices {
|
|
// TODO: Filter out devices that are in use by another cluster.
|
|
// We need to retain the devices in use for this cluster so the provisioner will continue to configure the same OSDs.
|
|
for _, device := range devicesInUse {
|
|
if nodeDevice.Name == device.Name {
|
|
break
|
|
}
|
|
}
|
|
nodeDevices = append(nodeDevices, nodeDevice)
|
|
}
|
|
claimedDevices := []sys.LocalDisk{}
|
|
// now those left are free to use
|
|
if len(devices) > 0 {
|
|
for i := range devices {
|
|
for j := range nodeDevices {
|
|
if devices[i].FullPath != "" && matchDeviceFullPath(nodeDevices[j].DevLinks, devices[i].FullPath) {
|
|
if devices[i].Name == "" {
|
|
devices[i].Name = nodeDevices[j].Name
|
|
}
|
|
results = append(results, devices[i])
|
|
claimedDevices = append(claimedDevices, nodeDevices[j])
|
|
} else if devices[i].Name == nodeDevices[j].Name {
|
|
results = append(results, devices[i])
|
|
claimedDevices = append(claimedDevices, nodeDevices[j])
|
|
}
|
|
}
|
|
}
|
|
} else if len(filter) >= 0 {
|
|
for i := range nodeDevices {
|
|
//TODO support filter based on other keys
|
|
matched, err := regexp.Match(filter, []byte(nodeDevices[i].Name))
|
|
if err == nil && matched {
|
|
d := cephv1.Device{
|
|
Name: nodeDevices[i].Name,
|
|
}
|
|
claimedDevices = append(claimedDevices, nodeDevices[i])
|
|
results = append(results, d)
|
|
}
|
|
}
|
|
} else if useAllDevices {
|
|
for i := range nodeDevices {
|
|
d := cephv1.Device{
|
|
Name: nodeDevices[i].Name,
|
|
}
|
|
results = append(results, d)
|
|
claimedDevices = append(claimedDevices, nodeDevices[i])
|
|
}
|
|
}
|
|
// mark these devices in use
|
|
if len(claimedDevices) > 0 {
|
|
deviceJson, err := json.Marshal(claimedDevices)
|
|
if err != nil {
|
|
logger.Infof("failed to marshal: %v", err)
|
|
return results, err
|
|
}
|
|
data := make(map[string]string, 1)
|
|
data[discoverDaemon.LocalDiskCMData] = string(deviceJson)
|
|
|
|
cm := &v1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: k8sutil.TruncateNodeName(fmt.Sprintf(deviceInUseCMName, clusterName, "%s"), nodeName),
|
|
Namespace: namespace,
|
|
Labels: map[string]string{
|
|
k8sutil.AppAttr: deviceInUseAppName,
|
|
discoverDaemon.NodeAttr: nodeName,
|
|
deviceInUseClusterAttr: clusterName,
|
|
},
|
|
},
|
|
Data: data,
|
|
}
|
|
_, err = clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Create(ctx, cm, metav1.CreateOptions{})
|
|
if err != nil {
|
|
if !k8serrors.IsAlreadyExists(err) {
|
|
return results, fmt.Errorf("failed to update device in use for cluster %s node %s: %v", clusterName, nodeName, err)
|
|
}
|
|
if _, err := clusterdContext.Clientset.CoreV1().ConfigMaps(namespace).Update(ctx, cm, metav1.UpdateOptions{}); err != nil {
|
|
return results, fmt.Errorf("failed to update devices in use. %+v", err)
|
|
}
|
|
}
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
// Stop the discover
|
|
func (d *Discover) Stop(ctx context.Context, namespace string) error {
|
|
err := d.clientset.AppsV1().DaemonSets(namespace).Delete(ctx, discoverDaemonsetName, metav1.DeleteOptions{})
|
|
if err != nil && !k8serrors.IsNotFound(err) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|