Files
my-rook-config/pkg/operator/ceph/cluster/mgr/spec.go
T
Sébastien Han cb7d600841 ceph: set process working dir to /var/log/ceph
By setting the working directory of a Ceph daemon to the log direcor
(which is bindmounted to the host), we can ensure that the coredumps
will be available.

On CentOS 7 **only**, the kernel is configured with:

```
cat /proc/sys/kernel/core_pattern
core
```

which means that the coredumps will end up in the process working
directory and will be named "core".

On CentOS 8, everything is different and this won't work but still
CentOS will be able to consume this and changing the working directory
is harmless.

Signed-off-by: Sébastien Han <seb@redhat.com>
2021-01-15 16:06:42 +01:00

397 lines
13 KiB
Go

/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mgr
import (
"fmt"
"os"
"strconv"
"strings"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookcephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/config"
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
"github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
const (
podIPEnvVar = "ROOK_POD_IP"
serviceMetricName = "http-metrics"
ExternalMgrAppName = "rook-ceph-mgr-external"
ServiceExternalMetricName = "http-external-metrics"
)
func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) {
logger.Debugf("mgrConfig: %+v", mgrConfig)
podSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: mgrConfig.ResourceName,
Labels: c.getPodLabels(mgrConfig.DaemonID, true),
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
c.makeChownInitContainer(mgrConfig),
},
Containers: []v1.Container{
c.makeMgrDaemonContainer(mgrConfig),
},
ServiceAccountName: serviceAccountName,
RestartPolicy: v1.RestartPolicyAlways,
Volumes: controller.DaemonVolumes(mgrConfig.DataPathMap, mgrConfig.ResourceName),
HostNetwork: c.spec.Network.IsHost(),
PriorityClassName: cephv1.GetMgrPriorityClassName(c.spec.PriorityClassNames),
},
}
// If the log collector is enabled we add the side-car container
if c.spec.LogCollector.Enabled {
shareProcessNamespace := true
podSpec.Spec.ShareProcessNamespace = &shareProcessNamespace
podSpec.Spec.Containers = append(podSpec.Spec.Containers, *controller.LogCollectorContainer(fmt.Sprintf("ceph-mgr.%s", mgrConfig.DaemonID), c.clusterInfo.Namespace, c.spec))
}
// Replace default unreachable node toleration
k8sutil.AddUnreachableNodeToleration(&podSpec.Spec)
// Only add the dashboard init container if dashboard is enabled
if c.spec.Dashboard.Enabled {
podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, []v1.Container{
c.makeSetServerAddrInitContainer(mgrConfig, "dashboard"),
}...)
}
podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, []v1.Container{
c.makeSetServerAddrInitContainer(mgrConfig, "prometheus"),
}...)
// ceph config set commands want admin keyring
podSpec.Spec.Volumes = append(podSpec.Spec.Volumes,
keyring.Volume().Admin())
if c.spec.Network.IsHost() {
podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
} else if c.spec.Network.NetworkSpec.IsMultus() {
if err := k8sutil.ApplyMultus(c.spec.Network.NetworkSpec, &podSpec.ObjectMeta); err != nil {
return nil, err
}
}
cephv1.GetMgrAnnotations(c.spec.Annotations).ApplyToObjectMeta(&podSpec.ObjectMeta)
c.applyPrometheusAnnotations(&podSpec.ObjectMeta)
cephv1.GetMgrLabels(c.spec.Labels).ApplyToObjectMeta(&podSpec.ObjectMeta)
cephv1.GetMgrPlacement(c.spec.Placement).ApplyToPodSpec(&podSpec.Spec)
replicas := int32(1)
d := &apps.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: mgrConfig.ResourceName,
Namespace: c.clusterInfo.Namespace,
Labels: c.getPodLabels(mgrConfig.DaemonID, true),
},
Spec: apps.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: c.getPodLabels(mgrConfig.DaemonID, false),
},
Template: podSpec,
Replicas: &replicas,
Strategy: apps.DeploymentStrategy{
Type: apps.RecreateDeploymentStrategyType,
},
},
}
k8sutil.AddRookVersionLabelToDeployment(d)
cephv1.GetMgrLabels(c.spec.Labels).ApplyToObjectMeta(&d.ObjectMeta)
controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d)
k8sutil.SetOwnerRef(&d.ObjectMeta, &c.clusterInfo.OwnerRef)
return d, nil
}
// if we do not need the http bind fix, then we need to be careful. if we are
// upgrading from a cluster that had the fix applied, then the fix is no longer
// needed, and furthermore, needs to be removed so that there is not a lingering
// ceph configuration option that contains an old ip. by clearing the option,
// we let ceph bind to its default ANYADDR address. However, since we don't
// know which version of Ceph we are may be upgrading _from_ we need to (a)
// always do this and (b) make sure that all forms of the configuration option
// are removed (see the init container factory method). Once the minimum
// supported version of Rook contains the fix, all of this can be removed.
func (c *Cluster) clearHTTPBindFix() error {
// We only need to apply these changes once. No harm in once each time the operator restarts.
if c.appliedHttpBind {
return nil
}
for _, daemonID := range c.getDaemonIDs() {
for _, module := range []string{"dashboard", "prometheus"} {
// there are two forms of the configuration key that might exist which
// depends not on the current version, but on the version that may be
// the version being upgraded from.
if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID,
fmt.Sprintf("mgr/%s/server_addr", module), "", false); err != nil {
return errors.Wrap(err, "failed to set config for an mgr daemon using v2 format")
}
// this is for the format used in v1.0
// https://github.com/rook/rook/commit/11d318fb2f77a6ac9a8f2b9be42c826d3b4a93c3
if _, err := client.MgrSetConfig(c.context, c.clusterInfo, daemonID,
fmt.Sprintf("mgr/%s/%s/server_addr", module, daemonID), "", false); err != nil {
return errors.Wrap(err, "failed to set config for an mgr daemon using v1 format")
}
}
}
c.appliedHttpBind = true
return nil
}
func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container {
return controller.ChownCephDataDirsInitContainer(
*mgrConfig.DataPathMap,
c.spec.CephVersion.Image,
controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName),
cephv1.GetMgrResources(c.spec.Resources),
controller.PodSecurityContext(),
)
}
func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule string) v1.Container {
// Commands produced for various Ceph major versions (differences highlighted)
// L: config-key set mgr/<mod>/server_addr $(ROOK_CEPH_<MOD>_SERVER_ADDR)
// M: config set mgr.a mgr/<mod>/server_addr $(ROOK_CEPH_<MOD>_SERVER_ADDR)
// N: config set mgr.a mgr/<mod>/server_addr $(ROOK_CEPH_<MOD>_SERVER_ADDR) --force
cfgSetArgs := []string{"config", "set"}
cfgSetArgs = append(cfgSetArgs, fmt.Sprintf("mgr.%s", mgrConfig.DaemonID))
cfgPath := fmt.Sprintf("mgr/%s/%s/server_addr", mgrModule, mgrConfig.DaemonID)
cfgSetArgs = append(cfgSetArgs, cfgPath, controller.ContainerEnvVarReference(podIPEnvVar))
cfgSetArgs = append(cfgSetArgs, "--force")
cfgSetArgs = append(cfgSetArgs, "--verbose")
container := v1.Container{
Name: "init-set-" + strings.ToLower(mgrModule) + "-server-addr",
Command: []string{
"ceph",
},
Args: append(
controller.AdminFlags(c.clusterInfo),
cfgSetArgs...,
),
Image: c.spec.CephVersion.Image,
VolumeMounts: append(
controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName),
keyring.VolumeMount().Admin(),
),
Env: append(
controller.DaemonEnvVars(c.spec.CephVersion.Image),
c.cephMgrOrchestratorModuleEnvs()...,
),
Resources: cephv1.GetMgrResources(c.spec.Resources),
}
return container
}
func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container {
container := v1.Container{
Name: "mgr",
Command: []string{
"ceph-mgr",
},
Args: append(
controller.DaemonFlags(c.clusterInfo, &c.spec, mgrConfig.DaemonID),
// for ceph-mgr cephfs
// see https://github.com/ceph/ceph-csi/issues/486 for more details
config.NewFlag("client-mount-uid", "0"),
config.NewFlag("client-mount-gid", "0"),
"--foreground",
),
Image: c.spec.CephVersion.Image,
VolumeMounts: controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName),
Ports: []v1.ContainerPort{
{
Name: "mgr",
ContainerPort: int32(6800),
Protocol: v1.ProtocolTCP,
},
{
Name: "http-metrics",
ContainerPort: int32(DefaultMetricsPort),
Protocol: v1.ProtocolTCP,
},
{
Name: "dashboard",
ContainerPort: int32(c.dashboardPort()),
Protocol: v1.ProtocolTCP,
},
},
Env: append(
controller.DaemonEnvVars(c.spec.CephVersion.Image),
c.cephMgrOrchestratorModuleEnvs()...,
),
Resources: cephv1.GetMgrResources(c.spec.Resources),
SecurityContext: controller.PodSecurityContext(),
LivenessProbe: getDefaultMgrLivenessProbe(),
WorkingDir: config.VarLogCephDir,
}
// If the liveness probe is enabled
container = config.ConfigureLivenessProbe(rookcephv1.KeyMgr, container, c.spec.HealthCheck)
// If host networking is enabled, we don't need a bind addr that is different from the public addr
if !c.spec.Network.IsHost() {
// Opposite of the above, --public-bind-addr will *not* still advertise on the previous
// port, which makes sense because this is the pod IP, which changes with every new pod.
container.Args = append(container.Args,
config.NewFlag("public-addr", controller.ContainerEnvVarReference(podIPEnvVar)))
}
return container
}
func getDefaultMgrLivenessProbe() *v1.Probe {
return &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Path: "/",
Port: intstr.FromInt(int(DefaultMetricsPort)),
},
},
InitialDelaySeconds: 60,
}
}
// MakeMetricsService generates the Kubernetes service object for the monitoring service
func (c *Cluster) MakeMetricsService(name, servicePortMetricName string) *v1.Service {
labels := controller.AppLabels(AppName, c.clusterInfo.Namespace)
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: c.clusterInfo.Namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Type: v1.ServiceTypeClusterIP,
Ports: []v1.ServicePort{
{
Name: servicePortMetricName,
Port: int32(DefaultMetricsPort),
Protocol: v1.ProtocolTCP,
},
},
},
}
// If the cluster is external we don't need to add the selector
if name != ExternalMgrAppName {
svc.Spec.Selector = labels
}
k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.clusterInfo.OwnerRef)
return svc
}
func (c *Cluster) makeDashboardService(name string) *v1.Service {
labels := controller.AppLabels(AppName, c.clusterInfo.Namespace)
portName := "https-dashboard"
if !c.spec.Dashboard.SSL {
portName = "http-dashboard"
}
svc := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-dashboard", name),
Namespace: c.clusterInfo.Namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Selector: labels,
Type: v1.ServiceTypeClusterIP,
Ports: []v1.ServicePort{
{
Name: portName,
Port: int32(c.dashboardPort()),
Protocol: v1.ProtocolTCP,
},
},
},
}
k8sutil.SetOwnerRef(&svc.ObjectMeta, &c.clusterInfo.OwnerRef)
return svc
}
func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string {
labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName, includeNewLabels)
// leave "instance" key for legacy usage
labels["instance"] = daemonName
return labels
}
func (c *Cluster) applyPrometheusAnnotations(objectMeta *metav1.ObjectMeta) {
if len(cephv1.GetMgrAnnotations(c.spec.Annotations)) == 0 {
t := rookv1.Annotations{
"prometheus.io/scrape": "true",
"prometheus.io/port": strconv.Itoa(int(DefaultMetricsPort)),
}
t.ApplyToObjectMeta(objectMeta)
}
}
func (c *Cluster) cephMgrOrchestratorModuleEnvs() []v1.EnvVar {
operatorNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
envVars := []v1.EnvVar{
{Name: "ROOK_OPERATOR_NAMESPACE", Value: operatorNamespace},
{Name: "ROOK_CEPH_CLUSTER_CRD_VERSION", Value: rookcephv1.Version},
{Name: "ROOK_CEPH_CLUSTER_CRD_NAME", Value: c.clusterInfo.NamespacedName().Name},
k8sutil.PodIPEnvVar(podIPEnvVar),
}
return envVars
}
// CreateExternalMetricsEndpoints creates external metric endpoint
func CreateExternalMetricsEndpoints(namespace string, monitoringSpec cephv1.MonitoringSpec, ownerRef metav1.OwnerReference) *v1.Endpoints {
labels := controller.AppLabels(AppName, namespace)
endpoints := &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: ExternalMgrAppName,
Namespace: namespace,
Labels: labels,
},
Subsets: []v1.EndpointSubset{
{
Addresses: monitoringSpec.ExternalMgrEndpoints,
Ports: []v1.EndpointPort{
{
Name: ServiceExternalMetricName,
Port: int32(monitoringSpec.ExternalMgrPrometheusPort),
Protocol: v1.ProtocolTCP,
},
},
},
},
}
k8sutil.SetOwnerRef(&endpoints.ObjectMeta, &ownerRef)
return endpoints
}