From 7dc36e0f4605cc9f3cf7d41f1815998f0e69d2ca Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 3 Feb 2021 19:32:44 -0700 Subject: [PATCH] ceph: use localhost for the mgr endpoint binding instead of podIP The pod IP address is no longer needed for port binding for the dashboard and prometheus endpoints anymore in the mgr modules. The default binding to localhost will be sufficient. This will allow the endpoints to be more reliable in some environments where today the podIP is not working. Signed-off-by: Travis Nielsen --- pkg/operator/ceph/cluster/mgr/spec.go | 51 ---------------------- pkg/operator/ceph/cluster/mgr/spec_test.go | 26 ----------- 2 files changed, 77 deletions(-) diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go index 6edc3ab3b..a357be13e 100644 --- a/pkg/operator/ceph/cluster/mgr/spec.go +++ b/pkg/operator/ceph/cluster/mgr/spec.go @@ -20,7 +20,6 @@ import ( "fmt" "os" "strconv" - "strings" "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -28,7 +27,6 @@ import ( rookv1 "github.com/rook/rook/pkg/apis/rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" - "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" @@ -76,20 +74,6 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error) // Replace default unreachable node toleration k8sutil.AddUnreachableNodeToleration(&podSpec.Spec) - // Only add the dashboard init container if dashboard is enabled - if c.spec.Dashboard.Enabled { - podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, []v1.Container{ - c.makeSetServerAddrInitContainer(mgrConfig, "dashboard"), - }...) - } - - podSpec.Spec.InitContainers = append(podSpec.Spec.InitContainers, []v1.Container{ - c.makeSetServerAddrInitContainer(mgrConfig, "prometheus"), - }...) - - // ceph config set commands want admin keyring - podSpec.Spec.Volumes = append(podSpec.Spec.Volumes, - keyring.Volume().Admin()) if c.spec.Network.IsHost() { podSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet } else if c.spec.Network.NetworkSpec.IsMultus() { @@ -175,41 +159,6 @@ func (c *Cluster) makeChownInitContainer(mgrConfig *mgrConfig) v1.Container { ) } -func (c *Cluster) makeSetServerAddrInitContainer(mgrConfig *mgrConfig, mgrModule string) v1.Container { - // Commands produced for various Ceph major versions (differences highlighted) - // L: config-key set mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) - // M: config set mgr.a mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) - // N: config set mgr.a mgr//server_addr $(ROOK_CEPH__SERVER_ADDR) --force - cfgSetArgs := []string{"config", "set"} - cfgSetArgs = append(cfgSetArgs, fmt.Sprintf("mgr.%s", mgrConfig.DaemonID)) - cfgPath := fmt.Sprintf("mgr/%s/%s/server_addr", mgrModule, mgrConfig.DaemonID) - cfgSetArgs = append(cfgSetArgs, cfgPath, "0.0.0.0") - cfgSetArgs = append(cfgSetArgs, "--force") - cfgSetArgs = append(cfgSetArgs, "--verbose") - - container := v1.Container{ - Name: "init-set-" + strings.ToLower(mgrModule) + "-server-addr", - Command: []string{ - "ceph", - }, - Args: append( - controller.AdminFlags(c.clusterInfo), - cfgSetArgs..., - ), - Image: c.spec.CephVersion.Image, - VolumeMounts: append( - controller.DaemonVolumeMounts(mgrConfig.DataPathMap, mgrConfig.ResourceName), - keyring.VolumeMount().Admin(), - ), - Env: append( - controller.DaemonEnvVars(c.spec.CephVersion.Image), - c.cephMgrOrchestratorModuleEnvs()..., - ), - Resources: cephv1.GetMgrResources(c.spec.Resources), - } - return container -} - func (c *Cluster) makeMgrDaemonContainer(mgrConfig *mgrConfig) v1.Container { container := v1.Container{ diff --git a/pkg/operator/ceph/cluster/mgr/spec_test.go b/pkg/operator/ceph/cluster/mgr/spec_test.go index 2ff7aa9d6..921d8c34d 100644 --- a/pkg/operator/ceph/cluster/mgr/spec_test.go +++ b/pkg/operator/ceph/cluster/mgr/spec_test.go @@ -25,7 +25,6 @@ import ( cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/config" cephtest "github.com/rook/rook/pkg/operator/ceph/test" - cephver "github.com/rook/rook/pkg/operator/ceph/version" optest "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -115,31 +114,6 @@ func TestHostNetwork(t *testing.T) { assert.Equal(t, v1.DNSClusterFirstWithHostNet, d.Spec.Template.Spec.DNSPolicy) } -func TestHttpBindFix(t *testing.T) { - clientset := optest.New(t, 1) - clusterInfo := &cephclient.ClusterInfo{Namespace: "ns", FSID: "myfsid"} - clusterInfo.SetName("test") - clusterSpec := cephv1.ClusterSpec{ - Dashboard: cephv1.DashboardSpec{Enabled: true, Port: 1234}, - DataDirHostPath: "/var/lib/rook/", - } - c := New(&clusterd.Context{Clientset: clientset}, clusterInfo, clusterSpec, "myversion") - - mgrTestConfig := mgrConfig{ - DaemonID: "a", - ResourceName: "mgr-a", - DataPathMap: config.NewStatelessDaemonDataPathMap(config.MgrType, "a", "rook-ceph", "/var/lib/rook/"), - } - - c.clusterInfo.CephVersion = cephver.Nautilus - expectedInitContainers := 3 - d, err := c.makeDeployment(&mgrTestConfig) - assert.NoError(t, err) - assert.NotNil(t, d) - assert.Equal(t, expectedInitContainers, - len(d.Spec.Template.Spec.InitContainers)) -} - func TestApplyPrometheusAnnotations(t *testing.T) { clientset := optest.New(t, 1) clusterSpec := cephv1.ClusterSpec{