diff --git a/pkg/daemon/ceph/mon/config.go b/pkg/daemon/ceph/mon/config.go index 65d6bfd07..5596c6b67 100644 --- a/pkg/daemon/ceph/mon/config.go +++ b/pkg/daemon/ceph/mon/config.go @@ -18,7 +18,6 @@ package mon import ( "fmt" - "net" "path" "path/filepath" "strings" @@ -66,14 +65,6 @@ func getMonKeyringPath(configDir, monName string) string { // generateConnectionConfigFile generates and writes the monitor config file to disk func generateConnectionConfigFile(context *clusterd.Context, config *Config, pathRoot, keyringPath string) (string, error) { - // public_bind_addr is set from the pod IP which can only be known at runtime, so set this - // at config init int the Ceph config file. - // See pkg/operator/ceph/cluster/mon/spec.go - makeMonDaemonContainer() comment notes for more - privateAddr := net.JoinHostPort(context.NetworkInfo.ClusterAddr, fmt.Sprintf("%d", config.Port)) - settings := map[string]string{ - "public bind addr": privateAddr, - } - // The user is "mon." for mon config items clientUser := fmt.Sprintf("mon.%s", config.Name) @@ -82,6 +73,6 @@ func generateConnectionConfigFile(context *clusterd.Context, config *Config, pat config.Cluster, pathRoot, clientUser, keyringPath, nil, - settings, + nil, ) } diff --git a/pkg/daemon/ceph/mon/endpoint_test.go b/pkg/daemon/ceph/mon/endpoint_test.go index 053845ba7..a64ef118e 100644 --- a/pkg/daemon/ceph/mon/endpoint_test.go +++ b/pkg/daemon/ceph/mon/endpoint_test.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package mon import ( diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go index e12b576ca..ca0275531 100644 --- a/pkg/operator/ceph/cluster/mon/spec.go +++ b/pkg/operator/ceph/cluster/mon/spec.go @@ -193,14 +193,6 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container { func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { return v1.Container{ - // The operator has set up the mon's service already, so the IP that the mon should - // broadcast as its own (--public-addr) is known. But the pod's IP, which the mon should - // bind to (--public-bind-addr) isn't known until runtime. 3 solutions were considered for - // resolving this issue: - // 1. Rook in config init sets "public_bind_addr" in the Ceph config file - // - Chosen solution, but is not as transparent to inspection as using commandline arg - // 2. Use bash to do variable substitution with the pod IP env var; but bash is a poor PID1 - // 3. Use tini to do var substitution as above; but tini doesn't exist in the ceph images. Name: "mon", Command: []string{ cephMonCommand, @@ -209,7 +201,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { []string{ "--foreground", "--public-addr", joinHostPort(monConfig.PublicIP, monConfig.Port), - // --public-bind-addr is set in the config file at init time + "--public-bind-addr", joinHostPort("$(ROOK_PRIVATE_IP)", monConfig.Port), // do not add the '--cluster/--conf/--keyring' flags; rook wants their default values }, c.cephMonCommonArgs(monConfig)..., @@ -224,7 +216,10 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container { Protocol: v1.ProtocolTCP, }, }, - Env: k8sutil.ClusterDaemonEnvVars(), + Env: append( + k8sutil.ClusterDaemonEnvVars(), + k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), + ), Resources: c.resources, } } diff --git a/pkg/operator/ceph/cluster/mon/spec_test.go b/pkg/operator/ceph/cluster/mon/spec_test.go index 7e801e228..c8a005b54 100644 --- a/pkg/operator/ceph/cluster/mon/spec_test.go +++ b/pkg/operator/ceph/cluster/mon/spec_test.go @@ -149,7 +149,7 @@ func testPodSpec(t *testing.T, dataDir string) { assert.Equal(t, "1337", cont.Resources.Requests.Memory().String()) // main mon daemon container - monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) + monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) + 1 monDaemonContDev := test_opceph.ContainerTestDefinition{ Image: &cephImage, Command: []string{ @@ -157,7 +157,8 @@ func testPodSpec(t *testing.T, dataDir string) { Args: append( monCommonExpectedArgs(name, c), []string{"--foreground"}, - []string{"--public-addr", "2.4.6.1:6790"}), + []string{"--public-addr", "2.4.6.1:6790"}, + []string{"--public-bind-addr", "$(ROOK_PRIVATE_IP):6790"}), VolumeMountNames: cephVolumeMountNames, EnvCount: &monDaemonEnvs, Ports: []v1.ContainerPort{