ceph mon: set public-bind-addr from commandline

Set each Ceph mon's `public bind addr` from the commandline instead of
by inserting it into the config file. This eliminates an extra step from
initializing mon configs.

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>

pkg/daemon/ceph/mon/config.go

@@ -18,7 +18,6 @@ package mon
 
 import (
 	"fmt"
-	"net"
 	"path"
 	"path/filepath"
 	"strings"
@@ -66,14 +65,6 @@ func getMonKeyringPath(configDir, monName string) string {
 
 // generateConnectionConfigFile generates and writes the monitor config file to disk
 func generateConnectionConfigFile(context *clusterd.Context, config *Config, pathRoot, keyringPath string) (string, error) {
-	// public_bind_addr is set from the pod IP which can only be known at runtime, so set this
-	// at config init int the Ceph config file.
-	// See pkg/operator/ceph/cluster/mon/spec.go - makeMonDaemonContainer() comment notes for more
-	privateAddr := net.JoinHostPort(context.NetworkInfo.ClusterAddr, fmt.Sprintf("%d", config.Port))
-	settings := map[string]string{
-		"public bind addr": privateAddr,
-	}
-
 	// The user is "mon.<mon-name>" for mon config items
 	clientUser := fmt.Sprintf("mon.%s", config.Name)
 
@@ -82,6 +73,6 @@ func generateConnectionConfigFile(context *clusterd.Context, config *Config, pat
 		config.Cluster,
 		pathRoot, clientUser, keyringPath,
 		nil,
-		settings,
+		nil,
 	)
 }

pkg/daemon/ceph/mon/endpoint_test.go

@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+
 package mon
 
 import (

pkg/operator/ceph/cluster/mon/spec.go

@@ -193,14 +193,6 @@ func (c *Cluster) makeMonFSInitContainer(monConfig *monConfig) v1.Container {
 
 func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
 	return v1.Container{
-		// The operator has set up the mon's service already, so the IP that the mon should
-		// broadcast as its own (--public-addr) is known. But the pod's IP, which the mon should
-		// bind to (--public-bind-addr) isn't known until runtime. 3 solutions were considered for
-		// resolving this issue:
-		// 1. Rook in config init sets "public_bind_addr" in the Ceph config file
-		//    - Chosen solution, but is not as transparent to inspection as using commandline arg
-		// 2. Use bash to do variable substitution with the pod IP env var; but bash is a poor PID1
-		// 3. Use tini to do var substitution as above; but tini doesn't exist in the ceph images.
 		Name: "mon",
 		Command: []string{
 			cephMonCommand,
@@ -209,7 +201,7 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
 			[]string{
 				"--foreground",
 				"--public-addr", joinHostPort(monConfig.PublicIP, monConfig.Port),
-				// --public-bind-addr is set in the config file at init time
+				"--public-bind-addr", joinHostPort("$(ROOK_PRIVATE_IP)", monConfig.Port),
 				// do not add the '--cluster/--conf/--keyring' flags; rook wants their default values
 			},
 			c.cephMonCommonArgs(monConfig)...,
@@ -224,7 +216,10 @@ func (c *Cluster) makeMonDaemonContainer(monConfig *monConfig) v1.Container {
 				Protocol:      v1.ProtocolTCP,
 			},
 		},
-		Env:       k8sutil.ClusterDaemonEnvVars(),
+		Env: append(
+			k8sutil.ClusterDaemonEnvVars(),
+			k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
+		),
 		Resources: c.resources,
 	}
 }

pkg/operator/ceph/cluster/mon/spec_test.go

@@ -149,7 +149,7 @@ func testPodSpec(t *testing.T, dataDir string) {
 	assert.Equal(t, "1337", cont.Resources.Requests.Memory().String())
 
 	// main mon daemon container
-	monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars())
+	monDaemonEnvs := len(k8sutil.ClusterDaemonEnvVars()) + 1
 	monDaemonContDev := test_opceph.ContainerTestDefinition{
 		Image: &cephImage,
 		Command: []string{
@@ -157,7 +157,8 @@ func testPodSpec(t *testing.T, dataDir string) {
 		Args: append(
 			monCommonExpectedArgs(name, c),
 			[]string{"--foreground"},
-			[]string{"--public-addr", "2.4.6.1:6790"}),
+			[]string{"--public-addr", "2.4.6.1:6790"},
+			[]string{"--public-bind-addr", "$(ROOK_PRIVATE_IP):6790"}),
 		VolumeMountNames: cephVolumeMountNames,
 		EnvCount:         &monDaemonEnvs,
 		Ports: []v1.ContainerPort{