network: add a new operator config setting ROOK_ENFORCE_HOSTNETWORK

This new setting is of Boolean type and defaults to "false".

    When set to "true", it changes the behavior of the
     rook operator to
    nable host network on all pods created by the cephcluster controller

     new method to check the setting:  opcontroller.EnForceHostNetwork()

Signed-off-by: Michael Adam <obnox@samba.org>
(cherry picked from commit e378588359)

pkg/apis/ceph.rook.io/v1/network.go

@@ -27,6 +27,11 @@ import (
 	"github.com/pkg/errors"
 )
 
+// enforceHostNetwork is a private package variable that can be set via the rook-operator-config
+// setting "ROOK_ENFORCE_HOST_NETWORK". when set to "true", it lets rook create all pods with host network enabled.
+// This can be used, for example, to run Rook in k8s clusters with no CNI where host networking is required
+var enforceHostNetwork bool = false
+
 // IsMultus get whether to use multus network provider
 func (n *NetworkSpec) IsMultus() bool {
 	return n.Provider == NetworkProviderMultus
@@ -40,7 +45,7 @@ func (n *NetworkSpec) IsMultus() bool {
 // together with an empty or unset network provider has the same effect as
 // network.Provider set to "host"
 func (n *NetworkSpec) IsHost() bool {
-	return (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost
+	return enforceHostNetwork || (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost
 }
 
 func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error {
@@ -62,7 +67,7 @@ func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error {
 
 	if !spec.AddressRanges.IsEmpty() {
 		if !spec.IsMultus() && !spec.IsHost() {
-			// TODO: be sure to update docs that AddressRanges can be specified for host networking  as
+			// TODO: be sure to update docs that AddressRanges can be specified for host networking as
 			// well as multus so that the override configmap doesn't need to be set
 			return errors.Errorf("network ranges can only be specified for %q and %q network providers", NetworkProviderHost, NetworkProviderMultus)
 		}
@@ -181,3 +186,11 @@ func (l *CIDRList) String() string {
 	}
 	return strings.Join(sl, ", ")
 }
+
+func SetEnforceHostNetwork(val bool) {
+	enforceHostNetwork = val
+}
+
+func EnforceHostNetwork() bool {
+	return enforceHostNetwork
+}

pkg/apis/ceph.rook.io/v1/network_test.go

@@ -75,6 +75,8 @@ func TestValidateNetworkSpec(t *testing.T) {
 // test the NetworkSpec.IsHost method with different network providers
 // Also test it in combination with the legacy
 // "HostNetwork" setting.
+// Also test the effect of the operator config setting
+// ROOK_ENFORCE_HOST_NETWORK.
 func TestNetworkCephIsHost(t *testing.T) {
 	net := NetworkSpec{HostNetwork: false}
 
@@ -85,6 +87,14 @@ func TestNetworkCephIsHost(t *testing.T) {
 	net.HostNetwork = true
 	assert.True(t, net.IsHost())
 
+	// enforcing does not change the result if host network is selected
+	// anyway in the cluster.
+	SetEnforceHostNetwork(true)
+	assert.True(t, net.IsHost())
+
+	SetEnforceHostNetwork(false)
+	assert.True(t, net.IsHost())
+
 	net = NetworkSpec{}
 	net.Provider = NetworkProviderDefault
 	net.HostNetwork = false
@@ -95,16 +105,28 @@ func TestNetworkCephIsHost(t *testing.T) {
 	net.HostNetwork = false
 	assert.False(t, net.IsHost())
 
+	// test that not enforcing does not change the result.
+	SetEnforceHostNetwork(false)
+	assert.False(t, net.IsHost())
+
+	// test enforcing of host network
+	SetEnforceHostNetwork(true)
+	assert.True(t, net.IsHost())
+
+	SetEnforceHostNetwork(false)
 	net = NetworkSpec{}
 	net.Provider = NetworkProviderMultus
 	net.HostNetwork = true
 	assert.False(t, net.IsHost())
 
-	// test with  nonempty but invalid provider
+	// test with nonempty but invalid provider
 	net = NetworkSpec{}
 	net.HostNetwork = true
 	net.Provider = "foo"
+	SetEnforceHostNetwork(false)
 	assert.False(t, net.IsHost())
+	SetEnforceHostNetwork(true)
+	assert.True(t, net.IsHost())
 
 }
 

pkg/operator/ceph/cluster/cleanup.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/ceph/file/mds"
 	"github.com/rook/rook/pkg/operator/ceph/file/mirror"
 	"github.com/rook/rook/pkg/operator/ceph/object"
@@ -162,6 +163,7 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster,
 			RestartPolicy:      v1.RestartPolicyOnFailure,
 			PriorityClassName:  cephv1.GetCleanupPriorityClassName(cluster.Spec.PriorityClassNames),
 			ServiceAccountName: k8sutil.DefaultServiceAccount,
+			HostNetwork:        opcontroller.EnforceHostNetwork(),
 		},
 	}
 

pkg/operator/ceph/cluster/osd/provision_spec.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
 	"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	batch "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
@@ -154,7 +155,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
 		},
 		RestartPolicy:     restart,
 		Volumes:           volumes,
-		HostNetwork:       c.spec.Network.IsHost(),
+		HostNetwork:       opcontroller.EnforceHostNetwork(),
 		PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames),
 		SchedulerName:     osdProps.schedulerName,
 	}

pkg/operator/ceph/controller.go

@@ -132,6 +132,7 @@ func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result
 	}
 
 	opcontroller.SetAllowLoopDevices(r.config.Parameters)
+	opcontroller.SetEnforceHostNetwork(r.config.Parameters)
 
 	logger.Infof("%s done reconciling", controllerName)
 	return reconcile.Result{}, nil

pkg/operator/ceph/controller/controller_utils.go

@@ -50,7 +50,9 @@ type ClusterHealth struct {
 
 const (
 	// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
-	OperatorSettingConfigMapName string = "rook-ceph-operator-config"
+	OperatorSettingConfigMapName   string = "rook-ceph-operator-config"
+	enforceHostNetworkSettingName  string = "ROOK_ENFORCE_HOST_NETWORK"
+	enforceHostNetworkDefaultValue string = "false"
 
 	// UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file
 	// This typically is raised when the operator has not finished initializing
@@ -116,6 +118,21 @@ func LoopDevicesAllowed() bool {
 	return loopDevicesAllowed
 }
 
+func SetEnforceHostNetwork(data map[string]string) {
+	strval := k8sutil.GetValue(data, enforceHostNetworkSettingName, enforceHostNetworkDefaultValue)
+	val, err := strconv.ParseBool(strval)
+	if err != nil {
+		logger.Warningf("failed to parse value %q for %q. assuming false value", strval, enforceHostNetworkSettingName)
+		cephv1.SetEnforceHostNetwork(false)
+		return
+	}
+	cephv1.SetEnforceHostNetwork(val)
+}
+
+func EnforceHostNetwork() bool {
+	return cephv1.EnforceHostNetwork()
+}
+
 // canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely.
 func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool {
 	// Get a list of all the keys causing the HEALTH_ERR status.
@@ -153,7 +170,6 @@ func IsReadyToReconcile(ctx context.Context, c client.Client, namespacedName typ
 		return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
 	}
 	cephCluster = clusterList.Items[0]
-
 	// If the cluster has a cleanup policy to destroy the cluster and it has been marked for deletion, treat it as if it does not exist
 	if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.DeletionTimestamp.IsZero() {
 		logger.Infof("%q: CephCluster has a destructive cleanup policy, allowing %q to be deleted", controllerName, namespacedName)

pkg/operator/ceph/controller/controller_utils_test.go

@@ -106,6 +106,33 @@ func TestSetAllowLoopDevices(t *testing.T) {
 	assert.True(t, LoopDevicesAllowed())
 }
 
+func TestSetEnforceHostNetwork(t *testing.T) {
+	logger.Infof("testing default value for %v", enforceHostNetworkSettingName)
+	opConfig := map[string]string{}
+	SetEnforceHostNetwork(opConfig)
+	assert.False(t, EnforceHostNetwork())
+
+	// test invalid setting
+	var value string = "foo"
+	logger.Infof("testing invalid value'%v' for %v", value, enforceHostNetworkSettingName)
+	opConfig[enforceHostNetworkSettingName] = value
+	SetEnforceHostNetwork(opConfig)
+	assert.False(t, EnforceHostNetwork())
+
+	// test valid settings
+	value = "true"
+	logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName)
+	opConfig[enforceHostNetworkSettingName] = value
+	SetEnforceHostNetwork(opConfig)
+	assert.True(t, EnforceHostNetwork())
+
+	value = "false"
+	logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName)
+	opConfig[enforceHostNetworkSettingName] = value
+	SetEnforceHostNetwork(opConfig)
+	assert.False(t, EnforceHostNetwork())
+}
+
 func TestIsReadyToReconcile(t *testing.T) {
 	scheme := scheme.Scheme
 	scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})

pkg/operator/ceph/csi/spec.go

@@ -382,6 +382,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if tp.CSILogRotation {
 			applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
 		}
+		rbdProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork()
 
 		// Create service if either liveness or GRPC metrics are enabled.
 		if CSIParam.EnableLiveness {
@@ -419,6 +420,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if tp.CSILogRotation {
 			applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp)
 		}
+		cephfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork()
 
 		// Create service if either liveness or GRPC metrics are enabled.
 		if CSIParam.EnableLiveness {
@@ -457,6 +459,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
 		if tp.CSILogRotation {
 			applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp)
 		}
+		nfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork()
 
 		enabledDrivers = append(enabledDrivers, driverDetails{
 			name:           NFSDriverShortName,

pkg/operator/ceph/object/cosi/spec.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -83,6 +84,7 @@ func createCOSIPodSpec(cephCOSIDriver *cephv1.CephCOSIDriver) (corev1.PodTemplat
 	cosiSideCarContainer := createCOSISideCarContainer(cephCOSIDriver)
 
 	podSpec := corev1.PodSpec{
+		HostNetwork: opcontroller.EnforceHostNetwork(),
 		Containers: []corev1.Container{
 			cosiDriverContainer,
 			cosiSideCarContainer,

pkg/operator/discover/discover.go

@@ -31,6 +31,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
 	"github.com/rook/rook/pkg/operator/ceph/controller"
+	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/sys"
 
@@ -173,7 +174,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco
 							},
 						},
 					},
-					HostNetwork:       false,
+					HostNetwork:       opcontroller.EnforceHostNetwork(),
 					PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""),
 				},
 			},

pkg/operator/k8sutil/cmdreporter/cmdreporter.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/daemon/util"
 	"github.com/rook/rook/pkg/operator/k8sutil"
 	batch "k8s.io/api/batch/v1"
@@ -302,6 +303,7 @@ func (cr *cmdReporterCfg) initJobSpec() (*batch.Job, error) {
 		},
 		RestartPolicy:      v1.RestartPolicyOnFailure,
 		ServiceAccountName: k8sutil.DefaultServiceAccount,
+		HostNetwork:        cephv1.EnforceHostNetwork(),
 	}
 	copyBinsVol, _ := copyBinariesVolAndMount()
 	podSpec.Volumes = []v1.Volume{copyBinsVol}