diff --git a/pkg/apis/ceph.rook.io/v1/network.go b/pkg/apis/ceph.rook.io/v1/network.go index c950eed22..debe0d39f 100644 --- a/pkg/apis/ceph.rook.io/v1/network.go +++ b/pkg/apis/ceph.rook.io/v1/network.go @@ -27,6 +27,11 @@ import ( "github.com/pkg/errors" ) +// enforceHostNetwork is a private package variable that can be set via the rook-operator-config +// setting "ROOK_ENFORCE_HOST_NETWORK". when set to "true", it lets rook create all pods with host network enabled. +// This can be used, for example, to run Rook in k8s clusters with no CNI where host networking is required +var enforceHostNetwork bool = false + // IsMultus get whether to use multus network provider func (n *NetworkSpec) IsMultus() bool { return n.Provider == NetworkProviderMultus @@ -40,7 +45,7 @@ func (n *NetworkSpec) IsMultus() bool { // together with an empty or unset network provider has the same effect as // network.Provider set to "host" func (n *NetworkSpec) IsHost() bool { - return (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost + return enforceHostNetwork || (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost } func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error { @@ -62,7 +67,7 @@ func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error { if !spec.AddressRanges.IsEmpty() { if !spec.IsMultus() && !spec.IsHost() { - // TODO: be sure to update docs that AddressRanges can be specified for host networking as + // TODO: be sure to update docs that AddressRanges can be specified for host networking as // well as multus so that the override configmap doesn't need to be set return errors.Errorf("network ranges can only be specified for %q and %q network providers", NetworkProviderHost, NetworkProviderMultus) } @@ -181,3 +186,11 @@ func (l *CIDRList) String() string { } return strings.Join(sl, ", ") } + +func SetEnforceHostNetwork(val bool) { + enforceHostNetwork = val +} + +func EnforceHostNetwork() bool { + return enforceHostNetwork +} diff --git a/pkg/apis/ceph.rook.io/v1/network_test.go b/pkg/apis/ceph.rook.io/v1/network_test.go index 6c6980f2c..7269a86df 100644 --- a/pkg/apis/ceph.rook.io/v1/network_test.go +++ b/pkg/apis/ceph.rook.io/v1/network_test.go @@ -75,6 +75,8 @@ func TestValidateNetworkSpec(t *testing.T) { // test the NetworkSpec.IsHost method with different network providers // Also test it in combination with the legacy // "HostNetwork" setting. +// Also test the effect of the operator config setting +// ROOK_ENFORCE_HOST_NETWORK. func TestNetworkCephIsHost(t *testing.T) { net := NetworkSpec{HostNetwork: false} @@ -85,6 +87,14 @@ func TestNetworkCephIsHost(t *testing.T) { net.HostNetwork = true assert.True(t, net.IsHost()) + // enforcing does not change the result if host network is selected + // anyway in the cluster. + SetEnforceHostNetwork(true) + assert.True(t, net.IsHost()) + + SetEnforceHostNetwork(false) + assert.True(t, net.IsHost()) + net = NetworkSpec{} net.Provider = NetworkProviderDefault net.HostNetwork = false @@ -95,16 +105,28 @@ func TestNetworkCephIsHost(t *testing.T) { net.HostNetwork = false assert.False(t, net.IsHost()) + // test that not enforcing does not change the result. + SetEnforceHostNetwork(false) + assert.False(t, net.IsHost()) + + // test enforcing of host network + SetEnforceHostNetwork(true) + assert.True(t, net.IsHost()) + + SetEnforceHostNetwork(false) net = NetworkSpec{} net.Provider = NetworkProviderMultus net.HostNetwork = true assert.False(t, net.IsHost()) - // test with nonempty but invalid provider + // test with nonempty but invalid provider net = NetworkSpec{} net.HostNetwork = true net.Provider = "foo" + SetEnforceHostNetwork(false) assert.False(t, net.IsHost()) + SetEnforceHostNetwork(true) + assert.True(t, net.IsHost()) } diff --git a/pkg/operator/ceph/cluster/cleanup.go b/pkg/operator/ceph/cluster/cleanup.go index 92b293359..eb50d1a33 100644 --- a/pkg/operator/ceph/cluster/cleanup.go +++ b/pkg/operator/ceph/cluster/cleanup.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/file/mirror" "github.com/rook/rook/pkg/operator/ceph/object" @@ -162,6 +163,7 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster, RestartPolicy: v1.RestartPolicyOnFailure, PriorityClassName: cephv1.GetCleanupPriorityClassName(cluster.Spec.PriorityClassNames), ServiceAccountName: k8sutil.DefaultServiceAccount, + HostNetwork: opcontroller.EnforceHostNetwork(), }, } diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 9e3673dfb..4839ea6a2 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" @@ -154,7 +155,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re }, RestartPolicy: restart, Volumes: volumes, - HostNetwork: c.spec.Network.IsHost(), + HostNetwork: opcontroller.EnforceHostNetwork(), PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames), SchedulerName: osdProps.schedulerName, } diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 0e32a898a..238dd29bd 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -132,6 +132,7 @@ func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result } opcontroller.SetAllowLoopDevices(r.config.Parameters) + opcontroller.SetEnforceHostNetwork(r.config.Parameters) logger.Infof("%s done reconciling", controllerName) return reconcile.Result{}, nil diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index b726e2e97..f5cc1ca1d 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -50,7 +50,9 @@ type ClusterHealth struct { const ( // OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator - OperatorSettingConfigMapName string = "rook-ceph-operator-config" + OperatorSettingConfigMapName string = "rook-ceph-operator-config" + enforceHostNetworkSettingName string = "ROOK_ENFORCE_HOST_NETWORK" + enforceHostNetworkDefaultValue string = "false" // UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file // This typically is raised when the operator has not finished initializing @@ -116,6 +118,21 @@ func LoopDevicesAllowed() bool { return loopDevicesAllowed } +func SetEnforceHostNetwork(data map[string]string) { + strval := k8sutil.GetValue(data, enforceHostNetworkSettingName, enforceHostNetworkDefaultValue) + val, err := strconv.ParseBool(strval) + if err != nil { + logger.Warningf("failed to parse value %q for %q. assuming false value", strval, enforceHostNetworkSettingName) + cephv1.SetEnforceHostNetwork(false) + return + } + cephv1.SetEnforceHostNetwork(val) +} + +func EnforceHostNetwork() bool { + return cephv1.EnforceHostNetwork() +} + // canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely. func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool { // Get a list of all the keys causing the HEALTH_ERR status. @@ -153,7 +170,6 @@ func IsReadyToReconcile(ctx context.Context, c client.Client, namespacedName typ return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady } cephCluster = clusterList.Items[0] - // If the cluster has a cleanup policy to destroy the cluster and it has been marked for deletion, treat it as if it does not exist if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.DeletionTimestamp.IsZero() { logger.Infof("%q: CephCluster has a destructive cleanup policy, allowing %q to be deleted", controllerName, namespacedName) diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index eadcb2376..532cc0bcb 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -106,6 +106,33 @@ func TestSetAllowLoopDevices(t *testing.T) { assert.True(t, LoopDevicesAllowed()) } +func TestSetEnforceHostNetwork(t *testing.T) { + logger.Infof("testing default value for %v", enforceHostNetworkSettingName) + opConfig := map[string]string{} + SetEnforceHostNetwork(opConfig) + assert.False(t, EnforceHostNetwork()) + + // test invalid setting + var value string = "foo" + logger.Infof("testing invalid value'%v' for %v", value, enforceHostNetworkSettingName) + opConfig[enforceHostNetworkSettingName] = value + SetEnforceHostNetwork(opConfig) + assert.False(t, EnforceHostNetwork()) + + // test valid settings + value = "true" + logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) + opConfig[enforceHostNetworkSettingName] = value + SetEnforceHostNetwork(opConfig) + assert.True(t, EnforceHostNetwork()) + + value = "false" + logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) + opConfig[enforceHostNetworkSettingName] = value + SetEnforceHostNetwork(opConfig) + assert.False(t, EnforceHostNetwork()) +} + func TestIsReadyToReconcile(t *testing.T) { scheme := scheme.Scheme scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 4be362ed2..d63e7bc9d 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -382,6 +382,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if tp.CSILogRotation { applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp) } + rbdProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() // Create service if either liveness or GRPC metrics are enabled. if CSIParam.EnableLiveness { @@ -419,6 +420,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if tp.CSILogRotation { applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp) } + cephfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() // Create service if either liveness or GRPC metrics are enabled. if CSIParam.EnableLiveness { @@ -457,6 +459,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if tp.CSILogRotation { applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp) } + nfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() enabledDrivers = append(enabledDrivers, driverDetails{ name: NFSDriverShortName, diff --git a/pkg/operator/ceph/object/cosi/spec.go b/pkg/operator/ceph/object/cosi/spec.go index f01ebabbd..50bf737c5 100644 --- a/pkg/operator/ceph/object/cosi/spec.go +++ b/pkg/operator/ceph/object/cosi/spec.go @@ -20,6 +20,7 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -83,6 +84,7 @@ func createCOSIPodSpec(cephCOSIDriver *cephv1.CephCOSIDriver) (corev1.PodTemplat cosiSideCarContainer := createCOSISideCarContainer(cephCOSIDriver) podSpec := corev1.PodSpec{ + HostNetwork: opcontroller.EnforceHostNetwork(), Containers: []corev1.Container{ cosiDriverContainer, cosiSideCarContainer, diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index bf543524b..ee4cdb3f6 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/clusterd" discoverDaemon "github.com/rook/rook/pkg/daemon/discover" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" k8sutil "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/sys" @@ -173,7 +174,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco }, }, }, - HostNetwork: false, + HostNetwork: opcontroller.EnforceHostNetwork(), PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""), }, }, diff --git a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go index affc87558..b651def6d 100644 --- a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go +++ b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/util" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" @@ -302,6 +303,7 @@ func (cr *cmdReporterCfg) initJobSpec() (*batch.Job, error) { }, RestartPolicy: v1.RestartPolicyOnFailure, ServiceAccountName: k8sutil.DefaultServiceAccount, + HostNetwork: cephv1.EnforceHostNetwork(), } copyBinsVol, _ := copyBinariesVolAndMount() podSpec.Volumes = []v1.Volume{copyBinsVol}