From d73f4ffb0fbb6fe49a8dd3c46f4bb747f9fbe434 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Fri, 16 Aug 2024 19:04:01 +0200 Subject: [PATCH] network: add a new operator config setting ROOK_ENFORCE_HOSTNETWORK This new setting is of Boolean type and defaults to "false". When set to "true", it changes the behavior of the rook operator to nable host network on all pods created by the cephcluster controller new method to check the setting: opcontroller.EnForceHostNetwork() Signed-off-by: Michael Adam (cherry picked from commit e3785883591ca6710a04e89bd4f0f88079775d81) --- pkg/apis/ceph.rook.io/v1/network.go | 17 ++++++++++-- pkg/apis/ceph.rook.io/v1/network_test.go | 24 ++++++++++++++++- pkg/operator/ceph/cluster/cleanup.go | 2 ++ .../ceph/cluster/osd/provision_spec.go | 3 ++- pkg/operator/ceph/controller.go | 1 + .../ceph/controller/controller_utils.go | 20 ++++++++++++-- .../ceph/controller/controller_utils_test.go | 27 +++++++++++++++++++ pkg/operator/ceph/csi/spec.go | 3 +++ pkg/operator/ceph/object/cosi/spec.go | 2 ++ pkg/operator/discover/discover.go | 3 ++- .../k8sutil/cmdreporter/cmdreporter.go | 2 ++ 11 files changed, 97 insertions(+), 7 deletions(-) diff --git a/pkg/apis/ceph.rook.io/v1/network.go b/pkg/apis/ceph.rook.io/v1/network.go index c950eed22..debe0d39f 100644 --- a/pkg/apis/ceph.rook.io/v1/network.go +++ b/pkg/apis/ceph.rook.io/v1/network.go @@ -27,6 +27,11 @@ import ( "github.com/pkg/errors" ) +// enforceHostNetwork is a private package variable that can be set via the rook-operator-config +// setting "ROOK_ENFORCE_HOST_NETWORK". when set to "true", it lets rook create all pods with host network enabled. +// This can be used, for example, to run Rook in k8s clusters with no CNI where host networking is required +var enforceHostNetwork bool = false + // IsMultus get whether to use multus network provider func (n *NetworkSpec) IsMultus() bool { return n.Provider == NetworkProviderMultus @@ -40,7 +45,7 @@ func (n *NetworkSpec) IsMultus() bool { // together with an empty or unset network provider has the same effect as // network.Provider set to "host" func (n *NetworkSpec) IsHost() bool { - return (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost + return enforceHostNetwork || (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost } func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error { @@ -62,7 +67,7 @@ func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error { if !spec.AddressRanges.IsEmpty() { if !spec.IsMultus() && !spec.IsHost() { - // TODO: be sure to update docs that AddressRanges can be specified for host networking as + // TODO: be sure to update docs that AddressRanges can be specified for host networking as // well as multus so that the override configmap doesn't need to be set return errors.Errorf("network ranges can only be specified for %q and %q network providers", NetworkProviderHost, NetworkProviderMultus) } @@ -181,3 +186,11 @@ func (l *CIDRList) String() string { } return strings.Join(sl, ", ") } + +func SetEnforceHostNetwork(val bool) { + enforceHostNetwork = val +} + +func EnforceHostNetwork() bool { + return enforceHostNetwork +} diff --git a/pkg/apis/ceph.rook.io/v1/network_test.go b/pkg/apis/ceph.rook.io/v1/network_test.go index 6c6980f2c..7269a86df 100644 --- a/pkg/apis/ceph.rook.io/v1/network_test.go +++ b/pkg/apis/ceph.rook.io/v1/network_test.go @@ -75,6 +75,8 @@ func TestValidateNetworkSpec(t *testing.T) { // test the NetworkSpec.IsHost method with different network providers // Also test it in combination with the legacy // "HostNetwork" setting. +// Also test the effect of the operator config setting +// ROOK_ENFORCE_HOST_NETWORK. func TestNetworkCephIsHost(t *testing.T) { net := NetworkSpec{HostNetwork: false} @@ -85,6 +87,14 @@ func TestNetworkCephIsHost(t *testing.T) { net.HostNetwork = true assert.True(t, net.IsHost()) + // enforcing does not change the result if host network is selected + // anyway in the cluster. + SetEnforceHostNetwork(true) + assert.True(t, net.IsHost()) + + SetEnforceHostNetwork(false) + assert.True(t, net.IsHost()) + net = NetworkSpec{} net.Provider = NetworkProviderDefault net.HostNetwork = false @@ -95,16 +105,28 @@ func TestNetworkCephIsHost(t *testing.T) { net.HostNetwork = false assert.False(t, net.IsHost()) + // test that not enforcing does not change the result. + SetEnforceHostNetwork(false) + assert.False(t, net.IsHost()) + + // test enforcing of host network + SetEnforceHostNetwork(true) + assert.True(t, net.IsHost()) + + SetEnforceHostNetwork(false) net = NetworkSpec{} net.Provider = NetworkProviderMultus net.HostNetwork = true assert.False(t, net.IsHost()) - // test with nonempty but invalid provider + // test with nonempty but invalid provider net = NetworkSpec{} net.HostNetwork = true net.Provider = "foo" + SetEnforceHostNetwork(false) assert.False(t, net.IsHost()) + SetEnforceHostNetwork(true) + assert.True(t, net.IsHost()) } diff --git a/pkg/operator/ceph/cluster/cleanup.go b/pkg/operator/ceph/cluster/cleanup.go index 92b293359..eb50d1a33 100644 --- a/pkg/operator/ceph/cluster/cleanup.go +++ b/pkg/operator/ceph/cluster/cleanup.go @@ -30,6 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/osd" "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/file/mirror" "github.com/rook/rook/pkg/operator/ceph/object" @@ -162,6 +163,7 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster, RestartPolicy: v1.RestartPolicyOnFailure, PriorityClassName: cephv1.GetCleanupPriorityClassName(cluster.Spec.PriorityClassNames), ServiceAccountName: k8sutil.DefaultServiceAccount, + HostNetwork: opcontroller.EnforceHostNetwork(), }, } diff --git a/pkg/operator/ceph/cluster/osd/provision_spec.go b/pkg/operator/ceph/cluster/osd/provision_spec.go index 9e3673dfb..4839ea6a2 100644 --- a/pkg/operator/ceph/cluster/osd/provision_spec.go +++ b/pkg/operator/ceph/cluster/osd/provision_spec.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/mon" "github.com/rook/rook/pkg/operator/ceph/cluster/osd/config" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" @@ -154,7 +155,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re }, RestartPolicy: restart, Volumes: volumes, - HostNetwork: c.spec.Network.IsHost(), + HostNetwork: opcontroller.EnforceHostNetwork(), PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames), SchedulerName: osdProps.schedulerName, } diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 0e32a898a..238dd29bd 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -132,6 +132,7 @@ func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result } opcontroller.SetAllowLoopDevices(r.config.Parameters) + opcontroller.SetEnforceHostNetwork(r.config.Parameters) logger.Infof("%s done reconciling", controllerName) return reconcile.Result{}, nil diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index b726e2e97..f5cc1ca1d 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -50,7 +50,9 @@ type ClusterHealth struct { const ( // OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator - OperatorSettingConfigMapName string = "rook-ceph-operator-config" + OperatorSettingConfigMapName string = "rook-ceph-operator-config" + enforceHostNetworkSettingName string = "ROOK_ENFORCE_HOST_NETWORK" + enforceHostNetworkDefaultValue string = "false" // UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file // This typically is raised when the operator has not finished initializing @@ -116,6 +118,21 @@ func LoopDevicesAllowed() bool { return loopDevicesAllowed } +func SetEnforceHostNetwork(data map[string]string) { + strval := k8sutil.GetValue(data, enforceHostNetworkSettingName, enforceHostNetworkDefaultValue) + val, err := strconv.ParseBool(strval) + if err != nil { + logger.Warningf("failed to parse value %q for %q. assuming false value", strval, enforceHostNetworkSettingName) + cephv1.SetEnforceHostNetwork(false) + return + } + cephv1.SetEnforceHostNetwork(val) +} + +func EnforceHostNetwork() bool { + return cephv1.EnforceHostNetwork() +} + // canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely. func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool { // Get a list of all the keys causing the HEALTH_ERR status. @@ -153,7 +170,6 @@ func IsReadyToReconcile(ctx context.Context, c client.Client, namespacedName typ return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady } cephCluster = clusterList.Items[0] - // If the cluster has a cleanup policy to destroy the cluster and it has been marked for deletion, treat it as if it does not exist if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.DeletionTimestamp.IsZero() { logger.Infof("%q: CephCluster has a destructive cleanup policy, allowing %q to be deleted", controllerName, namespacedName) diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index eadcb2376..532cc0bcb 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -106,6 +106,33 @@ func TestSetAllowLoopDevices(t *testing.T) { assert.True(t, LoopDevicesAllowed()) } +func TestSetEnforceHostNetwork(t *testing.T) { + logger.Infof("testing default value for %v", enforceHostNetworkSettingName) + opConfig := map[string]string{} + SetEnforceHostNetwork(opConfig) + assert.False(t, EnforceHostNetwork()) + + // test invalid setting + var value string = "foo" + logger.Infof("testing invalid value'%v' for %v", value, enforceHostNetworkSettingName) + opConfig[enforceHostNetworkSettingName] = value + SetEnforceHostNetwork(opConfig) + assert.False(t, EnforceHostNetwork()) + + // test valid settings + value = "true" + logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) + opConfig[enforceHostNetworkSettingName] = value + SetEnforceHostNetwork(opConfig) + assert.True(t, EnforceHostNetwork()) + + value = "false" + logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) + opConfig[enforceHostNetworkSettingName] = value + SetEnforceHostNetwork(opConfig) + assert.False(t, EnforceHostNetwork()) +} + func TestIsReadyToReconcile(t *testing.T) { scheme := scheme.Scheme scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{}) diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 4be362ed2..d63e7bc9d 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -382,6 +382,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if tp.CSILogRotation { applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp) } + rbdProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() // Create service if either liveness or GRPC metrics are enabled. if CSIParam.EnableLiveness { @@ -419,6 +420,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if tp.CSILogRotation { applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp) } + cephfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() // Create service if either liveness or GRPC metrics are enabled. if CSIParam.EnableLiveness { @@ -457,6 +459,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI if tp.CSILogRotation { applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp) } + nfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork() enabledDrivers = append(enabledDrivers, driverDetails{ name: NFSDriverShortName, diff --git a/pkg/operator/ceph/object/cosi/spec.go b/pkg/operator/ceph/object/cosi/spec.go index f01ebabbd..50bf737c5 100644 --- a/pkg/operator/ceph/object/cosi/spec.go +++ b/pkg/operator/ceph/object/cosi/spec.go @@ -20,6 +20,7 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -83,6 +84,7 @@ func createCOSIPodSpec(cephCOSIDriver *cephv1.CephCOSIDriver) (corev1.PodTemplat cosiSideCarContainer := createCOSISideCarContainer(cephCOSIDriver) podSpec := corev1.PodSpec{ + HostNetwork: opcontroller.EnforceHostNetwork(), Containers: []corev1.Container{ cosiDriverContainer, cosiSideCarContainer, diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index bf543524b..ee4cdb3f6 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -31,6 +31,7 @@ import ( "github.com/rook/rook/pkg/clusterd" discoverDaemon "github.com/rook/rook/pkg/daemon/discover" "github.com/rook/rook/pkg/operator/ceph/controller" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" k8sutil "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/sys" @@ -173,7 +174,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco }, }, }, - HostNetwork: false, + HostNetwork: opcontroller.EnforceHostNetwork(), PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""), }, }, diff --git a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go index affc87558..b651def6d 100644 --- a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go +++ b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go @@ -25,6 +25,7 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/util" "github.com/rook/rook/pkg/operator/k8sutil" batch "k8s.io/api/batch/v1" @@ -302,6 +303,7 @@ func (cr *cmdReporterCfg) initJobSpec() (*batch.Job, error) { }, RestartPolicy: v1.RestartPolicyOnFailure, ServiceAccountName: k8sutil.DefaultServiceAccount, + HostNetwork: cephv1.EnforceHostNetwork(), } copyBinsVol, _ := copyBinariesVolAndMount() podSpec.Volumes = []v1.Volume{copyBinsVol}