From d109dc9029294ea05d0446c35974f3fa74664776 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 21 Feb 2025 18:02:00 -0700 Subject: [PATCH] core: implement operator settings as env vars The operator settings loaded from the configmap have proven inefficient for load time and frequently checking the configmap. To avoid this ineffenciency, the configmap is only loaded once each time it is created or updated. The values in the configmap are applied as environment variables, which then are very efficient to query throughout the various controllers, without needing to be concerned about loading the configmap again. Co-authored-by: Dmitry Mishin Signed-off-by: Travis Nielsen --- deploy/examples/operator.yaml | 7 +- pkg/operator/ceph/cluster/controller.go | 5 +- pkg/operator/ceph/cluster/watcher.go | 12 +- pkg/operator/ceph/controller.go | 40 +++--- .../ceph/controller/controller_utils.go | 25 ++-- .../ceph/controller/controller_utils_test.go | 85 +++++++------ pkg/operator/ceph/controller_test.go | 40 +----- pkg/operator/ceph/cr_manager.go | 6 +- pkg/operator/ceph/csi/controller.go | 24 +--- pkg/operator/ceph/csi/csi.go | 114 +++++++++--------- pkg/operator/ceph/csi/operator_config.go | 8 +- pkg/operator/ceph/csi/operator_driver.go | 28 ++--- pkg/operator/ceph/csi/spec.go | 62 +++++----- pkg/operator/ceph/csi/util.go | 34 +++--- pkg/operator/ceph/csi/util_test.go | 80 ++++++------ pkg/operator/ceph/object/bucket/controller.go | 22 +--- .../ceph/object/bucket/provisioner_test.go | 39 +++--- pkg/operator/ceph/object/bucket/util.go | 4 +- pkg/operator/ceph/object/objectstore.go | 6 +- pkg/operator/ceph/object/objectstore_test.go | 27 +++-- pkg/operator/ceph/operator.go | 7 +- pkg/operator/discover/discover.go | 22 ++-- pkg/operator/discover/discover_test.go | 20 +-- pkg/operator/k8sutil/configmap.go | 75 ++++++------ pkg/operator/k8sutil/configmap_test.go | 48 ++------ 25 files changed, 372 insertions(+), 468 deletions(-) diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 8a44c36c4..b1fcd62b3 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -577,6 +577,9 @@ data: # RevisionHistoryLimit value for all deployments created by rook. # ROOK_REVISION_HISTORY_LIMIT: "3" + + # Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used + ROOK_CUSTOM_HOSTNAME_LABEL: "" --- # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 @@ -657,10 +660,6 @@ spec: - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS value: "5" - # Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used - - name: ROOK_CUSTOM_HOSTNAME_LABEL - value: "" - # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 11eecc632..f499cccce 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -187,10 +187,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // Watch for changes on the hotplug config map // TODO: to improve, can we run this against the operator namespace only? - disableVal, err := k8sutil.GetOperatorSetting(opManagerContext, context.Clientset, opcontroller.OperatorSettingConfigMapName, disableHotplugEnv, "false") - if err != nil { - logger.Errorf("failed to get %s setting %s. %v", disableHotplugEnv, opcontroller.OperatorSettingConfigMapName, err) - } + disableVal := k8sutil.GetOperatorSetting(disableHotplugEnv, "false") if disableVal != "true" { logger.Info("enabling hotplug orchestration") s := source.Kind[client.Object]( diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 57363bfa2..c5ee33184 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -159,27 +159,21 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, op } func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node, opNamespace string) error { - watchForNodeLoss, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_WATCH_FOR_NODE_FAILURE", "true") - if err != nil { - return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_WATCH_FOR_NODE_FAILURE`.") - } + watchForNodeLoss := k8sutil.GetOperatorSetting("ROOK_WATCH_FOR_NODE_FAILURE", "true") if strings.ToLower(watchForNodeLoss) != "true" { logger.Debugf("not watching for node failures since `ROOK_WATCH_FOR_NODE_FAILURE` is set to %q", watchForNodeLoss) return nil } - disabledCSI, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_CSI_DISABLE_DRIVER", "false") - if err != nil { - return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_CSI_DISABLE_DRIVER`.") - } + disabledCSI := k8sutil.GetOperatorSetting("ROOK_CSI_DISABLE_DRIVER", "false") if strings.ToLower(disabledCSI) != "false" { logger.Debugf("not watching for node failures since `ROOK_CSI_DISABLE_DRIVER` is set to %q, skip creating networkFence", disabledCSI) return nil } - _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{}) + _, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { logger.Debug("networkfences.csiaddons.openshift.io CRD not found, skip creating networkFence") diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 879a51b2e..5aa86f336 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -33,7 +33,6 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" v1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -103,52 +102,43 @@ func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.R func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the operator's configmap - opConfig := &v1.ConfigMap{} logger.Debugf("reconciling %s", request.NamespacedName) - err := r.client.Get(r.opManagerContext, request.NamespacedName, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("operator's configmap resource not found. will use default value or env var.") - } else { - // Error reading the object - requeue the request. - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") + if request.Name == opcontroller.OperatorSettingConfigMapName { + if err := k8sutil.ApplyOperatorSettingsConfigmap(r.opManagerContext, r.context.Clientset); err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator setting configmap") } - } else { - // Populate the operator's config - r.config.Parameters = opConfig.Data } - // Reconcile Ceph CLI timeout, since the clusterd context is passed to by pointer to all CRD // controllers they will receive the update - opcontroller.SetCephCommandsTimeout(r.config.Parameters) + opcontroller.SetCephCommandsTimeout() // Reconcile Operator's logging level - reconcileOperatorLogLevel(opConfig.Data) + reconcileOperatorLogLevel() // Reconcile discovery daemon - err = r.reconcileDiscoveryDaemon(opConfig.Data) + err := r.reconcileDiscoveryDaemon() if err != nil { return opcontroller.ImmediateRetryResult, err } - opcontroller.SetAllowLoopDevices(r.config.Parameters) - opcontroller.SetEnforceHostNetwork(r.config.Parameters) - opcontroller.SetRevisionHistoryLimit(r.config.Parameters) - opcontroller.SetObcAllowAdditionalConfigFields(r.config.Parameters) + opcontroller.SetAllowLoopDevices() + opcontroller.SetEnforceHostNetwork() + opcontroller.SetRevisionHistoryLimit() + opcontroller.SetObcAllowAdditionalConfigFields() logger.Infof("%s done reconciling", controllerName) return reconcile.Result{}, nil } -func reconcileOperatorLogLevel(data map[string]string) { - rookLogLevel := k8sutil.GetValue(data, "ROOK_LOG_LEVEL", util.DefaultLogLevel.String()) +func reconcileOperatorLogLevel() { + rookLogLevel := k8sutil.GetOperatorSetting("ROOK_LOG_LEVEL", util.DefaultLogLevel.String()) util.SetGlobalLogLevel(rookLogLevel, logger) } -func (r *ReconcileConfig) reconcileDiscoveryDaemon(data map[string]string) error { +func (r *ReconcileConfig) reconcileDiscoveryDaemon() error { rookDiscover := discover.New(r.context.Clientset) - if opcontroller.DiscoveryDaemonEnabled(r.config.Parameters) { - if err := rookDiscover.Start(r.opManagerContext, r.config.OperatorNamespace, r.config.Image, r.config.ServiceAccount, data, true); err != nil { + if opcontroller.DiscoveryDaemonEnabled() { + if err := rookDiscover.Start(r.opManagerContext, r.config.OperatorNamespace, r.config.Image, r.config.ServiceAccount, true); err != nil { return errors.Wrap(err, "failed to start device discovery daemonset") } } else { diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index 8cb661d74..d55d00124 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -40,7 +40,6 @@ type OperatorConfig struct { Image string ServiceAccount string NamespaceToWatch string - Parameters map[string]string } // ClusterHealth is passed to the various monitoring go routines to stop them when the context is cancelled @@ -99,13 +98,13 @@ var ( obcAllowAdditionalConfigFields = strings.Split(obcAllowAdditionalConfigFieldsDefaultValue, ",") ) -func DiscoveryDaemonEnabled(data map[string]string) bool { - return k8sutil.GetValue(data, "ROOK_ENABLE_DISCOVERY_DAEMON", "false") == "true" +func DiscoveryDaemonEnabled() bool { + return k8sutil.GetOperatorSetting("ROOK_ENABLE_DISCOVERY_DAEMON", "false") == "true" } // SetCephCommandsTimeout sets the timeout value of Ceph commands which are executed from Rook -func SetCephCommandsTimeout(data map[string]string) { - strTimeoutSeconds := k8sutil.GetValue(data, "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15") +func SetCephCommandsTimeout() { + strTimeoutSeconds := k8sutil.GetOperatorSetting("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15") timeoutSeconds, err := strconv.Atoi(strTimeoutSeconds) if err != nil || timeoutSeconds < 1 { logger.Warningf("ROOK_CEPH_COMMANDS_TIMEOUT is %q but it should be >= 1, set the default value 15", strTimeoutSeconds) @@ -114,8 +113,8 @@ func SetCephCommandsTimeout(data map[string]string) { exec.CephCommandsTimeout = time.Duration(timeoutSeconds) * time.Second } -func SetAllowLoopDevices(data map[string]string) { - strLoopDevicesAllowed := k8sutil.GetValue(data, "ROOK_CEPH_ALLOW_LOOP_DEVICES", "false") +func SetAllowLoopDevices() { + strLoopDevicesAllowed := k8sutil.GetOperatorSetting("ROOK_CEPH_ALLOW_LOOP_DEVICES", "false") var err error loopDevicesAllowed, err = strconv.ParseBool(strLoopDevicesAllowed) if err != nil { @@ -128,8 +127,8 @@ func LoopDevicesAllowed() bool { return loopDevicesAllowed } -func SetEnforceHostNetwork(data map[string]string) { - strval := k8sutil.GetValue(data, enforceHostNetworkSettingName, enforceHostNetworkDefaultValue) +func SetEnforceHostNetwork() { + strval := k8sutil.GetOperatorSetting(enforceHostNetworkSettingName, enforceHostNetworkDefaultValue) val, err := strconv.ParseBool(strval) if err != nil { logger.Warningf("failed to parse value %q for %q. assuming false value", strval, enforceHostNetworkSettingName) @@ -143,8 +142,8 @@ func EnforceHostNetwork() bool { return cephv1.EnforceHostNetwork() } -func SetRevisionHistoryLimit(data map[string]string) { - strval := k8sutil.GetValue(data, revisionHistoryLimitSettingName, "") +func SetRevisionHistoryLimit() { + strval := k8sutil.GetOperatorSetting(revisionHistoryLimitSettingName, "") var limit int32 if strval == "" { logger.Debugf("not parsing empty string to int for %q. assuming default value.", revisionHistoryLimitSettingName) @@ -166,8 +165,8 @@ func RevisionHistoryLimit() *int32 { return revisionHistoryLimit } -func SetObcAllowAdditionalConfigFields(data map[string]string) { - strval := k8sutil.GetValue(data, obcAllowAdditionalConfigFieldsSettingName, obcAllowAdditionalConfigFieldsDefaultValue) +func SetObcAllowAdditionalConfigFields() { + strval := k8sutil.GetOperatorSetting(obcAllowAdditionalConfigFieldsSettingName, obcAllowAdditionalConfigFieldsDefaultValue) obcAllowAdditionalConfigFields = strings.Split(strval, ",") } diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index cda45d991..d1eac8297 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -18,6 +18,7 @@ package controller import ( ctx "context" + "os" "testing" "time" @@ -80,83 +81,94 @@ func TestCanIgnoreHealthErrStatusInReconcile(t *testing.T) { } func TestSetCephCommandsTimeout(t *testing.T) { - SetCephCommandsTimeout(map[string]string{}) + defer os.Unsetenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS") + + SetCephCommandsTimeout() assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout) exec.CephCommandsTimeout = 0 - SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "0"}) + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "0") + SetCephCommandsTimeout() assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout) exec.CephCommandsTimeout = 0 - SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"}) + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "1") + SetCephCommandsTimeout() assert.Equal(t, 1*time.Second, exec.CephCommandsTimeout) } func TestSetAllowLoopDevices(t *testing.T) { - SetAllowLoopDevices(map[string]string{}) + defer os.Unsetenv("ROOK_CEPH_ALLOW_LOOP_DEVICES") + + SetAllowLoopDevices() assert.False(t, LoopDevicesAllowed()) - SetAllowLoopDevices(map[string]string{"ROOK_CEPH_ALLOW_LOOP_DEVICES": "foo"}) + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "foo") + SetAllowLoopDevices() assert.False(t, LoopDevicesAllowed()) - SetAllowLoopDevices(map[string]string{"ROOK_CEPH_ALLOW_LOOP_DEVICES": "false"}) + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "false") + SetAllowLoopDevices() assert.False(t, LoopDevicesAllowed()) - SetAllowLoopDevices(map[string]string{"ROOK_CEPH_ALLOW_LOOP_DEVICES": "true"}) + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "true") + SetAllowLoopDevices() assert.True(t, LoopDevicesAllowed()) } func TestSetEnforceHostNetwork(t *testing.T) { + defer os.Unsetenv(enforceHostNetworkSettingName) + logger.Infof("testing default value for %v", enforceHostNetworkSettingName) - opConfig := map[string]string{} - SetEnforceHostNetwork(opConfig) + SetEnforceHostNetwork() assert.False(t, EnforceHostNetwork()) // test invalid setting var value string = "foo" logger.Infof("testing invalid value'%v' for %v", value, enforceHostNetworkSettingName) - opConfig[enforceHostNetworkSettingName] = value - SetEnforceHostNetwork(opConfig) + os.Setenv(enforceHostNetworkSettingName, value) + SetEnforceHostNetwork() assert.False(t, EnforceHostNetwork()) // test valid settings value = "true" logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) - opConfig[enforceHostNetworkSettingName] = value - SetEnforceHostNetwork(opConfig) + os.Setenv(enforceHostNetworkSettingName, value) + SetEnforceHostNetwork() assert.True(t, EnforceHostNetwork()) value = "false" logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) - opConfig[enforceHostNetworkSettingName] = value - SetEnforceHostNetwork(opConfig) + os.Setenv(enforceHostNetworkSettingName, value) + SetEnforceHostNetwork() assert.False(t, EnforceHostNetwork()) } func TestSetRevisionHistoryLimit(t *testing.T) { - opConfig := map[string]string{} + defer os.Unsetenv(revisionHistoryLimitSettingName) + t.Run("ROOK_REVISION_HISTORY_LIMIT: test default value", func(t *testing.T) { - SetRevisionHistoryLimit(opConfig) + SetRevisionHistoryLimit() assert.Nil(t, RevisionHistoryLimit()) }) var value string = "foo" t.Run("ROOK_REVISION_HISTORY_LIMIT: test invalid value 'foo'", func(t *testing.T) { - opConfig[revisionHistoryLimitSettingName] = value - SetRevisionHistoryLimit(opConfig) + os.Setenv(revisionHistoryLimitSettingName, value) + SetRevisionHistoryLimit() assert.Nil(t, RevisionHistoryLimit()) }) t.Run("ROOK_REVISION_HISTORY_LIMIT: test empty string value", func(t *testing.T) { value = "" - opConfig[revisionHistoryLimitSettingName] = value - SetRevisionHistoryLimit(opConfig) + os.Setenv(revisionHistoryLimitSettingName, value) + SetRevisionHistoryLimit() assert.Nil(t, RevisionHistoryLimit()) }) t.Run("ROOK_REVISION_HISTORY_LIMIT: test valig value '10'", func(t *testing.T) { value = "10" - opConfig[revisionHistoryLimitSettingName] = value - SetRevisionHistoryLimit(opConfig) + os.Setenv(revisionHistoryLimitSettingName, value) + SetRevisionHistoryLimit() assert.Equal(t, int32(10), *RevisionHistoryLimit()) }) } @@ -249,36 +261,39 @@ func TestIsReadyToReconcile(t *testing.T) { func TestObcAllowAdditionalConfigFields(t *testing.T) { tests := []struct { - name string - data map[string]string - shouldAllow []string - shouldDisallow []string + name string + additionalConfig string + shouldAllow []string + shouldDisallow []string }{ - {"not set", map[string]string{}, + {"not set", "", []string{"maxObjects", "maxSize"}, // default allowlist is unlikely to need changing EVER []string{"bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"set to empty", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": ""}, + {"set to empty", "", []string{}, // admin can allow no quota options if desired []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"set to default", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize"}, + {"set to default", "maxObjects,maxSize", []string{"maxObjects", "maxSize"}, []string{"bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"all quota fields", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize"}, + {"all quota fields", "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize", []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize"}, []string{"bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"all fields", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"}, + {"all fields", "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner", []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner"}, []string{"random"}}, // this mechanism doesn't do any field checking - that isn't it's job - it merely handles // allow-listing essentially arbitrary config keys - {"all fields including unknown", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner,random"}, + {"all fields including unknown", "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner,random", []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}, []string{"otherRandom"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - SetObcAllowAdditionalConfigFields(tt.data) - + if tt.additionalConfig != "" { + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", tt.additionalConfig) + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + } + SetObcAllowAdditionalConfigFields() for _, toAllow := range tt.shouldAllow { assert.True(t, ObcAdditionalConfigKeyIsAllowed(toAllow)) } diff --git a/pkg/operator/ceph/controller_test.go b/pkg/operator/ceph/controller_test.go index ed4485611..a9f08b672 100644 --- a/pkg/operator/ceph/controller_test.go +++ b/pkg/operator/ceph/controller_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/coreos/pkg/capnslog" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" @@ -31,9 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/test" "github.com/rook/rook/pkg/util/exec" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -64,7 +61,6 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() @@ -95,7 +91,6 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() @@ -128,23 +123,12 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. // s := scheme.Scheme s := clientgoscheme.Scheme - // s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}) - opConfigCM := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.OperatorSettingConfigMapName, - Namespace: namespace, - }, - Data: map[string]string{ - "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "20", - }, - } + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "20") + defer os.Unsetenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS") - object := []runtime.Object{ - opConfigCM, - } // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects().Build() // Create a ReconcileCSI object with the scheme and fake client. r := &ReconcileConfig{ @@ -173,7 +157,6 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() @@ -209,23 +192,12 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. // s := scheme.Scheme s := clientgoscheme.Scheme - // s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}) - opConfigCM := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.OperatorSettingConfigMapName, - Namespace: namespace, - }, - Data: map[string]string{ - "ROOK_CEPH_ALLOW_LOOP_DEVICES": "true", - }, - } + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "true") + defer os.Unsetenv("ROOK_CEPH_ALLOW_LOOP_DEVICES") - object := []runtime.Object{ - opConfigCM, - } // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects().Build() // Create a ReconcileCSI object with the scheme and fake client. r := &ReconcileConfig{ diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go index b3398d681..21d9e4860 100644 --- a/pkg/operator/ceph/cr_manager.go +++ b/pkg/operator/ceph/cr_manager.go @@ -143,11 +143,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro } } - metricsBindAddress, err := k8sutil.GetOperatorSetting(context, o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_OPERATOR_METRICS_BIND_ADDRESS", "0") - if err != nil { - mgrErrorCh <- errors.Wrap(err, "failed to get configmap value `ROOK_OPERATOR_METRICS_BIND_ADDRESS`.") - return - } + metricsBindAddress := k8sutil.GetOperatorSetting("ROOK_OPERATOR_METRICS_BIND_ADDRESS", "0") skipNameValidation := true // Set up a manager mgrOpts := manager.Options{ diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index b9c6c5034..809b2da4d 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -42,7 +42,6 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ) const ( @@ -158,32 +157,17 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map") } - // Fetch the operator's configmap. We force the NamespaceName to the operator since the request - // could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the - // CM won't be found - opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace} - opConfig := &v1.ConfigMap{} - err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("operator's configmap resource not found. will use default value or env var.") - r.opConfig.Parameters = make(map[string]string) - } else { - // Error reading the object - requeue the request. - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") - } - } else { - // Populate the operator's config - r.opConfig.Parameters = opConfig.Data + if err := k8sutil.ApplyOperatorSettingsConfigmap(r.opManagerContext, r.context.Clientset); err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to apply operator settings configmap") } - enableCSIOperator, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_USE_CSI_OPERATOR", "false")) + enableCSIOperator, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_USE_CSI_OPERATOR", "false")) if err != nil { return reconcileResult, errors.Wrap(err, "unable to parse value for 'ROOK_USE_CSI_OPERATOR'") } // do not recocnile if csi driver is disabled - disableCSI, err := strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_DISABLE_DRIVER", "false")) + disableCSI, err := strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_DISABLE_DRIVER", "false")) if err != nil { return reconcile.Result{}, errors.Wrap(err, "unable to parse value for 'ROOK_CSI_DISABLE_DRIVER") } else if disableCSI { diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 1e66cf459..6919eaf54 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -56,31 +56,31 @@ func (r *ReconcileCSI) validateAndConfigureDrivers(ownerInfo *k8sutil.OwnerInfo) func (r *ReconcileCSI) setParams() error { var err error - if EnableRBD, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_RBD", "true")); err != nil { + if EnableRBD, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_RBD", "true")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_RBD'") } - if EnableCephFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil { + if EnableCephFS, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_CEPHFS'") } - if EnableNFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_NFS", "false")); err != nil { + if EnableNFS, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_NFS", "false")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_NFS'") } - if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil { + if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetOperatorSetting("CSI_ENABLE_HOST_NETWORK", "true")); err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") } // If not set or set to anything but "false", the kernel client will be enabled - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") { CSIParam.ForceCephFSKernelClient = "false" } else { CSIParam.ForceCephFSKernelClient = "true" } // parse RPC timeout - timeout := k8sutil.GetValue(r.opConfig.Parameters, grpcTimeout, "150") + timeout := k8sutil.GetOperatorSetting(grpcTimeout, "150") timeoutSeconds, err := strconv.Atoi(timeout) if err != nil { logger.Errorf("failed to parse %q. Defaulting to %d. %v", grpcTimeout, defaultGRPCTimeout, err) @@ -93,32 +93,32 @@ func (r *ReconcileCSI) setParams() error { CSIParam.GRPCTimeout = time.Duration(timeoutSeconds) * time.Second // parse Liveness port - CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) + CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") } - CSIParam.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort) + CSIParam.CSIAddonsPort, err = getPortFromConfig("CSIADDONS_PORT", DefaultCSIAddonsPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port") } - CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort) + CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig("CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port for RBD provisioner") } - CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort) + CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig("CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port for Ceph FS provisioner") } - CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) + CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig("CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") } - CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_LIVENESS", "false")) + CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetOperatorSetting("CSI_ENABLE_LIVENESS", "false")) if err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_LIVENESS'") } @@ -126,31 +126,31 @@ func (r *ReconcileCSI) setParams() error { CSIParam.Privileged = controller.HostPathRequiresPrivileged() // default value `system-node-critical` is the highest available priority - CSIParam.PluginPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") + CSIParam.PluginPriorityClassName = k8sutil.GetOperatorSetting("CSI_PLUGIN_PRIORITY_CLASSNAME", "") // default value `system-cluster-critical` is applied for some // critical pods in cluster but less priority than plugin pods - CSIParam.ProvisionerPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "") + CSIParam.ProvisionerPriorityClassName = k8sutil.GetOperatorSetting("CSI_PROVISIONER_PRIORITY_CLASSNAME", "") CSIParam.EnableOMAPGenerator = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_OMAP_GENERATOR", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_OMAP_GENERATOR", "false"), "true") { CSIParam.EnableOMAPGenerator = true } CSIParam.EnableCSIDriverSeLinuxMount = true CSIParam.EnableRBDSnapshotter = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { CSIParam.EnableRBDSnapshotter = false } CSIParam.EnableCephFSSnapshotter = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") { CSIParam.EnableCephFSSnapshotter = false } CSIParam.EnableNFSSnapshotter = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_NFS_SNAPSHOTTER", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_NFS_SNAPSHOTTER", "true"), "false") { CSIParam.EnableNFSSnapshotter = false } @@ -159,36 +159,36 @@ func (r *ReconcileCSI) setParams() error { if err == nil { CSIParam.EnableCSIAddonsSideCar = true } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", ""), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CSIADDONS", ""), "false") { CSIParam.EnableCSIAddonsSideCar = false } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", ""), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CSIADDONS", ""), "true") { CSIParam.EnableCSIAddonsSideCar = true } CSIParam.EnableCSITopology = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_TOPOLOGY", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_TOPOLOGY", "false"), "true") { CSIParam.EnableCSITopology = true } CSIParam.EnableCSIEncryption = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_ENCRYPTION", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_ENCRYPTION", "false"), "true") { CSIParam.EnableCSIEncryption = true } CSIParam.CSIEnableMetadata = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_METADATA", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_METADATA", "false"), "true") { CSIParam.CSIEnableMetadata = true } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.CephFSPluginUpdateStrategy = onDelete } else { CSIParam.CephFSPluginUpdateStrategy = rollingUpdate - CSIParam.CephFSPluginUpdateStrategyMaxUnavailable = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") + CSIParam.CephFSPluginUpdateStrategyMaxUnavailable = k8sutil.GetOperatorSetting("CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.NFSPluginUpdateStrategy = onDelete } else { CSIParam.NFSPluginUpdateStrategy = rollingUpdate @@ -196,19 +196,19 @@ func (r *ReconcileCSI) setParams() error { // Default values are based on Kubernetes official documentation. // https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.RBDPluginUpdateStrategy = onDelete } else { CSIParam.RBDPluginUpdateStrategy = rollingUpdate - CSIParam.RBDPluginUpdateStrategyMaxUnavailable = k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") + CSIParam.RBDPluginUpdateStrategyMaxUnavailable = k8sutil.GetOperatorSetting("CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") } CSIParam.EnablePluginSelinuxHostMount = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") { CSIParam.EnablePluginSelinuxHostMount = true } - logLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LOG_LEVEL", "") + logLevel := k8sutil.GetOperatorSetting("CSI_LOG_LEVEL", "") CSIParam.LogLevel = defaultLogLevel if logLevel != "" { l, err := strconv.ParseUint(logLevel, 10, 8) @@ -219,7 +219,7 @@ func (r *ReconcileCSI) setParams() error { } } - sidecarLogLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_SIDECAR_LOG_LEVEL", "") + sidecarLogLevel := k8sutil.GetOperatorSetting("CSI_SIDECAR_LOG_LEVEL", "") CSIParam.SidecarLogLevel = defaultSidecarLogLevel if sidecarLogLevel != "" { l, err := strconv.ParseUint(sidecarLogLevel, 10, 8) @@ -230,7 +230,7 @@ func (r *ReconcileCSI) setParams() error { } } - leaderElectionLeaseDuration := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_LEASE_DURATION", "") + leaderElectionLeaseDuration := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_LEASE_DURATION", "") CSIParam.LeaderElectionLeaseDuration = defaultLeaderElectionLeaseDuration if leaderElectionLeaseDuration != "" { d, err := time.ParseDuration(leaderElectionLeaseDuration) @@ -241,7 +241,7 @@ func (r *ReconcileCSI) setParams() error { } } - leaderElectionRenewDeadline := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RENEW_DEADLINE", "") + leaderElectionRenewDeadline := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_RENEW_DEADLINE", "") CSIParam.LeaderElectionRenewDeadline = defaultLeaderElectionRenewDeadline if leaderElectionRenewDeadline != "" { d, err := time.ParseDuration(leaderElectionRenewDeadline) @@ -252,7 +252,7 @@ func (r *ReconcileCSI) setParams() error { } } - leaderElectionRetryPeriod := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RETRY_PERIOD", "") + leaderElectionRetryPeriod := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_RETRY_PERIOD", "") CSIParam.LeaderElectionRetryPeriod = defaultLeaderElectionRetryPeriod if leaderElectionRetryPeriod != "" { d, err := time.ParseDuration(leaderElectionRetryPeriod) @@ -269,7 +269,7 @@ func (r *ReconcileCSI) setParams() error { if len(nodes.Items) == 1 { CSIParam.ProvisionerReplicas = 1 } else { - replicaStr := k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_REPLICAS", "2") + replicaStr := k8sutil.GetOperatorSetting("CSI_PROVISIONER_REPLICAS", "2") replicas, err := strconv.ParseInt(replicaStr, 10, 32) if err != nil { logger.Errorf("failed to parse CSI_PROVISIONER_REPLICAS. Defaulting to %d. %v", defaultProvisionerReplicas, err) @@ -281,39 +281,39 @@ func (r *ReconcileCSI) setParams() error { logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to %d. %v", CSIParam.ProvisionerReplicas, err) } - CSIParam.CSIPluginImage = getImage(r.opConfig.Parameters, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) - CSIParam.RegistrarImage = getImage(r.opConfig.Parameters, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) - CSIParam.ProvisionerImage = getImage(r.opConfig.Parameters, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) - CSIParam.AttacherImage = getImage(r.opConfig.Parameters, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) - CSIParam.SnapshotterImage = getImage(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) - CSIParam.ResizerImage = getImage(r.opConfig.Parameters, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) - CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) - CSIParam.CSIAddonsImage = getImage(r.opConfig.Parameters, "ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) - CSIParam.CSIDomainLabels = k8sutil.GetValue(r.opConfig.Parameters, "CSI_TOPOLOGY_DOMAIN_LABELS", "") - csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "") + CSIParam.CSIPluginImage = getImage("ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) + CSIParam.RegistrarImage = getImage("ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) + CSIParam.ProvisionerImage = getImage("ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) + CSIParam.AttacherImage = getImage("ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) + CSIParam.SnapshotterImage = getImage("ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) + CSIParam.ResizerImage = getImage("ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) + CSIParam.KubeletDirPath = k8sutil.GetOperatorSetting("ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) + CSIParam.CSIAddonsImage = getImage("ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) + CSIParam.CSIDomainLabels = k8sutil.GetOperatorSetting("CSI_TOPOLOGY_DOMAIN_LABELS", "") + csiCephFSPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_CEPHFS_POD_LABELS", "") CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels) - csiNFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_NFS_POD_LABELS", "") + csiNFSPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_NFS_POD_LABELS", "") CSIParam.CSINFSPodLabels = k8sutil.ParseStringToLabels(csiNFSPodLabels) - csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "") + csiRBDPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_RBD_POD_LABELS", "") CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels) - CSIParam.CSIClusterName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CLUSTER_NAME", "") - CSIParam.ImagePullPolicy = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_IMAGE_PULL_POLICY", DefaultCSIImagePullPolicy) - CSIParam.CephFSKernelMountOptions = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_KERNEL_MOUNT_OPTIONS", "") + CSIParam.CSIClusterName = k8sutil.GetOperatorSetting("CSI_CLUSTER_NAME", "") + CSIParam.ImagePullPolicy = k8sutil.GetOperatorSetting("ROOK_CSI_IMAGE_PULL_POLICY", DefaultCSIImagePullPolicy) + CSIParam.CephFSKernelMountOptions = k8sutil.GetOperatorSetting("CSI_CEPHFS_KERNEL_MOUNT_OPTIONS", "") CSIParam.CephFSAttachRequired = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_ATTACH_REQUIRED", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_CEPHFS_ATTACH_REQUIRED", "true"), "false") { CSIParam.CephFSAttachRequired = false } CSIParam.RBDAttachRequired = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_ATTACH_REQUIRED", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_RBD_ATTACH_REQUIRED", "true"), "false") { CSIParam.RBDAttachRequired = false } CSIParam.NFSAttachRequired = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_ATTACH_REQUIRED", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_NFS_ATTACH_REQUIRED", "true"), "false") { CSIParam.NFSAttachRequired = false } - CSIParam.DriverNamePrefix = k8sutil.GetValue(r.opConfig.Parameters, "CSI_DRIVER_NAME_PREFIX", r.opConfig.OperatorNamespace) + CSIParam.DriverNamePrefix = k8sutil.GetOperatorSetting("CSI_DRIVER_NAME_PREFIX", r.opConfig.OperatorNamespace) crd, err := r.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io", metav1.GetOptions{}) if err != nil && !kerrors.IsNotFound(err) { @@ -324,7 +324,7 @@ func (r *ReconcileCSI) setParams() error { if err == nil && len(crd.Spec.Versions) > 0 { ver := crd.Spec.Versions[0] // Determine if VolumeGroupSnapshot feature should be disabled - disableVGS := strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false") + disableVGS := strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false") const ( enableVolumeGroupSnapshotFlag = "--enable-volume-group-snapshots=" featureGateFlag = "--feature-gates=CSIVolumeGroupSnapshot=" @@ -346,7 +346,7 @@ func (r *ReconcileCSI) setParams() error { } } - kubeApiBurst := k8sutil.GetValue(r.opConfig.Parameters, "CSI_KUBE_API_BURST", "") + kubeApiBurst := k8sutil.GetOperatorSetting("CSI_KUBE_API_BURST", "") CSIParam.KubeApiBurst = 0 if kubeApiBurst != "" { k, err := strconv.ParseUint(kubeApiBurst, 10, 16) @@ -357,7 +357,7 @@ func (r *ReconcileCSI) setParams() error { } } - kubeApiQPS := k8sutil.GetValue(r.opConfig.Parameters, "CSI_KUBE_API_QPS", "") + kubeApiQPS := k8sutil.GetOperatorSetting("CSI_KUBE_API_QPS", "") CSIParam.KubeApiQPS = 0 if kubeApiQPS != "" { k, err := strconv.ParseFloat(kubeApiQPS, 32) diff --git a/pkg/operator/ceph/csi/operator_config.go b/pkg/operator/ceph/csi/operator_config.go index 638e512b2..7e013245c 100644 --- a/pkg/operator/ceph/csi/operator_config.go +++ b/pkg/operator/ceph/csi/operator_config.go @@ -94,9 +94,9 @@ func (r *ReconcileCSI) generateCSIOpConfigSpec(cluster cephv1.CephCluster, opCon PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.ProvisionerPriorityClassName, Affinity: &v1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &v1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(pluginNodeAffinityEnv, &v1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []v1.Toleration{}), + Tolerations: getToleration(pluginTolerationsEnv, []v1.Toleration{}), }, Resources: csiopv1a1.NodePluginResourcesSpec{}, KubeletDirPath: CSIParam.KubeletDirPath, @@ -106,9 +106,9 @@ func (r *ReconcileCSI) generateCSIOpConfigSpec(cluster cephv1.CephCluster, opCon PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.PluginPriorityClassName, Affinity: &v1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &v1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(provisionerNodeAffinityEnv, &v1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []v1.Toleration{}), + Tolerations: getToleration(provisionerTolerationsEnv, []v1.Toleration{}), }, Replicas: &CSIParam.ProvisionerReplicas, Resources: csiopv1a1.ControllerPluginResourcesSpec{}, diff --git a/pkg/operator/ceph/csi/operator_driver.go b/pkg/operator/ceph/csi/operator_driver.go index f5457be45..99c845a2b 100644 --- a/pkg/operator/ceph/csi/operator_driver.go +++ b/pkg/operator/ceph/csi/operator_driver.go @@ -92,11 +92,11 @@ func (r *ReconcileCSI) createOrUpdateRBDDriverResource(cluster cephv1.CephCluste Spec: spec, } - rbdDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(r.opConfig.Parameters, rbdPluginResource) + rbdDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(rbdPluginResource) rbdDriver.Spec.Liveness = &csiopv1a1.LivenessSpec{ MetricsPort: int(CSIParam.RBDLivenessMetricsPort), } - rbdDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(r.opConfig.Parameters, rbdProvisionerResource) + rbdDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(rbdProvisionerResource) rbdDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ Type: v1.RollingUpdateDaemonSetStrategyType, } @@ -143,12 +143,12 @@ func (r *ReconcileCSI) createOrUpdateCephFSDriverResource(cluster cephv1.CephClu cephFsDriver.Spec.SnapshotPolicy = csiopv1a1.VolumeGroupSnapshotPolicy } - cephFsDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(r.opConfig.Parameters, cephFSPluginResource) + cephFsDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(cephFSPluginResource) cephFsDriver.Spec.Liveness = &csiopv1a1.LivenessSpec{ MetricsPort: int(CSIParam.CephFSLivenessMetricsPort), } - cephFsDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(r.opConfig.Parameters, cephFSProvisionerResource) + cephFsDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(cephFSProvisionerResource) cephFsDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ Type: v1.RollingUpdateDaemonSetStrategyType, } @@ -190,9 +190,9 @@ func (r *ReconcileCSI) createOrUpdateNFSDriverResource(cluster cephv1.CephCluste Spec: spec, } - NFSDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(r.opConfig.Parameters, nfsPluginResource) + NFSDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(nfsPluginResource) - NFSDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(r.opConfig.Parameters, nfsProvisionerResource) + NFSDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(nfsProvisionerResource) NFSDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ Type: v1.RollingUpdateDaemonSetStrategyType, } @@ -270,9 +270,9 @@ func (r *ReconcileCSI) generateDriverSpec(clusterName string) (csiopv1a1.DriverS PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.ProvisionerPriorityClassName, Affinity: &corev1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &corev1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(pluginNodeAffinityEnv, &corev1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []corev1.Toleration{}), + Tolerations: getToleration(pluginTolerationsEnv, []corev1.Toleration{}), }, Resources: csiopv1a1.NodePluginResourcesSpec{}, KubeletDirPath: CSIParam.KubeletDirPath, @@ -282,9 +282,9 @@ func (r *ReconcileCSI) generateDriverSpec(clusterName string) (csiopv1a1.DriverS PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.PluginPriorityClassName, Affinity: &corev1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}), + Tolerations: getToleration(provisionerTolerationsEnv, []corev1.Toleration{}), }, Replicas: &CSIParam.ProvisionerReplicas, Resources: csiopv1a1.ControllerPluginResourcesSpec{}, @@ -294,9 +294,9 @@ func (r *ReconcileCSI) generateDriverSpec(clusterName string) (csiopv1a1.DriverS }, nil } -func createDriverControllerPluginResources(opConfig map[string]string, key string) csiopv1a1.ControllerPluginResourcesSpec { +func createDriverControllerPluginResources(key string) csiopv1a1.ControllerPluginResourcesSpec { controllerPluginResources := csiopv1a1.ControllerPluginResourcesSpec{} - resource := getComputeResource(opConfig, key) + resource := getComputeResource(key) for _, r := range resource { if !reflect.DeepEqual(r.Resource, corev1.ResourceRequirements{}) { @@ -347,9 +347,9 @@ func createDriverControllerPluginResources(opConfig map[string]string, key strin return controllerPluginResources } -func createDriverNodePluginResouces(opConfig map[string]string, key string) csiopv1a1.NodePluginResourcesSpec { +func createDriverNodePluginResouces(key string) csiopv1a1.NodePluginResourcesSpec { nodePluginResources := csiopv1a1.NodePluginResourcesSpec{} - resource := getComputeResource(opConfig, key) + resource := getComputeResource(key) for _, r := range resource { if !reflect.DeepEqual(r.Resource, corev1.ResourceRequirements{}) { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 72797b166..5b968453f 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -426,25 +426,25 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { } // get common provisioner tolerations and node affinity - provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}) - provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) + provisionerTolerations := getToleration(provisionerTolerationsEnv, []corev1.Toleration{}) + provisionerNodeAffinity := getNodeAffinity(provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) // get common plugin tolerations and node affinity - pluginTolerations := getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []corev1.Toleration{}) - pluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &corev1.NodeAffinity{}) + pluginTolerations := getToleration(pluginTolerationsEnv, []corev1.Toleration{}) + pluginNodeAffinity := getNodeAffinity(pluginNodeAffinityEnv, &corev1.NodeAffinity{}) if rbdPlugin != nil { // get RBD plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdPluginTolerations := getToleration(r.opConfig.Parameters, rbdPluginTolerationsEnv, pluginTolerations) - rbdPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdPluginNodeAffinityEnv, pluginNodeAffinity) + rbdPluginTolerations := getToleration(rbdPluginTolerationsEnv, pluginTolerations) + rbdPluginNodeAffinity := getNodeAffinity(rbdPluginNodeAffinityEnv, pluginNodeAffinity) // apply RBD plugin tolerations and node affinity applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations) // apply resource request and limit to rbdplugin containers - applyResourcesToContainers(r.opConfig.Parameters, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) + applyResourcesToContainers(rbdPluginResource, &rbdPlugin.Spec.Template.Spec) // apply custom mounts to volumes - applyVolumeToPodSpec(r.opConfig.Parameters, rbdPluginVolume, &rbdPlugin.Spec.Template.Spec) + applyVolumeToPodSpec(rbdPluginVolume, &rbdPlugin.Spec.Template.Spec) // apply custom mounts to volume mounts - applyVolumeMountToContainer(r.opConfig.Parameters, rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec) + applyVolumeMountToContainer(rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name) @@ -462,12 +462,12 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if rbdProvisionerDeployment != nil { // get RBD provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdProvisionerTolerations := getToleration(r.opConfig.Parameters, rbdProvisionerTolerationsEnv, provisionerTolerations) - rbdProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) + rbdProvisionerTolerations := getToleration(rbdProvisionerTolerationsEnv, provisionerTolerations) + rbdProvisionerNodeAffinity := getNodeAffinity(rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply RBD provisioner tolerations and node affinity applyToPodSpec(&rbdProvisionerDeployment.Spec.Template.Spec, rbdProvisionerNodeAffinity, rbdProvisionerTolerations) // apply resource request and limit to rbd provisioner containers - applyResourcesToContainers(r.opConfig.Parameters, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd provisioner deployment %q", rbdProvisionerDeployment.Name) @@ -504,16 +504,16 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if cephfsPlugin != nil { // get CephFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSPluginTolerations := getToleration(r.opConfig.Parameters, cephFSPluginTolerationsEnv, pluginTolerations) - cephFSPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSPluginNodeAffinityEnv, pluginNodeAffinity) + cephFSPluginTolerations := getToleration(cephFSPluginTolerationsEnv, pluginTolerations) + cephFSPluginNodeAffinity := getNodeAffinity(cephFSPluginNodeAffinityEnv, pluginNodeAffinity) // apply CephFS plugin tolerations and node affinity applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations) // apply resource request and limit to cephfs plugin containers - applyResourcesToContainers(r.opConfig.Parameters, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) + applyResourcesToContainers(cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) // apply custom mounts to volumes - applyVolumeToPodSpec(r.opConfig.Parameters, cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec) + applyVolumeToPodSpec(cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec) // apply custom mounts to volume mounts - applyVolumeMountToContainer(r.opConfig.Parameters, cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec) + applyVolumeMountToContainer(cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name) @@ -532,13 +532,13 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if cephfsProvisionerDeployment != nil { // get CephFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSProvisionerTolerations := getToleration(r.opConfig.Parameters, cephFSProvisionerTolerationsEnv, provisionerTolerations) - cephFSProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) + cephFSProvisionerTolerations := getToleration(cephFSProvisionerTolerationsEnv, provisionerTolerations) + cephFSProvisionerNodeAffinity := getNodeAffinity(cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply CephFS provisioner tolerations and node affinity applyToPodSpec(&cephfsProvisionerDeployment.Spec.Template.Spec, cephFSProvisionerNodeAffinity, cephFSProvisionerTolerations) // get resource details for cephfs provisioner // apply resource request and limit to cephfs provisioner containers - applyResourcesToContainers(r.opConfig.Parameters, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) @@ -574,16 +574,16 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if nfsPlugin != nil { // get NFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - nfsPluginTolerations := getToleration(r.opConfig.Parameters, nfsPluginTolerationsEnv, pluginTolerations) - nfsPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, nfsPluginNodeAffinityEnv, pluginNodeAffinity) + nfsPluginTolerations := getToleration(nfsPluginTolerationsEnv, pluginTolerations) + nfsPluginNodeAffinity := getNodeAffinity(nfsPluginNodeAffinityEnv, pluginNodeAffinity) // apply NFS plugin tolerations and node affinity applyToPodSpec(&nfsPlugin.Spec.Template.Spec, nfsPluginNodeAffinity, nfsPluginTolerations) // apply resource request and limit to nfs plugin containers - applyResourcesToContainers(r.opConfig.Parameters, nfsPluginResource, &nfsPlugin.Spec.Template.Spec) + applyResourcesToContainers(nfsPluginResource, &nfsPlugin.Spec.Template.Spec) // apply custom mounts to volumes - applyVolumeToPodSpec(r.opConfig.Parameters, nfsPluginVolume, &nfsPlugin.Spec.Template.Spec) + applyVolumeToPodSpec(nfsPluginVolume, &nfsPlugin.Spec.Template.Spec) // apply custom mounts to volume mounts - applyVolumeMountToContainer(r.opConfig.Parameters, nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec) + applyVolumeMountToContainer(nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(nfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to nfs plugin daemonset %q", nfsPlugin.Name) @@ -602,13 +602,13 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if nfsProvisionerDeployment != nil { // get NFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - nfsProvisionerTolerations := getToleration(r.opConfig.Parameters, nfsProvisionerTolerationsEnv, provisionerTolerations) - nfsProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, nfsProvisionerNodeAffinityEnv, provisionerNodeAffinity) + nfsProvisionerTolerations := getToleration(nfsProvisionerTolerationsEnv, provisionerTolerations) + nfsProvisionerNodeAffinity := getNodeAffinity(nfsProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply NFS provisioner tolerations and node affinity applyToPodSpec(&nfsProvisionerDeployment.Spec.Template.Spec, nfsProvisionerNodeAffinity, nfsProvisionerTolerations) // get resource details for nfs provisioner // apply resource request and limit to nfs provisioner containers - applyResourcesToContainers(r.opConfig.Parameters, nfsProvisionerResource, &nfsProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(nfsProvisionerResource, &nfsProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(nfsProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to nfs provisioner deployment %q", nfsProvisionerDeployment.Name) @@ -634,7 +634,7 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if EnableRBD { err = csiDriverobj.createCSIDriverInfo( r.opManagerContext, r.context.Clientset, - RBDDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), + RBDDriverName, k8sutil.GetOperatorSetting("CSI_RBD_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), tp.Param.RBDAttachRequired, CSIParam.EnableCSIDriverSeLinuxMount) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName) @@ -643,7 +643,7 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if EnableCephFS { err = csiDriverobj.createCSIDriverInfo( r.opManagerContext, r.context.Clientset, - CephFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), + CephFSDriverName, k8sutil.GetOperatorSetting("CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), tp.Param.CephFSAttachRequired, CSIParam.EnableCSIDriverSeLinuxMount) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName) @@ -651,7 +651,7 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { } if EnableNFS { err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, - NFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), + NFSDriverName, k8sutil.GetOperatorSetting("CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), tp.Param.NFSAttachRequired, CSIParam.EnableCSIDriverSeLinuxMount) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", NFSDriverName) diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 4c3e70827..60f549bb1 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -98,8 +98,8 @@ func applyLogrotateSidecar(specTemplate *corev1.PodTemplateSpec, name, templateD specTemplate.Spec.Containers = append(specTemplate.Spec.Containers, logrotateSidecarContainer) } -func applyResourcesToContainers(opConfig map[string]string, key string, podspec *corev1.PodSpec) { - resource := getComputeResource(opConfig, key) +func applyResourcesToContainers(key string, podspec *corev1.PodSpec) { + resource := getComputeResource(key) for _, r := range resource { for i, c := range podspec.Containers { @@ -110,12 +110,12 @@ func applyResourcesToContainers(opConfig map[string]string, key string, podspec } } -func getComputeResource(opConfig map[string]string, key string) []k8sutil.ContainerResource { +func getComputeResource(key string) []k8sutil.ContainerResource { // Add Resource list if any resource := []k8sutil.ContainerResource{} var err error - if resourceRaw := k8sutil.GetValue(opConfig, key, ""); resourceRaw != "" { + if resourceRaw := k8sutil.GetOperatorSetting(key, ""); resourceRaw != "" { resource, err = k8sutil.YamlToContainerResourceArray(resourceRaw) if err != nil { logger.Warningf("failed to parse %q. %v", resourceRaw, err) @@ -124,9 +124,9 @@ func getComputeResource(opConfig map[string]string, key string) []k8sutil.Contai return resource } -func getToleration(opConfig map[string]string, tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { +func getToleration(tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { // Add toleration if any, otherwise return defaultTolerations - tolerationsRaw := k8sutil.GetValue(opConfig, tolerationsName, "") + tolerationsRaw := k8sutil.GetOperatorSetting(tolerationsName, "") if tolerationsRaw == "" { return defaultTolerations } @@ -147,9 +147,9 @@ func getToleration(opConfig map[string]string, tolerationsName string, defaultTo return tolerations } -func getNodeAffinity(opConfig map[string]string, nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { +func getNodeAffinity(nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { // Add NodeAffinity if any, otherwise return defaultNodeAffinity - nodeAffinity := k8sutil.GetValue(opConfig, nodeAffinityName, "") + nodeAffinity := k8sutil.GetOperatorSetting(nodeAffinityName, "") if nodeAffinity == "" { return defaultNodeAffinity } @@ -168,9 +168,9 @@ func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Tole } } -func getPortFromConfig(data map[string]string, env string, defaultPort uint16) (uint16, error) { - port := k8sutil.GetValue(data, env, strconv.Itoa(int(defaultPort))) - if strings.TrimSpace(k8sutil.GetValue(data, env, strconv.Itoa(int(defaultPort)))) == "" { +func getPortFromConfig(env string, defaultPort uint16) (uint16, error) { + port := k8sutil.GetOperatorSetting(env, strconv.Itoa(int(defaultPort))) + if strings.TrimSpace(k8sutil.GetOperatorSetting(env, strconv.Itoa(int(defaultPort)))) == "" { return defaultPort, nil } p, err := strconv.ParseUint(port, 10, 64) @@ -203,8 +203,8 @@ func GetPodAntiAffinity(key, value string) corev1.PodAntiAffinity { } } -func applyVolumeToPodSpec(opConfig map[string]string, configName string, podspec *corev1.PodSpec) { - volumesRaw := k8sutil.GetValue(opConfig, configName, "") +func applyVolumeToPodSpec(configName string, podspec *corev1.PodSpec) { + volumesRaw := k8sutil.GetOperatorSetting(configName, "") if volumesRaw == "" { return } @@ -230,8 +230,8 @@ func applyVolumeToPodSpec(opConfig map[string]string, configName string, podspec } } -func applyVolumeMountToContainer(opConfig map[string]string, configName, containerName string, podspec *corev1.PodSpec) { - volumeMountsRaw := k8sutil.GetValue(opConfig, configName, "") +func applyVolumeMountToContainer(configName, containerName string, podspec *corev1.PodSpec) { + volumeMountsRaw := k8sutil.GetOperatorSetting(configName, "") if volumeMountsRaw == "" { return } @@ -267,8 +267,8 @@ func applyVolumeMountToContainer(opConfig map[string]string, configName, contain // getImage returns the image for the given setting name. If the image does not contain version, // the default version is appended from the default image. -func getImage(data map[string]string, settingName, defaultImage string) string { - image := k8sutil.GetValue(data, settingName, defaultImage) +func getImage(settingName, defaultImage string) string { + image := k8sutil.GetOperatorSetting(settingName, defaultImage) if !strings.Contains(image, ":") { version := strings.SplitN(defaultImage, ":", 2)[1] image = fmt.Sprintf("%s:%s", image, version) diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index 29f5cbced..74860696e 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -47,28 +47,27 @@ func TestDeploymentTemplate(t *testing.T) { func TestGetPortFromConfig(t *testing.T) { key := "TEST_CSI_PORT_ENV" var defaultPort uint16 = 8000 - data := map[string]string{} // empty env variable - port, err := getPortFromConfig(data, key, defaultPort) + port, err := getPortFromConfig(key, defaultPort) assert.Nil(t, err) assert.Equal(t, port, defaultPort) // valid port is set in env t.Setenv(key, "9000") - port, err = getPortFromConfig(data, key, defaultPort) + port, err = getPortFromConfig(key, defaultPort) assert.Nil(t, err) assert.Equal(t, port, uint16(9000)) // higher port value is set in env t.Setenv(key, "65536") - port, err = getPortFromConfig(data, key, defaultPort) + port, err = getPortFromConfig(key, defaultPort) assert.Error(t, err) assert.Equal(t, port, defaultPort) // negative port is set in env t.Setenv(key, "-1") - port, err = getPortFromConfig(data, key, defaultPort) + port, err = getPortFromConfig(key, defaultPort) assert.Error(t, err) assert.Equal(t, port, defaultPort) @@ -80,7 +79,6 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { tp := templateParam{} rbdPlugin, err := templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp) assert.Nil(t, err) - params := make(map[string]string) // need to build using map[string]interface{} because the following resource // doesn't serialise nicely @@ -103,8 +101,9 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { resourceRaw, err := yaml.Marshal(resource) assert.Nil(t, err) - params[rbdPluginResource] = string(resourceRaw) - applyResourcesToContainers(params, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) + os.Setenv(rbdPluginResource, string(resourceRaw)) + defer os.Unsetenv(rbdPluginResource) + applyResourcesToContainers(rbdPluginResource, &rbdPlugin.Spec.Template.Spec) assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String(), "128Mi") assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String(), "256Mi") assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String(), "100m") @@ -113,7 +112,6 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { func Test_applyVolumeToPodSpec(t *testing.T) { // when no volumes specified - config := make(map[string]string) configKey := "TEST_CSI_PLUGIN_VOLUME" dsName := "test-ds" tp := templateParam{ @@ -124,7 +122,7 @@ func Test_applyVolumeToPodSpec(t *testing.T) { defaultVolumes := 11 ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes) @@ -132,7 +130,7 @@ func Test_applyVolumeToPodSpec(t *testing.T) { tp.CSILogRotation = true ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) tp.CSILogRotation = false @@ -145,10 +143,11 @@ func Test_applyVolumeToPodSpec(t *testing.T) { } volumeRaw, err := yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) + defer os.Unsetenv(configKey) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) // add one more volume @@ -158,10 +157,10 @@ func Test_applyVolumeToPodSpec(t *testing.T) { }) volumeRaw, err = yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) // override existing volume configuration volumes[1].VolumeSource = corev1.VolumeSource{ @@ -169,10 +168,10 @@ func Test_applyVolumeToPodSpec(t *testing.T) { } volumeRaw, err = yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) // remove existing volume configuration volumes = []corev1.Volume{ @@ -183,16 +182,15 @@ func Test_applyVolumeToPodSpec(t *testing.T) { } volumeRaw, err = yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) } func Test_applyVolumeMountToContainer(t *testing.T) { // when no volumes specified - config := make(map[string]string) configKey := "TEST_CSI_PLUGIN_VOLUME_MOUNT" dsName := "test-ds" rbdContainerName := "csi-rbdplugin" @@ -204,7 +202,7 @@ func Test_applyVolumeMountToContainer(t *testing.T) { defaultVolumes := 10 ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes) @@ -212,7 +210,7 @@ func Test_applyVolumeMountToContainer(t *testing.T) { tp.CSILogRotation = true ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) tp.CSILogRotation = false @@ -225,10 +223,11 @@ func Test_applyVolumeMountToContainer(t *testing.T) { } volumeMountsRaw, err := yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) + defer os.Unsetenv(configKey) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) // add one more volumemount @@ -238,19 +237,19 @@ func Test_applyVolumeMountToContainer(t *testing.T) { }) volumeMountsRaw, err = yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) // override existing volume configuration volumeMounts[1].MountPath = "/run/test/run" volumeMountsRaw, err = yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) // remove existing volume configuration volumeMounts = []corev1.VolumeMount{ @@ -261,17 +260,16 @@ func Test_applyVolumeMountToContainer(t *testing.T) { } volumeMountsRaw, err = yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) } func Test_getImage(t *testing.T) { type args struct { - data map[string]string - settingName string + userImage string defaultImage string } tests := []struct { @@ -282,8 +280,6 @@ func Test_getImage(t *testing.T) { { name: "test with default image", args: args{ - data: map[string]string{}, - settingName: "ROOK_CSI_CEPH_IMAGE", defaultImage: "quay.io/cephcsi/cephcsi:v3.13.0", }, want: DefaultCSIPluginImage, @@ -291,10 +287,7 @@ func Test_getImage(t *testing.T) { { name: "test with user image", args: args{ - data: map[string]string{ - "ROOK_CSI_CEPH_IMAGE": "registry.io/private/cephcsi:v8", - }, - settingName: "ROOK_CSI_CEPH_IMAGE", + userImage: "registry.io/private/cephcsi:v8", defaultImage: "quay.io/cephcsi/cephcsi:v3.11.0", }, want: "registry.io/private/cephcsi:v8", @@ -302,10 +295,7 @@ func Test_getImage(t *testing.T) { { name: "test with user image without version", args: args{ - data: map[string]string{ - "ROOK_CSI_CEPH_IMAGE": "registry.io/private/cephcsi", - }, - settingName: "ROOK_CSI_CEPH_IMAGE", + userImage: "registry.io/private/cephcsi", defaultImage: "quay.io/cephcsi/cephcsi:v3.11.0", }, want: "registry.io/private/cephcsi:v3.11.0", @@ -313,7 +303,11 @@ func Test_getImage(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := getImage(tt.args.data, tt.args.settingName, tt.args.defaultImage) + if tt.args.userImage != "" { + os.Setenv("ROOK_CSI_CEPH_IMAGE", tt.args.userImage) + defer os.Unsetenv("ROOK_CSI_CEPH_IMAGE") + } + got := getImage("ROOK_CSI_CEPH_IMAGE", tt.args.defaultImage) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/operator/ceph/object/bucket/controller.go b/pkg/operator/ceph/object/bucket/controller.go index eed00adc9..b9a4007f7 100644 --- a/pkg/operator/ceph/object/bucket/controller.go +++ b/pkg/operator/ceph/object/bucket/controller.go @@ -36,7 +36,6 @@ import ( v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) const ( @@ -140,25 +139,6 @@ func (r *ReconcileBucket) reconcile(request reconcile.Request) (reconcile.Result return reconcile.Result{}, nil } - // Fetch the operator's configmap. We force the NamespaceName to the operator since the request - // could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the - // CM won't be found - opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace} - opConfig := &v1.ConfigMap{} - err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("operator's configmap resource not found. will use default value or env var.") - r.opConfig.Parameters = make(map[string]string) - } else { - // Error reading the object - requeue the request. - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") - } - } else { - // Populate the operator's config - r.opConfig.Parameters = opConfig.Data - } - // Populate clusterInfo during each reconcile clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace, &cephCluster.Spec) if err != nil { @@ -177,7 +157,7 @@ func (r *ReconcileBucket) reconcile(request reconcile.Request) (reconcile.Result // note: the error return below is ignored and is expected to be removed from the // bucket library's `NewProvisioner` function - bucketController, _ := NewBucketController(r.context.KubeConfig, bucketProvisioner, r.opConfig.Parameters) + bucketController, _ := NewBucketController(r.context.KubeConfig, bucketProvisioner) // We must run this in a go routine since RunWithContext() blocks and waits for the context to // be Done. However, since it has a context, the go routine will exit on reload with SIGHUP diff --git a/pkg/operator/ceph/object/bucket/provisioner_test.go b/pkg/operator/ceph/object/bucket/provisioner_test.go index a098b51f9..0e44132a4 100644 --- a/pkg/operator/ceph/object/bucket/provisioner_test.go +++ b/pkg/operator/ceph/object/bucket/provisioner_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "testing" @@ -501,7 +502,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("maxObjects field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"maxObjects": "2"}) assert.NoError(t, err) @@ -510,7 +511,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("maxSize field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"maxSize": "3"}) assert.NoError(t, err) @@ -519,8 +520,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketMaxObjects field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketMaxObjects"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketMaxObjects") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketMaxObjects": "4"}) assert.NoError(t, err) @@ -528,8 +531,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketMaxSize field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketMaxSize"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketMaxSize") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketMaxSize": "5"}) assert.NoError(t, err) @@ -537,8 +542,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketPolicy field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketPolicy"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketPolicy") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketPolicy": "foo"}) assert.NoError(t, err) @@ -546,8 +553,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketLifecycle field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketLifecycle"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketLifecycle") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketLifecycle": "foo"}) assert.NoError(t, err) @@ -555,8 +564,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketOwner field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketOwner"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketOwner") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketOwner": "foo"}) assert.NoError(t, err) @@ -564,7 +575,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("fields disallowed by default", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() for _, configKey := range []string{"bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner"} { _, err := additionalConfigSpecFromMap(map[string]string{configKey: "foo"}) @@ -573,7 +584,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("does not fail on empty map", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{}) assert.NoError(t, err) diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index 19e294cb9..7e8df47ea 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -43,9 +43,9 @@ const ( objectStoreEndpoint = "endpoint" ) -func NewBucketController(cfg *rest.Config, p *Provisioner, data map[string]string) (*provisioner.Provisioner, error) { +func NewBucketController(cfg *rest.Config, p *Provisioner) (*provisioner.Provisioner, error) { const allNamespaces = "" - provName, err := cephObject.GetObjectBucketProvisioner(data, p.clusterInfo.Namespace) + provName, err := cephObject.GetObjectBucketProvisioner(p.clusterInfo.Namespace) if err != nil { return nil, errors.Wrap(err, "failed to get provisioner name") } diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 080367731..67a41626a 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -1137,10 +1137,10 @@ func poolName(poolPrefix, poolName string) string { } // GetObjectBucketProvisioner returns the bucket provisioner name appended with operator namespace if OBC is watching on it -func GetObjectBucketProvisioner(data map[string]string, namespace string) (string, error) { +func GetObjectBucketProvisioner(namespace string) (string, error) { provName := bucketProvisionerName - obcWatchOnNamespace := k8sutil.GetValue(data, "ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") - obcProvisionerNamePrefix := k8sutil.GetValue(data, "ROOK_OBC_PROVISIONER_NAME_PREFIX", "") + obcWatchOnNamespace := k8sutil.GetOperatorSetting("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") + obcProvisionerNamePrefix := k8sutil.GetOperatorSetting("ROOK_OBC_PROVISIONER_NAME_PREFIX", "") if obcProvisionerNamePrefix != "" { errList := validation.IsDNS1123Label(obcProvisionerNamePrefix) if len(errList) > 0 { diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index f384d6c70..8359e9897 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -517,36 +517,43 @@ func TestGetObjectBucketProvisioner(t *testing.T) { t.Setenv(k8sutil.PodNamespaceEnvVar, testNamespace) t.Run("watch ceph cluster namespace", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "true") + defer os.Unsetenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, fmt.Sprintf("%s.%s", testNamespace, bucketProvisionerName), bktprovisioner) assert.NoError(t, err) }) t.Run("watch all namespaces", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "false"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") + defer os.Unsetenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, bucketProvisionerName, bktprovisioner) assert.NoError(t, err) }) t.Run("prefix object provisioner", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_PROVISIONER_NAME_PREFIX": "my-prefix"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_PROVISIONER_NAME_PREFIX", "my-prefix") + defer os.Unsetenv("ROOK_OBC_PROVISIONER_NAME_PREFIX") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, "my-prefix."+bucketProvisionerName, bktprovisioner) assert.NoError(t, err) }) t.Run("watch ceph cluster namespace and prefix object provisioner", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", "ROOK_OBC_PROVISIONER_NAME_PREFIX": "my-prefix"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "true") + os.Setenv("ROOK_OBC_PROVISIONER_NAME_PREFIX", "my-prefix") + defer os.Unsetenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE") + defer os.Unsetenv("ROOK_OBC_PROVISIONER_NAME_PREFIX") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, "my-prefix."+bucketProvisionerName, bktprovisioner) assert.NoError(t, err) }) t.Run("invalid prefix value for object provisioner", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_PROVISIONER_NAME_PREFIX": "my-prefix."} - _, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_PROVISIONER_NAME_PREFIX", "my-prefix.") + defer os.Unsetenv("ROOK_OBC_PROVISIONER_NAME_PREFIX") + _, err := GetObjectBucketProvisioner(testNamespace) assert.Error(t, err) }) diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index e43ba8c78..2674c41ab 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -129,6 +129,11 @@ func (o *Operator) runCRDManager() { // Create the context and the cancellation function opManagerContext, opManagerStop = context.WithCancel(context.Background()) + // Initialize the operator settings from the configmap + if err := k8sutil.ApplyOperatorSettingsConfigmap(opManagerContext, o.context.Clientset); err != nil { + panic("failed to load operator settings configmap. " + err.Error()) + } + // The operator config manager is also watching for changes here so if the operator config map // content changes for ROOK_CURRENT_NAMESPACE_ONLY we must reload the operator CRD manager o.namespaceToWatch() @@ -149,7 +154,7 @@ func (o *Operator) runCRDManager() { } func (o *Operator) namespaceToWatch() { - currentNamespaceOnly, _ := k8sutil.GetOperatorSetting(opManagerContext, o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_CURRENT_NAMESPACE_ONLY", "true") + currentNamespaceOnly := k8sutil.GetOperatorSetting("ROOK_CURRENT_NAMESPACE_ONLY", "true") if currentNamespaceOnly == "true" { o.config.NamespaceToWatch = o.config.OperatorNamespace logger.Infof("watching the current namespace %q for Ceph CRs", o.config.OperatorNamespace) diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index 5b73b8cc0..dca033d57 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -73,16 +73,16 @@ func New(clientset kubernetes.Interface) *Discover { } // Start the discover -func (d *Discover) Start(ctx context.Context, namespace, discoverImage, securityAccount string, data map[string]string, useCephVolume bool) error { - err := d.createDiscoverDaemonSet(ctx, namespace, discoverImage, securityAccount, data, useCephVolume) +func (d *Discover) Start(ctx context.Context, namespace, discoverImage, securityAccount string, useCephVolume bool) error { + err := d.createDiscoverDaemonSet(ctx, namespace, discoverImage, securityAccount, useCephVolume) if err != nil { return fmt.Errorf("failed to start discover daemonset. %v", err) } return nil } -func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, discoverImage, securityAccount string, data map[string]string, useCephVolume bool) error { - discoveryInterval := k8sutil.GetValue(data, discoverIntervalEnv, defaultDiscoverInterval) +func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, discoverImage, securityAccount string, useCephVolume bool) error { + discoveryInterval := k8sutil.GetOperatorSetting(discoverIntervalEnv, defaultDiscoverInterval) discoveryParameters := []string{"discover", "--discover-interval", discoveryInterval} @@ -90,7 +90,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco discoveryParameters = append(discoveryParameters, "--use-ceph-volume") } - discoverDaemonResourcesRaw := k8sutil.GetValue(data, discoverDaemonResourcesEnv, "") + discoverDaemonResourcesRaw := k8sutil.GetOperatorSetting(discoverDaemonResourcesEnv, "") discoverDaemonResources, err := k8sutil.YamlToContainerResource(discoverDaemonResourcesRaw) if err != nil { logger.Warningf("failed to parse.%s %v", discoverDaemonResourcesRaw, err) @@ -176,7 +176,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco }, }, HostNetwork: opcontroller.EnforceHostNetwork(), - PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""), + PriorityClassName: k8sutil.GetOperatorSetting(discoverDaemonsetPriorityClassNameEnv, ""), SecurityContext: &v1.PodSecurityContext{}, }, }, @@ -191,18 +191,18 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco } // Add toleration if any - tolerationValue := k8sutil.GetValue(data, discoverDaemonsetTolerationEnv, "") + tolerationValue := k8sutil.GetOperatorSetting(discoverDaemonsetTolerationEnv, "") if tolerationValue != "" { ds.Spec.Template.Spec.Tolerations = []v1.Toleration{ { Effect: v1.TaintEffect(tolerationValue), Operator: v1.TolerationOpExists, - Key: k8sutil.GetValue(data, discoverDaemonsetTolerationKeyEnv, ""), + Key: k8sutil.GetOperatorSetting(discoverDaemonsetTolerationKeyEnv, ""), }, } } - tolerationsRaw := k8sutil.GetValue(data, discoverDaemonsetTolerationsEnv, "") + tolerationsRaw := k8sutil.GetOperatorSetting(discoverDaemonsetTolerationsEnv, "") tolerations, err := k8sutil.YamlToTolerations(tolerationsRaw) if err != nil { logger.Warningf("failed to parse %s. %+v", tolerationsRaw, err) @@ -211,7 +211,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, tolerations...) // Add NodeAffinity if any - nodeAffinity := k8sutil.GetValue(data, discoverDaemonSetNodeAffinityEnv, "") + nodeAffinity := k8sutil.GetOperatorSetting(discoverDaemonSetNodeAffinityEnv, "") if nodeAffinity != "" { v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity) if err != nil { @@ -224,7 +224,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco logger.Infof("nodeAffinity: %s", v1NodeAffinity) } - podLabels := k8sutil.GetValue(data, discoverDaemonSetPodLabelsEnv, "") + podLabels := k8sutil.GetOperatorSetting(discoverDaemonSetPodLabelsEnv, "") if podLabels != "" { podLabels := k8sutil.ParseStringToLabels(podLabels) // Override / Set the app label even if set by the user as diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go index 667e88bdc..ec19cc16e 100644 --- a/pkg/operator/discover/discover_test.go +++ b/pkg/operator/discover/discover_test.go @@ -19,12 +19,12 @@ package discover import ( "context" + "os" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" discoverDaemon "github.com/rook/rook/pkg/daemon/discover" - "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" @@ -63,7 +63,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) { _, err := clientset.CoreV1().Pods("rook-system").Create(ctx, &pod, metav1.CreateOptions{}) assert.NoError(t, err) // start a basic cluster - err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", map[string]string{}, false) + err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", false) assert.Nil(t, err) // check daemonset parameters @@ -86,21 +86,11 @@ func TestStartDiscoveryDaemonset(t *testing.T) { assert.Nil(t, agentDS.Spec.Template.Spec.Tolerations) // Test with rook override configmap setting - opConfigCM := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.OperatorSettingConfigMapName, - Namespace: namespace, - }, - Data: map[string]string{ - "DISCOVER_TOLERATIONS": "- effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n operator: Exists\n- effect: NoExecute\n key: node-role.kubernetes.io/etcd\n operator: Exists", - }, - } - - cm, err := clientset.CoreV1().ConfigMaps(namespace).Create(ctx, opConfigCM, metav1.CreateOptions{}) - assert.NoError(t, err) + os.Setenv("DISCOVER_TOLERATIONS", "- effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n operator: Exists\n- effect: NoExecute\n key: node-role.kubernetes.io/etcd\n operator: Exists") + defer os.Unsetenv("DISCOVER_TOLERATIONS") // start a basic cluster - err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", cm.Data, false) + err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", false) assert.Nil(t, err) agentDS, err = clientset.AppsV1().DaemonSets(namespace).Get(ctx, "rook-discover", metav1.GetOptions{}) diff --git a/pkg/operator/k8sutil/configmap.go b/pkg/operator/k8sutil/configmap.go index 08b932430..7e57bb7d7 100644 --- a/pkg/operator/k8sutil/configmap.go +++ b/pkg/operator/k8sutil/configmap.go @@ -20,17 +20,17 @@ import ( "context" "fmt" "os" - "sync" "time" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) -var loggedOperatorSettings = sync.Map{} +var loadedOperatorSettings bool // DeleteConfigMap deletes a ConfigMap. func DeleteConfigMap(ctx context.Context, clientset kubernetes.Interface, cmName, namespace string, opts *DeleteOptions) error { @@ -70,50 +70,43 @@ func CreateOrUpdateConfigMap(ctx context.Context, clientset kubernetes.Interface return existingCm, nil } -// GetOperatorSetting gets the operator setting from ConfigMap or Env Var +// GetOperatorSetting gets the operator setting from Env Var merged with ConfigMap // returns defaultValue if setting is not found -func GetOperatorSetting(context context.Context, clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) { - // config must be in operator pod namespace - namespace := os.Getenv(PodNamespaceEnvVar) - cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(context, configMapName, metav1.GetOptions{}) +func GetOperatorSetting(settingName, defaultValue string) string { + if !loadedOperatorSettings { + logger.Warningf("WARNING: attempting to load operator setting %q before configmap is loaded", settingName) + } + if settingValue, ok := os.LookupEnv(settingName); ok { + return settingValue + } + return defaultValue +} + +func ApplyOperatorSettingsConfigmap(ctx context.Context, clientset kubernetes.Interface) error { + namespacedName := types.NamespacedName{Namespace: os.Getenv(PodNamespaceEnvVar), Name: "rook-ceph-operator-config"} + logger.Debugf("loading operator settings configmap from %v", namespacedName) + opConfig, err := clientset.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{}) + if err != nil { if kerrors.IsNotFound(err) { - if settingValue, ok := os.LookupEnv(settingName); ok { - logger.Infof("%s=%q (env var)", settingName, settingValue) - return settingValue, nil - } - logger.Infof("%s=%q (default)", settingName, defaultValue) - return defaultValue, nil + loadedOperatorSettings = true + logger.Debug("operator's configmap resource not found. will use default value or env var.") + return nil } - return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", configMapName, err) + return err } - return GetValue(cm.Data, settingName, defaultValue), nil -} - -func GetValue(data map[string]string, settingName, defaultValue string) string { - settingValue := defaultValue - settingSource := "default" - - if val, ok := data[settingName]; ok { - settingValue = val - settingSource = "configmap" - } else if val, ok := os.LookupEnv(settingName); ok { - settingValue = val - settingSource = "env var" + for key, value := range opConfig.Data { + currentValue := os.Getenv(key) + if currentValue == value { + continue + } + logger.Infof("operator setting %q = %q", key, value) + if err := os.Setenv(key, value); err != nil { + logger.Errorf("failed to set env var %q = %q. %v", key, value, err) + } } - - if logChangedSettings(settingName, settingValue) { - logger.Infof("%s=%q (%s)", settingName, settingValue, settingSource) - } - - return settingValue -} - -func logChangedSettings(settingName, settingValue string) bool { - if val, ok := loggedOperatorSettings.Load(settingName); !ok || val != settingValue { - loggedOperatorSettings.Store(settingName, settingValue) - return true - } - return false + loadedOperatorSettings = true + logger.Debug("done loading operator settings configmap") + return nil } diff --git a/pkg/operator/k8sutil/configmap_test.go b/pkg/operator/k8sutil/configmap_test.go index cc03f1fcc..d4577b0fc 100644 --- a/pkg/operator/k8sutil/configmap_test.go +++ b/pkg/operator/k8sutil/configmap_test.go @@ -63,7 +63,7 @@ func TestGetOperatorSetting(t *testing.T) { k8s := fake.NewSimpleClientset() ctx := context.TODO() - operatorSettingConfigMapName := "rook-operator-config" + operatorSettingConfigMapName := "rook-ceph-operator-config" testNamespace := "test-namespace" cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -81,63 +81,41 @@ func TestGetOperatorSetting(t *testing.T) { cmSettingValue := "storage=rook, worker" defaultValue := "" - // ConfigMap is not found - setting, err := GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) - assert.NoError(t, err) - - // Env Var doesn't exist + // ConfigMap is not yet applied so we expect the default value + setting := GetOperatorSetting(nodeAffinity, defaultValue) assert.Equal(t, defaultValue, setting) + // Env Var exists t.Setenv(nodeAffinity, envSettingValue) - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) - assert.NoError(t, err) + setting = GetOperatorSetting(nodeAffinity, defaultValue) assert.Equal(t, envSettingValue, setting) // ConfigMap is found t.Setenv("POD_NAMESPACE", testNamespace) - _, err = k8s.CoreV1().ConfigMaps(testNamespace).Create(ctx, cm, metav1.CreateOptions{}) + _, err := k8s.CoreV1().ConfigMaps(testNamespace).Create(ctx, cm, metav1.CreateOptions{}) + assert.NoError(t, err) + err = ApplyOperatorSettingsConfigmap(ctx, k8s) assert.NoError(t, err) // Setting exists in ConfigMap - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) - assert.NoError(t, err) - // Env Var exists + setting = GetOperatorSetting(nodeAffinity, defaultValue) assert.Equal(t, cmSettingValue, setting) + // Env Var doesn't exist err = os.Unsetenv(nodeAffinity) assert.NoError(t, err) assert.Equal(t, cmSettingValue, setting) - // Setting doesn't exist in ConfigMap - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, podAffinity, defaultValue) - assert.NoError(t, err) - // Env Var doesn't exist + // Setting doesn't exist in ConfigMap, expect the default value + setting = GetOperatorSetting(podAffinity, defaultValue) assert.Equal(t, defaultValue, setting) // Env Var exists t.Setenv(podAffinity, envSettingValue) - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, podAffinity, defaultValue) + setting = GetOperatorSetting(podAffinity, defaultValue) assert.NoError(t, err) assert.Equal(t, envSettingValue, setting) } -func TestLogChangedSettings(t *testing.T) { - key := "Node_Affinity" - value := "storage=rook, worker" - - // When the key is not in the map, it should be added and return true for logging - ok := logChangedSettings(key, value) - assert.True(t, true, ok) - - // When the key is already present in the map and value is same, it should return false for logging - ok = logChangedSettings(key, value) - assert.False(t, false, ok) - - // When the key is already present in the map and value is updated, it should return true for logging - value = "storage=rook, test" - ok = logChangedSettings(key, value) - assert.True(t, true, ok) -} - func TestCreateOrUpdateConfigMap(t *testing.T) { k8s := fake.NewSimpleClientset() ctx := context.TODO()