diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 8a44c36c4..b1fcd62b3 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -577,6 +577,9 @@ data: # RevisionHistoryLimit value for all deployments created by rook. # ROOK_REVISION_HISTORY_LIMIT: "3" + + # Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used + ROOK_CUSTOM_HOSTNAME_LABEL: "" --- # OLM: BEGIN OPERATOR DEPLOYMENT apiVersion: apps/v1 @@ -657,10 +660,6 @@ spec: - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS value: "5" - # Custom label to identify node hostname. If not set `kubernetes.io/hostname` will be used - - name: ROOK_CUSTOM_HOSTNAME_LABEL - value: "" - # The name of the node to pass with the downward API - name: NODE_NAME valueFrom: diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 11eecc632..f499cccce 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -187,10 +187,7 @@ func add(opManagerContext context.Context, mgr manager.Manager, r reconcile.Reco // Watch for changes on the hotplug config map // TODO: to improve, can we run this against the operator namespace only? - disableVal, err := k8sutil.GetOperatorSetting(opManagerContext, context.Clientset, opcontroller.OperatorSettingConfigMapName, disableHotplugEnv, "false") - if err != nil { - logger.Errorf("failed to get %s setting %s. %v", disableHotplugEnv, opcontroller.OperatorSettingConfigMapName, err) - } + disableVal := k8sutil.GetOperatorSetting(disableHotplugEnv, "false") if disableVal != "true" { logger.Info("enabling hotplug orchestration") s := source.Kind[client.Object]( diff --git a/pkg/operator/ceph/cluster/watcher.go b/pkg/operator/ceph/cluster/watcher.go index 57363bfa2..c5ee33184 100644 --- a/pkg/operator/ceph/cluster/watcher.go +++ b/pkg/operator/ceph/cluster/watcher.go @@ -159,27 +159,21 @@ func (c *clientCluster) onK8sNode(ctx context.Context, object runtime.Object, op } func (c *clientCluster) handleNodeFailure(ctx context.Context, cluster *cephv1.CephCluster, node *corev1.Node, opNamespace string) error { - watchForNodeLoss, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_WATCH_FOR_NODE_FAILURE", "true") - if err != nil { - return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_WATCH_FOR_NODE_FAILURE`.") - } + watchForNodeLoss := k8sutil.GetOperatorSetting("ROOK_WATCH_FOR_NODE_FAILURE", "true") if strings.ToLower(watchForNodeLoss) != "true" { logger.Debugf("not watching for node failures since `ROOK_WATCH_FOR_NODE_FAILURE` is set to %q", watchForNodeLoss) return nil } - disabledCSI, err := k8sutil.GetOperatorSetting(ctx, c.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_CSI_DISABLE_DRIVER", "false") - if err != nil { - return pkgerror.Wrapf(err, "failed to get configmap value `ROOK_CSI_DISABLE_DRIVER`.") - } + disabledCSI := k8sutil.GetOperatorSetting("ROOK_CSI_DISABLE_DRIVER", "false") if strings.ToLower(disabledCSI) != "false" { logger.Debugf("not watching for node failures since `ROOK_CSI_DISABLE_DRIVER` is set to %q, skip creating networkFence", disabledCSI) return nil } - _, err = c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{}) + _, err := c.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "networkfences.csiaddons.openshift.io", metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { logger.Debug("networkfences.csiaddons.openshift.io CRD not found, skip creating networkFence") diff --git a/pkg/operator/ceph/controller.go b/pkg/operator/ceph/controller.go index 879a51b2e..5aa86f336 100644 --- a/pkg/operator/ceph/controller.go +++ b/pkg/operator/ceph/controller.go @@ -33,7 +33,6 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util" v1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -103,52 +102,43 @@ func (r *ReconcileConfig) Reconcile(context context.Context, request reconcile.R func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the operator's configmap - opConfig := &v1.ConfigMap{} logger.Debugf("reconciling %s", request.NamespacedName) - err := r.client.Get(r.opManagerContext, request.NamespacedName, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("operator's configmap resource not found. will use default value or env var.") - } else { - // Error reading the object - requeue the request. - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") + if request.Name == opcontroller.OperatorSettingConfigMapName { + if err := k8sutil.ApplyOperatorSettingsConfigmap(r.opManagerContext, r.context.Clientset); err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator setting configmap") } - } else { - // Populate the operator's config - r.config.Parameters = opConfig.Data } - // Reconcile Ceph CLI timeout, since the clusterd context is passed to by pointer to all CRD // controllers they will receive the update - opcontroller.SetCephCommandsTimeout(r.config.Parameters) + opcontroller.SetCephCommandsTimeout() // Reconcile Operator's logging level - reconcileOperatorLogLevel(opConfig.Data) + reconcileOperatorLogLevel() // Reconcile discovery daemon - err = r.reconcileDiscoveryDaemon(opConfig.Data) + err := r.reconcileDiscoveryDaemon() if err != nil { return opcontroller.ImmediateRetryResult, err } - opcontroller.SetAllowLoopDevices(r.config.Parameters) - opcontroller.SetEnforceHostNetwork(r.config.Parameters) - opcontroller.SetRevisionHistoryLimit(r.config.Parameters) - opcontroller.SetObcAllowAdditionalConfigFields(r.config.Parameters) + opcontroller.SetAllowLoopDevices() + opcontroller.SetEnforceHostNetwork() + opcontroller.SetRevisionHistoryLimit() + opcontroller.SetObcAllowAdditionalConfigFields() logger.Infof("%s done reconciling", controllerName) return reconcile.Result{}, nil } -func reconcileOperatorLogLevel(data map[string]string) { - rookLogLevel := k8sutil.GetValue(data, "ROOK_LOG_LEVEL", util.DefaultLogLevel.String()) +func reconcileOperatorLogLevel() { + rookLogLevel := k8sutil.GetOperatorSetting("ROOK_LOG_LEVEL", util.DefaultLogLevel.String()) util.SetGlobalLogLevel(rookLogLevel, logger) } -func (r *ReconcileConfig) reconcileDiscoveryDaemon(data map[string]string) error { +func (r *ReconcileConfig) reconcileDiscoveryDaemon() error { rookDiscover := discover.New(r.context.Clientset) - if opcontroller.DiscoveryDaemonEnabled(r.config.Parameters) { - if err := rookDiscover.Start(r.opManagerContext, r.config.OperatorNamespace, r.config.Image, r.config.ServiceAccount, data, true); err != nil { + if opcontroller.DiscoveryDaemonEnabled() { + if err := rookDiscover.Start(r.opManagerContext, r.config.OperatorNamespace, r.config.Image, r.config.ServiceAccount, true); err != nil { return errors.Wrap(err, "failed to start device discovery daemonset") } } else { diff --git a/pkg/operator/ceph/controller/controller_utils.go b/pkg/operator/ceph/controller/controller_utils.go index 8cb661d74..d55d00124 100644 --- a/pkg/operator/ceph/controller/controller_utils.go +++ b/pkg/operator/ceph/controller/controller_utils.go @@ -40,7 +40,6 @@ type OperatorConfig struct { Image string ServiceAccount string NamespaceToWatch string - Parameters map[string]string } // ClusterHealth is passed to the various monitoring go routines to stop them when the context is cancelled @@ -99,13 +98,13 @@ var ( obcAllowAdditionalConfigFields = strings.Split(obcAllowAdditionalConfigFieldsDefaultValue, ",") ) -func DiscoveryDaemonEnabled(data map[string]string) bool { - return k8sutil.GetValue(data, "ROOK_ENABLE_DISCOVERY_DAEMON", "false") == "true" +func DiscoveryDaemonEnabled() bool { + return k8sutil.GetOperatorSetting("ROOK_ENABLE_DISCOVERY_DAEMON", "false") == "true" } // SetCephCommandsTimeout sets the timeout value of Ceph commands which are executed from Rook -func SetCephCommandsTimeout(data map[string]string) { - strTimeoutSeconds := k8sutil.GetValue(data, "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15") +func SetCephCommandsTimeout() { + strTimeoutSeconds := k8sutil.GetOperatorSetting("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "15") timeoutSeconds, err := strconv.Atoi(strTimeoutSeconds) if err != nil || timeoutSeconds < 1 { logger.Warningf("ROOK_CEPH_COMMANDS_TIMEOUT is %q but it should be >= 1, set the default value 15", strTimeoutSeconds) @@ -114,8 +113,8 @@ func SetCephCommandsTimeout(data map[string]string) { exec.CephCommandsTimeout = time.Duration(timeoutSeconds) * time.Second } -func SetAllowLoopDevices(data map[string]string) { - strLoopDevicesAllowed := k8sutil.GetValue(data, "ROOK_CEPH_ALLOW_LOOP_DEVICES", "false") +func SetAllowLoopDevices() { + strLoopDevicesAllowed := k8sutil.GetOperatorSetting("ROOK_CEPH_ALLOW_LOOP_DEVICES", "false") var err error loopDevicesAllowed, err = strconv.ParseBool(strLoopDevicesAllowed) if err != nil { @@ -128,8 +127,8 @@ func LoopDevicesAllowed() bool { return loopDevicesAllowed } -func SetEnforceHostNetwork(data map[string]string) { - strval := k8sutil.GetValue(data, enforceHostNetworkSettingName, enforceHostNetworkDefaultValue) +func SetEnforceHostNetwork() { + strval := k8sutil.GetOperatorSetting(enforceHostNetworkSettingName, enforceHostNetworkDefaultValue) val, err := strconv.ParseBool(strval) if err != nil { logger.Warningf("failed to parse value %q for %q. assuming false value", strval, enforceHostNetworkSettingName) @@ -143,8 +142,8 @@ func EnforceHostNetwork() bool { return cephv1.EnforceHostNetwork() } -func SetRevisionHistoryLimit(data map[string]string) { - strval := k8sutil.GetValue(data, revisionHistoryLimitSettingName, "") +func SetRevisionHistoryLimit() { + strval := k8sutil.GetOperatorSetting(revisionHistoryLimitSettingName, "") var limit int32 if strval == "" { logger.Debugf("not parsing empty string to int for %q. assuming default value.", revisionHistoryLimitSettingName) @@ -166,8 +165,8 @@ func RevisionHistoryLimit() *int32 { return revisionHistoryLimit } -func SetObcAllowAdditionalConfigFields(data map[string]string) { - strval := k8sutil.GetValue(data, obcAllowAdditionalConfigFieldsSettingName, obcAllowAdditionalConfigFieldsDefaultValue) +func SetObcAllowAdditionalConfigFields() { + strval := k8sutil.GetOperatorSetting(obcAllowAdditionalConfigFieldsSettingName, obcAllowAdditionalConfigFieldsDefaultValue) obcAllowAdditionalConfigFields = strings.Split(strval, ",") } diff --git a/pkg/operator/ceph/controller/controller_utils_test.go b/pkg/operator/ceph/controller/controller_utils_test.go index cda45d991..d1eac8297 100644 --- a/pkg/operator/ceph/controller/controller_utils_test.go +++ b/pkg/operator/ceph/controller/controller_utils_test.go @@ -18,6 +18,7 @@ package controller import ( ctx "context" + "os" "testing" "time" @@ -80,83 +81,94 @@ func TestCanIgnoreHealthErrStatusInReconcile(t *testing.T) { } func TestSetCephCommandsTimeout(t *testing.T) { - SetCephCommandsTimeout(map[string]string{}) + defer os.Unsetenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS") + + SetCephCommandsTimeout() assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout) exec.CephCommandsTimeout = 0 - SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "0"}) + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "0") + SetCephCommandsTimeout() assert.Equal(t, 15*time.Second, exec.CephCommandsTimeout) exec.CephCommandsTimeout = 0 - SetCephCommandsTimeout(map[string]string{"ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "1"}) + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "1") + SetCephCommandsTimeout() assert.Equal(t, 1*time.Second, exec.CephCommandsTimeout) } func TestSetAllowLoopDevices(t *testing.T) { - SetAllowLoopDevices(map[string]string{}) + defer os.Unsetenv("ROOK_CEPH_ALLOW_LOOP_DEVICES") + + SetAllowLoopDevices() assert.False(t, LoopDevicesAllowed()) - SetAllowLoopDevices(map[string]string{"ROOK_CEPH_ALLOW_LOOP_DEVICES": "foo"}) + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "foo") + SetAllowLoopDevices() assert.False(t, LoopDevicesAllowed()) - SetAllowLoopDevices(map[string]string{"ROOK_CEPH_ALLOW_LOOP_DEVICES": "false"}) + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "false") + SetAllowLoopDevices() assert.False(t, LoopDevicesAllowed()) - SetAllowLoopDevices(map[string]string{"ROOK_CEPH_ALLOW_LOOP_DEVICES": "true"}) + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "true") + SetAllowLoopDevices() assert.True(t, LoopDevicesAllowed()) } func TestSetEnforceHostNetwork(t *testing.T) { + defer os.Unsetenv(enforceHostNetworkSettingName) + logger.Infof("testing default value for %v", enforceHostNetworkSettingName) - opConfig := map[string]string{} - SetEnforceHostNetwork(opConfig) + SetEnforceHostNetwork() assert.False(t, EnforceHostNetwork()) // test invalid setting var value string = "foo" logger.Infof("testing invalid value'%v' for %v", value, enforceHostNetworkSettingName) - opConfig[enforceHostNetworkSettingName] = value - SetEnforceHostNetwork(opConfig) + os.Setenv(enforceHostNetworkSettingName, value) + SetEnforceHostNetwork() assert.False(t, EnforceHostNetwork()) // test valid settings value = "true" logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) - opConfig[enforceHostNetworkSettingName] = value - SetEnforceHostNetwork(opConfig) + os.Setenv(enforceHostNetworkSettingName, value) + SetEnforceHostNetwork() assert.True(t, EnforceHostNetwork()) value = "false" logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName) - opConfig[enforceHostNetworkSettingName] = value - SetEnforceHostNetwork(opConfig) + os.Setenv(enforceHostNetworkSettingName, value) + SetEnforceHostNetwork() assert.False(t, EnforceHostNetwork()) } func TestSetRevisionHistoryLimit(t *testing.T) { - opConfig := map[string]string{} + defer os.Unsetenv(revisionHistoryLimitSettingName) + t.Run("ROOK_REVISION_HISTORY_LIMIT: test default value", func(t *testing.T) { - SetRevisionHistoryLimit(opConfig) + SetRevisionHistoryLimit() assert.Nil(t, RevisionHistoryLimit()) }) var value string = "foo" t.Run("ROOK_REVISION_HISTORY_LIMIT: test invalid value 'foo'", func(t *testing.T) { - opConfig[revisionHistoryLimitSettingName] = value - SetRevisionHistoryLimit(opConfig) + os.Setenv(revisionHistoryLimitSettingName, value) + SetRevisionHistoryLimit() assert.Nil(t, RevisionHistoryLimit()) }) t.Run("ROOK_REVISION_HISTORY_LIMIT: test empty string value", func(t *testing.T) { value = "" - opConfig[revisionHistoryLimitSettingName] = value - SetRevisionHistoryLimit(opConfig) + os.Setenv(revisionHistoryLimitSettingName, value) + SetRevisionHistoryLimit() assert.Nil(t, RevisionHistoryLimit()) }) t.Run("ROOK_REVISION_HISTORY_LIMIT: test valig value '10'", func(t *testing.T) { value = "10" - opConfig[revisionHistoryLimitSettingName] = value - SetRevisionHistoryLimit(opConfig) + os.Setenv(revisionHistoryLimitSettingName, value) + SetRevisionHistoryLimit() assert.Equal(t, int32(10), *RevisionHistoryLimit()) }) } @@ -249,36 +261,39 @@ func TestIsReadyToReconcile(t *testing.T) { func TestObcAllowAdditionalConfigFields(t *testing.T) { tests := []struct { - name string - data map[string]string - shouldAllow []string - shouldDisallow []string + name string + additionalConfig string + shouldAllow []string + shouldDisallow []string }{ - {"not set", map[string]string{}, + {"not set", "", []string{"maxObjects", "maxSize"}, // default allowlist is unlikely to need changing EVER []string{"bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"set to empty", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": ""}, + {"set to empty", "", []string{}, // admin can allow no quota options if desired []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"set to default", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize"}, + {"set to default", "maxObjects,maxSize", []string{"maxObjects", "maxSize"}, []string{"bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"all quota fields", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize"}, + {"all quota fields", "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize", []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize"}, []string{"bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}}, - {"all fields", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner"}, + {"all fields", "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner", []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner"}, []string{"random"}}, // this mechanism doesn't do any field checking - that isn't it's job - it merely handles // allow-listing essentially arbitrary config keys - {"all fields including unknown", map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner,random"}, + {"all fields including unknown", "maxObjects,maxSize,bucketMaxObjects,bucketMaxSize,bucketPolicy,bucketLifecycle,bucketOwner,random", []string{"maxObjects", "maxSize", "bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner", "random"}, []string{"otherRandom"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - SetObcAllowAdditionalConfigFields(tt.data) - + if tt.additionalConfig != "" { + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", tt.additionalConfig) + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + } + SetObcAllowAdditionalConfigFields() for _, toAllow := range tt.shouldAllow { assert.True(t, ObcAdditionalConfigKeyIsAllowed(toAllow)) } diff --git a/pkg/operator/ceph/controller_test.go b/pkg/operator/ceph/controller_test.go index ed4485611..a9f08b672 100644 --- a/pkg/operator/ceph/controller_test.go +++ b/pkg/operator/ceph/controller_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/coreos/pkg/capnslog" - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" "github.com/rook/rook/pkg/clusterd" @@ -31,9 +30,7 @@ import ( "github.com/rook/rook/pkg/operator/test" "github.com/rook/rook/pkg/util/exec" "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -64,7 +61,6 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() @@ -95,7 +91,6 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() @@ -128,23 +123,12 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. // s := scheme.Scheme s := clientgoscheme.Scheme - // s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}) - opConfigCM := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.OperatorSettingConfigMapName, - Namespace: namespace, - }, - Data: map[string]string{ - "ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS": "20", - }, - } + os.Setenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS", "20") + defer os.Unsetenv("ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS") - object := []runtime.Object{ - opConfigCM, - } // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects().Build() // Create a ReconcileCSI object with the scheme and fake client. r := &ReconcileConfig{ @@ -173,7 +157,6 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. s := scheme.Scheme - s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}, &v1.ConfigMapList{}) // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithScheme(s).Build() @@ -209,23 +192,12 @@ func TestOperatorController(t *testing.T) { // Register operator types with the runtime scheme. // s := scheme.Scheme s := clientgoscheme.Scheme - // s.AddKnownTypes(cephv1.SchemeGroupVersion, &v1.ConfigMap{}) - opConfigCM := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.OperatorSettingConfigMapName, - Namespace: namespace, - }, - Data: map[string]string{ - "ROOK_CEPH_ALLOW_LOOP_DEVICES": "true", - }, - } + os.Setenv("ROOK_CEPH_ALLOW_LOOP_DEVICES", "true") + defer os.Unsetenv("ROOK_CEPH_ALLOW_LOOP_DEVICES") - object := []runtime.Object{ - opConfigCM, - } // Create a fake client to mock API calls. - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build() + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects().Build() // Create a ReconcileCSI object with the scheme and fake client. r := &ReconcileConfig{ diff --git a/pkg/operator/ceph/cr_manager.go b/pkg/operator/ceph/cr_manager.go index b3398d681..21d9e4860 100644 --- a/pkg/operator/ceph/cr_manager.go +++ b/pkg/operator/ceph/cr_manager.go @@ -143,11 +143,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro } } - metricsBindAddress, err := k8sutil.GetOperatorSetting(context, o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_OPERATOR_METRICS_BIND_ADDRESS", "0") - if err != nil { - mgrErrorCh <- errors.Wrap(err, "failed to get configmap value `ROOK_OPERATOR_METRICS_BIND_ADDRESS`.") - return - } + metricsBindAddress := k8sutil.GetOperatorSetting("ROOK_OPERATOR_METRICS_BIND_ADDRESS", "0") skipNameValidation := true // Set up a manager mgrOpts := manager.Options{ diff --git a/pkg/operator/ceph/csi/controller.go b/pkg/operator/ceph/csi/controller.go index b9c6c5034..809b2da4d 100644 --- a/pkg/operator/ceph/csi/controller.go +++ b/pkg/operator/ceph/csi/controller.go @@ -42,7 +42,6 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ) const ( @@ -158,32 +157,17 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map") } - // Fetch the operator's configmap. We force the NamespaceName to the operator since the request - // could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the - // CM won't be found - opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace} - opConfig := &v1.ConfigMap{} - err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("operator's configmap resource not found. will use default value or env var.") - r.opConfig.Parameters = make(map[string]string) - } else { - // Error reading the object - requeue the request. - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") - } - } else { - // Populate the operator's config - r.opConfig.Parameters = opConfig.Data + if err := k8sutil.ApplyOperatorSettingsConfigmap(r.opManagerContext, r.context.Clientset); err != nil { + return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to apply operator settings configmap") } - enableCSIOperator, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_USE_CSI_OPERATOR", "false")) + enableCSIOperator, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_USE_CSI_OPERATOR", "false")) if err != nil { return reconcileResult, errors.Wrap(err, "unable to parse value for 'ROOK_USE_CSI_OPERATOR'") } // do not recocnile if csi driver is disabled - disableCSI, err := strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_DISABLE_DRIVER", "false")) + disableCSI, err := strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_DISABLE_DRIVER", "false")) if err != nil { return reconcile.Result{}, errors.Wrap(err, "unable to parse value for 'ROOK_CSI_DISABLE_DRIVER") } else if disableCSI { diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index 1e66cf459..6919eaf54 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -56,31 +56,31 @@ func (r *ReconcileCSI) validateAndConfigureDrivers(ownerInfo *k8sutil.OwnerInfo) func (r *ReconcileCSI) setParams() error { var err error - if EnableRBD, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_RBD", "true")); err != nil { + if EnableRBD, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_RBD", "true")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_RBD'") } - if EnableCephFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil { + if EnableCephFS, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_CEPHFS", "true")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_CEPHFS'") } - if EnableNFS, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_NFS", "false")); err != nil { + if EnableNFS, err = strconv.ParseBool(k8sutil.GetOperatorSetting("ROOK_CSI_ENABLE_NFS", "false")); err != nil { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_NFS'") } - if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil { + if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetOperatorSetting("CSI_ENABLE_HOST_NETWORK", "true")); err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") } // If not set or set to anything but "false", the kernel client will be enabled - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_FORCE_CEPHFS_KERNEL_CLIENT", "true"), "false") { CSIParam.ForceCephFSKernelClient = "false" } else { CSIParam.ForceCephFSKernelClient = "true" } // parse RPC timeout - timeout := k8sutil.GetValue(r.opConfig.Parameters, grpcTimeout, "150") + timeout := k8sutil.GetOperatorSetting(grpcTimeout, "150") timeoutSeconds, err := strconv.Atoi(timeout) if err != nil { logger.Errorf("failed to parse %q. Defaulting to %d. %v", grpcTimeout, defaultGRPCTimeout, err) @@ -93,32 +93,32 @@ func (r *ReconcileCSI) setParams() error { CSIParam.GRPCTimeout = time.Duration(timeoutSeconds) * time.Second // parse Liveness port - CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) + CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig("CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") } - CSIParam.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort) + CSIParam.CSIAddonsPort, err = getPortFromConfig("CSIADDONS_PORT", DefaultCSIAddonsPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port") } - CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort) + CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig("CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port for RBD provisioner") } - CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort) + CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig("CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port for Ceph FS provisioner") } - CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) + CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig("CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI RBD liveness metrics port.") } - CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_LIVENESS", "false")) + CSIParam.EnableLiveness, err = strconv.ParseBool(k8sutil.GetOperatorSetting("CSI_ENABLE_LIVENESS", "false")) if err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_LIVENESS'") } @@ -126,31 +126,31 @@ func (r *ReconcileCSI) setParams() error { CSIParam.Privileged = controller.HostPathRequiresPrivileged() // default value `system-node-critical` is the highest available priority - CSIParam.PluginPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_PRIORITY_CLASSNAME", "") + CSIParam.PluginPriorityClassName = k8sutil.GetOperatorSetting("CSI_PLUGIN_PRIORITY_CLASSNAME", "") // default value `system-cluster-critical` is applied for some // critical pods in cluster but less priority than plugin pods - CSIParam.ProvisionerPriorityClassName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_PRIORITY_CLASSNAME", "") + CSIParam.ProvisionerPriorityClassName = k8sutil.GetOperatorSetting("CSI_PROVISIONER_PRIORITY_CLASSNAME", "") CSIParam.EnableOMAPGenerator = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_OMAP_GENERATOR", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_OMAP_GENERATOR", "false"), "true") { CSIParam.EnableOMAPGenerator = true } CSIParam.EnableCSIDriverSeLinuxMount = true CSIParam.EnableRBDSnapshotter = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_RBD_SNAPSHOTTER", "true"), "false") { CSIParam.EnableRBDSnapshotter = false } CSIParam.EnableCephFSSnapshotter = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CEPHFS_SNAPSHOTTER", "true"), "false") { CSIParam.EnableCephFSSnapshotter = false } CSIParam.EnableNFSSnapshotter = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_NFS_SNAPSHOTTER", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_NFS_SNAPSHOTTER", "true"), "false") { CSIParam.EnableNFSSnapshotter = false } @@ -159,36 +159,36 @@ func (r *ReconcileCSI) setParams() error { if err == nil { CSIParam.EnableCSIAddonsSideCar = true } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", ""), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CSIADDONS", ""), "false") { CSIParam.EnableCSIAddonsSideCar = false } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_CSIADDONS", ""), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_CSIADDONS", ""), "true") { CSIParam.EnableCSIAddonsSideCar = true } CSIParam.EnableCSITopology = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_TOPOLOGY", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_TOPOLOGY", "false"), "true") { CSIParam.EnableCSITopology = true } CSIParam.EnableCSIEncryption = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_ENCRYPTION", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_ENCRYPTION", "false"), "true") { CSIParam.EnableCSIEncryption = true } CSIParam.CSIEnableMetadata = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_METADATA", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_METADATA", "false"), "true") { CSIParam.CSIEnableMetadata = true } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.CephFSPluginUpdateStrategy = onDelete } else { CSIParam.CephFSPluginUpdateStrategy = rollingUpdate - CSIParam.CephFSPluginUpdateStrategyMaxUnavailable = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") + CSIParam.CephFSPluginUpdateStrategyMaxUnavailable = k8sutil.GetOperatorSetting("CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") } - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_NFS_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.NFSPluginUpdateStrategy = onDelete } else { CSIParam.NFSPluginUpdateStrategy = rollingUpdate @@ -196,19 +196,19 @@ func (r *ReconcileCSI) setParams() error { // Default values are based on Kubernetes official documentation. // https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_RBD_PLUGIN_UPDATE_STRATEGY", rollingUpdate), onDelete) { CSIParam.RBDPluginUpdateStrategy = onDelete } else { CSIParam.RBDPluginUpdateStrategy = rollingUpdate - CSIParam.RBDPluginUpdateStrategyMaxUnavailable = k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") + CSIParam.RBDPluginUpdateStrategyMaxUnavailable = k8sutil.GetOperatorSetting("CSI_RBD_PLUGIN_UPDATE_STRATEGY_MAX_UNAVAILABLE", "1") } CSIParam.EnablePluginSelinuxHostMount = false - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_PLUGIN_ENABLE_SELINUX_HOST_MOUNT", "false"), "true") { CSIParam.EnablePluginSelinuxHostMount = true } - logLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LOG_LEVEL", "") + logLevel := k8sutil.GetOperatorSetting("CSI_LOG_LEVEL", "") CSIParam.LogLevel = defaultLogLevel if logLevel != "" { l, err := strconv.ParseUint(logLevel, 10, 8) @@ -219,7 +219,7 @@ func (r *ReconcileCSI) setParams() error { } } - sidecarLogLevel := k8sutil.GetValue(r.opConfig.Parameters, "CSI_SIDECAR_LOG_LEVEL", "") + sidecarLogLevel := k8sutil.GetOperatorSetting("CSI_SIDECAR_LOG_LEVEL", "") CSIParam.SidecarLogLevel = defaultSidecarLogLevel if sidecarLogLevel != "" { l, err := strconv.ParseUint(sidecarLogLevel, 10, 8) @@ -230,7 +230,7 @@ func (r *ReconcileCSI) setParams() error { } } - leaderElectionLeaseDuration := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_LEASE_DURATION", "") + leaderElectionLeaseDuration := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_LEASE_DURATION", "") CSIParam.LeaderElectionLeaseDuration = defaultLeaderElectionLeaseDuration if leaderElectionLeaseDuration != "" { d, err := time.ParseDuration(leaderElectionLeaseDuration) @@ -241,7 +241,7 @@ func (r *ReconcileCSI) setParams() error { } } - leaderElectionRenewDeadline := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RENEW_DEADLINE", "") + leaderElectionRenewDeadline := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_RENEW_DEADLINE", "") CSIParam.LeaderElectionRenewDeadline = defaultLeaderElectionRenewDeadline if leaderElectionRenewDeadline != "" { d, err := time.ParseDuration(leaderElectionRenewDeadline) @@ -252,7 +252,7 @@ func (r *ReconcileCSI) setParams() error { } } - leaderElectionRetryPeriod := k8sutil.GetValue(r.opConfig.Parameters, "CSI_LEADER_ELECTION_RETRY_PERIOD", "") + leaderElectionRetryPeriod := k8sutil.GetOperatorSetting("CSI_LEADER_ELECTION_RETRY_PERIOD", "") CSIParam.LeaderElectionRetryPeriod = defaultLeaderElectionRetryPeriod if leaderElectionRetryPeriod != "" { d, err := time.ParseDuration(leaderElectionRetryPeriod) @@ -269,7 +269,7 @@ func (r *ReconcileCSI) setParams() error { if len(nodes.Items) == 1 { CSIParam.ProvisionerReplicas = 1 } else { - replicaStr := k8sutil.GetValue(r.opConfig.Parameters, "CSI_PROVISIONER_REPLICAS", "2") + replicaStr := k8sutil.GetOperatorSetting("CSI_PROVISIONER_REPLICAS", "2") replicas, err := strconv.ParseInt(replicaStr, 10, 32) if err != nil { logger.Errorf("failed to parse CSI_PROVISIONER_REPLICAS. Defaulting to %d. %v", defaultProvisionerReplicas, err) @@ -281,39 +281,39 @@ func (r *ReconcileCSI) setParams() error { logger.Errorf("failed to get nodes. Defaulting the number of replicas of provisioner pods to %d. %v", CSIParam.ProvisionerReplicas, err) } - CSIParam.CSIPluginImage = getImage(r.opConfig.Parameters, "ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) - CSIParam.RegistrarImage = getImage(r.opConfig.Parameters, "ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) - CSIParam.ProvisionerImage = getImage(r.opConfig.Parameters, "ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) - CSIParam.AttacherImage = getImage(r.opConfig.Parameters, "ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) - CSIParam.SnapshotterImage = getImage(r.opConfig.Parameters, "ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) - CSIParam.ResizerImage = getImage(r.opConfig.Parameters, "ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) - CSIParam.KubeletDirPath = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) - CSIParam.CSIAddonsImage = getImage(r.opConfig.Parameters, "ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) - CSIParam.CSIDomainLabels = k8sutil.GetValue(r.opConfig.Parameters, "CSI_TOPOLOGY_DOMAIN_LABELS", "") - csiCephFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_CEPHFS_POD_LABELS", "") + CSIParam.CSIPluginImage = getImage("ROOK_CSI_CEPH_IMAGE", DefaultCSIPluginImage) + CSIParam.RegistrarImage = getImage("ROOK_CSI_REGISTRAR_IMAGE", DefaultRegistrarImage) + CSIParam.ProvisionerImage = getImage("ROOK_CSI_PROVISIONER_IMAGE", DefaultProvisionerImage) + CSIParam.AttacherImage = getImage("ROOK_CSI_ATTACHER_IMAGE", DefaultAttacherImage) + CSIParam.SnapshotterImage = getImage("ROOK_CSI_SNAPSHOTTER_IMAGE", DefaultSnapshotterImage) + CSIParam.ResizerImage = getImage("ROOK_CSI_RESIZER_IMAGE", DefaultResizerImage) + CSIParam.KubeletDirPath = k8sutil.GetOperatorSetting("ROOK_CSI_KUBELET_DIR_PATH", DefaultKubeletDirPath) + CSIParam.CSIAddonsImage = getImage("ROOK_CSIADDONS_IMAGE", DefaultCSIAddonsImage) + CSIParam.CSIDomainLabels = k8sutil.GetOperatorSetting("CSI_TOPOLOGY_DOMAIN_LABELS", "") + csiCephFSPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_CEPHFS_POD_LABELS", "") CSIParam.CSICephFSPodLabels = k8sutil.ParseStringToLabels(csiCephFSPodLabels) - csiNFSPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_NFS_POD_LABELS", "") + csiNFSPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_NFS_POD_LABELS", "") CSIParam.CSINFSPodLabels = k8sutil.ParseStringToLabels(csiNFSPodLabels) - csiRBDPodLabels := k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_RBD_POD_LABELS", "") + csiRBDPodLabels := k8sutil.GetOperatorSetting("ROOK_CSI_RBD_POD_LABELS", "") CSIParam.CSIRBDPodLabels = k8sutil.ParseStringToLabels(csiRBDPodLabels) - CSIParam.CSIClusterName = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CLUSTER_NAME", "") - CSIParam.ImagePullPolicy = k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_IMAGE_PULL_POLICY", DefaultCSIImagePullPolicy) - CSIParam.CephFSKernelMountOptions = k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_KERNEL_MOUNT_OPTIONS", "") + CSIParam.CSIClusterName = k8sutil.GetOperatorSetting("CSI_CLUSTER_NAME", "") + CSIParam.ImagePullPolicy = k8sutil.GetOperatorSetting("ROOK_CSI_IMAGE_PULL_POLICY", DefaultCSIImagePullPolicy) + CSIParam.CephFSKernelMountOptions = k8sutil.GetOperatorSetting("CSI_CEPHFS_KERNEL_MOUNT_OPTIONS", "") CSIParam.CephFSAttachRequired = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_ATTACH_REQUIRED", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_CEPHFS_ATTACH_REQUIRED", "true"), "false") { CSIParam.CephFSAttachRequired = false } CSIParam.RBDAttachRequired = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_ATTACH_REQUIRED", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_RBD_ATTACH_REQUIRED", "true"), "false") { CSIParam.RBDAttachRequired = false } CSIParam.NFSAttachRequired = true - if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_ATTACH_REQUIRED", "true"), "false") { + if strings.EqualFold(k8sutil.GetOperatorSetting("CSI_NFS_ATTACH_REQUIRED", "true"), "false") { CSIParam.NFSAttachRequired = false } - CSIParam.DriverNamePrefix = k8sutil.GetValue(r.opConfig.Parameters, "CSI_DRIVER_NAME_PREFIX", r.opConfig.OperatorNamespace) + CSIParam.DriverNamePrefix = k8sutil.GetOperatorSetting("CSI_DRIVER_NAME_PREFIX", r.opConfig.OperatorNamespace) crd, err := r.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io", metav1.GetOptions{}) if err != nil && !kerrors.IsNotFound(err) { @@ -324,7 +324,7 @@ func (r *ReconcileCSI) setParams() error { if err == nil && len(crd.Spec.Versions) > 0 { ver := crd.Spec.Versions[0] // Determine if VolumeGroupSnapshot feature should be disabled - disableVGS := strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false") + disableVGS := strings.EqualFold(k8sutil.GetOperatorSetting("CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false") const ( enableVolumeGroupSnapshotFlag = "--enable-volume-group-snapshots=" featureGateFlag = "--feature-gates=CSIVolumeGroupSnapshot=" @@ -346,7 +346,7 @@ func (r *ReconcileCSI) setParams() error { } } - kubeApiBurst := k8sutil.GetValue(r.opConfig.Parameters, "CSI_KUBE_API_BURST", "") + kubeApiBurst := k8sutil.GetOperatorSetting("CSI_KUBE_API_BURST", "") CSIParam.KubeApiBurst = 0 if kubeApiBurst != "" { k, err := strconv.ParseUint(kubeApiBurst, 10, 16) @@ -357,7 +357,7 @@ func (r *ReconcileCSI) setParams() error { } } - kubeApiQPS := k8sutil.GetValue(r.opConfig.Parameters, "CSI_KUBE_API_QPS", "") + kubeApiQPS := k8sutil.GetOperatorSetting("CSI_KUBE_API_QPS", "") CSIParam.KubeApiQPS = 0 if kubeApiQPS != "" { k, err := strconv.ParseFloat(kubeApiQPS, 32) diff --git a/pkg/operator/ceph/csi/operator_config.go b/pkg/operator/ceph/csi/operator_config.go index 638e512b2..7e013245c 100644 --- a/pkg/operator/ceph/csi/operator_config.go +++ b/pkg/operator/ceph/csi/operator_config.go @@ -94,9 +94,9 @@ func (r *ReconcileCSI) generateCSIOpConfigSpec(cluster cephv1.CephCluster, opCon PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.ProvisionerPriorityClassName, Affinity: &v1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &v1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(pluginNodeAffinityEnv, &v1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []v1.Toleration{}), + Tolerations: getToleration(pluginTolerationsEnv, []v1.Toleration{}), }, Resources: csiopv1a1.NodePluginResourcesSpec{}, KubeletDirPath: CSIParam.KubeletDirPath, @@ -106,9 +106,9 @@ func (r *ReconcileCSI) generateCSIOpConfigSpec(cluster cephv1.CephCluster, opCon PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.PluginPriorityClassName, Affinity: &v1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &v1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(provisionerNodeAffinityEnv, &v1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []v1.Toleration{}), + Tolerations: getToleration(provisionerTolerationsEnv, []v1.Toleration{}), }, Replicas: &CSIParam.ProvisionerReplicas, Resources: csiopv1a1.ControllerPluginResourcesSpec{}, diff --git a/pkg/operator/ceph/csi/operator_driver.go b/pkg/operator/ceph/csi/operator_driver.go index f5457be45..99c845a2b 100644 --- a/pkg/operator/ceph/csi/operator_driver.go +++ b/pkg/operator/ceph/csi/operator_driver.go @@ -92,11 +92,11 @@ func (r *ReconcileCSI) createOrUpdateRBDDriverResource(cluster cephv1.CephCluste Spec: spec, } - rbdDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(r.opConfig.Parameters, rbdPluginResource) + rbdDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(rbdPluginResource) rbdDriver.Spec.Liveness = &csiopv1a1.LivenessSpec{ MetricsPort: int(CSIParam.RBDLivenessMetricsPort), } - rbdDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(r.opConfig.Parameters, rbdProvisionerResource) + rbdDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(rbdProvisionerResource) rbdDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ Type: v1.RollingUpdateDaemonSetStrategyType, } @@ -143,12 +143,12 @@ func (r *ReconcileCSI) createOrUpdateCephFSDriverResource(cluster cephv1.CephClu cephFsDriver.Spec.SnapshotPolicy = csiopv1a1.VolumeGroupSnapshotPolicy } - cephFsDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(r.opConfig.Parameters, cephFSPluginResource) + cephFsDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(cephFSPluginResource) cephFsDriver.Spec.Liveness = &csiopv1a1.LivenessSpec{ MetricsPort: int(CSIParam.CephFSLivenessMetricsPort), } - cephFsDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(r.opConfig.Parameters, cephFSProvisionerResource) + cephFsDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(cephFSProvisionerResource) cephFsDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ Type: v1.RollingUpdateDaemonSetStrategyType, } @@ -190,9 +190,9 @@ func (r *ReconcileCSI) createOrUpdateNFSDriverResource(cluster cephv1.CephCluste Spec: spec, } - NFSDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(r.opConfig.Parameters, nfsPluginResource) + NFSDriver.Spec.ControllerPlugin.Resources = createDriverControllerPluginResources(nfsPluginResource) - NFSDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(r.opConfig.Parameters, nfsProvisionerResource) + NFSDriver.Spec.NodePlugin.Resources = createDriverNodePluginResouces(nfsProvisionerResource) NFSDriver.Spec.NodePlugin.UpdateStrategy = &v1.DaemonSetUpdateStrategy{ Type: v1.RollingUpdateDaemonSetStrategyType, } @@ -270,9 +270,9 @@ func (r *ReconcileCSI) generateDriverSpec(clusterName string) (csiopv1a1.DriverS PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.ProvisionerPriorityClassName, Affinity: &corev1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &corev1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(pluginNodeAffinityEnv, &corev1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []corev1.Toleration{}), + Tolerations: getToleration(pluginTolerationsEnv, []corev1.Toleration{}), }, Resources: csiopv1a1.NodePluginResourcesSpec{}, KubeletDirPath: CSIParam.KubeletDirPath, @@ -282,9 +282,9 @@ func (r *ReconcileCSI) generateDriverSpec(clusterName string) (csiopv1a1.DriverS PodCommonSpec: csiopv1a1.PodCommonSpec{ PrioritylClassName: &CSIParam.PluginPriorityClassName, Affinity: &corev1.Affinity{ - NodeAffinity: getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), + NodeAffinity: getNodeAffinity(provisionerNodeAffinityEnv, &corev1.NodeAffinity{}), }, - Tolerations: getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}), + Tolerations: getToleration(provisionerTolerationsEnv, []corev1.Toleration{}), }, Replicas: &CSIParam.ProvisionerReplicas, Resources: csiopv1a1.ControllerPluginResourcesSpec{}, @@ -294,9 +294,9 @@ func (r *ReconcileCSI) generateDriverSpec(clusterName string) (csiopv1a1.DriverS }, nil } -func createDriverControllerPluginResources(opConfig map[string]string, key string) csiopv1a1.ControllerPluginResourcesSpec { +func createDriverControllerPluginResources(key string) csiopv1a1.ControllerPluginResourcesSpec { controllerPluginResources := csiopv1a1.ControllerPluginResourcesSpec{} - resource := getComputeResource(opConfig, key) + resource := getComputeResource(key) for _, r := range resource { if !reflect.DeepEqual(r.Resource, corev1.ResourceRequirements{}) { @@ -347,9 +347,9 @@ func createDriverControllerPluginResources(opConfig map[string]string, key strin return controllerPluginResources } -func createDriverNodePluginResouces(opConfig map[string]string, key string) csiopv1a1.NodePluginResourcesSpec { +func createDriverNodePluginResouces(key string) csiopv1a1.NodePluginResourcesSpec { nodePluginResources := csiopv1a1.NodePluginResourcesSpec{} - resource := getComputeResource(opConfig, key) + resource := getComputeResource(key) for _, r := range resource { if !reflect.DeepEqual(r.Resource, corev1.ResourceRequirements{}) { diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 72797b166..5b968453f 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -426,25 +426,25 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { } // get common provisioner tolerations and node affinity - provisionerTolerations := getToleration(r.opConfig.Parameters, provisionerTolerationsEnv, []corev1.Toleration{}) - provisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) + provisionerTolerations := getToleration(provisionerTolerationsEnv, []corev1.Toleration{}) + provisionerNodeAffinity := getNodeAffinity(provisionerNodeAffinityEnv, &corev1.NodeAffinity{}) // get common plugin tolerations and node affinity - pluginTolerations := getToleration(r.opConfig.Parameters, pluginTolerationsEnv, []corev1.Toleration{}) - pluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, pluginNodeAffinityEnv, &corev1.NodeAffinity{}) + pluginTolerations := getToleration(pluginTolerationsEnv, []corev1.Toleration{}) + pluginNodeAffinity := getNodeAffinity(pluginNodeAffinityEnv, &corev1.NodeAffinity{}) if rbdPlugin != nil { // get RBD plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdPluginTolerations := getToleration(r.opConfig.Parameters, rbdPluginTolerationsEnv, pluginTolerations) - rbdPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdPluginNodeAffinityEnv, pluginNodeAffinity) + rbdPluginTolerations := getToleration(rbdPluginTolerationsEnv, pluginTolerations) + rbdPluginNodeAffinity := getNodeAffinity(rbdPluginNodeAffinityEnv, pluginNodeAffinity) // apply RBD plugin tolerations and node affinity applyToPodSpec(&rbdPlugin.Spec.Template.Spec, rbdPluginNodeAffinity, rbdPluginTolerations) // apply resource request and limit to rbdplugin containers - applyResourcesToContainers(r.opConfig.Parameters, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) + applyResourcesToContainers(rbdPluginResource, &rbdPlugin.Spec.Template.Spec) // apply custom mounts to volumes - applyVolumeToPodSpec(r.opConfig.Parameters, rbdPluginVolume, &rbdPlugin.Spec.Template.Spec) + applyVolumeToPodSpec(rbdPluginVolume, &rbdPlugin.Spec.Template.Spec) // apply custom mounts to volume mounts - applyVolumeMountToContainer(r.opConfig.Parameters, rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec) + applyVolumeMountToContainer(rbdPluginVolumeMount, "csi-rbdplugin", &rbdPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd plugin daemonset %q", rbdPlugin.Name) @@ -462,12 +462,12 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if rbdProvisionerDeployment != nil { // get RBD provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - rbdProvisionerTolerations := getToleration(r.opConfig.Parameters, rbdProvisionerTolerationsEnv, provisionerTolerations) - rbdProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) + rbdProvisionerTolerations := getToleration(rbdProvisionerTolerationsEnv, provisionerTolerations) + rbdProvisionerNodeAffinity := getNodeAffinity(rbdProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply RBD provisioner tolerations and node affinity applyToPodSpec(&rbdProvisionerDeployment.Spec.Template.Spec, rbdProvisionerNodeAffinity, rbdProvisionerTolerations) // apply resource request and limit to rbd provisioner containers - applyResourcesToContainers(r.opConfig.Parameters, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(rbdProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to rbd provisioner deployment %q", rbdProvisionerDeployment.Name) @@ -504,16 +504,16 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if cephfsPlugin != nil { // get CephFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSPluginTolerations := getToleration(r.opConfig.Parameters, cephFSPluginTolerationsEnv, pluginTolerations) - cephFSPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSPluginNodeAffinityEnv, pluginNodeAffinity) + cephFSPluginTolerations := getToleration(cephFSPluginTolerationsEnv, pluginTolerations) + cephFSPluginNodeAffinity := getNodeAffinity(cephFSPluginNodeAffinityEnv, pluginNodeAffinity) // apply CephFS plugin tolerations and node affinity applyToPodSpec(&cephfsPlugin.Spec.Template.Spec, cephFSPluginNodeAffinity, cephFSPluginTolerations) // apply resource request and limit to cephfs plugin containers - applyResourcesToContainers(r.opConfig.Parameters, cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) + applyResourcesToContainers(cephFSPluginResource, &cephfsPlugin.Spec.Template.Spec) // apply custom mounts to volumes - applyVolumeToPodSpec(r.opConfig.Parameters, cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec) + applyVolumeToPodSpec(cephFSPluginVolume, &cephfsPlugin.Spec.Template.Spec) // apply custom mounts to volume mounts - applyVolumeMountToContainer(r.opConfig.Parameters, cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec) + applyVolumeMountToContainer(cephFSPluginVolumeMount, "csi-cephfsplugin", &cephfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs plugin daemonset %q", cephfsPlugin.Name) @@ -532,13 +532,13 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if cephfsProvisionerDeployment != nil { // get CephFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - cephFSProvisionerTolerations := getToleration(r.opConfig.Parameters, cephFSProvisionerTolerationsEnv, provisionerTolerations) - cephFSProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) + cephFSProvisionerTolerations := getToleration(cephFSProvisionerTolerationsEnv, provisionerTolerations) + cephFSProvisionerNodeAffinity := getNodeAffinity(cephFSProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply CephFS provisioner tolerations and node affinity applyToPodSpec(&cephfsProvisionerDeployment.Spec.Template.Spec, cephFSProvisionerNodeAffinity, cephFSProvisionerTolerations) // get resource details for cephfs provisioner // apply resource request and limit to cephfs provisioner containers - applyResourcesToContainers(r.opConfig.Parameters, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(cephfsProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to cephfs provisioner deployment %q", cephfsProvisionerDeployment.Name) @@ -574,16 +574,16 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if nfsPlugin != nil { // get NFS plugin tolerations and node affinity, defaults to common tolerations and node affinity if not specified - nfsPluginTolerations := getToleration(r.opConfig.Parameters, nfsPluginTolerationsEnv, pluginTolerations) - nfsPluginNodeAffinity := getNodeAffinity(r.opConfig.Parameters, nfsPluginNodeAffinityEnv, pluginNodeAffinity) + nfsPluginTolerations := getToleration(nfsPluginTolerationsEnv, pluginTolerations) + nfsPluginNodeAffinity := getNodeAffinity(nfsPluginNodeAffinityEnv, pluginNodeAffinity) // apply NFS plugin tolerations and node affinity applyToPodSpec(&nfsPlugin.Spec.Template.Spec, nfsPluginNodeAffinity, nfsPluginTolerations) // apply resource request and limit to nfs plugin containers - applyResourcesToContainers(r.opConfig.Parameters, nfsPluginResource, &nfsPlugin.Spec.Template.Spec) + applyResourcesToContainers(nfsPluginResource, &nfsPlugin.Spec.Template.Spec) // apply custom mounts to volumes - applyVolumeToPodSpec(r.opConfig.Parameters, nfsPluginVolume, &nfsPlugin.Spec.Template.Spec) + applyVolumeToPodSpec(nfsPluginVolume, &nfsPlugin.Spec.Template.Spec) // apply custom mounts to volume mounts - applyVolumeMountToContainer(r.opConfig.Parameters, nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec) + applyVolumeMountToContainer(nfsPluginVolumeMount, "csi-nfsplugin", &nfsPlugin.Spec.Template.Spec) err = ownerInfo.SetControllerReference(nfsPlugin) if err != nil { return errors.Wrapf(err, "failed to set owner reference to nfs plugin daemonset %q", nfsPlugin.Name) @@ -602,13 +602,13 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if nfsProvisionerDeployment != nil { // get NFS provisioner tolerations and node affinity, defaults to common tolerations and node affinity if not specified - nfsProvisionerTolerations := getToleration(r.opConfig.Parameters, nfsProvisionerTolerationsEnv, provisionerTolerations) - nfsProvisionerNodeAffinity := getNodeAffinity(r.opConfig.Parameters, nfsProvisionerNodeAffinityEnv, provisionerNodeAffinity) + nfsProvisionerTolerations := getToleration(nfsProvisionerTolerationsEnv, provisionerTolerations) + nfsProvisionerNodeAffinity := getNodeAffinity(nfsProvisionerNodeAffinityEnv, provisionerNodeAffinity) // apply NFS provisioner tolerations and node affinity applyToPodSpec(&nfsProvisionerDeployment.Spec.Template.Spec, nfsProvisionerNodeAffinity, nfsProvisionerTolerations) // get resource details for nfs provisioner // apply resource request and limit to nfs provisioner containers - applyResourcesToContainers(r.opConfig.Parameters, nfsProvisionerResource, &nfsProvisionerDeployment.Spec.Template.Spec) + applyResourcesToContainers(nfsProvisionerResource, &nfsProvisionerDeployment.Spec.Template.Spec) err = ownerInfo.SetControllerReference(nfsProvisionerDeployment) if err != nil { return errors.Wrapf(err, "failed to set owner reference to nfs provisioner deployment %q", nfsProvisionerDeployment.Name) @@ -634,7 +634,7 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if EnableRBD { err = csiDriverobj.createCSIDriverInfo( r.opManagerContext, r.context.Clientset, - RBDDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_RBD_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), + RBDDriverName, k8sutil.GetOperatorSetting("CSI_RBD_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), tp.Param.RBDAttachRequired, CSIParam.EnableCSIDriverSeLinuxMount) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", RBDDriverName) @@ -643,7 +643,7 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { if EnableCephFS { err = csiDriverobj.createCSIDriverInfo( r.opManagerContext, r.context.Clientset, - CephFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), + CephFSDriverName, k8sutil.GetOperatorSetting("CSI_CEPHFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), tp.Param.CephFSAttachRequired, CSIParam.EnableCSIDriverSeLinuxMount) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", CephFSDriverName) @@ -651,7 +651,7 @@ func (r *ReconcileCSI) startDrivers(ownerInfo *k8sutil.OwnerInfo) error { } if EnableNFS { err = csiDriverobj.createCSIDriverInfo(r.opManagerContext, r.context.Clientset, - NFSDriverName, k8sutil.GetValue(r.opConfig.Parameters, "CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), + NFSDriverName, k8sutil.GetOperatorSetting("CSI_NFS_FSGROUPPOLICY", string(k8scsi.FileFSGroupPolicy)), tp.Param.NFSAttachRequired, CSIParam.EnableCSIDriverSeLinuxMount) if err != nil { return errors.Wrapf(err, "failed to create CSI driver object for %q", NFSDriverName) diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 4c3e70827..60f549bb1 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -98,8 +98,8 @@ func applyLogrotateSidecar(specTemplate *corev1.PodTemplateSpec, name, templateD specTemplate.Spec.Containers = append(specTemplate.Spec.Containers, logrotateSidecarContainer) } -func applyResourcesToContainers(opConfig map[string]string, key string, podspec *corev1.PodSpec) { - resource := getComputeResource(opConfig, key) +func applyResourcesToContainers(key string, podspec *corev1.PodSpec) { + resource := getComputeResource(key) for _, r := range resource { for i, c := range podspec.Containers { @@ -110,12 +110,12 @@ func applyResourcesToContainers(opConfig map[string]string, key string, podspec } } -func getComputeResource(opConfig map[string]string, key string) []k8sutil.ContainerResource { +func getComputeResource(key string) []k8sutil.ContainerResource { // Add Resource list if any resource := []k8sutil.ContainerResource{} var err error - if resourceRaw := k8sutil.GetValue(opConfig, key, ""); resourceRaw != "" { + if resourceRaw := k8sutil.GetOperatorSetting(key, ""); resourceRaw != "" { resource, err = k8sutil.YamlToContainerResourceArray(resourceRaw) if err != nil { logger.Warningf("failed to parse %q. %v", resourceRaw, err) @@ -124,9 +124,9 @@ func getComputeResource(opConfig map[string]string, key string) []k8sutil.Contai return resource } -func getToleration(opConfig map[string]string, tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { +func getToleration(tolerationsName string, defaultTolerations []corev1.Toleration) []corev1.Toleration { // Add toleration if any, otherwise return defaultTolerations - tolerationsRaw := k8sutil.GetValue(opConfig, tolerationsName, "") + tolerationsRaw := k8sutil.GetOperatorSetting(tolerationsName, "") if tolerationsRaw == "" { return defaultTolerations } @@ -147,9 +147,9 @@ func getToleration(opConfig map[string]string, tolerationsName string, defaultTo return tolerations } -func getNodeAffinity(opConfig map[string]string, nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { +func getNodeAffinity(nodeAffinityName string, defaultNodeAffinity *corev1.NodeAffinity) *corev1.NodeAffinity { // Add NodeAffinity if any, otherwise return defaultNodeAffinity - nodeAffinity := k8sutil.GetValue(opConfig, nodeAffinityName, "") + nodeAffinity := k8sutil.GetOperatorSetting(nodeAffinityName, "") if nodeAffinity == "" { return defaultNodeAffinity } @@ -168,9 +168,9 @@ func applyToPodSpec(pod *corev1.PodSpec, n *corev1.NodeAffinity, t []corev1.Tole } } -func getPortFromConfig(data map[string]string, env string, defaultPort uint16) (uint16, error) { - port := k8sutil.GetValue(data, env, strconv.Itoa(int(defaultPort))) - if strings.TrimSpace(k8sutil.GetValue(data, env, strconv.Itoa(int(defaultPort)))) == "" { +func getPortFromConfig(env string, defaultPort uint16) (uint16, error) { + port := k8sutil.GetOperatorSetting(env, strconv.Itoa(int(defaultPort))) + if strings.TrimSpace(k8sutil.GetOperatorSetting(env, strconv.Itoa(int(defaultPort)))) == "" { return defaultPort, nil } p, err := strconv.ParseUint(port, 10, 64) @@ -203,8 +203,8 @@ func GetPodAntiAffinity(key, value string) corev1.PodAntiAffinity { } } -func applyVolumeToPodSpec(opConfig map[string]string, configName string, podspec *corev1.PodSpec) { - volumesRaw := k8sutil.GetValue(opConfig, configName, "") +func applyVolumeToPodSpec(configName string, podspec *corev1.PodSpec) { + volumesRaw := k8sutil.GetOperatorSetting(configName, "") if volumesRaw == "" { return } @@ -230,8 +230,8 @@ func applyVolumeToPodSpec(opConfig map[string]string, configName string, podspec } } -func applyVolumeMountToContainer(opConfig map[string]string, configName, containerName string, podspec *corev1.PodSpec) { - volumeMountsRaw := k8sutil.GetValue(opConfig, configName, "") +func applyVolumeMountToContainer(configName, containerName string, podspec *corev1.PodSpec) { + volumeMountsRaw := k8sutil.GetOperatorSetting(configName, "") if volumeMountsRaw == "" { return } @@ -267,8 +267,8 @@ func applyVolumeMountToContainer(opConfig map[string]string, configName, contain // getImage returns the image for the given setting name. If the image does not contain version, // the default version is appended from the default image. -func getImage(data map[string]string, settingName, defaultImage string) string { - image := k8sutil.GetValue(data, settingName, defaultImage) +func getImage(settingName, defaultImage string) string { + image := k8sutil.GetOperatorSetting(settingName, defaultImage) if !strings.Contains(image, ":") { version := strings.SplitN(defaultImage, ":", 2)[1] image = fmt.Sprintf("%s:%s", image, version) diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go index 29f5cbced..74860696e 100644 --- a/pkg/operator/ceph/csi/util_test.go +++ b/pkg/operator/ceph/csi/util_test.go @@ -47,28 +47,27 @@ func TestDeploymentTemplate(t *testing.T) { func TestGetPortFromConfig(t *testing.T) { key := "TEST_CSI_PORT_ENV" var defaultPort uint16 = 8000 - data := map[string]string{} // empty env variable - port, err := getPortFromConfig(data, key, defaultPort) + port, err := getPortFromConfig(key, defaultPort) assert.Nil(t, err) assert.Equal(t, port, defaultPort) // valid port is set in env t.Setenv(key, "9000") - port, err = getPortFromConfig(data, key, defaultPort) + port, err = getPortFromConfig(key, defaultPort) assert.Nil(t, err) assert.Equal(t, port, uint16(9000)) // higher port value is set in env t.Setenv(key, "65536") - port, err = getPortFromConfig(data, key, defaultPort) + port, err = getPortFromConfig(key, defaultPort) assert.Error(t, err) assert.Equal(t, port, defaultPort) // negative port is set in env t.Setenv(key, "-1") - port, err = getPortFromConfig(data, key, defaultPort) + port, err = getPortFromConfig(key, defaultPort) assert.Error(t, err) assert.Equal(t, port, defaultPort) @@ -80,7 +79,6 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { tp := templateParam{} rbdPlugin, err := templateToDaemonSet("rbdplugin", RBDPluginTemplatePath, tp) assert.Nil(t, err) - params := make(map[string]string) // need to build using map[string]interface{} because the following resource // doesn't serialise nicely @@ -103,8 +101,9 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { resourceRaw, err := yaml.Marshal(resource) assert.Nil(t, err) - params[rbdPluginResource] = string(resourceRaw) - applyResourcesToContainers(params, rbdPluginResource, &rbdPlugin.Spec.Template.Spec) + os.Setenv(rbdPluginResource, string(resourceRaw)) + defer os.Unsetenv(rbdPluginResource) + applyResourcesToContainers(rbdPluginResource, &rbdPlugin.Spec.Template.Spec) assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String(), "128Mi") assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String(), "256Mi") assert.Equal(t, rbdPlugin.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String(), "100m") @@ -113,7 +112,6 @@ func TestApplyingResourcesToRBDPlugin(t *testing.T) { func Test_applyVolumeToPodSpec(t *testing.T) { // when no volumes specified - config := make(map[string]string) configKey := "TEST_CSI_PLUGIN_VOLUME" dsName := "test-ds" tp := templateParam{ @@ -124,7 +122,7 @@ func Test_applyVolumeToPodSpec(t *testing.T) { defaultVolumes := 11 ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes) @@ -132,7 +130,7 @@ func Test_applyVolumeToPodSpec(t *testing.T) { tp.CSILogRotation = true ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) tp.CSILogRotation = false @@ -145,10 +143,11 @@ func Test_applyVolumeToPodSpec(t *testing.T) { } volumeRaw, err := yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) + defer os.Unsetenv(configKey) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) // add one more volume @@ -158,10 +157,10 @@ func Test_applyVolumeToPodSpec(t *testing.T) { }) volumeRaw, err = yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) // override existing volume configuration volumes[1].VolumeSource = corev1.VolumeSource{ @@ -169,10 +168,10 @@ func Test_applyVolumeToPodSpec(t *testing.T) { } volumeRaw, err = yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+2) // remove existing volume configuration volumes = []corev1.Volume{ @@ -183,16 +182,15 @@ func Test_applyVolumeToPodSpec(t *testing.T) { } volumeRaw, err = yaml.Marshal(volumes) assert.Nil(t, err) - config[configKey] = string(volumeRaw) + os.Setenv(configKey, string(volumeRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeToPodSpec(config, configKey, &ds.Spec.Template.Spec) + applyVolumeToPodSpec(configKey, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Volumes, defaultVolumes+1) } func Test_applyVolumeMountToContainer(t *testing.T) { // when no volumes specified - config := make(map[string]string) configKey := "TEST_CSI_PLUGIN_VOLUME_MOUNT" dsName := "test-ds" rbdContainerName := "csi-rbdplugin" @@ -204,7 +202,7 @@ func Test_applyVolumeMountToContainer(t *testing.T) { defaultVolumes := 10 ds, err := templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes) @@ -212,7 +210,7 @@ func Test_applyVolumeMountToContainer(t *testing.T) { tp.CSILogRotation = true ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) tp.CSILogRotation = false @@ -225,10 +223,11 @@ func Test_applyVolumeMountToContainer(t *testing.T) { } volumeMountsRaw, err := yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) + defer os.Unsetenv(configKey) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) // add one more volumemount @@ -238,19 +237,19 @@ func Test_applyVolumeMountToContainer(t *testing.T) { }) volumeMountsRaw, err = yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) // override existing volume configuration volumeMounts[1].MountPath = "/run/test/run" volumeMountsRaw, err = yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+2) // remove existing volume configuration volumeMounts = []corev1.VolumeMount{ @@ -261,17 +260,16 @@ func Test_applyVolumeMountToContainer(t *testing.T) { } volumeMountsRaw, err = yaml.Marshal(volumeMounts) assert.Nil(t, err) - config[configKey] = string(volumeMountsRaw) + os.Setenv(configKey, string(volumeMountsRaw)) ds, err = templateToDaemonSet(dsName, RBDPluginTemplatePath, tp) assert.Nil(t, err) - applyVolumeMountToContainer(config, configKey, rbdContainerName, &ds.Spec.Template.Spec) + applyVolumeMountToContainer(configKey, rbdContainerName, &ds.Spec.Template.Spec) assert.Len(t, ds.Spec.Template.Spec.Containers[1].VolumeMounts, defaultVolumes+1) } func Test_getImage(t *testing.T) { type args struct { - data map[string]string - settingName string + userImage string defaultImage string } tests := []struct { @@ -282,8 +280,6 @@ func Test_getImage(t *testing.T) { { name: "test with default image", args: args{ - data: map[string]string{}, - settingName: "ROOK_CSI_CEPH_IMAGE", defaultImage: "quay.io/cephcsi/cephcsi:v3.13.0", }, want: DefaultCSIPluginImage, @@ -291,10 +287,7 @@ func Test_getImage(t *testing.T) { { name: "test with user image", args: args{ - data: map[string]string{ - "ROOK_CSI_CEPH_IMAGE": "registry.io/private/cephcsi:v8", - }, - settingName: "ROOK_CSI_CEPH_IMAGE", + userImage: "registry.io/private/cephcsi:v8", defaultImage: "quay.io/cephcsi/cephcsi:v3.11.0", }, want: "registry.io/private/cephcsi:v8", @@ -302,10 +295,7 @@ func Test_getImage(t *testing.T) { { name: "test with user image without version", args: args{ - data: map[string]string{ - "ROOK_CSI_CEPH_IMAGE": "registry.io/private/cephcsi", - }, - settingName: "ROOK_CSI_CEPH_IMAGE", + userImage: "registry.io/private/cephcsi", defaultImage: "quay.io/cephcsi/cephcsi:v3.11.0", }, want: "registry.io/private/cephcsi:v3.11.0", @@ -313,7 +303,11 @@ func Test_getImage(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := getImage(tt.args.data, tt.args.settingName, tt.args.defaultImage) + if tt.args.userImage != "" { + os.Setenv("ROOK_CSI_CEPH_IMAGE", tt.args.userImage) + defer os.Unsetenv("ROOK_CSI_CEPH_IMAGE") + } + got := getImage("ROOK_CSI_CEPH_IMAGE", tt.args.defaultImage) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/operator/ceph/object/bucket/controller.go b/pkg/operator/ceph/object/bucket/controller.go index eed00adc9..b9a4007f7 100644 --- a/pkg/operator/ceph/object/bucket/controller.go +++ b/pkg/operator/ceph/object/bucket/controller.go @@ -36,7 +36,6 @@ import ( v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" ) const ( @@ -140,25 +139,6 @@ func (r *ReconcileBucket) reconcile(request reconcile.Request) (reconcile.Result return reconcile.Result{}, nil } - // Fetch the operator's configmap. We force the NamespaceName to the operator since the request - // could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the - // CM won't be found - opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace} - opConfig := &v1.ConfigMap{} - err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig) - if err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("operator's configmap resource not found. will use default value or env var.") - r.opConfig.Parameters = make(map[string]string) - } else { - // Error reading the object - requeue the request. - return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap") - } - } else { - // Populate the operator's config - r.opConfig.Parameters = opConfig.Data - } - // Populate clusterInfo during each reconcile clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cephCluster.Namespace, &cephCluster.Spec) if err != nil { @@ -177,7 +157,7 @@ func (r *ReconcileBucket) reconcile(request reconcile.Request) (reconcile.Result // note: the error return below is ignored and is expected to be removed from the // bucket library's `NewProvisioner` function - bucketController, _ := NewBucketController(r.context.KubeConfig, bucketProvisioner, r.opConfig.Parameters) + bucketController, _ := NewBucketController(r.context.KubeConfig, bucketProvisioner) // We must run this in a go routine since RunWithContext() blocks and waits for the context to // be Done. However, since it has a context, the go routine will exit on reload with SIGHUP diff --git a/pkg/operator/ceph/object/bucket/provisioner_test.go b/pkg/operator/ceph/object/bucket/provisioner_test.go index a098b51f9..0e44132a4 100644 --- a/pkg/operator/ceph/object/bucket/provisioner_test.go +++ b/pkg/operator/ceph/object/bucket/provisioner_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "testing" @@ -501,7 +502,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("maxObjects field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"maxObjects": "2"}) assert.NoError(t, err) @@ -510,7 +511,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("maxSize field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"maxSize": "3"}) assert.NoError(t, err) @@ -519,8 +520,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketMaxObjects field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketMaxObjects"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketMaxObjects") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketMaxObjects": "4"}) assert.NoError(t, err) @@ -528,8 +531,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketMaxSize field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketMaxSize"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketMaxSize") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketMaxSize": "5"}) assert.NoError(t, err) @@ -537,8 +542,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketPolicy field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketPolicy"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketPolicy") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketPolicy": "foo"}) assert.NoError(t, err) @@ -546,8 +553,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketLifecycle field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketLifecycle"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketLifecycle") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketLifecycle": "foo"}) assert.NoError(t, err) @@ -555,8 +564,10 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("bucketOwner field should be set", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{"ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS": "bucketOwner"}) - defer opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + os.Setenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS", "bucketOwner") + defer os.Unsetenv("ROOK_OBC_ALLOW_ADDITIONAL_CONFIG_FIELDS") + opcontroller.SetObcAllowAdditionalConfigFields() + defer opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{"bucketOwner": "foo"}) assert.NoError(t, err) @@ -564,7 +575,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("fields disallowed by default", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() for _, configKey := range []string{"bucketMaxObjects", "bucketMaxSize", "bucketPolicy", "bucketLifecycle", "bucketOwner"} { _, err := additionalConfigSpecFromMap(map[string]string{configKey: "foo"}) @@ -573,7 +584,7 @@ func TestProvisioner_additionalConfigSpecFromMap(t *testing.T) { }) t.Run("does not fail on empty map", func(t *testing.T) { - opcontroller.SetObcAllowAdditionalConfigFields(map[string]string{}) + opcontroller.SetObcAllowAdditionalConfigFields() spec, err := additionalConfigSpecFromMap(map[string]string{}) assert.NoError(t, err) diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index 19e294cb9..7e8df47ea 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -43,9 +43,9 @@ const ( objectStoreEndpoint = "endpoint" ) -func NewBucketController(cfg *rest.Config, p *Provisioner, data map[string]string) (*provisioner.Provisioner, error) { +func NewBucketController(cfg *rest.Config, p *Provisioner) (*provisioner.Provisioner, error) { const allNamespaces = "" - provName, err := cephObject.GetObjectBucketProvisioner(data, p.clusterInfo.Namespace) + provName, err := cephObject.GetObjectBucketProvisioner(p.clusterInfo.Namespace) if err != nil { return nil, errors.Wrap(err, "failed to get provisioner name") } diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 080367731..67a41626a 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -1137,10 +1137,10 @@ func poolName(poolPrefix, poolName string) string { } // GetObjectBucketProvisioner returns the bucket provisioner name appended with operator namespace if OBC is watching on it -func GetObjectBucketProvisioner(data map[string]string, namespace string) (string, error) { +func GetObjectBucketProvisioner(namespace string) (string, error) { provName := bucketProvisionerName - obcWatchOnNamespace := k8sutil.GetValue(data, "ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") - obcProvisionerNamePrefix := k8sutil.GetValue(data, "ROOK_OBC_PROVISIONER_NAME_PREFIX", "") + obcWatchOnNamespace := k8sutil.GetOperatorSetting("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") + obcProvisionerNamePrefix := k8sutil.GetOperatorSetting("ROOK_OBC_PROVISIONER_NAME_PREFIX", "") if obcProvisionerNamePrefix != "" { errList := validation.IsDNS1123Label(obcProvisionerNamePrefix) if len(errList) > 0 { diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index f384d6c70..8359e9897 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -517,36 +517,43 @@ func TestGetObjectBucketProvisioner(t *testing.T) { t.Setenv(k8sutil.PodNamespaceEnvVar, testNamespace) t.Run("watch ceph cluster namespace", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "true") + defer os.Unsetenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, fmt.Sprintf("%s.%s", testNamespace, bucketProvisionerName), bktprovisioner) assert.NoError(t, err) }) t.Run("watch all namespaces", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "false"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "false") + defer os.Unsetenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, bucketProvisionerName, bktprovisioner) assert.NoError(t, err) }) t.Run("prefix object provisioner", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_PROVISIONER_NAME_PREFIX": "my-prefix"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_PROVISIONER_NAME_PREFIX", "my-prefix") + defer os.Unsetenv("ROOK_OBC_PROVISIONER_NAME_PREFIX") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, "my-prefix."+bucketProvisionerName, bktprovisioner) assert.NoError(t, err) }) t.Run("watch ceph cluster namespace and prefix object provisioner", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_WATCH_OPERATOR_NAMESPACE": "true", "ROOK_OBC_PROVISIONER_NAME_PREFIX": "my-prefix"} - bktprovisioner, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE", "true") + os.Setenv("ROOK_OBC_PROVISIONER_NAME_PREFIX", "my-prefix") + defer os.Unsetenv("ROOK_OBC_WATCH_OPERATOR_NAMESPACE") + defer os.Unsetenv("ROOK_OBC_PROVISIONER_NAME_PREFIX") + bktprovisioner, err := GetObjectBucketProvisioner(testNamespace) assert.Equal(t, "my-prefix."+bucketProvisionerName, bktprovisioner) assert.NoError(t, err) }) t.Run("invalid prefix value for object provisioner", func(t *testing.T) { - data := map[string]string{"ROOK_OBC_PROVISIONER_NAME_PREFIX": "my-prefix."} - _, err := GetObjectBucketProvisioner(data, testNamespace) + os.Setenv("ROOK_OBC_PROVISIONER_NAME_PREFIX", "my-prefix.") + defer os.Unsetenv("ROOK_OBC_PROVISIONER_NAME_PREFIX") + _, err := GetObjectBucketProvisioner(testNamespace) assert.Error(t, err) }) diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index e43ba8c78..2674c41ab 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -129,6 +129,11 @@ func (o *Operator) runCRDManager() { // Create the context and the cancellation function opManagerContext, opManagerStop = context.WithCancel(context.Background()) + // Initialize the operator settings from the configmap + if err := k8sutil.ApplyOperatorSettingsConfigmap(opManagerContext, o.context.Clientset); err != nil { + panic("failed to load operator settings configmap. " + err.Error()) + } + // The operator config manager is also watching for changes here so if the operator config map // content changes for ROOK_CURRENT_NAMESPACE_ONLY we must reload the operator CRD manager o.namespaceToWatch() @@ -149,7 +154,7 @@ func (o *Operator) runCRDManager() { } func (o *Operator) namespaceToWatch() { - currentNamespaceOnly, _ := k8sutil.GetOperatorSetting(opManagerContext, o.context.Clientset, opcontroller.OperatorSettingConfigMapName, "ROOK_CURRENT_NAMESPACE_ONLY", "true") + currentNamespaceOnly := k8sutil.GetOperatorSetting("ROOK_CURRENT_NAMESPACE_ONLY", "true") if currentNamespaceOnly == "true" { o.config.NamespaceToWatch = o.config.OperatorNamespace logger.Infof("watching the current namespace %q for Ceph CRs", o.config.OperatorNamespace) diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go index 5b73b8cc0..dca033d57 100644 --- a/pkg/operator/discover/discover.go +++ b/pkg/operator/discover/discover.go @@ -73,16 +73,16 @@ func New(clientset kubernetes.Interface) *Discover { } // Start the discover -func (d *Discover) Start(ctx context.Context, namespace, discoverImage, securityAccount string, data map[string]string, useCephVolume bool) error { - err := d.createDiscoverDaemonSet(ctx, namespace, discoverImage, securityAccount, data, useCephVolume) +func (d *Discover) Start(ctx context.Context, namespace, discoverImage, securityAccount string, useCephVolume bool) error { + err := d.createDiscoverDaemonSet(ctx, namespace, discoverImage, securityAccount, useCephVolume) if err != nil { return fmt.Errorf("failed to start discover daemonset. %v", err) } return nil } -func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, discoverImage, securityAccount string, data map[string]string, useCephVolume bool) error { - discoveryInterval := k8sutil.GetValue(data, discoverIntervalEnv, defaultDiscoverInterval) +func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, discoverImage, securityAccount string, useCephVolume bool) error { + discoveryInterval := k8sutil.GetOperatorSetting(discoverIntervalEnv, defaultDiscoverInterval) discoveryParameters := []string{"discover", "--discover-interval", discoveryInterval} @@ -90,7 +90,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco discoveryParameters = append(discoveryParameters, "--use-ceph-volume") } - discoverDaemonResourcesRaw := k8sutil.GetValue(data, discoverDaemonResourcesEnv, "") + discoverDaemonResourcesRaw := k8sutil.GetOperatorSetting(discoverDaemonResourcesEnv, "") discoverDaemonResources, err := k8sutil.YamlToContainerResource(discoverDaemonResourcesRaw) if err != nil { logger.Warningf("failed to parse.%s %v", discoverDaemonResourcesRaw, err) @@ -176,7 +176,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco }, }, HostNetwork: opcontroller.EnforceHostNetwork(), - PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""), + PriorityClassName: k8sutil.GetOperatorSetting(discoverDaemonsetPriorityClassNameEnv, ""), SecurityContext: &v1.PodSecurityContext{}, }, }, @@ -191,18 +191,18 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco } // Add toleration if any - tolerationValue := k8sutil.GetValue(data, discoverDaemonsetTolerationEnv, "") + tolerationValue := k8sutil.GetOperatorSetting(discoverDaemonsetTolerationEnv, "") if tolerationValue != "" { ds.Spec.Template.Spec.Tolerations = []v1.Toleration{ { Effect: v1.TaintEffect(tolerationValue), Operator: v1.TolerationOpExists, - Key: k8sutil.GetValue(data, discoverDaemonsetTolerationKeyEnv, ""), + Key: k8sutil.GetOperatorSetting(discoverDaemonsetTolerationKeyEnv, ""), }, } } - tolerationsRaw := k8sutil.GetValue(data, discoverDaemonsetTolerationsEnv, "") + tolerationsRaw := k8sutil.GetOperatorSetting(discoverDaemonsetTolerationsEnv, "") tolerations, err := k8sutil.YamlToTolerations(tolerationsRaw) if err != nil { logger.Warningf("failed to parse %s. %+v", tolerationsRaw, err) @@ -211,7 +211,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations, tolerations...) // Add NodeAffinity if any - nodeAffinity := k8sutil.GetValue(data, discoverDaemonSetNodeAffinityEnv, "") + nodeAffinity := k8sutil.GetOperatorSetting(discoverDaemonSetNodeAffinityEnv, "") if nodeAffinity != "" { v1NodeAffinity, err := k8sutil.GenerateNodeAffinity(nodeAffinity) if err != nil { @@ -224,7 +224,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco logger.Infof("nodeAffinity: %s", v1NodeAffinity) } - podLabels := k8sutil.GetValue(data, discoverDaemonSetPodLabelsEnv, "") + podLabels := k8sutil.GetOperatorSetting(discoverDaemonSetPodLabelsEnv, "") if podLabels != "" { podLabels := k8sutil.ParseStringToLabels(podLabels) // Override / Set the app label even if set by the user as diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go index 667e88bdc..ec19cc16e 100644 --- a/pkg/operator/discover/discover_test.go +++ b/pkg/operator/discover/discover_test.go @@ -19,12 +19,12 @@ package discover import ( "context" + "os" "testing" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" discoverDaemon "github.com/rook/rook/pkg/daemon/discover" - "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/test" @@ -63,7 +63,7 @@ func TestStartDiscoveryDaemonset(t *testing.T) { _, err := clientset.CoreV1().Pods("rook-system").Create(ctx, &pod, metav1.CreateOptions{}) assert.NoError(t, err) // start a basic cluster - err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", map[string]string{}, false) + err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", false) assert.Nil(t, err) // check daemonset parameters @@ -86,21 +86,11 @@ func TestStartDiscoveryDaemonset(t *testing.T) { assert.Nil(t, agentDS.Spec.Template.Spec.Tolerations) // Test with rook override configmap setting - opConfigCM := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: controller.OperatorSettingConfigMapName, - Namespace: namespace, - }, - Data: map[string]string{ - "DISCOVER_TOLERATIONS": "- effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n operator: Exists\n- effect: NoExecute\n key: node-role.kubernetes.io/etcd\n operator: Exists", - }, - } - - cm, err := clientset.CoreV1().ConfigMaps(namespace).Create(ctx, opConfigCM, metav1.CreateOptions{}) - assert.NoError(t, err) + os.Setenv("DISCOVER_TOLERATIONS", "- effect: NoSchedule\n key: node-role.kubernetes.io/control-plane\n operator: Exists\n- effect: NoExecute\n key: node-role.kubernetes.io/etcd\n operator: Exists") + defer os.Unsetenv("DISCOVER_TOLERATIONS") // start a basic cluster - err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", cm.Data, false) + err = a.Start(ctx, namespace, "rook/rook:myversion", "mysa", false) assert.Nil(t, err) agentDS, err = clientset.AppsV1().DaemonSets(namespace).Get(ctx, "rook-discover", metav1.GetOptions{}) diff --git a/pkg/operator/k8sutil/configmap.go b/pkg/operator/k8sutil/configmap.go index 08b932430..7e57bb7d7 100644 --- a/pkg/operator/k8sutil/configmap.go +++ b/pkg/operator/k8sutil/configmap.go @@ -20,17 +20,17 @@ import ( "context" "fmt" "os" - "sync" "time" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) -var loggedOperatorSettings = sync.Map{} +var loadedOperatorSettings bool // DeleteConfigMap deletes a ConfigMap. func DeleteConfigMap(ctx context.Context, clientset kubernetes.Interface, cmName, namespace string, opts *DeleteOptions) error { @@ -70,50 +70,43 @@ func CreateOrUpdateConfigMap(ctx context.Context, clientset kubernetes.Interface return existingCm, nil } -// GetOperatorSetting gets the operator setting from ConfigMap or Env Var +// GetOperatorSetting gets the operator setting from Env Var merged with ConfigMap // returns defaultValue if setting is not found -func GetOperatorSetting(context context.Context, clientset kubernetes.Interface, configMapName, settingName, defaultValue string) (string, error) { - // config must be in operator pod namespace - namespace := os.Getenv(PodNamespaceEnvVar) - cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(context, configMapName, metav1.GetOptions{}) +func GetOperatorSetting(settingName, defaultValue string) string { + if !loadedOperatorSettings { + logger.Warningf("WARNING: attempting to load operator setting %q before configmap is loaded", settingName) + } + if settingValue, ok := os.LookupEnv(settingName); ok { + return settingValue + } + return defaultValue +} + +func ApplyOperatorSettingsConfigmap(ctx context.Context, clientset kubernetes.Interface) error { + namespacedName := types.NamespacedName{Namespace: os.Getenv(PodNamespaceEnvVar), Name: "rook-ceph-operator-config"} + logger.Debugf("loading operator settings configmap from %v", namespacedName) + opConfig, err := clientset.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{}) + if err != nil { if kerrors.IsNotFound(err) { - if settingValue, ok := os.LookupEnv(settingName); ok { - logger.Infof("%s=%q (env var)", settingName, settingValue) - return settingValue, nil - } - logger.Infof("%s=%q (default)", settingName, defaultValue) - return defaultValue, nil + loadedOperatorSettings = true + logger.Debug("operator's configmap resource not found. will use default value or env var.") + return nil } - return defaultValue, fmt.Errorf("error reading ConfigMap %q. %v", configMapName, err) + return err } - return GetValue(cm.Data, settingName, defaultValue), nil -} - -func GetValue(data map[string]string, settingName, defaultValue string) string { - settingValue := defaultValue - settingSource := "default" - - if val, ok := data[settingName]; ok { - settingValue = val - settingSource = "configmap" - } else if val, ok := os.LookupEnv(settingName); ok { - settingValue = val - settingSource = "env var" + for key, value := range opConfig.Data { + currentValue := os.Getenv(key) + if currentValue == value { + continue + } + logger.Infof("operator setting %q = %q", key, value) + if err := os.Setenv(key, value); err != nil { + logger.Errorf("failed to set env var %q = %q. %v", key, value, err) + } } - - if logChangedSettings(settingName, settingValue) { - logger.Infof("%s=%q (%s)", settingName, settingValue, settingSource) - } - - return settingValue -} - -func logChangedSettings(settingName, settingValue string) bool { - if val, ok := loggedOperatorSettings.Load(settingName); !ok || val != settingValue { - loggedOperatorSettings.Store(settingName, settingValue) - return true - } - return false + loadedOperatorSettings = true + logger.Debug("done loading operator settings configmap") + return nil } diff --git a/pkg/operator/k8sutil/configmap_test.go b/pkg/operator/k8sutil/configmap_test.go index cc03f1fcc..d4577b0fc 100644 --- a/pkg/operator/k8sutil/configmap_test.go +++ b/pkg/operator/k8sutil/configmap_test.go @@ -63,7 +63,7 @@ func TestGetOperatorSetting(t *testing.T) { k8s := fake.NewSimpleClientset() ctx := context.TODO() - operatorSettingConfigMapName := "rook-operator-config" + operatorSettingConfigMapName := "rook-ceph-operator-config" testNamespace := "test-namespace" cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -81,63 +81,41 @@ func TestGetOperatorSetting(t *testing.T) { cmSettingValue := "storage=rook, worker" defaultValue := "" - // ConfigMap is not found - setting, err := GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) - assert.NoError(t, err) - - // Env Var doesn't exist + // ConfigMap is not yet applied so we expect the default value + setting := GetOperatorSetting(nodeAffinity, defaultValue) assert.Equal(t, defaultValue, setting) + // Env Var exists t.Setenv(nodeAffinity, envSettingValue) - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) - assert.NoError(t, err) + setting = GetOperatorSetting(nodeAffinity, defaultValue) assert.Equal(t, envSettingValue, setting) // ConfigMap is found t.Setenv("POD_NAMESPACE", testNamespace) - _, err = k8s.CoreV1().ConfigMaps(testNamespace).Create(ctx, cm, metav1.CreateOptions{}) + _, err := k8s.CoreV1().ConfigMaps(testNamespace).Create(ctx, cm, metav1.CreateOptions{}) + assert.NoError(t, err) + err = ApplyOperatorSettingsConfigmap(ctx, k8s) assert.NoError(t, err) // Setting exists in ConfigMap - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, nodeAffinity, defaultValue) - assert.NoError(t, err) - // Env Var exists + setting = GetOperatorSetting(nodeAffinity, defaultValue) assert.Equal(t, cmSettingValue, setting) + // Env Var doesn't exist err = os.Unsetenv(nodeAffinity) assert.NoError(t, err) assert.Equal(t, cmSettingValue, setting) - // Setting doesn't exist in ConfigMap - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, podAffinity, defaultValue) - assert.NoError(t, err) - // Env Var doesn't exist + // Setting doesn't exist in ConfigMap, expect the default value + setting = GetOperatorSetting(podAffinity, defaultValue) assert.Equal(t, defaultValue, setting) // Env Var exists t.Setenv(podAffinity, envSettingValue) - setting, err = GetOperatorSetting(ctx, k8s, operatorSettingConfigMapName, podAffinity, defaultValue) + setting = GetOperatorSetting(podAffinity, defaultValue) assert.NoError(t, err) assert.Equal(t, envSettingValue, setting) } -func TestLogChangedSettings(t *testing.T) { - key := "Node_Affinity" - value := "storage=rook, worker" - - // When the key is not in the map, it should be added and return true for logging - ok := logChangedSettings(key, value) - assert.True(t, true, ok) - - // When the key is already present in the map and value is same, it should return false for logging - ok = logChangedSettings(key, value) - assert.False(t, false, ok) - - // When the key is already present in the map and value is updated, it should return true for logging - value = "storage=rook, test" - ok = logChangedSettings(key, value) - assert.True(t, true, ok) -} - func TestCreateOrUpdateConfigMap(t *testing.T) { k8s := fake.NewSimpleClientset() ctx := context.TODO()