my-rook-config/pkg/operator/ceph/controller/cluster_info_test.go

/*
Copyright 2022 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"os"
	"testing"

	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/operator/test"
	exectest "github.com/rook/rook/pkg/util/exec/test"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestCreateClusterSecrets(t *testing.T) {
	ctx := context.TODO()
	clientset := test.New(t, 1)
	configDir := "ns"
	err := os.MkdirAll(configDir, 0o755)
	assert.NoError(t, err)
	defer os.RemoveAll(configDir)
	adminSecret := "AQDkLIBd9vLGJxAAnXsIKPrwvUXAmY+D1g0X1Q==" //nolint:gosec // This is just a var name, not a real secret
	executor := &exectest.MockExecutor{
		MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
			logger.Infof("COMMAND: %s %v", command, args)
			if command == "ceph-authtool" && args[0] == "--create-keyring" {
				filename := args[1]
				assert.NoError(t, os.WriteFile(filename, []byte(fmt.Sprintf("key = %s", adminSecret)), 0o600))
			}
			return "", nil
		},
	}
	context := &clusterd.Context{
		Clientset: clientset,
		Executor:  executor,
	}
	cephClusterSpec := &cephv1.ClusterSpec{
		Network: cephv1.NetworkSpec{
			Provider: cephv1.NetworkProviderMultus,
		},
	}
	namespace := "ns"
	ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
	info, maxID, mapping, err := CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	assert.Equal(t, -1, maxID)
	require.NotNil(t, info)
	assert.Equal(t, "client.admin", info.CephCred.Username)
	assert.Equal(t, adminSecret, info.CephCred.Secret)
	assert.NotEqual(t, "", info.FSID)
	assert.NotNil(t, mapping)

	// check for the cluster secret
	secret, err := clientset.CoreV1().Secrets(namespace).Get(ctx, "rook-ceph-mon", metav1.GetOptions{})
	assert.NoError(t, err)
	assert.Equal(t, adminSecret, string(secret.Data["ceph-secret"]))

	// ensure secret owner info can be loaded and is useful
	// this is what the owner info looks like in a live cluster
	ownerController := true
	blockOwnerDel := true
	secret.OwnerReferences[0] = metav1.OwnerReference{
		APIVersion:         "ceph.rook.io/v1",
		Kind:               "CephCluster",
		Name:               "my-cluster",
		UID:                "e55604f2-710c-4353-9a3e-9d23ea2d6eb9", // random uuid
		Controller:         &ownerController,
		BlockOwnerDeletion: &blockOwnerDel,
	}
	_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
	assert.NoError(t, err)
	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	// use the SetOwnerReference() method to ensure that the loaded OwnerInfo is usable and correct
	cm := corev1.ConfigMap{}
	cm.Name = "bob"
	cm.Namespace = namespace
	err = info.OwnerInfo.SetOwnerReference(&cm)
	assert.NoError(t, err)
	cmOwner := cm.OwnerReferences[0]
	assert.Equal(t, "ceph.rook.io/v1", cmOwner.APIVersion)
	assert.Equal(t, "CephCluster", cmOwner.Kind)
	assert.Equal(t, "my-cluster", cmOwner.Name)
	assert.Equal(t, "e55604f2-710c-4353-9a3e-9d23ea2d6eb9", string(cmOwner.UID))
	assert.True(t, *cmOwner.Controller)
	assert.True(t, *cmOwner.BlockOwnerDeletion)

	// For backward compatibility check that the admin secret can be loaded as previously specified
	// Update the secret as if created in an old cluster
	delete(secret.Data, CephUserSecretKey)
	delete(secret.Data, CephUsernameKey)
	secret.Data[AdminSecretNameKey] = []byte(adminSecret)
	_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
	assert.NoError(t, err)

	// Check that the cluster info can now be loaded
	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	assert.Equal(t, "client.admin", info.CephCred.Username)
	assert.Equal(t, adminSecret, info.CephCred.Secret)

	// Fail to load the external cluster if the admin placeholder is specified
	secret.Data[AdminSecretNameKey] = []byte(AdminSecretNameKey)
	_, err = clientset.CoreV1().Secrets(namespace).Update(ctx, secret, metav1.UpdateOptions{})
	assert.NoError(t, err)
	_, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.Error(t, err)

	// Load the external cluster with the legacy external creds
	secret.Name = OperatorCreds
	secret.Data = map[string][]byte{
		"userID":  []byte("testid"),
		"userKey": []byte("testkey"),
	}
	_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
	assert.NoError(t, err)
	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	assert.Equal(t, "testid", info.CephCred.Username)
	assert.Equal(t, "testkey", info.CephCred.Secret)
	assert.Equal(t, "", info.CSIDriverSpec.CephFS.KernelMountOptions)

	// Verify the csi settings default to secure if the network is encrypted
	cephClusterSpec.Network = cephv1.NetworkSpec{
		Connections: &cephv1.ConnectionsSpec{
			Encryption: &cephv1.EncryptionSpec{
				Enabled: true,
			},
		},
	}
	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	assert.Equal(t, "ms_mode=secure", info.CSIDriverSpec.CephFS.KernelMountOptions)

	// Verify the csi settings are loaded from an env var
	cephClusterSpec.Network = cephv1.NetworkSpec{}
	t.Setenv("CSI_CEPHFS_KERNEL_MOUNT_OPTIONS", "ms_mode=crc")
	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	assert.Equal(t, "ms_mode=crc", info.CSIDriverSpec.CephFS.KernelMountOptions)

	// Verify the csi settings are loaded from the cephClusterSpec if specified and override the env var
	cephClusterSpec.CSI = cephv1.CSIDriverSpec{
		ReadAffinity: cephv1.ReadAffinitySpec{
			Enabled:             true,
			CrushLocationLabels: []string{"kubernetes.io/hostname"},
		},
		CephFS: cephv1.CSICephFSSpec{
			KernelMountOptions: "ms_mode=secure",
		},
	}
	info, _, _, err = CreateOrLoadClusterInfo(context, ctx, namespace, ownerInfo, cephClusterSpec)
	assert.NoError(t, err)
	assert.Equal(t, "ms_mode=secure", info.CSIDriverSpec.CephFS.KernelMountOptions)
}