Files
my-rook-config/pkg/operator/ceph/cluster/osd/migrate_test.go
T
Anas Khan 439b8c84d9 osd: skip upgrades for OSDs pending migration when the last migration is incomplete
startOSDMigration guards against upgrading OSDs while an OSD migration is
still in progress by returning the set of OSDs pending migration, which the
caller then removes from the update queue.

When the last migrated OSD's deployment has not been recreated yet,
isLastOSDMigrationComplete returns (false, nil). startOSDMigration then ran
return nil, errors.Wrapf(err, ...) with err already nil, and
errors.Wrapf(nil, ...) returns nil, so it returned (nil, nil), the same
value it uses to signal that no migration was requested. The caller
therefore skipped removing the pending OSDs from the update queue and could
upgrade and restart them mid-migration, which the pending-migration guard
was meant to prevent.

Return the freshly computed migration config in this case so the caller
still prunes the pending OSDs, without starting a new migration and without
aborting the reconcile. Aborting here would wedge the cluster because the
code that recreates the interrupted OSD runs later in Start, so an early
error return would leave that OSD deleted and block every later reconcile.
Letting the reconcile continue recreates the interrupted OSD and lets the
migration resume on a subsequent reconcile.

Also construct a non-nil error for the sibling unhealthy-PGs guard, which
wrapped a nil err in the same way.

Add a regression test asserting that startOSDMigration returns the pending
migration set, and does not delete another OSD deployment, while a previous
migration is still in progress.

Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>
Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
2026-07-20 17:51:30 -07:00

281 lines
8.3 KiB
Go

/*
Copyright 2024 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package osd for the Ceph OSDs.
package osd
import (
"context"
"testing"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestMigrateForEncryption(t *testing.T) {
namespace := "rook-ceph"
namespace2 := "rook-ceph2"
clientset := fake.NewClientset()
ctx := &clusterd.Context{
Clientset: clientset,
}
clusterInfo := &cephclient.ClusterInfo{
Namespace: namespace,
Context: context.TODO(),
}
clusterInfo.SetName("mycluster")
clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master")
t.Run("no OSD migration is required due to encryption", func(t *testing.T) {
c.spec.Storage.StorageClassDeviceSets = []cephv1.StorageClassDeviceSet{
{
Name: "set1",
Encrypted: true,
},
{
Name: "set2",
Encrypted: true,
},
}
d1 := getDummyDeploymentOnNode(clientset, c, "node2", 1)
d1.Labels["encrypted"] = "true"
d1.Labels["ceph.rook.io/DeviceSet"] = "set1"
createDeploymentOrPanic(clientset, d1)
d2 := getDummyDeploymentOnNode(clientset, c, "node2", 2)
d2.Labels["encrypted"] = "true"
d2.Labels["ceph.rook.io/DeviceSet"] = "set1"
createDeploymentOrPanic(clientset, d2)
deployments, err := c.getOSDDeployments()
assert.NoError(t, err)
mc := migrationConfig{
osds: map[int]*OSDInfo{},
}
err = mc.migrateForEncryption(c, deployments)
assert.NoError(t, err)
assert.Equal(t, 0, len(mc.osds))
})
t.Run("osd.1 on set1 needs migration", func(t *testing.T) {
c.clusterInfo.Namespace = namespace2
c.spec.Storage.StorageClassDeviceSets = []cephv1.StorageClassDeviceSet{
{
Name: "set1",
Encrypted: true,
},
}
d1 := getDummyDeploymentOnNode(clientset, c, "node2", 1)
d1.Labels["encrypted"] = "false"
d1.Labels["ceph.rook.io/DeviceSet"] = "set1"
createDeploymentOrPanic(clientset, d1)
deployments, err := c.getOSDDeployments()
assert.NoError(t, err)
mc := migrationConfig{
osds: map[int]*OSDInfo{},
}
err = mc.migrateForEncryption(c, deployments)
assert.NoError(t, err)
assert.Equal(t, 1, len(mc.osds))
assert.Equal(t, 1, mc.osds[1].ID)
})
}
func TestMigrationForOSDStore(t *testing.T) {
namespace := "rook-ceph"
namespace2 := "rook-ceph2"
clientset := fake.NewClientset()
ctx := &clusterd.Context{
Clientset: clientset,
}
clusterInfo := &cephclient.ClusterInfo{
Namespace: namespace,
Context: context.TODO(),
}
clusterInfo.SetName("mycluster")
clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master")
t.Run("no OSD migration is required due to OSD store change", func(t *testing.T) {
c.spec.Storage.Store.Type = "store1"
d1 := getDummyDeploymentOnNode(clientset, c, "node2", 1)
d1.Labels[osdStore] = "store1"
createDeploymentOrPanic(clientset, d1)
d2 := getDummyDeploymentOnNode(clientset, c, "node2", 2)
d2.Labels[osdStore] = "store1"
createDeploymentOrPanic(clientset, d2)
deployments, err := c.getOSDDeployments()
assert.NoError(t, err)
mc := migrationConfig{
osds: map[int]*OSDInfo{},
}
err = mc.migrateForEncryption(c, deployments)
assert.NoError(t, err)
assert.Equal(t, 0, len(mc.osds))
})
t.Run("osd.1 needs migration due to change is OSD store type", func(t *testing.T) {
c.clusterInfo.Namespace = namespace2
c.spec.Storage.Store.Type = "newStore"
d1 := getDummyDeploymentOnNode(clientset, c, "node2", 1)
d1.Labels[osdStore] = "store1" // store type is set to `store1` but spec has `newStore`
createDeploymentOrPanic(clientset, d1)
d2 := getDummyDeploymentOnNode(clientset, c, "node2", 2)
d2.Labels[osdStore] = "newStore" // store type label matches with the spec
createDeploymentOrPanic(clientset, d2)
deployments, err := c.getOSDDeployments()
assert.NoError(t, err)
mc := migrationConfig{
osds: map[int]*OSDInfo{},
}
err = mc.migrateForOSDStore(c, deployments)
assert.NoError(t, err)
assert.Equal(t, 1, len(mc.osds))
assert.Equal(t, 1, mc.osds[1].ID)
})
}
func createMigrationConfigmap(osdID, ns string, clientset *fake.Clientset) error {
ctx := context.TODO()
data := make(map[string]string, 1)
data[OSDIdKey] = osdID
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: OSDMigrationConfigName,
Namespace: ns,
},
Data: data,
}
_, err := clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{})
return err
}
func TestIsLastOSDMigrationComplete(t *testing.T) {
namespace := "rook-ceph"
clientset := fake.NewClientset()
ctx := &clusterd.Context{
Clientset: clientset,
}
clusterInfo := &cephclient.ClusterInfo{
Namespace: namespace,
Context: context.TODO(),
}
clusterInfo.SetName("mycluster")
clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master")
t.Run("no OSD migration config found", func(t *testing.T) {
result, err := isLastOSDMigrationComplete(c)
assert.NoError(t, err)
assert.Equal(t, true, result)
})
t.Run("osd.1 was migrated but its not up yet", func(t *testing.T) {
err := createMigrationConfigmap("1", namespace, clientset)
assert.NoError(t, err)
result, err := isLastOSDMigrationComplete(c)
assert.NoError(t, err)
assert.Equal(t, false, result)
})
t.Run("migrated osd.1 is up", func(t *testing.T) {
d1 := getDummyDeploymentOnNode(clientset, c, "node2", 1)
createDeploymentOrPanic(clientset, d1)
result, err := isLastOSDMigrationComplete(c)
assert.NoError(t, err)
assert.Equal(t, true, result)
})
}
func TestStartOSDMigration(t *testing.T) {
namespace := "rook-ceph"
clientset := fake.NewClientset()
// PGs report as clean so migration is not blocked on PG health.
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[0] == "status" {
return "{}", nil
}
return "", nil
},
}
ctx := &clusterd.Context{
Clientset: clientset,
Executor: executor,
}
clusterInfo := &cephclient.ClusterInfo{
Namespace: namespace,
Context: context.TODO(),
}
clusterInfo.SetName("mycluster")
clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t)
c := New(ctx, clusterInfo, cephv1.ClusterSpec{}, "rook/rook:master")
c.spec.Storage.Migration.Confirmation = OSDMigrationConfirmation
c.spec.Storage.Store.Type = "newStore"
t.Run("pending OSDs are returned without starting a new migration when the previous one is incomplete", func(t *testing.T) {
// osd.2 still runs on the old store, so it is pending migration.
d2 := getDummyDeploymentOnNode(clientset, c, "node2", 2)
d2.Labels[osdStore] = "oldStore"
createDeploymentOrPanic(clientset, d2)
// osd.1 was already migrated but its deployment has not been recreated yet, so the
// previous migration is still in progress.
err := createMigrationConfigmap("1", namespace, clientset)
assert.NoError(t, err)
migrationConfig, err := c.startOSDMigration()
// The reconcile must keep flowing (no error) so the interrupted OSD can be recreated
// downstream, while still returning the pending OSDs so the caller removes them from
// the update queue.
assert.NoError(t, err)
assert.NotNil(t, migrationConfig)
assert.Equal(t, []int{2}, migrationConfig.getOSDIds())
// No new migration must be started while the previous one is incomplete: osd.2's
// deployment must still exist and no OSD may be recorded for migration.
assert.Nil(t, c.migrateOSD)
_, err = clientset.AppsV1().Deployments(namespace).Get(context.TODO(), "rook-ceph-osd-2", metav1.GetOptions{})
assert.NoError(t, err)
})
}