2026-06-30 16:07:17 +02:00
|
|
|
/*
|
|
|
|
|
Copyright 2026 The Rook Authors. All rights reserved.
|
|
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
|
limitations under the License.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
package osd
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
2026-06-30 16:08:03 +02:00
|
|
|
"fmt"
|
|
|
|
|
"strconv"
|
|
|
|
|
"testing"
|
2026-06-30 16:07:17 +02:00
|
|
|
|
|
|
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
|
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
|
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
2026-06-30 16:08:03 +02:00
|
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
|
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2026-06-30 16:07:17 +02:00
|
|
|
"k8s.io/client-go/kubernetes/fake"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func newTestReplaceCluster(clientset *fake.Clientset) *Cluster {
|
|
|
|
|
return newTestReplaceClusterWithSpec(clientset, cephv1.ClusterSpec{})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func newTestReplaceClusterWithSpec(clientset *fake.Clientset, spec cephv1.ClusterSpec) *Cluster {
|
|
|
|
|
clusterInfo := &cephclient.ClusterInfo{
|
|
|
|
|
Namespace: "rook-ceph",
|
|
|
|
|
Context: context.TODO(),
|
|
|
|
|
OwnerInfo: cephclient.NewMinimumOwnerInfoWithOwnerRef(),
|
|
|
|
|
}
|
|
|
|
|
clusterInfo.SetName("test")
|
|
|
|
|
c := &clusterd.Context{Clientset: clientset}
|
|
|
|
|
return &Cluster{
|
|
|
|
|
context: c,
|
|
|
|
|
clusterInfo: clusterInfo,
|
|
|
|
|
rookVersion: "rook/ceph:test",
|
|
|
|
|
spec: spec,
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-06-30 16:08:03 +02:00
|
|
|
|
|
|
|
|
// osdTreeJSON builds an `osd tree` response containing one osd-type node per given id->status pair.
|
|
|
|
|
func osdTreeJSON(osds map[int]string) string {
|
|
|
|
|
nodes := ""
|
|
|
|
|
for id, status := range osds {
|
|
|
|
|
if nodes != "" {
|
|
|
|
|
nodes += ","
|
|
|
|
|
}
|
|
|
|
|
nodes += fmt.Sprintf(`{"id":%d,"name":"osd.%d","type":"osd","type_id":0,"exists":1,"status":%q}`, id, id, status)
|
|
|
|
|
}
|
|
|
|
|
return fmt.Sprintf(`{"nodes":[%s],"stray":[]}`, nodes)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func newReplaceClusterWithTree(clientset *fake.Clientset, osds map[int]string) *Cluster {
|
|
|
|
|
c := newTestReplaceCluster(clientset)
|
|
|
|
|
c.context.Executor = &exectest.MockExecutor{
|
|
|
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
|
|
|
if len(args) >= 2 && args[0] == "osd" && args[1] == "tree" {
|
|
|
|
|
return osdTreeJSON(osds), nil
|
|
|
|
|
}
|
|
|
|
|
return "", nil
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
return c
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func osdDeployment(osdID int, annotations, labels map[string]string) *appsv1.Deployment {
|
|
|
|
|
if labels == nil {
|
|
|
|
|
labels = map[string]string{}
|
|
|
|
|
}
|
|
|
|
|
labels[k8sutil.AppAttr] = AppName
|
|
|
|
|
labels[OsdIdLabelKey] = strconv.Itoa(osdID)
|
|
|
|
|
return &appsv1.Deployment{
|
|
|
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
|
|
|
Name: fmt.Sprintf(osdAppNameFmt, osdID),
|
|
|
|
|
Namespace: "rook-ceph",
|
|
|
|
|
Labels: labels,
|
|
|
|
|
Annotations: annotations,
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestValidateAndStartOSDReplacement(t *testing.T) {
|
|
|
|
|
getDep := func(c *Cluster, osdID int) *appsv1.Deployment {
|
|
|
|
|
d, err := c.context.Clientset.AppsV1().Deployments("rook-ceph").Get(context.TODO(), fmt.Sprintf(osdAppNameFmt, osdID), metav1.GetOptions{})
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
return d
|
|
|
|
|
}
|
|
|
|
|
|
2026-07-28 11:47:25 +02:00
|
|
|
t.Run("valid request gets fenced and marked in progress", func(t *testing.T) {
|
2026-06-30 16:08:03 +02:00
|
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"}, nil)
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
|
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
2026-07-28 11:47:25 +02:00
|
|
|
got := getDep(c, 5)
|
|
|
|
|
assert.Equal(t, "true", got.Labels[cephv1.SkipReconcileLabelKey])
|
|
|
|
|
assert.Equal(t, "true", got.Annotations[cephv1.ReplaceInProgressOSDAnnotationKey])
|
2026-06-30 16:08:03 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("id mismatch is rejected and not fenced", func(t *testing.T) {
|
|
|
|
|
// annotation says osd 6 but the deployment is osd 5
|
|
|
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-6"}, nil)
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
|
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
2026-07-28 11:47:25 +02:00
|
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
|
|
|
assert.NotContains(t, getDep(c, 5).Annotations, cephv1.ReplaceInProgressOSDAnnotationKey)
|
2026-06-30 16:08:03 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("PVC-backed OSD is rejected and not fenced", func(t *testing.T) {
|
|
|
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"},
|
|
|
|
|
map[string]string{OSDOverPVCLabelKey: "set1-0"})
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
|
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
2026-07-28 11:47:25 +02:00
|
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
|
|
|
assert.NotContains(t, getDep(c, 5).Annotations, cephv1.ReplaceInProgressOSDAnnotationKey)
|
2026-06-30 16:08:03 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("already-destroyed OSD is accepted and fenced", func(t *testing.T) {
|
|
|
|
|
// A rare state (manual destroy, or fence cleared mid-flow): accept it so the goroutine can
|
|
|
|
|
// resume idempotently from its destroyed phase rather than rejecting a partway teardown.
|
|
|
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"}, nil)
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "destroyed"})
|
|
|
|
|
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
2026-07-28 11:47:25 +02:00
|
|
|
got := getDep(c, 5)
|
|
|
|
|
assert.Equal(t, "true", got.Labels[cephv1.SkipReconcileLabelKey])
|
|
|
|
|
assert.Equal(t, "true", got.Annotations[cephv1.ReplaceInProgressOSDAnnotationKey])
|
2026-06-30 16:08:03 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("nonexistent OSD is rejected and not fenced", func(t *testing.T) {
|
|
|
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"}, nil)
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{7: "up"})
|
|
|
|
|
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
2026-07-28 11:47:25 +02:00
|
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
|
|
|
assert.NotContains(t, getDep(c, 5).Annotations, cephv1.ReplaceInProgressOSDAnnotationKey)
|
2026-06-30 16:08:03 +02:00
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("deployment without the annotation is ignored", func(t *testing.T) {
|
|
|
|
|
dep := osdDeployment(5, nil, nil)
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
|
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
|
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
|
|
|
})
|
|
|
|
|
|
2026-07-28 11:47:25 +02:00
|
|
|
t.Run("already-marked deployment is left untouched", func(t *testing.T) {
|
|
|
|
|
dep := osdDeployment(5, map[string]string{
|
|
|
|
|
cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5",
|
|
|
|
|
cephv1.ReplaceInProgressOSDAnnotationKey: "true",
|
|
|
|
|
}, map[string]string{cephv1.SkipReconcileLabelKey: "true"})
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
|
|
|
|
c := newTestReplaceCluster(clientset)
|
|
|
|
|
// no executor: an OSD the goroutine already owns must not trigger an osd tree lookup
|
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
|
|
|
assert.Equal(t, "true", getDep(c, 5).Labels[cephv1.SkipReconcileLabelKey])
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("already-fenced deployment is still validated", func(t *testing.T) {
|
|
|
|
|
// An OSD an admin (or the maintenance plugin) already fenced carries the same label without having
|
|
|
|
|
// been validated. The label must not be mistaken for a completed validation, or the goroutine would
|
|
|
|
|
// take over an unvalidated request.
|
2026-06-30 16:08:03 +02:00
|
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"},
|
|
|
|
|
map[string]string{cephv1.SkipReconcileLabelKey: "true"})
|
|
|
|
|
clientset := fake.NewClientset(dep)
|
2026-07-28 11:47:25 +02:00
|
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
|
|
2026-06-30 16:08:03 +02:00
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
2026-07-28 11:47:25 +02:00
|
|
|
assert.Equal(t, "true", getDep(c, 5).Annotations[cephv1.ReplaceInProgressOSDAnnotationKey])
|
2026-06-30 16:08:03 +02:00
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestReplacementReadyToRecreate(t *testing.T) {
|
|
|
|
|
withReplicas := func(d *appsv1.Deployment, n int32) *appsv1.Deployment {
|
|
|
|
|
d.Spec.Replicas = &n
|
|
|
|
|
return d
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
t.Run("ready-for-swap annotation present and scaled to zero", func(t *testing.T) {
|
|
|
|
|
dep := withReplicas(osdDeployment(5, map[string]string{cephv1.ReadyForSwapOSDAnnotationKey: ""}, nil), 0)
|
|
|
|
|
c := newTestReplaceCluster(fake.NewClientset(dep))
|
|
|
|
|
ready, err := c.replacementReadyForSwap(5)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.True(t, ready)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("annotation absent", func(t *testing.T) {
|
|
|
|
|
dep := withReplicas(osdDeployment(5, nil, nil), 0)
|
|
|
|
|
c := newTestReplaceCluster(fake.NewClientset(dep))
|
|
|
|
|
ready, err := c.replacementReadyForSwap(5)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.False(t, ready)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("deployment missing is not an error", func(t *testing.T) {
|
|
|
|
|
c := newTestReplaceCluster(fake.NewClientset())
|
|
|
|
|
ready, err := c.replacementReadyForSwap(5)
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.False(t, ready)
|
|
|
|
|
})
|
|
|
|
|
}
|