osd-replacement was relying on exising ceph.rook.io/do-not-reconcile label for fencing osd destroy process owned by osd health goroutine from controller. However, this label can be used but other components like rook krew maintenance plugin and cannot be owned by osd-replacement process. Added a separate osd.rook.io/replace-in-progress annotation for that purpose. Signed-off-by: Artem Torubarov <artem.torubarov@sap.com>
225 lines
8.6 KiB
Go
225 lines
8.6 KiB
Go
/*
|
|
Copyright 2026 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package osd
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strconv"
|
|
"testing"
|
|
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
exectest "github.com/rook/rook/pkg/util/exec/test"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/kubernetes/fake"
|
|
)
|
|
|
|
func newTestReplaceCluster(clientset *fake.Clientset) *Cluster {
|
|
return newTestReplaceClusterWithSpec(clientset, cephv1.ClusterSpec{})
|
|
}
|
|
|
|
func newTestReplaceClusterWithSpec(clientset *fake.Clientset, spec cephv1.ClusterSpec) *Cluster {
|
|
clusterInfo := &cephclient.ClusterInfo{
|
|
Namespace: "rook-ceph",
|
|
Context: context.TODO(),
|
|
OwnerInfo: cephclient.NewMinimumOwnerInfoWithOwnerRef(),
|
|
}
|
|
clusterInfo.SetName("test")
|
|
c := &clusterd.Context{Clientset: clientset}
|
|
return &Cluster{
|
|
context: c,
|
|
clusterInfo: clusterInfo,
|
|
rookVersion: "rook/ceph:test",
|
|
spec: spec,
|
|
}
|
|
}
|
|
|
|
// osdTreeJSON builds an `osd tree` response containing one osd-type node per given id->status pair.
|
|
func osdTreeJSON(osds map[int]string) string {
|
|
nodes := ""
|
|
for id, status := range osds {
|
|
if nodes != "" {
|
|
nodes += ","
|
|
}
|
|
nodes += fmt.Sprintf(`{"id":%d,"name":"osd.%d","type":"osd","type_id":0,"exists":1,"status":%q}`, id, id, status)
|
|
}
|
|
return fmt.Sprintf(`{"nodes":[%s],"stray":[]}`, nodes)
|
|
}
|
|
|
|
func newReplaceClusterWithTree(clientset *fake.Clientset, osds map[int]string) *Cluster {
|
|
c := newTestReplaceCluster(clientset)
|
|
c.context.Executor = &exectest.MockExecutor{
|
|
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
|
|
if len(args) >= 2 && args[0] == "osd" && args[1] == "tree" {
|
|
return osdTreeJSON(osds), nil
|
|
}
|
|
return "", nil
|
|
},
|
|
}
|
|
return c
|
|
}
|
|
|
|
func osdDeployment(osdID int, annotations, labels map[string]string) *appsv1.Deployment {
|
|
if labels == nil {
|
|
labels = map[string]string{}
|
|
}
|
|
labels[k8sutil.AppAttr] = AppName
|
|
labels[OsdIdLabelKey] = strconv.Itoa(osdID)
|
|
return &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: fmt.Sprintf(osdAppNameFmt, osdID),
|
|
Namespace: "rook-ceph",
|
|
Labels: labels,
|
|
Annotations: annotations,
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestValidateAndStartOSDReplacement(t *testing.T) {
|
|
getDep := func(c *Cluster, osdID int) *appsv1.Deployment {
|
|
d, err := c.context.Clientset.AppsV1().Deployments("rook-ceph").Get(context.TODO(), fmt.Sprintf(osdAppNameFmt, osdID), metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
return d
|
|
}
|
|
|
|
t.Run("valid request gets fenced and marked in progress", func(t *testing.T) {
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"}, nil)
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
got := getDep(c, 5)
|
|
assert.Equal(t, "true", got.Labels[cephv1.SkipReconcileLabelKey])
|
|
assert.Equal(t, "true", got.Annotations[cephv1.ReplaceInProgressOSDAnnotationKey])
|
|
})
|
|
|
|
t.Run("id mismatch is rejected and not fenced", func(t *testing.T) {
|
|
// annotation says osd 6 but the deployment is osd 5
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-6"}, nil)
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
assert.NotContains(t, getDep(c, 5).Annotations, cephv1.ReplaceInProgressOSDAnnotationKey)
|
|
})
|
|
|
|
t.Run("PVC-backed OSD is rejected and not fenced", func(t *testing.T) {
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"},
|
|
map[string]string{OSDOverPVCLabelKey: "set1-0"})
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
assert.NotContains(t, getDep(c, 5).Annotations, cephv1.ReplaceInProgressOSDAnnotationKey)
|
|
})
|
|
|
|
t.Run("already-destroyed OSD is accepted and fenced", func(t *testing.T) {
|
|
// A rare state (manual destroy, or fence cleared mid-flow): accept it so the goroutine can
|
|
// resume idempotently from its destroyed phase rather than rejecting a partway teardown.
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"}, nil)
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "destroyed"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
got := getDep(c, 5)
|
|
assert.Equal(t, "true", got.Labels[cephv1.SkipReconcileLabelKey])
|
|
assert.Equal(t, "true", got.Annotations[cephv1.ReplaceInProgressOSDAnnotationKey])
|
|
})
|
|
|
|
t.Run("nonexistent OSD is rejected and not fenced", func(t *testing.T) {
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"}, nil)
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{7: "up"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
assert.NotContains(t, getDep(c, 5).Annotations, cephv1.ReplaceInProgressOSDAnnotationKey)
|
|
})
|
|
|
|
t.Run("deployment without the annotation is ignored", func(t *testing.T) {
|
|
dep := osdDeployment(5, nil, nil)
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
assert.NotContains(t, getDep(c, 5).Labels, cephv1.SkipReconcileLabelKey)
|
|
})
|
|
|
|
t.Run("already-marked deployment is left untouched", func(t *testing.T) {
|
|
dep := osdDeployment(5, map[string]string{
|
|
cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5",
|
|
cephv1.ReplaceInProgressOSDAnnotationKey: "true",
|
|
}, map[string]string{cephv1.SkipReconcileLabelKey: "true"})
|
|
clientset := fake.NewClientset(dep)
|
|
c := newTestReplaceCluster(clientset)
|
|
// no executor: an OSD the goroutine already owns must not trigger an osd tree lookup
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
assert.Equal(t, "true", getDep(c, 5).Labels[cephv1.SkipReconcileLabelKey])
|
|
})
|
|
|
|
t.Run("already-fenced deployment is still validated", func(t *testing.T) {
|
|
// An OSD an admin (or the maintenance plugin) already fenced carries the same label without having
|
|
// been validated. The label must not be mistaken for a completed validation, or the goroutine would
|
|
// take over an unvalidated request.
|
|
dep := osdDeployment(5, map[string]string{cephv1.ReplaceOSDAnnotationKey: "yes-really-replace-osd-5"},
|
|
map[string]string{cephv1.SkipReconcileLabelKey: "true"})
|
|
clientset := fake.NewClientset(dep)
|
|
c := newReplaceClusterWithTree(clientset, map[int]string{5: "up"})
|
|
|
|
require.NoError(t, c.validateAndStartOSDReplacement())
|
|
assert.Equal(t, "true", getDep(c, 5).Annotations[cephv1.ReplaceInProgressOSDAnnotationKey])
|
|
})
|
|
}
|
|
|
|
func TestReplacementReadyToRecreate(t *testing.T) {
|
|
withReplicas := func(d *appsv1.Deployment, n int32) *appsv1.Deployment {
|
|
d.Spec.Replicas = &n
|
|
return d
|
|
}
|
|
|
|
t.Run("ready-for-swap annotation present and scaled to zero", func(t *testing.T) {
|
|
dep := withReplicas(osdDeployment(5, map[string]string{cephv1.ReadyForSwapOSDAnnotationKey: ""}, nil), 0)
|
|
c := newTestReplaceCluster(fake.NewClientset(dep))
|
|
ready, err := c.replacementReadyForSwap(5)
|
|
require.NoError(t, err)
|
|
assert.True(t, ready)
|
|
})
|
|
|
|
t.Run("annotation absent", func(t *testing.T) {
|
|
dep := withReplicas(osdDeployment(5, nil, nil), 0)
|
|
c := newTestReplaceCluster(fake.NewClientset(dep))
|
|
ready, err := c.replacementReadyForSwap(5)
|
|
require.NoError(t, err)
|
|
assert.False(t, ready)
|
|
})
|
|
|
|
t.Run("deployment missing is not an error", func(t *testing.T) {
|
|
c := newTestReplaceCluster(fake.NewClientset())
|
|
ready, err := c.replacementReadyForSwap(5)
|
|
require.NoError(t, err)
|
|
assert.False(t, ready)
|
|
})
|
|
}
|