feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

namespace: blocklist ip:nonce in cleanup job

Browse Source 
blocklisting IP can cause problems when the cleanup
job pod runs on the same host or gets the same ip
as that of the rbd client ip that is present in
the rbd status/watchers. instead of blocklisting
the whole ip of the node or the client blocklist
only the unique ip:nonce to allow the node to not
have any effect because of the cleanup job.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna committed 2025-09-24 12:47:48 +02:00
1 parent 96d631168c
commit 5ae49cf7a0
5 files changed
+40 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/daemon/ceph/cleanup/radosnamespace.go
+23 -23
View File
@@ -54,9 +54,9 @@ func cleanupImages(context *clusterd.Context, clusterInfo *cephclient.ClusterInf
return errors.Wrapf(err, "failed to list images in %s", msg)
}
err = blocklistClientIPs(context, clusterInfo, images, poolName, radosNamespace)
err = blocklistClients(context, clusterInfo, images, poolName, radosNamespace)
if err != nil {
return errors.Wrap(err, "failed to add client IPs to the blocklist")
return errors.Wrap(err, "failed to add clients to the blocklist")
}
var retErr error
@@ -104,47 +104,47 @@ func BlockPoolCleanup(context *clusterd.Context, clusterInfo *cephclient.Cluster
return nil
}
func blocklistClientIPs(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, images []cephclient.CephBlockImage, poolName, radosNamespace string) error {
ips, err := getClientIPs(context, clusterInfo, images, poolName, radosNamespace)
func blocklistClients(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, images []cephclient.CephBlockImage, poolName, radosNamespace string) error {
clients, err := getClients(context, clusterInfo, images, poolName, radosNamespace)
if err != nil {
return errors.Wrapf(err, "failed to get client IPs for all the images in pool %q in namespace %q", poolName, radosNamespace)
return errors.Wrapf(err, "failed to get clients for all the images in pool %q in namespace %q", poolName, radosNamespace)
}
if len(ips) == 0 {
logger.Info("no client IPs found for images in pool %q in rados namespace %q", poolName, radosNamespace)
if len(clients) == 0 {
logger.Infof("no clients found for images in pool %q in rados namespace %q", poolName, radosNamespace)
}
for ip := range ips {
logger.Infof("blocklist client IP %q for images in pool %q in namespace %q", ip, poolName, radosNamespace)
err = cephclient.BlocklistIP(context, clusterInfo, ip, ClientBlocklistDuration)
for client := range clients {
logger.Infof("blocklist client %q for images in pool %q in namespace %q", client, poolName, radosNamespace)
err = cephclient.Blocklist(context, clusterInfo, client, ClientBlocklistDuration)
if err != nil {
return errors.Wrapf(err, "failed to blocklist IP %q in pool %q in namespace %q", ip, poolName, radosNamespace)
return errors.Wrapf(err, "failed to blocklist client %q in pool %q in namespace %q", client, poolName, radosNamespace)
}
logger.Infof("successfully blocklisted client IP %q in pool %q in namespace %q", ip, poolName, radosNamespace)
logger.Infof("successfully blocklisted client %q in pool %q in namespace %q", client, poolName, radosNamespace)
}
return nil
}
func getClientIPs(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, images []cephclient.CephBlockImage, poolName, radosNamespace string) (sets.Set[string], error) {
clientIPs := sets.New[string]()
func getClients(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, images []cephclient.CephBlockImage, poolName, radosNamespace string) (sets.Set[string], error) {
clientSet := sets.New[string]()
for _, image := range images {
rbdStatus, err := cephclient.GetRBDImageStatus(context, clusterInfo, poolName, image.Name, radosNamespace)
if err != nil {
return clientIPs, errors.Wrapf(err, "failed to list watchers for the image %q in %s", image.Name, radosNamespace)
return clientSet, errors.Wrapf(err, "failed to list watchers for the image %q in %s", image.Name, radosNamespace)
}
ips := rbdStatus.GetWatcherIPs()
if len(ips) == 0 {
logger.Infof("no watcher IPs found for image %q in pool %q in namespace %q", image.Name, poolName, radosNamespace)
clients := rbdStatus.GetWatchers()
if len(clients) == 0 {
logger.Infof("no watchers found for image %q in pool %q in namespace %q", image.Name, poolName, radosNamespace)
continue
}
logger.Infof("watcher IPs for image %q in pool %q in namespace %q: %v", image.Name, poolName, radosNamespace, ips)
for _, ip := range ips {
clientIPs.Insert(ip)
logger.Infof("watchers for image %q in pool %q in namespace %q: %v", image.Name, poolName, radosNamespace, clients)
for _, client := range clients {
clientSet.Insert(client)
}
}
logger.Infof("client IPs : %v", clientIPs)
return clientIPs, nil
logger.Infof("clients: %v", clientSet)
return clientSet, nil
}
pkg/daemon/ceph/cleanup/radosnamespace_test.go
+4 -6
View File
@@ -167,18 +167,16 @@ func TestGetClientIPs(t *testing.T) {
executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[0] == "status" && args[1] == "test-pool/image1" {
return `{"watchers":[{"address":"192.168.39.137:0/3762982934"}, {"address":"192.168.39.137:0/3762982934"}]}`, nil
return `{"watchers":[{"address":"192.168.39.137:0/3762982931"}, {"address":"192.168.39.137:0/3762982933"}]}`, nil
}
if args[0] == "status" && args[1] == "test-pool/image2" {
return `{"watchers":[{"address":"192.168.39.137:0/3762982934"}, {"address":"192.168.39.137:0/3762982934"}]}`, nil
return `{"watchers":[{"address":"192.168.39.137:0/3762982932"}, {"address":"192.168.39.137:0/3762982934"}]}`, nil
}
return "", errors.New("unknown command")
}
context := &clusterd.Context{Executor: executor}
clusterInfo := cephclient.AdminTestClusterInfo("mycluster")
ips, err := getClientIPs(context, clusterInfo, images, "test-pool", "testrns")
clients, err := getClients(context, clusterInfo, images, "test-pool", "testrns")
assert.NoError(t, err)
assert.Equal(t, 1, len(ips))
actualIP, _ := ips.PopAny()
assert.Equal(t, "192.168.39.137", actualIP)
assert.Equal(t, 4, len(clients))
}
pkg/daemon/ceph/client/image.go
+7 -9
View File
@@ -19,7 +19,6 @@ import (
"encoding/json"
"fmt"
"regexp"
"strings"
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
@@ -172,16 +171,15 @@ type RBDStatus struct {
} `json:"watchers"`
}
// GetWatcherIPs returns a list of watcher IPs of the RBD image.
// The RBD status output is parsed to get the desired IPs.
// For example: `192.168.39.137:0/3762982934“ IP format returned in the RBD status output is parsed as 192.168.39.137
func (r RBDStatus) GetWatcherIPs() []string {
watcherIPList := []string{}
// GetWatchers returns a list of watchers of the RBD image.
// The RBD status output is parsed to get the desired IP+nonce.
// For example: `192.168.39.137:0/3762982934“ is returned.
func (r RBDStatus) GetWatchers() []string {
watcherList := []string{}
for _, watcher := range r.Watchers {
watcherIP := strings.Split(watcher.Address, ":0")[0]
watcherIPList = append(watcherIPList, watcherIP)
watcherList = append(watcherList, watcher.Address)
}
return watcherIPList
return watcherList
}
// GetRBDImageStatus returns the status of the RDB image.
pkg/daemon/ceph/client/image_test.go
+2 -4
View File
@@ -127,7 +127,7 @@ func TestListImageLogLevelDebug(t *testing.T) {
listCalled = false
}
func TestGetWatcherIPs(t *testing.T) {
func TestGetWatchers(t *testing.T) {
rbdStatus := RBDStatus{
Watchers: []struct {
Address string "json:\"address\""
@@ -141,8 +141,6 @@ func TestGetWatcherIPs(t *testing.T) {
},
}
res := rbdStatus.GetWatcherIPs()
res := rbdStatus.GetWatchers()
assert.Equal(t, 2, len(res))
assert.Equal(t, "192.168.39.137", res[0])
assert.Equal(t, "192.168.39.136", res[1])
}
pkg/daemon/ceph/client/osd.go
+4 -4
View File
@@ -467,12 +467,12 @@ func GetOSDMetadata(context *clusterd.Context, clusterInfo *ClusterInfo) (*[]OSD
return &osdMetadata, nil
}
// BlocklistIP blocklists the IP for predefined duration
func BlocklistIP(context *clusterd.Context, clusterInfo *ClusterInfo, ip, duration string) error {
args := []string{"osd", "blocklist", "add", ip, duration}
// Blocklist blocklists the client address for predefined duration
func Blocklist(context *clusterd.Context, clusterInfo *ClusterInfo, address, duration string) error {
args := []string{"osd", "blocklist", "add", address, duration}
_, err := NewCephCommand(context, clusterInfo, args).Run()
if err != nil {
return errors.Wrapf(err, "failed to blockist IP %q", ip)
return errors.Wrapf(err, "failed to blockist client %q", address)
}
return nil
}