Merge pull request #15631 from rook/mergify/bp/release-1.17/pr-15459

rbdmirror: mirroring b/w implicit and defined ns (backport #15459)
This commit is contained in:
Blaine Gardner authored and GitHub committed 2025-04-02 10:21:24 -06:00
commit cf0444a73b
12 files changed
+186 -35

No files matched your search

@@ -220,7 +220,7 @@ stretched) then you will have 2 replicas per datacenter where each replica ends
* `mirroring`: Sets up mirroring of the pool
* `enabled`: whether mirroring is enabled on that pool (default: false)
* `mode`: mirroring mode to run, possible values are "pool" or "image" (required). Refer to the [mirroring modes Ceph documentation](https://docs.ceph.com/docs/master/rbd/rbd-mirroring/#enable-mirroring) for more details.
* `mode`: mirroring mode to run, possible values are "pool", "image" or "init-only" (required). Refer to the [mirroring modes Ceph documentation](https://docs.ceph.com/en/latest/rbd/rbd-mirroring/#enable-mirroring) for more details.
* `snapshotSchedules`: schedule(s) snapshot at the **pool** level. One or more schedules are supported.
* `interval`: frequency of the snapshots. The interval can be specified in days, hours, or minutes using d, h, m suffix respectively.
* `startTime`: optional, determines at what time the snapshot process starts, specified using the ISO 8601 time format.
@@ -51,7 +51,7 @@ If any setting is unspecified, a suitable default will be used automatically.
- `blockPoolName`: The metadata name of the CephBlockPool CR where the rados namespace will be created.
- `mirroring`: Sets up mirroring of the rados namespace (requires Ceph v20 or newer)
- `mode`: mirroring mode to run, possible values are "pool" or "image" (required). Refer to the [mirroring modes Ceph documentation](https://docs.ceph.com/docs/master/rbd/rbd-mirroring/#enable-mirroring) for more details
- `mode`: mirroring mode to run, possible values are "pool" or "image" (required). Refer to the [mirroring modes Ceph documentation](https://docs.ceph.com/en/latest/rbd/rbd-mirroring/#namespace-configuration) for more details
- `remoteNamespace`: Name of the rados namespace on the peer cluster where the namespace should get mirrored. The default is the same rados namespace.
- `snapshotSchedules`: schedule(s) snapshot at the **rados namespace** level. It is an array and one or more schedules are supported.
- `interval`: frequency of the snapshots. The interval can be specified in days, hours, or minutes using d, h, m suffix respectively.
+2 -2
View File
@@ -8426,7 +8426,7 @@ string
</td>
<td>
<em>(Optional)</em>
<p>Mode is the mirroring mode: either pool or image</p>
<p>Mode is the mirroring mode: pool, image or init-only.</p>
</td>
</tr>
<tr>
@@ -11860,7 +11860,7 @@ RadosNamespaceMirroringMode
</em>
</td>
<td>
<p>Mode is the mirroring mode; either pool or image</p>
<p>Mode is the mirroring mode; either pool or image.</p>
</td>
</tr>
<tr>
@@ -64,7 +64,7 @@ spec:
description: Mirroring configuration of CephBlockPoolRadosNamespace
properties:
mode:
description: Mode is the mirroring mode; either pool or image
description: Mode is the mirroring mode; either pool or image.
enum:
- ""
- pool
@@ -388,7 +388,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -7013,7 +7017,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -7197,7 +7205,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -10853,7 +10865,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -12449,7 +12465,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -13298,7 +13318,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -13477,7 +13501,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
+36 -8
View File
@@ -67,7 +67,7 @@ spec:
description: Mirroring configuration of CephBlockPoolRadosNamespace
properties:
mode:
description: Mode is the mirroring mode; either pool or image
description: Mode is the mirroring mode; either pool or image.
enum:
- ""
- pool
@@ -390,7 +390,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -7008,7 +7012,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -7192,7 +7200,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -10844,7 +10856,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -12440,7 +12456,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -13286,7 +13306,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
@@ -13465,7 +13489,11 @@ spec:
description: Enabled whether this pool is mirrored or not
type: boolean
mode:
description: 'Mode is the mirroring mode: either pool or image'
description: 'Mode is the mirroring mode: pool, image or init-only.'
enum:
- pool
- image
- init-only
type: string
peers:
description: Peers represents the peers spec
+3 -2
View File
@@ -1087,7 +1087,8 @@ type MirroringSpec struct {
// +optional
Enabled bool `json:"enabled,omitempty"`
// Mode is the mirroring mode: either pool or image
// Mode is the mirroring mode: pool, image or init-only.
// +kubebuilder:validation:Enum=pool;image;init-only
// +optional
Mode string `json:"mode,omitempty"`
@@ -3444,7 +3445,7 @@ type RadosNamespaceMirroring struct {
// RemoteNamespace is the name of the CephBlockPoolRadosNamespace on the secondary cluster CephBlockPool
// +optional
RemoteNamespace *string `json:"remoteNamespace"`
// Mode is the mirroring mode; either pool or image
// Mode is the mirroring mode; either pool or image.
// +kubebuilder:validation:Enum="";pool;image
Mode RadosNamespaceMirroringMode `json:"mode"`
// SnapshotSchedules is the scheduling of snapshot for mirrored images
+26
View File
@@ -26,6 +26,7 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/ceph/version"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/util/exec"
"k8s.io/apimachinery/pkg/util/sets"
@@ -51,6 +52,13 @@ type Images struct {
Name string
}
const (
mirrorModeDisabled = "disabled"
mirrorModeInitOnly = "init-only"
ImplicitNamespaceKey = "<implicit>"
ImplicitNamespaceVal = ""
)
var (
rbdMirrorPeerCaps = []string{"mon", "profile rbd-mirror-peer", "osd", "profile rbd"}
rbdMirrorPeerKeyringID = "rbd-mirror-peer"
@@ -119,6 +127,20 @@ func CreateRBDMirrorBootstrapPeer(context *clusterd.Context, clusterInfo *Cluste
func enablePoolMirroring(context *clusterd.Context, clusterInfo *ClusterInfo, pool cephv1.NamedPoolSpec) error {
logger.Infof("enabling mirroring type %q for pool %q", pool.Mirroring.Mode, pool.Name)
if pool.Mirroring.Mode == mirrorModeInitOnly && !clusterInfo.CephVersion.IsAtLeastTentacle() {
return fmt.Errorf("ceph version %q does not support mirroring mode %s, minimum ceph version required is %q", clusterInfo.CephVersion.String(), pool.Mirroring.Mode, version.Tentacle.String())
}
mirrorInfo, err := GetPoolMirroringInfo(context, clusterInfo, pool.Name)
if err != nil {
return errors.Wrapf(err, "failed to get mirroring info for the pool %q", pool.Name)
}
if pool.Mirroring.Mode == mirrorInfo.Mode {
logger.Debugf("mirroring is already enabled on the pool %s with mode %s", pool.Name, mirrorInfo.Mode)
return nil
}
// Build command
args := []string{"mirror", "pool", "enable", pool.Name, pool.Mirroring.Mode}
cmd := NewRBDCommand(context, clusterInfo, args)
@@ -376,6 +398,10 @@ func EnableRBDRadosNamespaceMirroring(context *clusterd.Context, clusterInfo *Cl
return errors.Errorf("ceph version %q does not support mirroring in rados namespace %q with --remote-namespace flag, supported version are v20 and above.", clusterInfo.CephVersion.String(), poolAndRadosNamespaceName)
}
if remoteNamespace != nil && *remoteNamespace == ImplicitNamespaceKey {
*remoteNamespace = ImplicitNamespaceVal
}
args := []string{"mirror", "pool", "enable", poolAndRadosNamespaceName, mode}
if remoteNamespace != nil {
args = []string{"mirror", "pool", "enable", poolAndRadosNamespaceName, mode, "--remote-namespace", *remoteNamespace}
+11 -4
View File
@@ -60,6 +60,7 @@ func TestCreateRBDMirrorBootstrapPeer(t *testing.T) {
}
func TestEnablePoolMirroring(t *testing.T) {
poolInfo := `{"mode":"image","site_name":"9ea62548-53fa-4e43-b4b4-c5d1a43ae570","mirror_uuid":"c0336a10-612b-4e0d-bd7e-f0723812b22b","remote_namespace":"namespace2","peers":[{"uuid":"bea16887-9d44-432f-8f80-24259ad6022c","direction":"rx-tx","site_name":"3530db15-c9f6-41ab-9a9b-ce0c9c9fc331","mirror_uuid":"cb144924-37d8-41d7-84d6-badb76d500b1","client_name":"client.rbd-mirror-peer"}]}`
pool := cephv1.NamedPoolSpec{
Name: "pool-test",
PoolSpec: cephv1.PoolSpec{
@@ -69,10 +70,16 @@ func TestEnablePoolMirroring(t *testing.T) {
executor := &exectest.MockExecutor{}
executor.MockExecuteCommandWithOutput = func(command string, args ...string) (string, error) {
if args[0] == "mirror" {
assert.Equal(t, "pool", args[1])
assert.Equal(t, "enable", args[2])
assert.Equal(t, pool.Name, args[3])
assert.Equal(t, pool.Mirroring.Mode, args[4])
if args[2] == "enable" {
assert.Equal(t, "pool", args[1])
assert.Equal(t, pool.Name, args[3])
assert.Equal(t, pool.Mirroring.Mode, args[4])
return "", nil
} else if args[2] == "info" {
assert.Equal(t, "pool", args[1])
assert.Equal(t, pool.Name, args[3])
return poolInfo, nil
}
return "", nil
}
return "", errors.New("unknown command")
+6 -3
View File
@@ -350,9 +350,12 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile
if r.blockPoolContexts[blockPoolChannelKey].started {
logger.Debug("pool monitoring go routine already running!")
} else {
r.blockPoolContexts[blockPoolChannelKey].started = true
// Run the goroutine to update the mirroring status
go checker.CheckMirroring(r.blockPoolContexts[blockPoolChannelKey].internalCtx)
if cephBlockPool.Spec.Mirroring.Mode != "init-only" {
r.blockPoolContexts[blockPoolChannelKey].started = true
// Run the goroutine to update the mirroring status and skip when blockpool mirroing mode in init-only as radosnamespace mirroring is the right place to check
// mirroring status when blockpool mirroring mode is init-only.
go checker.CheckMirroring(r.blockPoolContexts[blockPoolChannelKey].internalCtx)
}
}
}
@@ -161,7 +161,10 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque
r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing)
}
poolAndRadosNamespaceName := fmt.Sprintf("%s/%s", cephBlockPoolRadosNamespace.Spec.BlockPoolName, getRadosNamespaceName(cephBlockPoolRadosNamespace))
poolAndRadosNamespaceName := cephBlockPoolRadosNamespace.Spec.BlockPoolName
if rns := getRadosNamespaceName(cephBlockPoolRadosNamespace); rns != "" {
poolAndRadosNamespaceName = fmt.Sprintf("%s/%s", cephBlockPoolRadosNamespace.Spec.BlockPoolName, rns)
}
// Make sure a CephCluster is present otherwise do nothing
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
@@ -322,7 +325,9 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque
}
func getRadosNamespaceName(cephBlockPoolRadosNamespace *cephv1.CephBlockPoolRadosNamespace) string {
if cephBlockPoolRadosNamespace.Spec.Name != "" {
if cephBlockPoolRadosNamespace.Spec.Name == cephclient.ImplicitNamespaceKey {
return cephclient.ImplicitNamespaceVal
} else if cephBlockPoolRadosNamespace.Spec.Name != "" {
return cephBlockPoolRadosNamespace.Spec.Name
}
return cephBlockPoolRadosNamespace.Name
@@ -366,6 +371,10 @@ func (r *ReconcileCephBlockPoolRadosNamespace) createOrUpdateRadosNamespace(ceph
namespacedName := fmt.Sprintf("%s/%s", cephBlockPoolRadosNamespace.Namespace, cephBlockPoolRadosNamespace.Name)
logger.Infof("creating ceph blockpool rados namespace %q", namespacedName)
if getRadosNamespaceName(cephBlockPoolRadosNamespace) == "" {
logger.Info("can't create empty radosnamepace as it is already present.")
return nil
}
err := cephclient.CreateRadosNamespace(r.context, r.clusterInfo, cephBlockPoolRadosNamespace.Spec.BlockPoolName, getRadosNamespaceName(cephBlockPoolRadosNamespace))
if err != nil {
return errors.Wrapf(err, "failed to create ceph blockpool rados namespace %q", cephBlockPoolRadosNamespace.Name)
@@ -379,6 +388,11 @@ func (r *ReconcileCephBlockPoolRadosNamespace) deleteRadosNamespace(cephBlockPoo
namespacedName := fmt.Sprintf("%s/%s", cephBlockPoolRadosNamespace.Namespace, cephBlockPoolRadosNamespace.Name)
logger.Infof("deleting ceph blockpool rados namespace object %q", namespacedName)
if getRadosNamespaceName(cephBlockPoolRadosNamespace) == "" {
logger.Info("can't delete empty radosnamepace as it present by default.")
return nil
}
if err := cephclient.DeleteRadosNamespace(r.context, r.clusterInfo, cephBlockPoolRadosNamespace.Spec.BlockPoolName, getRadosNamespaceName(cephBlockPoolRadosNamespace)); err != nil {
return errors.Wrapf(err, "failed to delete ceph blockpool rados namespace %q", cephBlockPoolRadosNamespace.Name)
}
@@ -437,6 +451,10 @@ func checkBlockPoolMirroring(cephBlockPool *cephv1.CephBlockPool) bool {
func (r *ReconcileCephBlockPoolRadosNamespace) reconcileMirroring(cephBlockPoolRadosNamespace *cephv1.CephBlockPoolRadosNamespace, cephBlockPool *cephv1.CephBlockPool) error {
poolAndRadosNamespaceName := fmt.Sprintf("%s/%s", cephBlockPool.Name, getRadosNamespaceName(cephBlockPoolRadosNamespace))
if getRadosNamespaceName(cephBlockPoolRadosNamespace) == "" {
poolAndRadosNamespaceName = cephBlockPool.Name
}
mirrorInfo, err := cephclient.GetPoolMirroringInfo(r.context, r.clusterInfo, poolAndRadosNamespaceName)
if err != nil {
return errors.Wrapf(err, "failed to get mirroring info for the radosnamespace %q", poolAndRadosNamespaceName)
@@ -22,18 +22,17 @@ import (
"testing"
csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
"github.com/coreos/pkg/capnslog"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
"github.com/rook/rook/pkg/clusterd"
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/k8sutil"
testop "github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
"github.com/coreos/pkg/capnslog"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -615,3 +614,44 @@ func Test_buildClusterID(t *testing.T) {
clusterID := buildClusterID(cephBlockPoolRadosNamespace)
assert.Equal(t, "2a74e5201e6ff9d15916ce2109c4f868", clusterID)
}
func TestGetRadosNamespaceName(t *testing.T) {
tests := []struct {
name string
args cephv1.CephBlockPoolRadosNamespace
want string
}{
{
"implicit-namespace",
cephv1.CephBlockPoolRadosNamespace{
ObjectMeta: metav1.ObjectMeta{Name: "cr-name1"},
Spec: cephv1.CephBlockPoolRadosNamespaceSpec{Name: cephclient.ImplicitNamespaceKey},
},
cephclient.ImplicitNamespaceVal,
},
{
"valid-namespace",
cephv1.CephBlockPoolRadosNamespace{
ObjectMeta: metav1.ObjectMeta{Name: "cr-name2"},
Spec: cephv1.CephBlockPoolRadosNamespaceSpec{Name: "valid-ns"},
},
"valid-ns",
},
{
"empty-namespace",
cephv1.CephBlockPoolRadosNamespace{
ObjectMeta: metav1.ObjectMeta{Name: "cr-name3"},
Spec: cephv1.CephBlockPoolRadosNamespaceSpec{},
},
"cr-name3",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getRadosNamespaceName(&tt.args); got != tt.want {
t.Errorf("getRadosNamespaceName() = %v, want %v", got, tt.want)
}
})
}
}
+1 -1
View File
@@ -161,7 +161,7 @@ func ValidatePoolSpec(context *clusterd.Context, clusterInfo *cephclient.Cluster
// Validate mirroring settings
if p.Mirroring.Enabled {
switch p.Mirroring.Mode {
case "image", "pool":
case "image", "pool", "init-only":
break
default:
return errors.Errorf("unrecognized mirroring mode %q. only 'image and 'pool' are supported", p.Mirroring.Mode)