From 9785ef5c8d93a4a85bf656998079306e80b59585 Mon Sep 17 00:00:00 2001 From: parth-gr Date: Wed, 23 Oct 2024 19:42:41 +0530 Subject: [PATCH] rbdmirror: enable periodic monitoring for rados namespace enable monitoring for rados namespace Signed-off-by: parth-gr --- .../ceph-block-pool-rados-namespace-crd.md | 3 + Documentation/CRDs/specification.md | 387 ++++++++++-------- PendingReleaseNotes.md | 1 + .../charts/rook-ceph/templates/resources.yaml | 137 ++++++- deploy/examples/crds.yaml | 137 ++++++- pkg/apis/ceph.rook.io/v1/types.go | 30 +- .../ceph.rook.io/v1/zz_generated.deepcopy.go | 141 ++++--- pkg/daemon/ceph/client/mirror.go | 16 +- pkg/daemon/ceph/client/mirror_health.go | 225 ++++++++++ pkg/daemon/ceph/client/mirror_health_test.go | 43 ++ pkg/operator/ceph/cluster/rbd/controller.go | 2 +- pkg/operator/ceph/pool/controller.go | 6 +- pkg/operator/ceph/pool/health.go | 117 ------ .../ceph/pool/radosnamespace/controller.go | 93 ++++- .../pool/radosnamespace/controller_test.go | 79 ++-- pkg/operator/ceph/pool/status.go | 74 ---- pkg/operator/ceph/pool/status_test.go | 35 -- 17 files changed, 989 insertions(+), 537 deletions(-) create mode 100644 pkg/daemon/ceph/client/mirror_health.go create mode 100644 pkg/daemon/ceph/client/mirror_health_test.go delete mode 100644 pkg/operator/ceph/pool/health.go diff --git a/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md b/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md index 1354048de..104e2045f 100644 --- a/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md +++ b/Documentation/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd.md @@ -57,6 +57,9 @@ If any setting is unspecified, a suitable default will be used automatically. - `interval`: frequency of the snapshots. The interval can be specified in days, hours, or minutes using d, h, m suffix respectively. - `startTime`: optional, determines at what time the snapshot process starts, specified using the ISO 8601 time format. +!!! note + If mirroring is enabled, whether to monitor the status and the interval of status updates is based on the `statusCheck` spec values of the parent CephBlockPool CR. + ## Creating a Storage Class Once the RADOS namespace is created, an RBD-based StorageClass can be created to diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index dd6949d64..317219f63 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -3297,6 +3297,45 @@ map[string]string (Optional) + + +mirroringStatus
+ + +MirroringStatusSpec + + + + +(Optional) + + + + +mirroringInfo
+ + +MirroringInfoSpec + + + + +(Optional) + + + + +snapshotScheduleStatus
+ + +SnapshotScheduleStatusSpec + + + + +(Optional) + +

CephBlockPoolStatus @@ -8030,13 +8069,13 @@ HealthCheckSpec -

MirroringInfoSpec +

MirroringInfo

-(Appears on:CephBlockPoolStatus) +(Appears on:MirroringInfoSpec)

-

MirroringInfoSpec is the status of the pool mirroring

+

MirroringInfo is the mirroring info of a given pool/radosnamespace

@@ -8048,16 +8087,72 @@ HealthCheckSpec + + + + + + + + + + + +
-PoolMirroringInfo
+mode
- -PoolMirroringInfo +string + +
+(Optional) +

Mode is the mirroring mode

+
+site_name
+ +string + +
+(Optional) +

SiteName is the current site name

+
+peers
+ + +[]PeersSpec + + +
+(Optional) +

Peers are the list of peer sites connected to that cluster

+
+

MirroringInfoSpec +

+

+(Appears on:CephBlockPoolRadosNamespaceStatus, CephBlockPoolStatus) +

+
+

MirroringInfoSpec is the status of the pool/radosnamespace mirroring

+
+ + + + + + + + + + @@ -8197,13 +8292,13 @@ MirroringPeerSpec
FieldDescription
+MirroringInfo
+ + +MirroringInfo

-(Members of PoolMirroringInfo are embedded into this type.) +(Members of MirroringInfo are embedded into this type.)

(Optional)
-

MirroringStatusSpec +

MirroringStatus

-(Appears on:CephBlockPoolStatus) +(Appears on:MirroringStatusSpec)

-

MirroringStatusSpec is the status of the pool mirroring

+

MirroringStatus is the pool/radosNamespace mirror status

@@ -8215,19 +8310,51 @@ MirroringPeerSpec + + + +
-PoolMirroringStatus
+summary
- -PoolMirroringStatus + +MirroringStatusSummarySpec + + +
+(Optional) +

Summary is the mirroring status summary

+
+

MirroringStatusSpec +

+

+(Appears on:CephBlockPoolRadosNamespaceStatus, CephBlockPoolStatus) +

+
+

MirroringStatusSpec is the status of the pool/radosNamespace mirroring

+
+ + + + + + + + + + @@ -8268,6 +8395,74 @@ string
FieldDescription
+MirroringStatus
+ + +MirroringStatus

-(Members of PoolMirroringStatus are embedded into this type.) +(Members of MirroringStatus are embedded into this type.)

(Optional) -

PoolMirroringStatus is the mirroring status of a pool

+

MirroringStatus is the mirroring status of a pool/radosNamespace

+

MirroringStatusSummarySpec +

+

+(Appears on:MirroringStatus) +

+
+

MirroringStatusSummarySpec is the summary output of the command

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+health
+ +string + +
+(Optional) +

Health is the mirroring health

+
+daemon_health
+ +string + +
+(Optional) +

DaemonHealth is the health of the mirroring daemon

+
+image_health
+ +string + +
+(Optional) +

ImageHealth is the health of the mirrored image

+
+states
+ + +StatesSpec + + +
+(Optional) +

States is the various state for all mirrored images

+

Module

@@ -10560,7 +10755,7 @@ int

PeersSpec

-(Appears on:PoolMirroringInfo) +(Appears on:MirroringInfo)

PeersSpec contains peer details

@@ -10775,162 +10970,6 @@ string -

PoolMirroringInfo -

-

-(Appears on:MirroringInfoSpec) -

-
-

PoolMirroringInfo is the mirroring info of a given pool

-
- - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-mode
- -string - -
-(Optional) -

Mode is the mirroring mode

-
-site_name
- -string - -
-(Optional) -

SiteName is the current site name

-
-peers
- - -[]PeersSpec - - -
-(Optional) -

Peers are the list of peer sites connected to that cluster

-
-

PoolMirroringStatus -

-

-(Appears on:MirroringStatusSpec) -

-
-

PoolMirroringStatus is the pool mirror status

-
- - - - - - - - - - - - - -
FieldDescription
-summary
- - -PoolMirroringStatusSummarySpec - - -
-(Optional) -

Summary is the mirroring status summary

-
-

PoolMirroringStatusSummarySpec -

-

-(Appears on:PoolMirroringStatus) -

-
-

PoolMirroringStatusSummarySpec is the summary output of the command

-
- - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-health
- -string - -
-(Optional) -

Health is the mirroring health

-
-daemon_health
- -string - -
-(Optional) -

DaemonHealth is the health of the mirroring daemon

-
-image_health
- -string - -
-(Optional) -

ImageHealth is the health of the mirrored image

-
-states
- - -StatesSpec - - -
-(Optional) -

States is the various state for all mirrored images

-

PoolPlacementSpec

@@ -12317,7 +12356,7 @@ string

SnapshotScheduleStatusSpec

-(Appears on:CephBlockPoolStatus) +(Appears on:CephBlockPoolRadosNamespaceStatus, CephBlockPoolStatus)

SnapshotScheduleStatusSpec is the status of the snapshot schedule

@@ -12453,7 +12492,7 @@ string

StatesSpec

-(Appears on:PoolMirroringStatusSummarySpec) +(Appears on:MirroringStatusSummarySpec)

StatesSpec are rbd images mirroring state

diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 116f32932..c4e63de81 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -7,3 +7,4 @@ ## Features - Enable mirroring for CephBlockPoolRadosNamespaces (see [#14701](https://github.com/rook/rook/pull/14701)). +- Enable periodic monitoring for CephBlockPoolRadosNamespaces mirroring (see [#14896](https://github.com/rook/rook/pull/14896)). diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index 12c2d1ffb..d6f2613dd 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -109,9 +109,142 @@ spec: type: string nullable: true type: object + mirroringInfo: + description: MirroringInfoSpec is the status of the pool/radosnamespace mirroring + properties: + details: + type: string + lastChanged: + type: string + lastChecked: + type: string + mode: + description: Mode is the mirroring mode + type: string + peers: + description: Peers are the list of peer sites connected to that cluster + items: + description: PeersSpec contains peer details + properties: + client_name: + description: ClientName is the CephX user used to connect to the peer + type: string + direction: + description: Direction is the peer mirroring direction + type: string + mirror_uuid: + description: MirrorUUID is the mirror UUID + type: string + site_name: + description: SiteName is the current site name + type: string + uuid: + description: UUID is the peer UUID + type: string + type: object + type: array + site_name: + description: SiteName is the current site name + type: string + type: object + mirroringStatus: + description: MirroringStatusSpec is the status of the pool/radosNamespace mirroring + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last changed + type: string + lastChecked: + description: LastChecked is the last time time the status was checked + type: string + summary: + description: Summary is the mirroring status summary + properties: + daemon_health: + description: DaemonHealth is the health of the mirroring daemon + type: string + health: + description: Health is the mirroring health + type: string + image_health: + description: ImageHealth is the health of the mirrored image + type: string + states: + description: States is the various state for all mirrored images + nullable: true + properties: + error: + description: Error is when the mirroring state is errored + type: integer + replaying: + description: Replaying is when the replay of the mirroring journal is on-going + type: integer + starting_replay: + description: StartingReplay is when the replay of the mirroring journal starts + type: integer + stopped: + description: Stopped is when the mirroring state is stopped + type: integer + stopping_replay: + description: StopReplaying is when the replay of the mirroring journal stops + type: integer + syncing: + description: Syncing is when the image is syncing + type: integer + unknown: + description: Unknown is when the mirroring state is unknown + type: integer + type: object + type: object + type: object phase: description: ConditionType represent a resource's status type: string + snapshotScheduleStatus: + description: SnapshotScheduleStatusSpec is the status of the snapshot schedule + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last changed + type: string + lastChecked: + description: LastChecked is the last time time the status was checked + type: string + snapshotSchedules: + description: SnapshotSchedules is the list of snapshots scheduled + items: + description: SnapshotSchedulesSpec is the list of snapshot scheduled for images in a pool + properties: + image: + description: Image is the mirrored image + type: string + items: + description: Items is the list schedules times for a given snapshot + items: + description: SnapshotSchedule is a schedule + properties: + interval: + description: Interval is the interval in which snapshots will be taken + type: string + start_time: + description: StartTime is the snapshot starting time + type: string + type: object + type: array + namespace: + description: Namespace is the RADOS namespace the image is part of + type: string + pool: + description: Pool is the pool name + type: string + type: object + nullable: true + type: array + type: object type: object x-kubernetes-preserve-unknown-fields: true required: @@ -405,7 +538,7 @@ spec: nullable: true type: object mirroringInfo: - description: MirroringInfoSpec is the status of the pool mirroring + description: MirroringInfoSpec is the status of the pool/radosnamespace mirroring properties: details: type: string @@ -443,7 +576,7 @@ spec: type: string type: object mirroringStatus: - description: MirroringStatusSpec is the status of the pool mirroring + description: MirroringStatusSpec is the status of the pool/radosNamespace mirroring properties: details: description: Details contains potential status errors diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index 1ffc720b8..53550a10b 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -112,9 +112,142 @@ spec: type: string nullable: true type: object + mirroringInfo: + description: MirroringInfoSpec is the status of the pool/radosnamespace mirroring + properties: + details: + type: string + lastChanged: + type: string + lastChecked: + type: string + mode: + description: Mode is the mirroring mode + type: string + peers: + description: Peers are the list of peer sites connected to that cluster + items: + description: PeersSpec contains peer details + properties: + client_name: + description: ClientName is the CephX user used to connect to the peer + type: string + direction: + description: Direction is the peer mirroring direction + type: string + mirror_uuid: + description: MirrorUUID is the mirror UUID + type: string + site_name: + description: SiteName is the current site name + type: string + uuid: + description: UUID is the peer UUID + type: string + type: object + type: array + site_name: + description: SiteName is the current site name + type: string + type: object + mirroringStatus: + description: MirroringStatusSpec is the status of the pool/radosNamespace mirroring + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last changed + type: string + lastChecked: + description: LastChecked is the last time time the status was checked + type: string + summary: + description: Summary is the mirroring status summary + properties: + daemon_health: + description: DaemonHealth is the health of the mirroring daemon + type: string + health: + description: Health is the mirroring health + type: string + image_health: + description: ImageHealth is the health of the mirrored image + type: string + states: + description: States is the various state for all mirrored images + nullable: true + properties: + error: + description: Error is when the mirroring state is errored + type: integer + replaying: + description: Replaying is when the replay of the mirroring journal is on-going + type: integer + starting_replay: + description: StartingReplay is when the replay of the mirroring journal starts + type: integer + stopped: + description: Stopped is when the mirroring state is stopped + type: integer + stopping_replay: + description: StopReplaying is when the replay of the mirroring journal stops + type: integer + syncing: + description: Syncing is when the image is syncing + type: integer + unknown: + description: Unknown is when the mirroring state is unknown + type: integer + type: object + type: object + type: object phase: description: ConditionType represent a resource's status type: string + snapshotScheduleStatus: + description: SnapshotScheduleStatusSpec is the status of the snapshot schedule + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last changed + type: string + lastChecked: + description: LastChecked is the last time time the status was checked + type: string + snapshotSchedules: + description: SnapshotSchedules is the list of snapshots scheduled + items: + description: SnapshotSchedulesSpec is the list of snapshot scheduled for images in a pool + properties: + image: + description: Image is the mirrored image + type: string + items: + description: Items is the list schedules times for a given snapshot + items: + description: SnapshotSchedule is a schedule + properties: + interval: + description: Interval is the interval in which snapshots will be taken + type: string + start_time: + description: StartTime is the snapshot starting time + type: string + type: object + type: array + namespace: + description: Namespace is the RADOS namespace the image is part of + type: string + pool: + description: Pool is the pool name + type: string + type: object + nullable: true + type: array + type: object type: object x-kubernetes-preserve-unknown-fields: true required: @@ -407,7 +540,7 @@ spec: nullable: true type: object mirroringInfo: - description: MirroringInfoSpec is the status of the pool mirroring + description: MirroringInfoSpec is the status of the pool/radosnamespace mirroring properties: details: type: string @@ -445,7 +578,7 @@ spec: type: string type: object mirroringStatus: - description: MirroringStatusSpec is the status of the pool mirroring + description: MirroringStatusSpec is the status of the pool/radosNamespace mirroring properties: details: description: Details contains potential status errors diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 61812212b..7d9ee31fe 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -861,11 +861,11 @@ type CephBlockPoolStatus struct { Conditions []Condition `json:"conditions,omitempty"` } -// MirroringStatusSpec is the status of the pool mirroring +// MirroringStatusSpec is the status of the pool/radosNamespace mirroring type MirroringStatusSpec struct { - // PoolMirroringStatus is the mirroring status of a pool + // MirroringStatus is the mirroring status of a pool/radosNamespace // +optional - PoolMirroringStatus `json:",inline"` + MirroringStatus `json:",inline"` // LastChecked is the last time time the status was checked // +optional LastChecked string `json:"lastChecked,omitempty"` @@ -877,15 +877,15 @@ type MirroringStatusSpec struct { Details string `json:"details,omitempty"` } -// PoolMirroringStatus is the pool mirror status -type PoolMirroringStatus struct { +// MirroringStatus is the pool/radosNamespace mirror status +type MirroringStatus struct { // Summary is the mirroring status summary // +optional - Summary *PoolMirroringStatusSummarySpec `json:"summary,omitempty"` + Summary *MirroringStatusSummarySpec `json:"summary,omitempty"` } -// PoolMirroringStatusSummarySpec is the summary output of the command -type PoolMirroringStatusSummarySpec struct { +// MirroringStatusSummarySpec is the summary output of the command +type MirroringStatusSummarySpec struct { // Health is the mirroring health // +optional Health string `json:"health,omitempty"` @@ -926,10 +926,10 @@ type StatesSpec struct { Error int `json:"error,omitempty"` } -// MirroringInfoSpec is the status of the pool mirroring +// MirroringInfoSpec is the status of the pool/radosnamespace mirroring type MirroringInfoSpec struct { // +optional - *PoolMirroringInfo `json:",inline"` + *MirroringInfo `json:",inline"` // +optional LastChecked string `json:"lastChecked,omitempty"` // +optional @@ -938,8 +938,8 @@ type MirroringInfoSpec struct { Details string `json:"details,omitempty"` } -// PoolMirroringInfo is the mirroring info of a given pool -type PoolMirroringInfo struct { +// MirroringInfo is the mirroring info of a given pool/radosnamespace +type MirroringInfo struct { // Mode is the mirroring mode // +optional Mode string `json:"mode,omitempty"` @@ -3386,6 +3386,12 @@ type CephBlockPoolRadosNamespaceStatus struct { // +optional // +nullable Info map[string]string `json:"info,omitempty"` + // +optional + MirroringStatus *MirroringStatusSpec `json:"mirroringStatus,omitempty"` + // +optional + MirroringInfo *MirroringInfoSpec `json:"mirroringInfo,omitempty"` + // +optional + SnapshotScheduleStatus *SnapshotScheduleStatusSpec `json:"snapshotScheduleStatus,omitempty"` } // Represents the source of a volume to mount. diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index e653d3869..92e443c3e 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -482,6 +482,21 @@ func (in *CephBlockPoolRadosNamespaceStatus) DeepCopyInto(out *CephBlockPoolRado (*out)[key] = val } } + if in.MirroringStatus != nil { + in, out := &in.MirroringStatus, &out.MirroringStatus + *out = new(MirroringStatusSpec) + (*in).DeepCopyInto(*out) + } + if in.MirroringInfo != nil { + in, out := &in.MirroringInfo, &out.MirroringInfo + *out = new(MirroringInfoSpec) + (*in).DeepCopyInto(*out) + } + if in.SnapshotScheduleStatus != nil { + in, out := &in.SnapshotScheduleStatus, &out.SnapshotScheduleStatus + *out = new(SnapshotScheduleStatusSpec) + (*in).DeepCopyInto(*out) + } return } @@ -3083,12 +3098,33 @@ func (in *MirrorHealthCheckSpec) DeepCopy() *MirrorHealthCheckSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MirroringInfo) DeepCopyInto(out *MirroringInfo) { + *out = *in + if in.Peers != nil { + in, out := &in.Peers, &out.Peers + *out = make([]PeersSpec, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MirroringInfo. +func (in *MirroringInfo) DeepCopy() *MirroringInfo { + if in == nil { + return nil + } + out := new(MirroringInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MirroringInfoSpec) DeepCopyInto(out *MirroringInfoSpec) { *out = *in - if in.PoolMirroringInfo != nil { - in, out := &in.PoolMirroringInfo, &out.PoolMirroringInfo - *out = new(PoolMirroringInfo) + if in.MirroringInfo != nil { + in, out := &in.MirroringInfo, &out.MirroringInfo + *out = new(MirroringInfo) (*in).DeepCopyInto(*out) } return @@ -3151,10 +3187,31 @@ func (in *MirroringSpec) DeepCopy() *MirroringSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MirroringStatus) DeepCopyInto(out *MirroringStatus) { + *out = *in + if in.Summary != nil { + in, out := &in.Summary, &out.Summary + *out = new(MirroringStatusSummarySpec) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MirroringStatus. +func (in *MirroringStatus) DeepCopy() *MirroringStatus { + if in == nil { + return nil + } + out := new(MirroringStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MirroringStatusSpec) DeepCopyInto(out *MirroringStatusSpec) { *out = *in - in.PoolMirroringStatus.DeepCopyInto(&out.PoolMirroringStatus) + in.MirroringStatus.DeepCopyInto(&out.MirroringStatus) return } @@ -3168,6 +3225,23 @@ func (in *MirroringStatusSpec) DeepCopy() *MirroringStatusSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MirroringStatusSummarySpec) DeepCopyInto(out *MirroringStatusSummarySpec) { + *out = *in + out.States = in.States + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MirroringStatusSummarySpec. +func (in *MirroringStatusSummarySpec) DeepCopy() *MirroringStatusSummarySpec { + if in == nil { + return nil + } + out := new(MirroringStatusSummarySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Module) DeepCopyInto(out *Module) { *out = *in @@ -4059,65 +4133,6 @@ func (in *PlacementStorageClassSpec) DeepCopy() *PlacementStorageClassSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PoolMirroringInfo) DeepCopyInto(out *PoolMirroringInfo) { - *out = *in - if in.Peers != nil { - in, out := &in.Peers, &out.Peers - *out = make([]PeersSpec, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolMirroringInfo. -func (in *PoolMirroringInfo) DeepCopy() *PoolMirroringInfo { - if in == nil { - return nil - } - out := new(PoolMirroringInfo) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PoolMirroringStatus) DeepCopyInto(out *PoolMirroringStatus) { - *out = *in - if in.Summary != nil { - in, out := &in.Summary, &out.Summary - *out = new(PoolMirroringStatusSummarySpec) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolMirroringStatus. -func (in *PoolMirroringStatus) DeepCopy() *PoolMirroringStatus { - if in == nil { - return nil - } - out := new(PoolMirroringStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PoolMirroringStatusSummarySpec) DeepCopyInto(out *PoolMirroringStatusSummarySpec) { - *out = *in - out.States = in.States - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PoolMirroringStatusSummarySpec. -func (in *PoolMirroringStatusSummarySpec) DeepCopy() *PoolMirroringStatusSummarySpec { - if in == nil { - return nil - } - out := new(PoolMirroringStatusSummarySpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PoolPlacementSpec) DeepCopyInto(out *PoolPlacementSpec) { *out = *in diff --git a/pkg/daemon/ceph/client/mirror.go b/pkg/daemon/ceph/client/mirror.go index 30b4f3c24..a37a95d87 100644 --- a/pkg/daemon/ceph/client/mirror.go +++ b/pkg/daemon/ceph/client/mirror.go @@ -165,7 +165,7 @@ func RemoveClusterPeer(context *clusterd.Context, clusterInfo *ClusterInfo, pool } // GetPoolMirroringStatus prints the pool mirroring status -func GetPoolMirroringStatus(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) (*cephv1.PoolMirroringStatus, error) { +func GetPoolMirroringStatus(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) (*cephv1.MirroringStatus, error) { logger.Debugf("retrieving mirroring pool %q status", poolName) // Build command @@ -179,7 +179,7 @@ func GetPoolMirroringStatus(context *clusterd.Context, clusterInfo *ClusterInfo, return nil, errors.Wrapf(err, "failed to retrieve mirroring pool %q status", poolName) } - var poolMirroringStatus cephv1.PoolMirroringStatus + var poolMirroringStatus cephv1.MirroringStatus if err := json.Unmarshal([]byte(buf), &poolMirroringStatus); err != nil { return nil, errors.Wrap(err, "failed to unmarshal mirror pool status response") } @@ -211,7 +211,8 @@ func GetMirroredPoolImages(context *clusterd.Context, clusterInfo *ClusterInfo, } // GetPoolMirroringInfo prints the pool mirroring information -func GetPoolMirroringInfo(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) (*cephv1.PoolMirroringInfo, error) { +// `poolName` is the name of the pool or the pool/radosNamespace +func GetPoolMirroringInfo(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) (*cephv1.MirroringInfo, error) { logger.Debugf("retrieving mirroring pool %q info", poolName) // Build command @@ -226,7 +227,7 @@ func GetPoolMirroringInfo(context *clusterd.Context, clusterInfo *ClusterInfo, p } // Unmarshal JSON into Go struct - var poolMirroringInfo cephv1.PoolMirroringInfo + var poolMirroringInfo cephv1.MirroringInfo if err := json.Unmarshal(buf, &poolMirroringInfo); err != nil { return nil, errors.Wrap(err, "failed to unmarshal mirror pool info response") } @@ -280,17 +281,18 @@ func removeSnapshotSchedule(context *clusterd.Context, clusterInfo *ClusterInfo, return nil } -func EnableSnapshotSchedules(context *clusterd.Context, clusterInfo *ClusterInfo, pool string, snapshotSchedules []cephv1.SnapshotScheduleSpec) error { +// `poolName` is the name of the pool or the pool/radosNamespace +func EnableSnapshotSchedules(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string, snapshotSchedules []cephv1.SnapshotScheduleSpec) error { logger.Info("resetting current snapshot schedules in cluster namespace %q", clusterInfo.Namespace) // Reset any existing schedules - err := removeSnapshotSchedules(context, clusterInfo, pool) + err := removeSnapshotSchedules(context, clusterInfo, poolName) if err != nil { logger.Errorf("failed to remove snapshot schedules. %v", err) } // Enable all the snap schedules for _, snapSchedule := range snapshotSchedules { - err := enableSnapshotSchedule(context, clusterInfo, snapSchedule, pool) + err := enableSnapshotSchedule(context, clusterInfo, snapSchedule, poolName) if err != nil { return errors.Wrap(err, "failed to enable snapshot schedule") } diff --git a/pkg/daemon/ceph/client/mirror_health.go b/pkg/daemon/ceph/client/mirror_health.go new file mode 100644 index 000000000..9bb99d296 --- /dev/null +++ b/pkg/daemon/ceph/client/mirror_health.go @@ -0,0 +1,225 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "time" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/reporting" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + defaultHealthCheckInterval = 1 * time.Minute +) + +type mirrorChecker struct { + context *clusterd.Context + interval *time.Duration + client client.Client + clusterInfo *ClusterInfo + namespacedName types.NamespacedName + monitoringSpec *cephv1.NamedPoolSpec + objectType client.Object +} + +// newMirrorChecker creates a new HealthChecker object +func NewMirrorChecker(context *clusterd.Context, client client.Client, clusterInfo *ClusterInfo, namespacedName types.NamespacedName, monitoringSpec *cephv1.NamedPoolSpec, object client.Object) *mirrorChecker { + c := &mirrorChecker{ + context: context, + interval: &defaultHealthCheckInterval, + clusterInfo: clusterInfo, + namespacedName: namespacedName, + client: client, + monitoringSpec: monitoringSpec, + objectType: object, + } + + // allow overriding the check interval + checkInterval := monitoringSpec.StatusCheck.Mirror.Interval + if checkInterval != nil { + logger.Infof("mirroring status check interval for %q is %q", namespacedName.Name, checkInterval.Duration.String()) + c.interval = &checkInterval.Duration + } + + return c +} + +// checkMirroring periodically checks the health of the cluster +func (c *mirrorChecker) CheckMirroring(context context.Context) { + // check the mirroring health immediately before starting the loop + err := c.CheckMirroringHealth() + if err != nil { + c.UpdateStatusMirroring(nil, nil, nil, err.Error()) + logger.Debugf("failed to check mirroring status for %q. %v", c.namespacedName.Name, err) + } + + for { + select { + case <-context.Done(): + logger.Infof("stopping monitoring mirroring status for %q", c.namespacedName.Name) + return + + case <-time.After(*c.interval): + logger.Debugf("checking mirroring status for %q", c.namespacedName.Name) + err := c.CheckMirroringHealth() + if err != nil { + c.UpdateStatusMirroring(nil, nil, nil, err.Error()) + logger.Debugf("failed to check mirroring status for %q. %v", c.namespacedName.Name, err) + } + } + } +} + +func (c *mirrorChecker) CheckMirroringHealth() error { + // Check mirroring status + mirrorStatus, err := GetPoolMirroringStatus(c.context, c.clusterInfo, c.monitoringSpec.Name) + if err != nil { + c.UpdateStatusMirroring(nil, nil, nil, err.Error()) + } + + // Check mirroring info + mirrorInfo, err := GetPoolMirroringInfo(c.context, c.clusterInfo, c.monitoringSpec.Name) + if err != nil { + c.UpdateStatusMirroring(nil, nil, nil, err.Error()) + } + + // If snapshot scheduling is enabled let's add it to the status + // snapSchedStatus := cephclient.SnapshotScheduleStatus{} + snapSchedStatus := []cephv1.SnapshotSchedulesSpec{} + if c.monitoringSpec.Mirroring.SnapshotSchedulesEnabled() { + snapSchedStatus, err = ListSnapshotSchedulesRecursively(c.context, c.clusterInfo, c.monitoringSpec.Name) + if err != nil { + c.UpdateStatusMirroring(nil, nil, nil, err.Error()) + } + } + + // On success + c.UpdateStatusMirroring(mirrorStatus.Summary, mirrorInfo, snapSchedStatus, "") + + return nil +} + +// updateStatusBucket updates an object with a given status +func (c *mirrorChecker) UpdateStatusMirroring(mirrorStatus *cephv1.MirroringStatusSummarySpec, mirrorInfo *cephv1.MirroringInfo, snapSchedStatus []cephv1.SnapshotSchedulesSpec, details string) { + switch c.objectType.(type) { + case *cephv1.CephBlockPool: + updatePoolStatusMirroring(c, mirrorStatus, mirrorInfo, snapSchedStatus, details) + + case *cephv1.CephBlockPoolRadosNamespace: + updateRadosNamespaceStatusMirroring(c, mirrorStatus, mirrorInfo, snapSchedStatus, details) + } +} + +func updatePoolStatusMirroring(c *mirrorChecker, mirrorStatus *cephv1.MirroringStatusSummarySpec, mirrorInfo *cephv1.MirroringInfo, snapSchedStatus []cephv1.SnapshotSchedulesSpec, details string) { + blockPool := &cephv1.CephBlockPool{} + if err := c.client.Get(c.clusterInfo.Context, c.namespacedName, blockPool); err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("CephBlockPool %q resource not found for updating the mirroring status, ignoring.", blockPool) + return + } + logger.Warningf("failed to retrieve ceph block pool %q to update mirroring status. %v", c.namespacedName.Name, err) + return + } + if blockPool.Status == nil { + blockPool.Status = &cephv1.CephBlockPoolStatus{} + } + + // Update the CephBlockPool CR status field + blockPool.Status.MirroringStatus, blockPool.Status.MirroringInfo, blockPool.Status.SnapshotScheduleStatus = toCustomResourceStatus(blockPool.Status.MirroringStatus, mirrorStatus, blockPool.Status.MirroringInfo, mirrorInfo, blockPool.Status.SnapshotScheduleStatus, snapSchedStatus, details) + if err := reporting.UpdateStatus(c.client, blockPool); err != nil { + logger.Errorf("failed to set ceph block pool %q mirroring status. %v", c.namespacedName.Name, err) + return + } + + logger.Debugf("ceph block pool %q mirroring status updated", c.namespacedName.Name) +} + +func updateRadosNamespaceStatusMirroring(c *mirrorChecker, mirrorStatus *cephv1.MirroringStatusSummarySpec, mirrorInfo *cephv1.MirroringInfo, snapSchedStatus []cephv1.SnapshotSchedulesSpec, details string) { + radosNamespace := &cephv1.CephBlockPoolRadosNamespace{} + if err := c.client.Get(c.clusterInfo.Context, c.namespacedName, radosNamespace); err != nil { + if kerrors.IsNotFound(err) { + logger.Debug("CephBlockPoolRadosNamespace %q resource not found for updating the mirroring status, ignoring.", radosNamespace) + return + } + logger.Warningf("failed to retrieve ceph block pool rados namespace %q to update mirroring status. %v", c.namespacedName.Name, err) + return + } + if radosNamespace.Status == nil { + radosNamespace.Status = &cephv1.CephBlockPoolRadosNamespaceStatus{} + } + + // Update the CephBlockPoolRadosNamespace CR status field + radosNamespace.Status.MirroringStatus, radosNamespace.Status.MirroringInfo, radosNamespace.Status.SnapshotScheduleStatus = toCustomResourceStatus(radosNamespace.Status.MirroringStatus, mirrorStatus, radosNamespace.Status.MirroringInfo, mirrorInfo, radosNamespace.Status.SnapshotScheduleStatus, snapSchedStatus, details) + if err := reporting.UpdateStatus(c.client, radosNamespace); err != nil { + logger.Errorf("failed to set ceph block pool rados namespace %q mirroring status. %v", c.namespacedName.Name, err) + return + } +} + +func toCustomResourceStatus(currentStatus *cephv1.MirroringStatusSpec, mirroringStatus *cephv1.MirroringStatusSummarySpec, + currentInfo *cephv1.MirroringInfoSpec, mirroringInfo *cephv1.MirroringInfo, + currentSnapSchedStatus *cephv1.SnapshotScheduleStatusSpec, snapSchedStatus []cephv1.SnapshotSchedulesSpec, + details string) (*cephv1.MirroringStatusSpec, *cephv1.MirroringInfoSpec, *cephv1.SnapshotScheduleStatusSpec) { + mirroringStatusSpec := &cephv1.MirroringStatusSpec{} + mirroringInfoSpec := &cephv1.MirroringInfoSpec{} + snapshotScheduleStatusSpec := &cephv1.SnapshotScheduleStatusSpec{} + + // mirroringStatus will be nil in case of an error to fetch it + if mirroringStatus != nil { + mirroringStatusSpec.LastChecked = time.Now().UTC().Format(time.RFC3339) + mirroringStatusSpec.MirroringStatus.Summary = mirroringStatus + } + + // Always display the details, typically an error + mirroringStatusSpec.Details = details + + if currentStatus != nil { + mirroringStatusSpec.LastChanged = currentStatus.LastChanged + } + + // mirroringInfo will be nil in case of an error to fetch it + if mirroringInfo != nil { + mirroringInfoSpec.LastChecked = time.Now().UTC().Format(time.RFC3339) + mirroringInfoSpec.MirroringInfo = mirroringInfo + } + // Always display the details, typically an error + mirroringInfoSpec.Details = details + + if currentInfo != nil { + mirroringInfoSpec.LastChanged = currentInfo.LastChecked + } + + // snapSchedStatus will be nil in case of an error to fetch it + if len(snapSchedStatus) != 0 { + snapshotScheduleStatusSpec.LastChecked = time.Now().UTC().Format(time.RFC3339) + snapshotScheduleStatusSpec.SnapshotSchedules = snapSchedStatus + } + // Always display the details, typically an error + snapshotScheduleStatusSpec.Details = details + + if currentSnapSchedStatus != nil { + snapshotScheduleStatusSpec.LastChanged = currentSnapSchedStatus.LastChecked + } + + return mirroringStatusSpec, mirroringInfoSpec, snapshotScheduleStatusSpec +} diff --git a/pkg/daemon/ceph/client/mirror_health_test.go b/pkg/daemon/ceph/client/mirror_health_test.go new file mode 100644 index 000000000..429517439 --- /dev/null +++ b/pkg/daemon/ceph/client/mirror_health_test.go @@ -0,0 +1,43 @@ +package client + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/stretchr/testify/assert" +) + +func TestToCustomResourceStatus(t *testing.T) { + mirroringStatus := &cephv1.MirroringStatusSummarySpec{} + mirroringStatus.Health = "HEALTH_OK" + mirroringInfo := &cephv1.MirroringInfo{ + Mode: "pool", + SiteName: "rook-ceph-emea", + Peers: []cephv1.PeersSpec{ + {UUID: "82656994-3314-4996-ac4c-263c2c9fd081"}, + }, + } + + // Test 1: Empty so it's disabled + { + newMirroringStatus, newMirroringInfo, _ := toCustomResourceStatus(&cephv1.MirroringStatusSpec{}, mirroringStatus, &cephv1.MirroringInfoSpec{}, mirroringInfo, &cephv1.SnapshotScheduleStatusSpec{}, []cephv1.SnapshotSchedulesSpec{}, "") + assert.NotEmpty(t, newMirroringStatus.MirroringStatus) + assert.Equal(t, "HEALTH_OK", newMirroringStatus.MirroringStatus.Summary.Health) + assert.Equal(t, "pool", newMirroringInfo.Mode) + } + + // Test 2: snap sched + { + snapSchedStatus := []cephv1.SnapshotSchedulesSpec{ + { + Pool: "my-pool", + Image: "pool/image", + }, + } + newMirroringStatus, newMirroringInfo, newSnapshotScheduleStatus := toCustomResourceStatus(&cephv1.MirroringStatusSpec{}, mirroringStatus, &cephv1.MirroringInfoSpec{}, mirroringInfo, &cephv1.SnapshotScheduleStatusSpec{}, snapSchedStatus, "") + assert.NotEmpty(t, newMirroringStatus.MirroringStatus) + assert.Equal(t, "HEALTH_OK", newMirroringStatus.MirroringStatus.Summary.Health) + assert.NotEmpty(t, newMirroringInfo.Mode, "pool") + assert.NotEmpty(t, newSnapshotScheduleStatus) + } +} diff --git a/pkg/operator/ceph/cluster/rbd/controller.go b/pkg/operator/ceph/cluster/rbd/controller.go index 9a8c6b594..09dc827bb 100644 --- a/pkg/operator/ceph/cluster/rbd/controller.go +++ b/pkg/operator/ceph/cluster/rbd/controller.go @@ -83,7 +83,7 @@ type ReconcileCephRBDMirror struct { // peerSpec represents peer details type peerSpec struct { - info *cephv1.PoolMirroringInfo + info *cephv1.MirroringInfo poolName string direction string } diff --git a/pkg/operator/ceph/pool/controller.go b/pkg/operator/ceph/pool/controller.go index 015d59dea..df42c9265 100644 --- a/pkg/operator/ceph/pool/controller.go +++ b/pkg/operator/ceph/pool/controller.go @@ -297,7 +297,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile if err := configureRBDStats(r.context, clusterInfo, ""); err != nil { return reconcile.Result{}, *cephBlockPool, errors.Wrap(err, "failed to enable/disable stats collection for pool(s)") } - checker := newMirrorChecker(r.context, r.client, r.clusterInfo, request.NamespacedName, &poolSpec) + checker := cephclient.NewMirrorChecker(r.context, r.client, r.clusterInfo, request.NamespacedName, &poolSpec, cephBlockPool) // ADD PEERS logger.Debug("reconciling create rbd mirror peer configuration") if cephBlockPool.Spec.Mirroring.Enabled { @@ -316,8 +316,8 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile if r.blockPoolContexts[blockPoolChannelKey].started { logger.Debug("pool monitoring go routine already running!") } else { - go checker.checkMirroring(r.blockPoolContexts[blockPoolChannelKey].internalCtx) r.blockPoolContexts[blockPoolChannelKey].started = true + go checker.CheckMirroring(r.blockPoolContexts[blockPoolChannelKey].internalCtx) } } @@ -353,7 +353,7 @@ func (r *ReconcileCephBlockPool) reconcile(request reconcile.Request) (reconcile if blockPoolContextsExists && r.blockPoolContexts[blockPoolChannelKey].started { r.cancelMirrorMonitoring(cephBlockPool) // Reset the MirrorHealthCheckSpec - checker.updateStatusMirroring(nil, nil, nil, "") + checker.UpdateStatusMirroring(nil, nil, nil, "") } } diff --git a/pkg/operator/ceph/pool/health.go b/pkg/operator/ceph/pool/health.go deleted file mode 100644 index 90731bf91..000000000 --- a/pkg/operator/ceph/pool/health.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2020 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pool - -import ( - "context" - "time" - - cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/clusterd" - cephclient "github.com/rook/rook/pkg/daemon/ceph/client" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - defaultHealthCheckInterval = 1 * time.Minute -) - -type mirrorChecker struct { - context *clusterd.Context - interval *time.Duration - client client.Client - clusterInfo *cephclient.ClusterInfo - namespacedName types.NamespacedName - poolSpec *cephv1.NamedPoolSpec -} - -// newMirrorChecker creates a new HealthChecker object -func newMirrorChecker(context *clusterd.Context, client client.Client, clusterInfo *cephclient.ClusterInfo, namespacedName types.NamespacedName, poolSpec *cephv1.NamedPoolSpec) *mirrorChecker { - c := &mirrorChecker{ - context: context, - interval: &defaultHealthCheckInterval, - clusterInfo: clusterInfo, - namespacedName: namespacedName, - client: client, - poolSpec: poolSpec, - } - - // allow overriding the check interval - checkInterval := poolSpec.StatusCheck.Mirror.Interval - if checkInterval != nil { - logger.Infof("pool mirroring status check interval for block pool %q is %q", namespacedName.Name, checkInterval.Duration.String()) - c.interval = &checkInterval.Duration - } - - return c -} - -// checkMirroring periodically checks the health of the cluster -func (c *mirrorChecker) checkMirroring(context context.Context) { - // check the mirroring health immediately before starting the loop - err := c.checkMirroringHealth() - if err != nil { - c.updateStatusMirroring(nil, nil, nil, err.Error()) - logger.Debugf("failed to check pool mirroring status for ceph block pool %q. %v", c.namespacedName.Name, err) - } - - for { - select { - case <-context.Done(): - logger.Infof("stopping monitoring pool mirroring status %q", c.namespacedName.Name) - return - - case <-time.After(*c.interval): - logger.Debugf("checking pool mirroring status %q", c.namespacedName.Name) - err := c.checkMirroringHealth() - if err != nil { - c.updateStatusMirroring(nil, nil, nil, err.Error()) - logger.Debugf("failed to check pool mirroring status for ceph block pool %q. %v", c.namespacedName.Name, err) - } - } - } -} - -func (c *mirrorChecker) checkMirroringHealth() error { - // Check mirroring status - mirrorStatus, err := cephclient.GetPoolMirroringStatus(c.context, c.clusterInfo, c.poolSpec.Name) - if err != nil { - c.updateStatusMirroring(nil, nil, nil, err.Error()) - } - - // Check mirroring info - mirrorInfo, err := cephclient.GetPoolMirroringInfo(c.context, c.clusterInfo, c.poolSpec.Name) - if err != nil { - c.updateStatusMirroring(nil, nil, nil, err.Error()) - } - - // If snapshot scheduling is enabled let's add it to the status - // snapSchedStatus := cephclient.SnapshotScheduleStatus{} - snapSchedStatus := []cephv1.SnapshotSchedulesSpec{} - if c.poolSpec.Mirroring.SnapshotSchedulesEnabled() { - snapSchedStatus, err = cephclient.ListSnapshotSchedulesRecursively(c.context, c.clusterInfo, c.poolSpec.Name) - if err != nil { - c.updateStatusMirroring(nil, nil, nil, err.Error()) - } - } - - // On success - c.updateStatusMirroring(mirrorStatus.Summary, mirrorInfo, snapSchedStatus, "") - - return nil -} diff --git a/pkg/operator/ceph/pool/radosnamespace/controller.go b/pkg/operator/ceph/pool/radosnamespace/controller.go index 7bae0b595..d67c39520 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller.go @@ -67,12 +67,19 @@ var controllerTypeMeta = metav1.TypeMeta{ // ReconcileCephBlockPoolRadosNamespace reconciles a CephBlockPoolRadosNamespace object type ReconcileCephBlockPoolRadosNamespace struct { - client client.Client - scheme *runtime.Scheme - context *clusterd.Context - clusterInfo *cephclient.ClusterInfo - opManagerContext context.Context - opConfig opcontroller.OperatorConfig + client client.Client + scheme *runtime.Scheme + context *clusterd.Context + clusterInfo *cephclient.ClusterInfo + radosNamespaceContexts map[string]*mirrorHealth + opManagerContext context.Context + opConfig opcontroller.OperatorConfig +} + +type mirrorHealth struct { + internalCtx context.Context + internalCancel context.CancelFunc + started bool } // Add creates a new CephBlockPoolRadosNamespace Controller and adds it to the @@ -85,11 +92,12 @@ func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext contex // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler { return &ReconcileCephBlockPoolRadosNamespace{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - context: context, - opManagerContext: opManagerContext, - opConfig: opConfig, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + context: context, + radosNamespaceContexts: make(map[string]*mirrorHealth), + opManagerContext: opManagerContext, + opConfig: opConfig, } } @@ -155,6 +163,8 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque r.updateStatus(r.client, request.NamespacedName, cephv1.ConditionProgressing) } + poolAndRadosNamespaceName := fmt.Sprintf("%s/%s", cephBlockPoolRadosNamespace.Spec.BlockPoolName, getRadosNamespaceName(cephBlockPoolRadosNamespace)) + // Make sure a CephCluster is present otherwise do nothing cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName) if !isReadyToReconcile { @@ -165,6 +175,8 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque // If not, we should wait for it to be ready // This handles the case where the operator is not ready to accept Ceph command but the cluster exists if !cephBlockPoolRadosNamespace.GetDeletionTimestamp().IsZero() && !cephClusterExists { + // don't leak the health checker routine if we are force-deleting + r.cancelMirrorMonitoring(radosNamespaceChannelKeyName(cephBlockPoolRadosNamespace.Namespace, poolAndRadosNamespaceName)) // Remove finalizer err = opcontroller.RemoveFinalizer(r.opManagerContext, r.client, cephBlockPoolRadosNamespace) if err != nil { @@ -219,6 +231,9 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcile(request reconcile.Reque } return reconcile.Result{}, errors.Wrapf(err, "failed to delete ceph blockpool rados namespace %q", cephBlockPoolRadosNamespace.Name) } + // If the ceph block pool is still in the map, we must remove it during CR deletion + // We must remove it first otherwise the checker will panic since the status/info will be nil + r.cancelMirrorMonitoring(radosNamespaceChannelKeyName(cephBlockPoolRadosNamespace.Namespace, poolAndRadosNamespaceName)) } err = csi.SaveClusterConfig(r.context.Clientset, buildClusterID(cephBlockPoolRadosNamespace), cephCluster.Namespace, r.clusterInfo, nil) if err != nil { @@ -425,6 +440,23 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcileMirroring(cephBlockPoolR return errors.Wrapf(err, "failed to get mirroring info for the radosnamespace %q", poolAndRadosNamespaceName) } + // Initialize the channel for radosNamespace + // This allows us to track multiple radosNamespace in the same namespace + radosNamespaceChannelKey := radosNamespaceChannelKeyName(cephBlockPool.Namespace, poolAndRadosNamespaceName) + _, radosNamespaceContextsExists := r.radosNamespaceContexts[radosNamespaceChannelKey] + if !radosNamespaceContextsExists { + internalCtx, internalCancel := context.WithCancel(r.opManagerContext) + r.radosNamespaceContexts[radosNamespaceChannelKey] = &mirrorHealth{ + internalCtx: internalCtx, + internalCancel: internalCancel, + } + } + monitoringSpec := cephv1.NamedPoolSpec{ + Name: poolAndRadosNamespaceName, // use the name of the blockpool/radosNamespace + PoolSpec: cephBlockPool.Spec.PoolSpec, + } + checker := cephclient.NewMirrorChecker(r.context, r.client, r.clusterInfo, types.NamespacedName{Name: cephBlockPoolRadosNamespace.Name, Namespace: cephBlockPoolRadosNamespace.Namespace}, &monitoringSpec, cephBlockPoolRadosNamespace) + if cephBlockPoolRadosNamespace.Spec.Mirroring != nil { mirroringDisabled := checkBlockPoolMirroring(cephBlockPool) if mirroringDisabled { @@ -441,6 +473,19 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcileMirroring(cephBlockPoolR if err != nil { return errors.Wrapf(err, "failed to enable snapshot scheduling for rbd rados namespace %q", poolAndRadosNamespaceName) } + + // Run the goroutine to update the mirroring status + // use the monitoring settings from the cephBlockPool CR + if !cephBlockPool.Spec.StatusCheck.Mirror.Disabled { + logger.Debugf("starting mirror monitoring for radosnamespace %q", poolAndRadosNamespaceName) + // Start monitoring of the radosNamespace + if r.radosNamespaceContexts[radosNamespaceChannelKey].started { + logger.Debug("radosnamespace monitoring go routine already running!") + } else { + r.radosNamespaceContexts[radosNamespaceChannelKey].started = true + go checker.CheckMirroring(r.radosNamespaceContexts[radosNamespaceChannelKey].internalCtx) + } + } } if cephBlockPoolRadosNamespace.Spec.Mirroring == nil && mirrorInfo.Mode != "disabled" { @@ -460,5 +505,31 @@ func (r *ReconcileCephBlockPoolRadosNamespace) reconcileMirroring(cephBlockPoolR return errors.Wrap(err, "failed to disable rbd rados namespace mirroring") } } + + if cephBlockPool.Spec.StatusCheck.Mirror.Disabled { + // Stop monitoring the mirroring status of this radosNamespace + if radosNamespaceContextsExists && r.radosNamespaceContexts[radosNamespaceChannelKey].started { + r.cancelMirrorMonitoring(radosNamespaceChannelKey) + // Reset the MirrorHealthCheckSpec + checker.UpdateStatusMirroring(nil, nil, nil, "") + } + } + return nil } + +func radosNamespaceChannelKeyName(poolAndRadosNamespaceName, namespace string) string { + return types.NamespacedName{Namespace: namespace, Name: poolAndRadosNamespaceName}.String() +} + +// cancel mirror monitoring. This is a noop if monitoring is not running. +func (r *ReconcileCephBlockPoolRadosNamespace) cancelMirrorMonitoring(channelKey string) { + _, poolContextExists := r.radosNamespaceContexts[channelKey] + if poolContextExists { + // Cancel the context to stop the go routine + r.radosNamespaceContexts[channelKey].internalCancel() + + // Remove ceph radosNamespace from the map + delete(r.radosNamespaceContexts, channelKey) + } +} diff --git a/pkg/operator/ceph/pool/radosnamespace/controller_test.go b/pkg/operator/ceph/pool/radosnamespace/controller_test.go index 1446c94a5..d940c3497 100644 --- a/pkg/operator/ceph/pool/radosnamespace/controller_test.go +++ b/pkg/operator/ceph/pool/radosnamespace/controller_test.go @@ -167,7 +167,7 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { Phase: "", }, } - + cephBlockPool.Spec.StatusCheck.Mirror.Disabled = true t.Run("error - ceph blockpool not ready", func(t *testing.T) { res, err := r.Reconcile(ctx, req) assert.NoError(t, err) @@ -218,11 +218,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &version.Reef, nil @@ -337,11 +338,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &version.Reef, nil @@ -385,11 +387,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &version.Reef, nil @@ -436,11 +439,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &version.Reef, nil @@ -487,11 +491,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &cephver.CephVersion{Major: 20, Minor: 0, Extra: 0}, nil @@ -541,11 +546,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &cephver.CephVersion{Major: 20, Minor: 0, Extra: 0}, nil @@ -599,11 +605,12 @@ func TestCephBlockPoolRadosNamespaceController(t *testing.T) { s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephBlockPoolList{}) // Create a ReconcileCephBlockPoolRadosNamespace object with the scheme and fake client. r = &ReconcileCephBlockPoolRadosNamespace{ - client: cl, - scheme: s, - context: c, - opManagerContext: context.TODO(), - opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + client: cl, + scheme: s, + context: c, + opManagerContext: context.TODO(), + opConfig: opcontroller.OperatorConfig{Image: "ceph/ceph:v14.2.9"}, + radosNamespaceContexts: make(map[string]*mirrorHealth), } detectCephVersion = func(ctx context.Context, rookImage, namespace, jobName string, ownerInfo *k8sutil.OwnerInfo, clientset kubernetes.Interface, cephClusterSpec *cephv1.ClusterSpec) (*cephver.CephVersion, error) { return &version.Reef, nil diff --git a/pkg/operator/ceph/pool/status.go b/pkg/operator/ceph/pool/status.go index e9de744fd..666efd5d1 100644 --- a/pkg/operator/ceph/pool/status.go +++ b/pkg/operator/ceph/pool/status.go @@ -19,7 +19,6 @@ package pool import ( "context" - "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" @@ -59,79 +58,6 @@ func updateStatus(ctx context.Context, client client.Client, poolName types.Name logger.Debugf("pool %q status updated to %q", poolName, status) } -// updateStatusBucket updates an object with a given status -func (c *mirrorChecker) updateStatusMirroring(mirrorStatus *cephv1.PoolMirroringStatusSummarySpec, mirrorInfo *cephv1.PoolMirroringInfo, snapSchedStatus []cephv1.SnapshotSchedulesSpec, details string) { - blockPool := &cephv1.CephBlockPool{} - if err := c.client.Get(c.clusterInfo.Context, c.namespacedName, blockPool); err != nil { - if kerrors.IsNotFound(err) { - logger.Debug("CephBlockPool resource not found. Ignoring since object must be deleted.") - return - } - logger.Warningf("failed to retrieve ceph block pool %q to update mirroring status. %v", c.namespacedName.Name, err) - return - } - if blockPool.Status == nil { - blockPool.Status = &cephv1.CephBlockPoolStatus{} - } - - // Update the CephBlockPool CR status field - blockPool.Status.MirroringStatus, blockPool.Status.MirroringInfo, blockPool.Status.SnapshotScheduleStatus = toCustomResourceStatus(blockPool.Status.MirroringStatus, mirrorStatus, blockPool.Status.MirroringInfo, mirrorInfo, blockPool.Status.SnapshotScheduleStatus, snapSchedStatus, details) - if err := reporting.UpdateStatus(c.client, blockPool); err != nil { - logger.Errorf("failed to set ceph block pool %q mirroring status. %v", c.namespacedName.Name, err) - return - } - - logger.Debugf("ceph block pool %q mirroring status updated", c.namespacedName.Name) -} - -func toCustomResourceStatus(currentStatus *cephv1.MirroringStatusSpec, mirroringStatus *cephv1.PoolMirroringStatusSummarySpec, - currentInfo *cephv1.MirroringInfoSpec, mirroringInfo *cephv1.PoolMirroringInfo, - currentSnapSchedStatus *cephv1.SnapshotScheduleStatusSpec, snapSchedStatus []cephv1.SnapshotSchedulesSpec, - details string) (*cephv1.MirroringStatusSpec, *cephv1.MirroringInfoSpec, *cephv1.SnapshotScheduleStatusSpec) { - mirroringStatusSpec := &cephv1.MirroringStatusSpec{} - mirroringInfoSpec := &cephv1.MirroringInfoSpec{} - snapshotScheduleStatusSpec := &cephv1.SnapshotScheduleStatusSpec{} - - // mirroringStatus will be nil in case of an error to fetch it - if mirroringStatus != nil { - mirroringStatusSpec.LastChecked = time.Now().UTC().Format(time.RFC3339) - mirroringStatusSpec.Summary = mirroringStatus - } - - // Always display the details, typically an error - mirroringStatusSpec.Details = details - - if currentStatus != nil { - mirroringStatusSpec.LastChanged = currentStatus.LastChanged - } - - // mirroringInfo will be nil in case of an error to fetch it - if mirroringInfo != nil { - mirroringInfoSpec.LastChecked = time.Now().UTC().Format(time.RFC3339) - mirroringInfoSpec.PoolMirroringInfo = mirroringInfo - } - // Always display the details, typically an error - mirroringInfoSpec.Details = details - - if currentInfo != nil { - mirroringInfoSpec.LastChanged = currentInfo.LastChecked - } - - // snapSchedStatus will be nil in case of an error to fetch it - if len(snapSchedStatus) != 0 { - snapshotScheduleStatusSpec.LastChecked = time.Now().UTC().Format(time.RFC3339) - snapshotScheduleStatusSpec.SnapshotSchedules = snapSchedStatus - } - // Always display the details, typically an error - snapshotScheduleStatusSpec.Details = details - - if currentSnapSchedStatus != nil { - snapshotScheduleStatusSpec.LastChanged = currentSnapSchedStatus.LastChecked - } - - return mirroringStatusSpec, mirroringInfoSpec, snapshotScheduleStatusSpec -} - func updateStatusInfo(cephBlockPool *cephv1.CephBlockPool) { m := make(map[string]string) if cephBlockPool.Status.Phase == cephv1.ConditionReady && cephBlockPool.Spec.Mirroring.Enabled { diff --git a/pkg/operator/ceph/pool/status_test.go b/pkg/operator/ceph/pool/status_test.go index e34b58282..cdcba99e4 100644 --- a/pkg/operator/ceph/pool/status_test.go +++ b/pkg/operator/ceph/pool/status_test.go @@ -25,41 +25,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestToCustomResourceStatus(t *testing.T) { - mirroringStatus := &cephv1.PoolMirroringStatusSummarySpec{} - mirroringStatus.Health = "HEALTH_OK" - mirroringInfo := &cephv1.PoolMirroringInfo{ - Mode: "pool", - SiteName: "rook-ceph-emea", - Peers: []cephv1.PeersSpec{ - {UUID: "82656994-3314-4996-ac4c-263c2c9fd081"}, - }, - } - - // Test 1: Empty so it's disabled - { - newMirroringStatus, newMirroringInfo, _ := toCustomResourceStatus(&cephv1.MirroringStatusSpec{}, mirroringStatus, &cephv1.MirroringInfoSpec{}, mirroringInfo, &cephv1.SnapshotScheduleStatusSpec{}, []cephv1.SnapshotSchedulesSpec{}, "") - assert.NotEmpty(t, newMirroringStatus.Summary) - assert.Equal(t, "HEALTH_OK", newMirroringStatus.Summary.Health) - assert.Equal(t, "pool", newMirroringInfo.Mode) - } - - // Test 2: snap sched - { - snapSchedStatus := []cephv1.SnapshotSchedulesSpec{ - { - Pool: "my-pool", - Image: "pool/image", - }, - } - newMirroringStatus, newMirroringInfo, newSnapshotScheduleStatus := toCustomResourceStatus(&cephv1.MirroringStatusSpec{}, mirroringStatus, &cephv1.MirroringInfoSpec{}, mirroringInfo, &cephv1.SnapshotScheduleStatusSpec{}, snapSchedStatus, "") - assert.NotEmpty(t, newMirroringStatus.Summary) - assert.Equal(t, "HEALTH_OK", newMirroringStatus.Summary.Health) - assert.NotEmpty(t, newMirroringInfo.Mode, "pool") - assert.NotEmpty(t, newSnapshotScheduleStatus) - } -} - func TestUpdateStatusInfo(t *testing.T) { cephBlockPoolReplicated := &cephv1.CephBlockPool{