Files
my-rook-config/pkg/daemon/ceph/client/mirror.go
T
Sébastien Han 451622a955 ceph: add rbd-mirror configuration
Rook is now capable of configuring mirroring between sites. The
implementation works at different levels:

* CephBlockPool: which introduces a new `mirroring` configuration as well
as `statusCheck`. When turned on, Rook will enable mirroring on the
pool. It will also create a bootstrap peer token and store it in a
Kubernetes Secret. The name of that Secret can be found in the Status
field of the CephBlockPool CRD. This token can be fetched and used by
other clusters to configure the site as a peer. Mirroring can be
configured either at the pool or the image level.

* CephRBDMirror: which introduces a new `peers` configuration allowing
Rook to connect to peers by passing a Secret name. The administrator will
create a Kubernetes Secret with 2 keys: 'token' for the bootstrap peer
token and 'pool' for the name of pool. Once detected the rbd-mirror
controller will go ahead and import the peer configuration.

Pool mirroring status example:

```
status:
  info:
    rbdMirrorBootstrapPeerSecretName: pool-peer-token-test
  mirroringInfo:
    lastChanged: "2020-09-17T14:47:27Z"
    lastChecked: "2020-09-17T14:48:27Z"
    summary:
      summary:
        mode: image
        peers:
        - client_name: client.rbd-mirror-peer
          direction: rx-tx
          mirror_uuid: ""
          site_name: rhcs
          uuid: c50522a4-28a4-4bd3-ba68-e11780308882
        site_name: 91eae0dd-06b1-4d2c-91f3-1311c9df382b-rook-ceph
  mirroringStatus:
    lastChecked: "2020-09-17T14:48:27Z"
    summary:
      summary:
        daemon_health: OK
        health: OK
        image_health: OK
        states:
          replaying: 1
```

Signed-off-by: Sébastien Han <seb@redhat.com>
2020-09-17 19:05:04 +02:00

171 lines
5.5 KiB
Go

/*
Copyright 2020 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/clusterd"
)
// PoolMirroringStatus is the mirroring status of a given pool
type PoolMirroringStatus struct {
Summary struct {
Health string `json:"health"`
DaemonHealth string `json:"daemon_health"`
ImageHealth string `json:"image_health"`
States interface{} `json:"states"`
} `json:"summary"`
}
// PoolMirroringInfo is the mirroring info of a given pool
type PoolMirroringInfo struct {
Mode string `json:"mode"`
SiteName string `json:"site_name"`
Peers []PeersSpec `json:"peers"`
}
// PeersSpec contains peer details
type PeersSpec struct {
UUID string `json:"uuid"`
Direction string `json:"direction"`
SiteName string `json:"site_name"`
MirrorUUID string `json:"mirror_uuid"`
ClientName string `json:"client_name"`
}
// ImportRBDMirrorBootstrapPeer add a mirror peer in the rbd-mirror configuration
func ImportRBDMirrorBootstrapPeer(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, direction string, token []byte) error {
logger.Infof("add rbd-mirror bootstrap peer token for pool %q", poolName)
// Token file
tokenFilePath := fmt.Sprintf("/tmp/rbd-mirror-token-%s", poolName)
// Write token into a file
err := ioutil.WriteFile(tokenFilePath, token, 0400)
if err != nil {
return errors.Wrapf(err, "failed to write token to file %q", tokenFilePath)
}
// Remove token once we exit, we don't need it anymore
defer func() error {
err := os.Remove(tokenFilePath)
return err
}()
// Build command
args := []string{"mirror", "pool", "peer", "bootstrap", "import", poolName, tokenFilePath}
if direction != "" {
args = append(args, "--direction", direction)
}
cmd := NewRBDCommand(context, clusterInfo, args)
// Run command
output, err := cmd.Run()
if err != nil {
return errors.Wrapf(err, "failed to add rbd-mirror peer token for pool %q. %s", poolName, output)
}
return nil
}
// CreateRBDMirrorBootstrapPeer add a mirror peer in the rbd-mirror configuration
func CreateRBDMirrorBootstrapPeer(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) ([]byte, error) {
logger.Infof("create rbd-mirror bootstrap peer token for pool %q", poolName)
// Build command
args := []string{"mirror", "pool", "peer", "bootstrap", "create", poolName, "--site-name", fmt.Sprintf("%s-%s", clusterInfo.FSID, clusterInfo.Namespace)}
cmd := NewRBDCommand(context, clusterInfo, args)
// Run command
output, err := cmd.Run()
if err != nil {
return nil, errors.Wrapf(err, "failed to create rbd-mirror peer token for pool %q. %s", poolName, output)
}
return output, nil
}
// EnablePoolMirroring turns on mirroring on that pool by specifying the mirroring type
func EnablePoolMirroring(context *clusterd.Context, clusterInfo *ClusterInfo, pool cephv1.PoolSpec, poolName string) error {
logger.Infof("enabling mirroring type %q for pool %q", pool.Mirroring.Mode, poolName)
// Build command
args := []string{"mirror", "pool", "enable", poolName, pool.Mirroring.Mode}
cmd := NewRBDCommand(context, clusterInfo, args)
// Run command
output, err := cmd.Run()
if err != nil {
return errors.Wrapf(err, "failed to enable mirroring type %q for pool %q. %s", pool.Mirroring.Mode, poolName, output)
}
return nil
}
// GetPoolMirroringStatus prints the pool mirroring status
func GetPoolMirroringStatus(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) (*PoolMirroringStatus, error) {
logger.Debugf("retrieving mirroring pool %q status", poolName)
// Build command
args := []string{"mirror", "pool", "status", poolName}
cmd := NewRBDCommand(context, clusterInfo, args)
cmd.JsonOutput = true
// Run command
buf, err := cmd.Run()
if err != nil {
return nil, errors.Wrapf(err, "failed to retrieve mirroring pool %q status", poolName)
}
var poolMirroringStatus PoolMirroringStatus
if err := json.Unmarshal([]byte(buf), &poolMirroringStatus); err != nil {
return nil, errors.Wrap(err, "failed to unmarshal mirror pool status response")
}
return &poolMirroringStatus, nil
}
// GetPoolMirroringInfo prints the pool mirroring information
func GetPoolMirroringInfo(context *clusterd.Context, clusterInfo *ClusterInfo, poolName string) (*PoolMirroringInfo, error) {
logger.Debugf("retrieving mirroring pool %q info", poolName)
// Build command
args := []string{"mirror", "pool", "info", poolName}
cmd := NewRBDCommand(context, clusterInfo, args)
cmd.JsonOutput = true
// Run command
buf, err := cmd.Run()
if err != nil {
return nil, errors.Wrapf(err, "failed to retrieve mirroring pool %q info. %s", poolName, string(buf))
}
// Unmarshal JSON into Go struct
var poolMirroringInfo PoolMirroringInfo
if err := json.Unmarshal(buf, &poolMirroringInfo); err != nil {
return nil, errors.Wrap(err, "failed to unmarshal mirror pool info response")
}
return &poolMirroringInfo, nil
}