Merge pull request #8501 from josephsawaya/ceph-nfs-remove-rados

ceph: remove RADOS options from CephNFS and use .nfs pool
This commit is contained in:
Sébastien Han authored and GitHub committed 2021-10-13 10:41:49 +02:00
commit e33d905b59
8 files changed
+196 -4

No files matched your search

+4
View File
@@ -25,6 +25,8 @@ metadata:
name: my-nfs
namespace: rook-ceph
spec:
# rados property is not used in versions of Ceph equal to or greater than
# 16.2.7, see note in RADOS settings section below.
rados:
# RADOS pool where NFS client recovery data and per-daemon configs are
# stored. In this example the data pool for the "myfs" filesystem is used.
@@ -91,6 +93,8 @@ ceph dashboard set-ganesha-clusters-rados-pool-namespace <cluster_id>:<pool_name
* `pool`: The pool where ganesha recovery backend and supplemental configuration objects will be stored
* `namespace`: The namespace in `pool` where ganesha recovery backend and supplemental configuration objects will be stored
> **NOTE**: The RADOS settings aren't used in Ceph versions equal to or greater than Pacific 16.2.7, default values are used instead ".nfs" for the RADOS pool and the CephNFS CR's name for the RADOS namespace. However, RADOS settings are mandatory for versions preceding Pacific 16.2.7.
> **NOTE**: Don't use EC pools for NFS because ganesha uses omap in the recovery objects and grace db. EC pools do not support omap.
## EXPORT Block Configuration
@@ -5655,6 +5655,7 @@ spec:
properties:
rados:
description: RADOS is the Ganesha RADOS specification
nullable: true
properties:
namespace:
description: Namespace is the RADOS namespace where NFS client recovery data is stored.
@@ -6257,7 +6258,6 @@ spec:
- active
type: object
required:
- rados
- server
type: object
status:
+1 -1
View File
@@ -5653,6 +5653,7 @@ spec:
properties:
rados:
description: RADOS is the Ganesha RADOS specification
nullable: true
properties:
namespace:
description: Namespace is the RADOS namespace where NFS client recovery data is stored.
@@ -6255,7 +6256,6 @@ spec:
- active
type: object
required:
- rados
- server
type: object
status:
@@ -4,6 +4,7 @@ metadata:
name: my-nfs
namespace: rook-ceph # namespace:cluster
spec:
# rados settings aren't necessary in Ceph Versions equal to or greater than Pacific 16.2.7
rados:
# RADOS pool where NFS client recovery data is stored.
# In this example the data pool for the "myfs" filesystem is used.
+3 -1
View File
@@ -1619,7 +1619,9 @@ type CephNFSList struct {
// NFSGaneshaSpec represents the spec of an nfs ganesha server
type NFSGaneshaSpec struct {
// RADOS is the Ganesha RADOS specification
RADOS GaneshaRADOSSpec `json:"rados"`
// +nullable
// +optional
RADOS GaneshaRADOSSpec `json:"rados,omitempty"`
// Server is the Ganesha Server specification
Server GaneshaServerSpec `json:"server"`
+21
View File
@@ -31,6 +31,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/config"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/reporting"
"github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@@ -50,6 +51,9 @@ const (
controllerName = "ceph-nfs-controller"
)
// Version of Ceph where NFS default pool name changes to ".nfs"
var cephNFSChangeVersion = version.CephVersion{Major: 16, Minor: 2, Extra: 7}
var logger = capnslog.NewPackageLogger("github.com/rook/rook", controllerName)
// List of object resources to watch by the controller
@@ -247,10 +251,27 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul
}
r.clusterInfo.CephVersion = *runningCephVersion
// Octopus: Customization is allowed, so don't change the pool and namespace
// Pacific before 16.2.7: No customization, default pool name is nfs-ganesha
// Pacific after 16.2.7: No customization, default pool name is .nfs
// This code is changes the pool and namespace to the correct values if the version is Pacific.
// If the version precedes Pacific it doesn't change it at all and the values used are what the user provided in the spec.
if r.clusterInfo.CephVersion.IsAtLeastPacific() {
if r.clusterInfo.CephVersion.IsAtLeast(cephNFSChangeVersion) {
cephNFS.Spec.RADOS.Pool = postNFSChangeDefaultPoolName
} else {
cephNFS.Spec.RADOS.Pool = preNFSChangeDefaultPoolName
}
cephNFS.Spec.RADOS.Namespace = cephNFS.Name
}
// validate the store settings
if err := validateGanesha(r.context, r.clusterInfo, cephNFS); err != nil {
return reconcile.Result{}, errors.Wrapf(err, "invalid ceph nfs %q arguments", cephNFS.Name)
}
if err := fetchOrCreatePool(r.context, r.clusterInfo, cephNFS); err != nil {
return reconcile.Result{}, errors.Wrap(err, "failed to fetch or create RADOS pool")
}
// CREATE/UPDATE
logger.Debug("reconciling ceph nfs deployments")
+130
View File
@@ -29,7 +29,9 @@ import (
"github.com/rook/rook/pkg/client/clientset/versioned/scheme"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/version"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/operator/test"
exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -261,3 +263,131 @@ func TestGetGaneshaConfigObject(t *testing.T) {
logger.Infof("Config Object for Nautilus is %s", res)
assert.Equal(t, "conf-my-nfs.a", res)
}
func TestFetchOrCreatePool(t *testing.T) {
ctx := context.TODO()
cephNFS := &cephv1.CephNFS{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: cephv1.NFSGaneshaSpec{
Server: cephv1.GaneshaServerSpec{
Active: 1,
},
},
TypeMeta: controllerTypeMeta,
}
executor := &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
return "", nil
},
}
clientset := test.New(t, 3)
c := &clusterd.Context{
Executor: executor,
RookClientset: rookclient.NewSimpleClientset(),
Clientset: clientset,
}
// Mock clusterInfo
secrets := map[string][]byte{
"fsid": []byte(name),
"mon-secret": []byte("monsecret"),
"admin-secret": []byte("adminsecret"),
}
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "rook-ceph-mon",
Namespace: namespace,
},
Data: secrets,
Type: k8sutil.RookType,
}
_, err := c.Clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
assert.NoError(t, err)
clusterInfo, _, _, err := mon.LoadClusterInfo(c, ctx, namespace)
if err != nil {
return
}
err = fetchOrCreatePool(c, clusterInfo, cephNFS)
assert.NoError(t, err)
executor = &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[1] == "pool" && args[2] == "get" {
return "Error", errors.New("failed to get pool")
}
return "", nil
},
}
c.Executor = executor
err = fetchOrCreatePool(c, clusterInfo, cephNFS)
assert.Error(t, err)
executor = &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[1] == "pool" && args[2] == "get" {
return "Error", errors.New("failed to get pool: unrecognized pool")
}
return "", nil
},
}
c.Executor = executor
err = fetchOrCreatePool(c, clusterInfo, cephNFS)
assert.Error(t, err)
clusterInfo.CephVersion = cephver.CephVersion{
Major: 16,
Minor: 2,
Extra: 6,
}
executor = &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[1] == "pool" && args[2] == "get" {
return "Error", errors.New("failed to get pool: unrecognized pool")
}
return "", nil
},
}
c.Executor = executor
err = fetchOrCreatePool(c, clusterInfo, cephNFS)
assert.NoError(t, err)
executor = &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[1] == "pool" && args[2] == "get" {
return "Error", errors.New("failed to get pool: unrecognized pool")
}
if args[1] == "pool" && args[2] == "create" {
return "Error", errors.New("creating pool failed")
}
return "", nil
},
}
c.Executor = executor
err = fetchOrCreatePool(c, clusterInfo, cephNFS)
assert.Error(t, err)
executor = &exectest.MockExecutor{
MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) {
if args[1] == "pool" && args[2] == "get" {
return "Error", errors.New("unrecognized pool")
}
if args[1] == "pool" && args[2] == "application" {
return "Error", errors.New("enabling pool failed")
}
return "", nil
},
}
c.Executor = executor
err = fetchOrCreatePool(c, clusterInfo, cephNFS)
assert.Error(t, err)
}
+35 -1
View File
@@ -19,6 +19,7 @@ package nfs
import (
"fmt"
"strings"
"github.com/banzaicloud/k8s-objectmatcher/patch"
"github.com/pkg/errors"
@@ -36,6 +37,10 @@ import (
const (
ganeshaRadosGraceCmd = "ganesha-rados-grace"
// Default RADOS pool name after the NFS changes in Ceph
postNFSChangeDefaultPoolName = ".nfs"
// Default RADOS pool name before the NFS changes in Ceph
preNFSChangeDefaultPoolName = "nfs-ganesha"
)
var updateDeploymentAndWait = opmon.UpdateCephDeploymentAndWait
@@ -264,16 +269,45 @@ func validateGanesha(context *clusterd.Context, clusterInfo *cephclient.ClusterI
return errors.New("missing RADOS.pool")
}
if n.Spec.RADOS.Namespace == "" {
return errors.New("missing RADOS.namespace")
}
// Ganesha server properties
if n.Spec.Server.Active == 0 {
return errors.New("at least one active server required")
}
return nil
}
// create and enable default RADOS pool
func createDefaultNFSRADOSPool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, defaultRadosPoolName string) error {
args := []string{"osd", "pool", "create", defaultRadosPoolName}
_, err := cephclient.NewCephCommand(context, clusterInfo, args).Run()
if err != nil {
return err
}
args = []string{"osd", "pool", "application", "enable", defaultRadosPoolName, "nfs"}
_, err = cephclient.NewCephCommand(context, clusterInfo, args).Run()
if err != nil {
return err
}
return nil
}
func fetchOrCreatePool(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, n *cephv1.CephNFS) error {
// The existence of the pool provided in n.Spec.RADOS.Pool is necessary otherwise addRADOSConfigFile() will fail
_, err := cephclient.GetPoolDetails(context, clusterInfo, n.Spec.RADOS.Pool)
if err != nil {
if strings.Contains(err.Error(), "unrecognized pool") && clusterInfo.CephVersion.IsAtLeastPacific() {
err := createDefaultNFSRADOSPool(context, clusterInfo, n.Spec.RADOS.Pool)
if err != nil {
return errors.Wrapf(err, "failed to find %q pool and unable to create it", n.Spec.RADOS.Pool)
}
return nil
}
return errors.Wrapf(err, "pool %q not found", n.Spec.RADOS.Pool)
}
return nil
}