forked from rook/rook
Travis Nielsen
023608e6fd
For all of the controllers besides the cluster controller, the logging now includes the namespaced name of the resource that is being reconciled. This will help with log troubleshooting to help analyze logs consistently for the resource being reconciled. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
398 lines
14 KiB
Go
398 lines
14 KiB
Go
/*
|
|
Copyright 2018 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package nfs
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/pkg/errors"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
|
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
|
|
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
"github.com/rook/rook/pkg/util/exec"
|
|
"github.com/rook/rook/pkg/util/log"
|
|
)
|
|
|
|
const kerberosRadosObjectName = "kerberos"
|
|
|
|
const (
|
|
keyringTemplate = `
|
|
[%s]
|
|
key = %s
|
|
caps mon = "allow r"
|
|
caps osd = "%s"
|
|
`
|
|
)
|
|
|
|
func getNFSUserID(nodeID string) string {
|
|
return fmt.Sprintf("nfs-ganesha.%s", nodeID)
|
|
}
|
|
|
|
func getNFSClientID(n *cephv1.CephNFS, name string) string {
|
|
return fmt.Sprintf("client.%s", getNFSUserID(getNFSNodeID(n, name)))
|
|
}
|
|
|
|
func getNFSNodeID(n *cephv1.CephNFS, name string) string {
|
|
return fmt.Sprintf("%s.%s", n.Name, name)
|
|
}
|
|
|
|
func getGaneshaConfigObject(n *cephv1.CephNFS) string {
|
|
return fmt.Sprintf("conf-nfs.%s", n.Name)
|
|
}
|
|
|
|
func getRadosURL(n *cephv1.CephNFS) string {
|
|
url := fmt.Sprintf("rados://%s/", n.Spec.RADOS.Pool)
|
|
|
|
if n.Spec.RADOS.Namespace != "" {
|
|
url += n.Spec.RADOS.Namespace + "/"
|
|
}
|
|
|
|
url += getGaneshaConfigObject(n)
|
|
return url
|
|
}
|
|
|
|
func (r *ReconcileCephNFS) generateKeyring(n *cephv1.CephNFS, name string) error {
|
|
nsName := opcontroller.NsName(n.Namespace, n.Name)
|
|
osdCaps := fmt.Sprintf("allow rw pool=%s", n.Spec.RADOS.Pool)
|
|
if n.Spec.RADOS.Namespace != "" {
|
|
osdCaps = fmt.Sprintf("%s namespace=%s", osdCaps, n.Spec.RADOS.Namespace)
|
|
}
|
|
|
|
caps := []string{"mon", "allow r", "osd", osdCaps}
|
|
user := getNFSClientID(n, name)
|
|
|
|
ownerInfo := k8sutil.NewOwnerInfo(n, r.scheme)
|
|
s := keyring.GetSecretStore(r.context, r.clusterInfo, ownerInfo)
|
|
|
|
key, err := s.GenerateKey(user, caps)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create user %s", user)
|
|
}
|
|
|
|
if r.shouldRotateCephxKeys {
|
|
log.NamedInfo(nsName, logger, "rotating cephx key for nfs daemon %q", instanceName(n, name))
|
|
newKey, err := s.RotateKey(user)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to rotate cephx key for nfs daemon %q", instanceName(n, name))
|
|
}
|
|
key = newKey
|
|
}
|
|
|
|
keyring := fmt.Sprintf(keyringTemplate, user, key, osdCaps)
|
|
_, err = s.CreateOrUpdate(instanceName(n, name), keyring)
|
|
return err
|
|
}
|
|
|
|
func getGaneshaConfig(n *cephv1.CephNFS, version cephver.CephVersion, name string) string {
|
|
nodeID := getNFSNodeID(n, name)
|
|
userID := getNFSUserID(nodeID)
|
|
url := getRadosURL(n)
|
|
return `
|
|
NFS_CORE_PARAM {
|
|
Enable_NLM = false;
|
|
Enable_RQUOTA = false;
|
|
Protocols = 4;
|
|
allow_set_io_flusher_fail = true;
|
|
}
|
|
|
|
MDCACHE {
|
|
Dir_Chunk = 0;
|
|
}
|
|
|
|
EXPORT_DEFAULTS {
|
|
Attr_Expiration_Time = 0;
|
|
}
|
|
|
|
NFSv4 {
|
|
Delegations = false;
|
|
RecoveryBackend = "rados_cluster";
|
|
Minor_Versions = 1, 2;
|
|
}
|
|
|
|
RADOS_KV {
|
|
ceph_conf = "` + cephclient.DefaultConfigFilePath() + `";
|
|
userid = ` + userID + `;
|
|
nodeid = ` + nodeID + `;
|
|
pool = "` + n.Spec.RADOS.Pool + `";
|
|
namespace = "` + n.Spec.RADOS.Namespace + `";
|
|
}
|
|
|
|
RADOS_URLS {
|
|
ceph_conf = "` + cephclient.DefaultConfigFilePath() + `";
|
|
userid = ` + userID + `;
|
|
watch_url = "` + url + `";
|
|
}
|
|
|
|
RGW {
|
|
name = "client.` + userID + `";
|
|
}
|
|
|
|
%url ` + url + `
|
|
`
|
|
}
|
|
|
|
func ganeshaKrbConfigBlock(kerberosSpec *cephv1.KerberosSpec) string {
|
|
return fmt.Sprintf(`NFS_KRB5 {
|
|
PrincipalName = "%s" ;
|
|
KeytabPath = /etc/krb5.keytab ;
|
|
Active_krb5 = YES ;
|
|
}
|
|
`, kerberosSpec.GetPrincipalName())
|
|
}
|
|
|
|
func ganeshaConfigIncludeKrbBlock(nfs *cephv1.CephNFS, radosObjectName string) string {
|
|
// don't use sprintf b/c %u on front makes compiler confused
|
|
return `%url "rados://` + nfs.Spec.RADOS.Pool + `/` + nfs.Spec.RADOS.Namespace + `/` + radosObjectName + `"` + "\n\n"
|
|
}
|
|
|
|
func (r *ReconcileCephNFS) setRadosConfig(nfs *cephv1.CephNFS) error {
|
|
if nfs.Spec.Security.KerberosEnabled() {
|
|
return setKerberosRadosConfig(r.context, r.clusterInfo, nfs)
|
|
}
|
|
return removeKerberosRadosConfig(r.context, r.clusterInfo, nfs)
|
|
}
|
|
|
|
func setKerberosRadosConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, nfs *cephv1.CephNFS) error {
|
|
radosPool := nfs.Spec.RADOS.Pool
|
|
radosNs := nfs.Spec.RADOS.Namespace
|
|
radosInfoStr := fmt.Sprintf("rados://%s/%s/", radosPool, radosNs)
|
|
|
|
nsName := opcontroller.NsName(nfs.Namespace, nfs.Name)
|
|
log.NamedInfo(nsName, logger, "ensuring kerberos configuration exists in rados namespace %s", radosInfoStr)
|
|
|
|
// write ganesha kerberos configuration block into a temp file
|
|
krbBlockFile, err := os.CreateTemp("", "krb-block-file")
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create temp file for ganesha kerberos configuration block for %s", radosInfoStr)
|
|
}
|
|
defer krbBlockFile.Close()
|
|
_, err = krbBlockFile.WriteString(ganeshaKrbConfigBlock(nfs.Spec.Security.Kerberos))
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed write ganesha kerberos configuration block temp file for %s", radosInfoStr)
|
|
}
|
|
|
|
radosFlags := []string{
|
|
"--pool", radosPool,
|
|
"--namespace", radosNs,
|
|
}
|
|
|
|
// write ganesha kerberos configuration block to rados object from temp file
|
|
cmd := cephclient.NewRadosCommand(context, clusterInfo,
|
|
append(radosFlags, "put", kerberosRadosObjectName, krbBlockFile.Name()))
|
|
_, err = cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create or update the ganesha kerberos config object %s/%s",
|
|
radosInfoStr, kerberosRadosObjectName)
|
|
}
|
|
|
|
// prepend the config block that includes the kerberos config object to the ganesha config object
|
|
ganeshaConfigObjName := getGaneshaConfigObject(nfs)
|
|
krbIncludeBlock := ganeshaConfigIncludeKrbBlock(nfs, kerberosRadosObjectName)
|
|
err = atomicPrependToConfigObject(context, clusterInfo, radosPool, radosNs, ganeshaConfigObjName, krbIncludeBlock)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to update the ganesha config object to include the kerberos object config")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func removeKerberosRadosConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, nfs *cephv1.CephNFS) error {
|
|
radosPool := nfs.Spec.RADOS.Pool
|
|
radosNs := nfs.Spec.RADOS.Namespace
|
|
radosInfoStr := fmt.Sprintf("rados://%s/%s/", radosPool, radosNs)
|
|
|
|
nsName := opcontroller.NsName(nfs.Namespace, nfs.Name)
|
|
log.NamedInfo(nsName, logger, "ensuring kerberos configuration is removed from rados namespace %s", radosInfoStr)
|
|
|
|
// remove config block that includes the kerberos config object from the ganesha config object
|
|
ganeshaConfigObjName := getGaneshaConfigObject(nfs)
|
|
krbIncludeBlock := ganeshaConfigIncludeKrbBlock(nfs, kerberosRadosObjectName)
|
|
err := atomicRemoveFromConfigObject(context, clusterInfo, radosPool, radosNs, ganeshaConfigObjName, krbIncludeBlock)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to update the ganesha config object to remove the kerberos object config")
|
|
}
|
|
|
|
// remove the kerberos config rados object
|
|
err = cephclient.RadosRemoveObject(context, clusterInfo, radosPool, radosNs, kerberosRadosObjectName)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to remove the ganesha kerberos config object")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// prepend a config block to the rados object in an atomic function
|
|
// uses rados locks to ensure the changes we make here aren't going to race with changes that might
|
|
// be made by ceph mgr, ceph itself, or a user manually
|
|
func atomicPrependToConfigObject(
|
|
context *clusterd.Context, clusterInfo *cephclient.ClusterInfo,
|
|
radosPool, radosNamespace, objectName, configBlock string,
|
|
) error {
|
|
tmpFilePattern := fmt.Sprintf("%s_%s_%s_prepend", radosPool, radosNamespace, objectName)
|
|
objInfoString := fmt.Sprintf("rados://%s/%s/%s", radosPool, radosNamespace, objectName)
|
|
nsName := opcontroller.NsName(clusterInfo.Namespace, radosPool)
|
|
|
|
// read object into temp file
|
|
tempFile, err := os.CreateTemp("", tmpFilePattern)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create temp file for %s", objInfoString)
|
|
}
|
|
defer tempFile.Close()
|
|
|
|
radosFlags := []string{
|
|
"--pool", radosPool,
|
|
"--namespace", radosNamespace,
|
|
}
|
|
|
|
// acquire lock to ensure no other processes (user, ceph, or rook) are racing each other
|
|
// most common contender will be CSI when creating/removing NFS exports
|
|
lockName := AppName
|
|
|
|
// planning to perform 2 commands after the lock: get object, and optionally write object, so
|
|
// use lock timeout of 2x the normal ceph command timeout
|
|
log.NamedInfo(nsName, logger, "locking rados object %q", objInfoString)
|
|
cookie, err := cephclient.RadosLockObject(context, clusterInfo,
|
|
radosPool, radosNamespace, objectName, lockName, exec.CephCommandsTimeout*2)
|
|
if err != nil {
|
|
return err // already a good err message
|
|
}
|
|
log.NamedInfo(nsName, logger, "successfully locked rados object %q", objInfoString)
|
|
defer func() {
|
|
log.NamedInfo(nsName, logger, "unlocking rados object %q", objInfoString)
|
|
err := cephclient.RadosUnlockObject(context, clusterInfo,
|
|
radosPool, radosNamespace, objectName, lockName, cookie)
|
|
if err != nil {
|
|
log.NamedInfo(nsName, logger, "failed to unlock rados object %q, but since the lock has a timeout, we will continue. %v", objInfoString, err)
|
|
}
|
|
log.NamedInfo(nsName, logger, "successfully unlocked rados object %q", objInfoString)
|
|
}()
|
|
|
|
cmd := cephclient.NewRadosCommand(context, clusterInfo,
|
|
append(radosFlags, "get", objectName, tempFile.Name()))
|
|
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
|
return errors.Wrapf(err, "failed to get object %s", objInfoString)
|
|
}
|
|
|
|
rawObj, err := io.ReadAll(tempFile)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to read object %s from temp file", objInfoString)
|
|
}
|
|
|
|
if strings.Contains(string(rawObj), configBlock) {
|
|
log.NamedDebug(nsName, logger, "rados object %s already has config block: %s", objInfoString, configBlock)
|
|
return nil
|
|
}
|
|
log.NamedDebug(nsName, logger, "rados object %s will have config block prepended: %s", objInfoString, configBlock)
|
|
|
|
newConfig := fmt.Sprintf("%s%s", configBlock, string(rawObj))
|
|
if err := os.WriteFile(tempFile.Name(), []byte(newConfig), fs.FileMode(0o644)); err != nil {
|
|
return errors.Wrapf(err, "failed to write new config content for object %s to temp file", objInfoString)
|
|
}
|
|
|
|
cmd = cephclient.NewRadosCommand(context, clusterInfo,
|
|
append(radosFlags, "put", objectName, tempFile.Name()))
|
|
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
|
return errors.Wrapf(err, "failed to set new config content on object %s", objInfoString)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// remove a config block from the rados object in an atomic function
|
|
// uses rados locks to ensure the changes we make here aren't going to race with changes that might
|
|
// be made by ceph mgr, ceph itself, or a user manually
|
|
func atomicRemoveFromConfigObject(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo,
|
|
radosPool, radosNamespace, objectName, configBlock string,
|
|
) error {
|
|
tmpFilePattern := fmt.Sprintf("%s_%s_%s_remove", radosPool, radosNamespace, objectName)
|
|
objInfoString := fmt.Sprintf("rados://%s/%s/%s", radosPool, radosNamespace, objectName)
|
|
nsName := opcontroller.NsName(clusterInfo.Namespace, radosPool)
|
|
|
|
// read object into temp file
|
|
tempFile, err := os.CreateTemp("", tmpFilePattern)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create temp file for %s", objInfoString)
|
|
}
|
|
defer tempFile.Close()
|
|
|
|
radosFlags := []string{
|
|
"--pool", radosPool,
|
|
"--namespace", radosNamespace,
|
|
}
|
|
|
|
// acquire lock to ensure no other processes (user, ceph, or rook) are racing each other
|
|
// most common contender will be CSI when creating/removing NFS exports
|
|
lockName := AppName
|
|
|
|
// planning to perform 2 commands after the lock: get object, and optionally write object, so
|
|
// use lock timeout of 2x the normal ceph command timeout
|
|
log.NamedInfo(nsName, logger, "locking rados object %q", objInfoString)
|
|
cookie, err := cephclient.RadosLockObject(context, clusterInfo,
|
|
radosPool, radosNamespace, objectName, lockName, exec.CephCommandsTimeout*2)
|
|
if err != nil {
|
|
return err // already a good err message
|
|
}
|
|
log.NamedInfo(nsName, logger, "successfully locked rados object %q", objInfoString)
|
|
defer func() {
|
|
log.NamedInfo(nsName, logger, "unlocking rados object %q", objInfoString)
|
|
err := cephclient.RadosUnlockObject(context, clusterInfo,
|
|
radosPool, radosNamespace, objectName, lockName, cookie)
|
|
if err != nil {
|
|
log.NamedInfo(nsName, logger, "failed to unlock rados object %q, but since the lock has a timeout, we will continue. %v", objInfoString, err)
|
|
}
|
|
log.NamedInfo(nsName, logger, "successfully unlocked rados object %q", objInfoString)
|
|
}()
|
|
|
|
cmd := cephclient.NewRadosCommand(context, clusterInfo,
|
|
append(radosFlags, "get", objectName, tempFile.Name()))
|
|
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
|
return errors.Wrapf(err, "failed to get object %s", objInfoString)
|
|
}
|
|
|
|
rawObj, err := io.ReadAll(tempFile)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to read object %s from temp file", objInfoString)
|
|
}
|
|
|
|
if !strings.Contains(string(rawObj), configBlock) {
|
|
log.NamedDebug(nsName, logger, "rados object %q already does not have config block: %s", objInfoString, configBlock)
|
|
return nil
|
|
}
|
|
log.NamedDebug(nsName, logger, "rados object %s will have config block removed: %s", objInfoString, configBlock)
|
|
|
|
newConfig := strings.ReplaceAll(string(rawObj), configBlock, "")
|
|
if err := os.WriteFile(tempFile.Name(), []byte(newConfig), fs.FileMode(0o644)); err != nil {
|
|
return errors.Wrapf(err, "failed to write new config content for object %s to temp file", objInfoString)
|
|
}
|
|
|
|
cmd = cephclient.NewRadosCommand(context, clusterInfo,
|
|
append(radosFlags, "put", objectName, tempFile.Name()))
|
|
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
|
return errors.Wrapf(err, "failed to set new config content on object %s", objInfoString)
|
|
}
|
|
|
|
return nil
|
|
}
|