my-rook-config/pkg/operator/ceph/object/rgw.go

/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package rgw for the Ceph object store.
package object

import (
	"fmt"

	cephv1beta1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1beta1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/daemon/ceph/client"
	rgwdaemon "github.com/rook/rook/pkg/daemon/ceph/rgw"
	"github.com/rook/rook/pkg/operator/ceph/pool"
	"github.com/rook/rook/pkg/operator/k8sutil"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	appName        = "rook-ceph-rgw"
	keyringName    = "keyring"
	certVolumeName = "rook-rgw-cert"
	certMountPath  = "/etc/rook/private"
	certKeyName    = "cert"
	certFilename   = "rgw-cert.pem"
)

// Start the rgw manager
func CreateStore(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error {
	return createOrUpdate(context, store, version, hostNetwork, false, ownerRefs)
}

func UpdateStore(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork bool, ownerRefs []metav1.OwnerReference) error {
	return createOrUpdate(context, store, version, hostNetwork, true, ownerRefs)
}

func createOrUpdate(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error {
	// validate the object store settings
	if err := validateStore(context, store); err != nil {
		return fmt.Errorf("invalid object store %s arguments. %+v", store.Name, err)
	}

	// check if the object store already exists
	exists, err := storeExists(context, store)
	if err == nil && exists {
		if !update {
			logger.Infof("object store %s exists in namespace %s", store.Name, store.Namespace)
			return nil
		}
		logger.Infof("object store %s exists in namespace %s. checking for updates", store.Name, store.Namespace)
	}

	logger.Infof("creating object store %s in namespace %s", store.Name, store.Namespace)
	err = createKeyring(context, store, ownerRefs)
	if err != nil {
		return fmt.Errorf("failed to create rgw keyring. %+v", err)
	}

	// start the service
	serviceIP, err := startService(context, store, hostNetwork, ownerRefs)
	if err != nil {
		return fmt.Errorf("failed to start rgw service. %+v", err)
	}

	// create the ceph artifacts for the object store
	objContext := rgwdaemon.NewContext(context, store.Name, store.Namespace)
	err = rgwdaemon.CreateObjectStore(objContext, *store.Spec.MetadataPool.ToModel(""), *store.Spec.DataPool.ToModel(""), serviceIP, store.Spec.Gateway.Port)
	if err != nil {
		return fmt.Errorf("failed to create pools. %+v", err)
	}

	if err := startRGWPods(context, store, version, hostNetwork, update, ownerRefs); err != nil {
		return fmt.Errorf("failed to start pods. %+v", err)
	}

	logger.Infof("created object store %s", store.Name)
	return nil
}

func startRGWPods(context *clusterd.Context, store cephv1beta1.ObjectStore, version string, hostNetwork, update bool, ownerRefs []metav1.OwnerReference) error {

	// if intended to update, remove the old pods so they can be created with the new spec settings
	if update {
		err := k8sutil.DeleteDeployment(context.Clientset, store.Namespace, instanceName(store))
		if err != nil {
			logger.Warning(err.Error())
		}
		err = k8sutil.DeleteDaemonset(context.Clientset, store.Namespace, instanceName(store))
		if err != nil {
			logger.Warning(err.Error())
		}
	}

	// start the deployment or daemonset
	var rgwType string
	var err error
	if store.Spec.Gateway.AllNodes {
		rgwType = "daemonset"
		err = startDaemonset(context, store, version, hostNetwork, ownerRefs)
	} else {
		rgwType = "deployment"
		err = startDeployment(context, store, version, store.Spec.Gateway.Instances, hostNetwork, ownerRefs)
	}

	if err != nil {
		if !errors.IsAlreadyExists(err) {
			return fmt.Errorf("failed to create rgw %store. %+v", rgwType, err)
		}
		logger.Infof("rgw %s already exists", rgwType)
	} else {
		logger.Infof("rgw %s started", rgwType)
	}

	return nil
}

// Delete the object store.
// WARNING: This is a very destructive action that deletes all metadata and data pools.
func DeleteStore(context *clusterd.Context, store cephv1beta1.ObjectStore) error {
	// check if the object store  exists
	exists, err := storeExists(context, store)
	if err != nil {
		return fmt.Errorf("failed to detect if there is an object store to delete. %+v", err)
	}
	if !exists {
		logger.Infof("Object store %s does not exist in namespace %s", store.Name, store.Namespace)
		return nil
	}

	logger.Infof("Deleting object store %s from namespace %s", store.Name, store.Namespace)

	var gracePeriod int64
	propagation := metav1.DeletePropagationForeground
	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}

	// Delete the rgw service
	err = context.Clientset.CoreV1().Services(store.Namespace).Delete(instanceName(store), options)
	if err != nil && !errors.IsNotFound(err) {
		logger.Warningf("failed to delete rgw service. %+v", err)
	}

	// Make a best effort to delete the rgw pods
	err = k8sutil.DeleteDeployment(context.Clientset, store.Namespace, instanceName(store))
	if err != nil {
		logger.Warning(err.Error())
	}
	err = k8sutil.DeleteDaemonset(context.Clientset, store.Namespace, instanceName(store))
	if err != nil {
		logger.Warning(err.Error())
	}

	// Delete the rgw keyring
	err = context.Clientset.CoreV1().Secrets(store.Namespace).Delete(instanceName(store), options)
	if err != nil && !errors.IsNotFound(err) {
		logger.Warningf("failed to delete rgw secret. %+v", err)
	}

	// Delete the realm and pools
	objContext := rgwdaemon.NewContext(context, store.Name, store.Namespace)
	err = rgwdaemon.DeleteObjectStore(objContext)
	if err != nil {
		return fmt.Errorf("failed to delete the realm and pools. %+v", err)
	}

	logger.Infof("Completed deleting object store %s", store.Name)
	return nil
}

// Check if the object store exists depending on either the deployment or the daemonset
func storeExists(context *clusterd.Context, store cephv1beta1.ObjectStore) (bool, error) {
	_, err := context.Clientset.ExtensionsV1beta1().Deployments(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
	if err == nil {
		// the deployment was found
		return true, nil
	}
	if err != nil && !errors.IsNotFound(err) {
		return false, err
	}

	_, err = context.Clientset.ExtensionsV1beta1().DaemonSets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
	if err == nil {
		//  the daemonset was found
		return true, nil
	}
	if err != nil && !errors.IsNotFound(err) {
		return false, err
	}

	// neither one was found
	return false, nil
}

func createKeyring(context *clusterd.Context, store cephv1beta1.ObjectStore, ownerRefs []metav1.OwnerReference) error {
	_, err := context.Clientset.CoreV1().Secrets(store.Namespace).Get(instanceName(store), metav1.GetOptions{})
	if err == nil {
		logger.Infof("the rgw keyring was already generated")
		return nil
	}
	if !errors.IsNotFound(err) {
		return fmt.Errorf("failed to get rgw secrets. %+v", err)
	}

	// create the keyring
	logger.Infof("generating rgw keyring")
	keyring, err := createRGWKeyring(context, store.Namespace)
	if err != nil {
		return fmt.Errorf("failed to create keyring. %+v", err)
	}

	// store the secrets
	secrets := map[string]string{
		keyringName: keyring,
	}
	secret := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      instanceName(store),
			Namespace: store.Namespace,
		},
		StringData: secrets,
		Type:       k8sutil.RookType,
	}
	k8sutil.SetOwnerRefs(context.Clientset, store.Namespace, &secret.ObjectMeta, ownerRefs)
	_, err = context.Clientset.CoreV1().Secrets(store.Namespace).Create(secret)
	if err != nil {
		return fmt.Errorf("failed to save rgw secrets. %+v", err)
	}

	return nil
}

func instanceName(store cephv1beta1.ObjectStore) string {
	return fmt.Sprintf("%s-%s", appName, store.Name)
}

// create a keyring for the rgw client with a limited set of privileges
func createRGWKeyring(context *clusterd.Context, clusterName string) (string, error) {
	username := "client.radosgw.gateway"
	access := []string{"osd", "allow rwx", "mon", "allow rw"}

	// get-or-create-key for the user account
	key, err := client.AuthGetOrCreateKey(context, clusterName, username, access)
	if err != nil {
		return "", fmt.Errorf("failed to get or create auth key for %store. %+v", username, err)
	}

	return key, err
}

// Validate the object store arguments
func validateStore(context *clusterd.Context, s cephv1beta1.ObjectStore) error {
	if s.Name == "" {
		return fmt.Errorf("missing name")
	}
	if s.Namespace == "" {
		return fmt.Errorf("missing namespace")
	}
	if err := pool.ValidatePoolSpec(context, s.Namespace, &s.Spec.MetadataPool); err != nil {
		return fmt.Errorf("invalid metadata pool spec. %+v", err)
	}
	if err := pool.ValidatePoolSpec(context, s.Namespace, &s.Spec.DataPool); err != nil {
		return fmt.Errorf("invalid data pool spec. %+v", err)
	}

	return nil
}