my-rook-config/pkg/operator/nfs/controller.go

/*
Copyright 2018 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package nfs

import (
	"context"
	"fmt"
	"path"
	"strings"
	"time"

	nfsv1alpha1 "github.com/rook/rook/pkg/apis/nfs.rook.io/v1alpha1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/operator/k8sutil"

	"github.com/coreos/pkg/capnslog"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/pointer"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

const (
	nfsConfigMapPath = "/nfs-ganesha/config"
	nfsPort          = 2049
	rpcPort          = 111
)

type NFSServerReconciler struct {
	client.Client
	Context  *clusterd.Context
	Scheme   *runtime.Scheme
	Log      *capnslog.PackageLogger
	Recorder record.EventRecorder
}

func (r *NFSServerReconciler) Reconcile(context context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) {

	instance := &nfsv1alpha1.NFSServer{}
	if err := r.Client.Get(context, req.NamespacedName, instance); err != nil {
		if errors.IsNotFound(err) {
			return reconcile.Result{}, nil
		}

		return reconcile.Result{}, err
	}

	// Initialize patcher utility and store the initial cr object state to be compare later.
	patcher, err := k8sutil.NewPatcher(instance, r.Client)
	if err != nil {
		return reconcile.Result{}, err
	}

	defer func() {
		// Always patch the cr object if any changes at the end of each reconciliation.
		if err := patcher.Patch(context, instance); err != nil && reterr == nil {
			reterr = err
		}
	}()

	// Add Finalizer if not present
	controllerutil.AddFinalizer(instance, nfsv1alpha1.Finalizer)

	// Handle for deletion. Just remove finalizer
	if !instance.DeletionTimestamp.IsZero() {
		r.Log.Infof("Deleting NFSServer %s in %s namespace", instance.Name, instance.Namespace)

		// no operation since we don't need do anything when nfsserver deleted.
		controllerutil.RemoveFinalizer(instance, nfsv1alpha1.Finalizer)
	}

	// Check status state. if it's empty then initialize it
	// otherwise if has error state then skip reconciliation to prevent requeue on error.
	switch instance.Status.State {
	case "":
		instance.Status.State = nfsv1alpha1.StateInitializing
		r.Log.Info("Initialize status state")
		return reconcile.Result{Requeue: true}, nil
	case nfsv1alpha1.StateError:
		r.Log.Info("Error state detected, skip reconciliation")
		return reconcile.Result{Requeue: false}, nil
	}

	// Validate cr spec and give warning event when validation fail.
	if err := instance.ValidateSpec(); err != nil {
		r.Recorder.Eventf(instance, corev1.EventTypeWarning, nfsv1alpha1.EventFailed, "Invalid NFSServer spec: %+v", err)
		r.Log.Errorf("Invalid NFSServer spec: %+v", err)
		instance.Status.State = nfsv1alpha1.StateError
		return reconcile.Result{}, err
	}

	if err := r.reconcileNFSServerConfig(context, instance); err != nil {
		r.Recorder.Eventf(instance, corev1.EventTypeWarning, nfsv1alpha1.EventFailed, "Failed reconciling nfsserver config: %+v", err)
		r.Log.Errorf("Error reconciling nfsserver config: %+v", err)
		return reconcile.Result{}, err
	}

	if err := r.reconcileNFSServer(context, instance); err != nil {
		r.Recorder.Eventf(instance, corev1.EventTypeWarning, nfsv1alpha1.EventFailed, "Failed reconciling nfsserver: %+v", err)
		r.Log.Errorf("Error reconciling nfsserver: %+v", err)
		return reconcile.Result{}, err
	}

	// Reconcile status state based on statefulset ready replicas.
	sts := &appsv1.StatefulSet{}
	if err := r.Client.Get(context, req.NamespacedName, sts); err != nil {
		return reconcile.Result{}, client.IgnoreNotFound(err)
	}

	switch int(sts.Status.ReadyReplicas) {
	case instance.Spec.Replicas:
		instance.Status.State = nfsv1alpha1.StateRunning
		return reconcile.Result{}, nil
	default:
		instance.Status.State = nfsv1alpha1.StatePending
		return reconcile.Result{RequeueAfter: 10 * time.Second}, nil
	}
}

func (r *NFSServerReconciler) reconcileNFSServerConfig(ctx context.Context, cr *nfsv1alpha1.NFSServer) error {
	var exportsList []string

	id := 10
	for _, export := range cr.Spec.Exports {
		claimName := export.PersistentVolumeClaim.ClaimName
		var accessType string
		// validateNFSServerSpec guarantees `access` will be one of these values at this point
		switch strings.ToLower(export.Server.AccessMode) {
		case "readwrite":
			accessType = "RW"
		case "readonly":
			accessType = "RO"
		case "none":
			accessType = "None"
		}

		nfsGaneshaConfig := `
EXPORT {
	Export_Id = ` + fmt.Sprintf("%v", id) + `;
	Path = ` + path.Join("/", claimName) + `;
	Pseudo = ` + path.Join("/", claimName) + `;
	Protocols = 4;
	Transports = TCP;
	Sectype = sys;
	Access_Type = ` + accessType + `;
	Squash = ` + strings.ToLower(export.Server.Squash) + `;
	FSAL {
		Name = VFS;
	}
}`

		exportsList = append(exportsList, nfsGaneshaConfig)
		id++
	}

	nfsGaneshaAdditionalConfig := `
NFS_Core_Param {
	fsid_device = true;
}
`

	exportsList = append(exportsList, nfsGaneshaAdditionalConfig)
	configdata := make(map[string]string)
	configdata[cr.Name] = strings.Join(exportsList, "\n")
	cm := newConfigMapForNFSServer(cr)
	cmop, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error {
		if err := controllerutil.SetOwnerReference(cr, cm, r.Scheme); err != nil {
			return err
		}

		cm.Data = configdata
		return nil
	})

	if err != nil {
		return err
	}

	r.Log.Info("Reconciling NFSServer ConfigMap", "Operation.Result ", cmop)
	switch cmop {
	case controllerutil.OperationResultCreated:
		r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventCreated, "%s nfs-server config configmap: %s", strings.Title(string(cmop)), cm.Name)
		return nil
	case controllerutil.OperationResultUpdated:
		r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventUpdated, "%s nfs-server config configmap: %s", strings.Title(string(cmop)), cm.Name)
		return nil
	default:
		return nil
	}
}

func (r *NFSServerReconciler) reconcileNFSServer(ctx context.Context, cr *nfsv1alpha1.NFSServer) error {
	svc := newServiceForNFSServer(cr)
	svcop, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error {
		if !svc.ObjectMeta.CreationTimestamp.IsZero() {
			return nil
		}

		if err := controllerutil.SetControllerReference(cr, svc, r.Scheme); err != nil {
			return err
		}

		return nil
	})

	if err != nil {
		return err
	}

	r.Log.Info("Reconciling NFSServer Service", "Operation.Result ", svcop)
	switch svcop {
	case controllerutil.OperationResultCreated:
		r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventCreated, "%s nfs-server service: %s", strings.Title(string(svcop)), svc.Name)
	case controllerutil.OperationResultUpdated:
		r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventUpdated, "%s nfs-server service: %s", strings.Title(string(svcop)), svc.Name)
	}

	sts := newStatefulSetForNFSServer(cr)
	stsop, err := controllerutil.CreateOrUpdate(ctx, r.Client, sts, func() error {
		if sts.ObjectMeta.CreationTimestamp.IsZero() {
			sts.Spec.Selector = &metav1.LabelSelector{
				MatchLabels: newLabels(cr),
			}
		}

		if err := controllerutil.SetControllerReference(cr, sts, r.Scheme); err != nil {
			return err
		}

		volumes := []corev1.Volume{
			{
				Name: cr.Name,
				VolumeSource: corev1.VolumeSource{
					ConfigMap: &corev1.ConfigMapVolumeSource{
						LocalObjectReference: corev1.LocalObjectReference{
							Name: cr.Name,
						},
						Items: []corev1.KeyToPath{
							{
								Key:  cr.Name,
								Path: cr.Name,
							},
						},
						DefaultMode: pointer.Int32Ptr(corev1.ConfigMapVolumeSourceDefaultMode),
					},
				},
			},
		}
		volumeMounts := []corev1.VolumeMount{
			{
				Name:      cr.Name,
				MountPath: nfsConfigMapPath,
			},
		}
		for _, export := range cr.Spec.Exports {
			shareName := export.Name
			claimName := export.PersistentVolumeClaim.ClaimName
			volumes = append(volumes, corev1.Volume{
				Name: shareName,
				VolumeSource: corev1.VolumeSource{
					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
						ClaimName: claimName,
					},
				},
			})

			volumeMounts = append(volumeMounts, corev1.VolumeMount{
				Name:      shareName,
				MountPath: path.Join("/", claimName),
			})
		}

		sts.Spec.Template.Spec.Volumes = volumes
		for i, container := range sts.Spec.Template.Spec.Containers {
			if container.Name == "nfs-server" || container.Name == "nfs-provisioner" {
				sts.Spec.Template.Spec.Containers[i].VolumeMounts = volumeMounts
			}
		}

		return nil
	})

	if err != nil {
		return err
	}

	r.Log.Info("Reconciling NFSServer StatefulSet", "Operation.Result ", stsop)
	switch stsop {
	case controllerutil.OperationResultCreated:
		r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventCreated, "%s nfs-server statefulset: %s", strings.Title(string(stsop)), sts.Name)
		return nil
	case controllerutil.OperationResultUpdated:
		r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventUpdated, "%s nfs-server statefulset: %s", strings.Title(string(stsop)), sts.Name)
		return nil
	default:
		return nil
	}
}