nfs: make provisioner as sidecar of nfs server

Signed-off-by: Ahmad Nurus S <prksu.sh@gmail.com>
This commit is contained in:
Ahmad Nurus S committed 2020-08-21 16:40:33 +07:00
1 parent 92fffca1e4
commit 5cc2ceda2d
7 files changed
+37 -202

No files matched your search

+2 -8
View File
@@ -9,12 +9,6 @@ metadata:
name: rook-nfs-server
namespace: rook-nfs
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-nfs-provisioner
namespace: rook-nfs
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
@@ -52,10 +46,10 @@ rules:
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-provisioner
name: rook-nfs-provisioner-runner
subjects:
- kind: ServiceAccount
name: rook-nfs-provisioner
name: rook-nfs-server
# replace with namespace where provisioner is deployed
namespace: rook-nfs
roleRef:
@@ -67,7 +67,6 @@ rules:
- apiGroups:
- apps
resources:
- deployments
- statefulsets
verbs:
- create
+9 -75
View File
@@ -19,7 +19,7 @@ package nfs
import (
"context"
"fmt"
"path/filepath"
"path"
"strings"
"time"
@@ -33,7 +33,6 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
ctrl "sigs.k8s.io/controller-runtime"
@@ -124,12 +123,6 @@ func (r *NFSServerReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr
return reconcile.Result{}, err
}
if err := r.reconcileNFSProvisioner(ctx, instance); err != nil {
r.Recorder.Eventf(instance, corev1.EventTypeWarning, nfsv1alpha1.EventFailed, "Failed reconciling nfsprovisioner: %+v", err)
r.Log.Errorf("Error reconciling nfsprovisioner: %+v", err)
return reconcile.Result{}, err
}
// Reconcile status state based on statefulset ready replicas.
sts := &appsv1.StatefulSet{}
if err := r.Client.Get(ctx, req.NamespacedName, sts); err != nil {
@@ -166,8 +159,8 @@ func (r *NFSServerReconciler) reconcileNFSServerConfig(ctx context.Context, cr *
nfsGaneshaConfig := `
EXPORT {
Export_Id = ` + fmt.Sprintf("%v", id) + `;
Path = ` + filepath.Join(mountPath, claimName) + `;
Pseudo = ` + filepath.Join(mountPath, claimName) + `;
Path = ` + path.Join("/", claimName) + `;
Pseudo = ` + path.Join("/", claimName) + `;
Protocols = 4;
Transports = TCP;
Sectype = sys;
@@ -295,12 +288,16 @@ func (r *NFSServerReconciler) reconcileNFSServer(ctx context.Context, cr *nfsv1a
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: shareName,
MountPath: filepath.Join(mountPath, claimName),
MountPath: path.Join("/", claimName),
})
}
sts.Spec.Template.Spec.Volumes = volumes
sts.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
for i, container := range sts.Spec.Template.Spec.Containers {
if container.Name == "nfs-server" || container.Name == "nfs-provisioner" {
sts.Spec.Template.Spec.Containers[i].VolumeMounts = volumeMounts
}
}
return nil
})
@@ -321,66 +318,3 @@ func (r *NFSServerReconciler) reconcileNFSServer(ctx context.Context, cr *nfsv1a
return nil
}
}
func (r *NFSServerReconciler) reconcileNFSProvisioner(ctx context.Context, cr *nfsv1alpha1.NFSServer) error {
nfsserversvc := &corev1.Service{}
if err := r.Client.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.Name}, nfsserversvc); err != nil {
return err
}
dep := newDeploymentForNFSProvisioner(cr)
depop, err := controllerutil.CreateOrUpdate(ctx, r.Client, dep, func() error {
if dep.ObjectMeta.CreationTimestamp.IsZero() {
dep.Spec.Selector = &metav1.LabelSelector{
MatchLabels: newLabels(cr),
}
}
if err := controllerutil.SetControllerReference(cr, dep, r.Scheme); err != nil {
return err
}
var volumes []corev1.Volume
var volumeMounts []corev1.VolumeMount
for _, export := range cr.Spec.Exports {
shareName := export.Name
claimName := export.PersistentVolumeClaim.ClaimName
volumes = append(volumes, corev1.Volume{
Name: shareName,
VolumeSource: corev1.VolumeSource{
NFS: &corev1.NFSVolumeSource{
Server: nfsserversvc.Spec.ClusterIP,
Path: filepath.Join(mountPath, claimName),
},
},
})
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: shareName,
MountPath: filepath.Join(mountPath, claimName),
})
}
dep.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
dep.Spec.Template.Spec.Volumes = volumes
return nil
})
if err != nil {
return err
}
r.Log.Info("Reconciling NFSProvisioner Deployment", "Operation.Result ", depop)
switch depop {
case controllerutil.OperationResultCreated:
r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventCreated, "%s nfs-provisioner deployment: %s", strings.Title(string(depop)), dep.Name)
return nil
case controllerutil.OperationResultUpdated:
r.Recorder.Eventf(cr, corev1.EventTypeNormal, nfsv1alpha1.EventUpdated, "%s nfs-provisioner deployment: %s", strings.Title(string(depop)), dep.Name)
return nil
default:
return nil
}
}
+7 -58
View File
@@ -18,14 +18,13 @@ package nfs
import (
"context"
"path/filepath"
"path"
"reflect"
"testing"
nfsv1alpha1 "github.com/rook/rook/pkg/apis/nfs.rook.io/v1alpha1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
@@ -33,7 +32,6 @@ import (
"k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -141,12 +139,16 @@ func TestNFSServerReconciler_Reconcile(t *testing.T) {
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: shareName,
MountPath: filepath.Join(mountPath, claimName),
MountPath: path.Join("/", claimName),
})
}
sts.Status.ReadyReplicas = int32(cr.Spec.Replicas)
sts.Spec.Template.Spec.Volumes = volumes
sts.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
for i, container := range sts.Spec.Template.Spec.Containers {
if container.Name == "nfs-server" || container.Name == "nfs-provisioner" {
sts.Spec.Template.Spec.Containers[i].VolumeMounts = volumeMounts
}
}
return sts
}
@@ -157,42 +159,6 @@ func TestNFSServerReconciler_Reconcile(t *testing.T) {
return svc
}
expectedProvisionerFunc := func(scheme *runtime.Scheme, cr *nfsv1alpha1.NFSServer) *appsv1.Deployment {
dep := newDeploymentForNFSProvisioner(cr)
dep.Spec.Selector = &metav1.LabelSelector{
MatchLabels: newLabels(cr),
}
_ = controllerutil.SetControllerReference(cr, dep, scheme)
var volumes []corev1.Volume
var volumeMounts []corev1.VolumeMount
for _, export := range cr.Spec.Exports {
shareName := export.Name
claimName := export.PersistentVolumeClaim.ClaimName
volumes = append(volumes, v1.Volume{
Name: shareName,
VolumeSource: v1.VolumeSource{
NFS: &v1.NFSVolumeSource{
Server: expectedServerServiceFunc(scheme, cr).Spec.ClusterIP,
Path: filepath.Join(mountPath, claimName),
},
},
})
volumeMounts = append(volumeMounts, v1.VolumeMount{
Name: shareName,
MountPath: filepath.Join(mountPath, claimName),
})
}
dep.Status.ReadyReplicas = int32(cr.Spec.Replicas)
dep.Spec.Template.Spec.Volumes = volumes
dep.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
return dep
}
rr := reconcile.Request{
NamespacedName: types.NamespacedName{
Name: "nfs-server",
@@ -256,18 +222,15 @@ func TestNFSServerReconciler_Reconcile(t *testing.T) {
expectedServer := expectedServerFunc(scheme, tt.cr)
expectedServerService := expectedServerServiceFunc(scheme, tt.cr)
expectedProvisioner := expectedProvisionerFunc(scheme, tt.cr)
objs := []runtime.Object{
tt.cr,
expectedServer,
expectedServerService,
expectedProvisioner,
}
expectedServer.GetObjectKind().SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("StatefulSet"))
expectedServerService.GetObjectKind().SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service"))
expectedProvisioner.GetObjectKind().SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment"))
fc := fake.NewFakeClient(objs...)
fr := record.NewFakeRecorder(2)
@@ -304,20 +267,6 @@ func TestNFSServerReconciler_Reconcile(t *testing.T) {
if !reflect.DeepEqual(gotServerService, expectedServerService) {
t.Errorf("NFSServerReconciler.Reconcile() = %v, want %v", gotServerService, expectedServerService)
}
gotProvisioner := &appsv1.Deployment{}
provisionerObjKey := client.ObjectKey{
Name: tt.args.req.Name + "-provisioner",
Namespace: tt.args.req.Namespace,
}
if err := fc.Get(context.Background(), provisionerObjKey, gotProvisioner); err != nil {
t.Errorf("NFSServerReconciler.Reconcile() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotProvisioner, expectedProvisioner) {
t.Errorf("NFSServerReconciler.Reconcile() = %v, want %v", gotProvisioner, expectedProvisioner)
}
})
}
}
+9 -11
View File
@@ -19,7 +19,7 @@ package nfs
import (
"fmt"
"os"
"path/filepath"
"path"
"strings"
"github.com/pkg/errors"
@@ -40,7 +40,7 @@ const (
)
var (
mountPath = "/export"
mountPath = "/"
)
type Provisioner struct {
@@ -99,7 +99,7 @@ func (p *Provisioner) Provision(options controller.ProvisionOptions) (*v1.Persis
for _, export := range nfsserver.Spec.Exports {
if export.Name == exportName {
exportPath = filepath.Join(mountPath, export.PersistentVolumeClaim.ClaimName)
exportPath = path.Join(mountPath, export.PersistentVolumeClaim.ClaimName)
found = true
}
}
@@ -109,13 +109,11 @@ func (p *Provisioner) Provision(options controller.ProvisionOptions) (*v1.Persis
}
pvName := strings.Join([]string{options.PVC.Namespace, options.PVC.Name, options.PVName}, "-")
fullPath := filepath.Join(exportPath, pvName)
if err := os.MkdirAll(fullPath, 0777); err != nil {
fullPath := path.Join(exportPath, pvName)
if err := os.MkdirAll(fullPath, 0700); err != nil {
return nil, errors.New("unable to create directory to provision new pv: " + err.Error())
}
_ = os.Chmod(fullPath, 0777)
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: options.PVName,
@@ -141,8 +139,8 @@ func (p *Provisioner) Provision(options controller.ProvisionOptions) (*v1.Persis
}
func (p *Provisioner) Delete(volume *v1.PersistentVolume) error {
path := volume.Spec.PersistentVolumeSource.NFS.Path
pvName := filepath.Base(path)
nfsPath := volume.Spec.PersistentVolumeSource.NFS.Path
pvName := path.Base(nfsPath)
sc, err := p.storageClassForPV(volume)
if err != nil {
@@ -176,7 +174,7 @@ func (p *Provisioner) Delete(volume *v1.PersistentVolume) error {
for _, export := range nfsserver.Spec.Exports {
if export.Name == exportName {
exportPath = filepath.Join(mountPath, export.PersistentVolumeClaim.ClaimName)
exportPath = path.Join(mountPath, export.PersistentVolumeClaim.ClaimName)
found = true
}
}
@@ -185,7 +183,7 @@ func (p *Provisioner) Delete(volume *v1.PersistentVolume) error {
return fmt.Errorf("No export name from storageclass is match with NFSServer %s in namespace %s", nfsserver.Name, nfsserver.Namespace)
}
fullPath := filepath.Join(exportPath, pvName)
fullPath := path.Join(exportPath, pvName)
return os.RemoveAll(fullPath)
}
+9 -41
View File
@@ -67,6 +67,7 @@ func newServiceForNFSServer(cr *nfsv1alpha1.NFSServer) *corev1.Service {
}
func newStatefulSetForNFSServer(cr *nfsv1alpha1.NFSServer) *appsv1.StatefulSet {
privilaged := true
replicas := int32(cr.Spec.Replicas)
return &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
@@ -87,10 +88,9 @@ func newStatefulSetForNFSServer(cr *nfsv1alpha1.NFSServer) *appsv1.StatefulSet {
ServiceAccountName: "rook-nfs-server",
Containers: []corev1.Container{
{
ImagePullPolicy: "IfNotPresent",
Name: cr.Name,
Image: "rook/nfs:master",
Args: []string{"nfs", "server", "--ganeshaConfigPath=" + nfsConfigMapPath + "/" + cr.Name},
Name: "nfs-server",
Image: "rook/nfs:master",
Args: []string{"nfs", "server", "--ganeshaConfigPath=" + nfsConfigMapPath + "/" + cr.Name},
Ports: []corev1.ContainerPort{
{
Name: "nfs-port",
@@ -110,49 +110,17 @@ func newStatefulSetForNFSServer(cr *nfsv1alpha1.NFSServer) *appsv1.StatefulSet {
},
},
},
},
},
},
},
}
}
func newDeploymentForNFSProvisioner(cr *nfsv1alpha1.NFSServer) *appsv1.Deployment {
name := cr.Name + "-provisioner"
provisionerName := "nfs.rook.io/" + name
replicas := int32(1)
defaultTerminationGracePeriodSeconds := int64(30)
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: cr.Namespace,
Labels: newLabels(cr),
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: cr.Name,
Namespace: cr.Namespace,
Labels: newLabels(cr),
},
Spec: corev1.PodSpec{
ServiceAccountName: "rook-nfs-provisioner",
Containers: []corev1.Container{
{
Name: name,
Name: "nfs-provisioner",
Image: "rook/nfs:master",
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{"nfs", "provisioner", "--provisioner=" + provisionerName},
Args: []string{"nfs", "provisioner", "--provisioner=" + "nfs.rook.io/" + cr.Name + "-provisioner"},
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
SecurityContext: &corev1.SecurityContext{
Privileged: &privilaged,
},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
TerminationGracePeriodSeconds: &defaultTerminationGracePeriodSeconds,
DNSPolicy: corev1.DNSClusterFirst,
SecurityContext: &corev1.PodSecurityContext{},
SchedulerName: corev1.DefaultSchedulerName,
},
},
},
+1 -8
View File
@@ -240,7 +240,6 @@ rules:
- apiGroups:
- apps
resources:
- deployments
- statefulsets
verbs:
- create
@@ -429,12 +428,6 @@ metadata:
name: rook-nfs-server
namespace: ` + namespace + `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-nfs-provisioner
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
@@ -475,7 +468,7 @@ metadata:
name: run-nfs-provisioner
subjects:
- kind: ServiceAccount
name: rook-nfs-provisioner
name: rook-nfs-server
namespace: ` + namespace + `
roleRef:
kind: ClusterRole