From 23efbf658bc04bca8d5efac1d6f6f811d0a78648 Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Tue, 5 Mar 2019 02:46:52 +0530 Subject: [PATCH] enhance(NFS): adds dynamic provisoning for nfs Signed-off-by: Ashish Ranjan This commit adds dynamic provisioning for nfs --- cluster/examples/kubernetes/nfs/nfs.yaml | 63 ++++++++++ cluster/examples/kubernetes/nfs/operator.yaml | 3 +- cluster/examples/kubernetes/nfs/pvc.yaml | 5 +- cmd/rook/nfs/nfs.go | 1 + cmd/rook/nfs/server.go | 103 +++++++++++++++ images/nfs/Dockerfile | 38 +++--- images/nfs/Makefile | 3 +- images/nfs/start.sh | 57 --------- pkg/apis/nfs.rook.io/v1alpha1/types.go | 7 +- pkg/operator/nfs/controller.go | 109 ++++++++++------ pkg/operator/nfs/controller_test.go | 12 +- pkg/operator/nfs/provisioner.go | 118 ++++++++++++++++++ pkg/operator/nfs/server.go | 100 +++++++++++++++ tests/framework/installer/nfs_installer.go | 28 +---- tests/framework/installer/nfs_manifests.go | 79 ++++++++++-- tests/integration/nfs_test.go | 2 +- 16 files changed, 566 insertions(+), 162 deletions(-) create mode 100644 cmd/rook/nfs/server.go delete mode 100755 images/nfs/start.sh create mode 100644 pkg/operator/nfs/provisioner.go create mode 100644 pkg/operator/nfs/server.go diff --git a/cluster/examples/kubernetes/nfs/nfs.yaml b/cluster/examples/kubernetes/nfs/nfs.yaml index 29d42de47..8f9e980b5 100644 --- a/cluster/examples/kubernetes/nfs/nfs.yaml +++ b/cluster/examples/kubernetes/nfs/nfs.yaml @@ -3,6 +3,68 @@ kind: Namespace metadata: name: rook-nfs --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: rook-nfs + namespace: rook-nfs +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: rook-nfs +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: rook-nfs +subjects: + - kind: ServiceAccount + name: rook-nfs + # replace with namespace where provisioner is deployed + namespace: rook-nfs +roleRef: + kind: Role + name: leader-locking-nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-nfs + namespace: rook-nfs +--- # A default storageclass must be present apiVersion: v1 kind: PersistentVolumeClaim @@ -22,6 +84,7 @@ metadata: name: rook-nfs namespace: rook-nfs spec: + serviceAccountName: rook-nfs replicas: 1 exports: - name: nfs-share diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index 95c6d08b2..2464251c5 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -23,12 +23,13 @@ metadata: name: rook-nfs-operator rules: - apiGroups: - - "" + - "*" resources: - namespaces - configmaps - pods - services + - storageclasses verbs: - get - watch diff --git a/cluster/examples/kubernetes/nfs/pvc.yaml b/cluster/examples/kubernetes/nfs/pvc.yaml index 84d70ec8d..721d442bb 100644 --- a/cluster/examples/kubernetes/nfs/pvc.yaml +++ b/cluster/examples/kubernetes/nfs/pvc.yaml @@ -2,10 +2,11 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: rook-nfs-pv-claim + annotations: + volume.beta.kubernetes.io/storage-class: "nfs-share" spec: - storageClassName: rook-nfs-sc accessModes: - ReadWriteMany resources: requests: - storage: 1Mi + storage: 1Mi \ No newline at end of file diff --git a/cmd/rook/nfs/nfs.go b/cmd/rook/nfs/nfs.go index cbd21276d..5740d009d 100644 --- a/cmd/rook/nfs/nfs.go +++ b/cmd/rook/nfs/nfs.go @@ -36,6 +36,7 @@ var ( func init() { Cmd.AddCommand(operatorCmd) + Cmd.AddCommand(serverCmd) } func createContext() *clusterd.Context { diff --git a/cmd/rook/nfs/server.go b/cmd/rook/nfs/server.go new file mode 100644 index 000000000..3dcec60ba --- /dev/null +++ b/cmd/rook/nfs/server.go @@ -0,0 +1,103 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/operator/nfs" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" +) + +var serverCmd = &cobra.Command{ + Use: "server", + Short: "Runs the NFS server to deploy and manage NFS server in kubernetes clusters", + Long: `Runs the NFS operator to deploy and manage NFS server in kubernetes clusters. +https://github.com/rook/rook`, +} + +var ( + provisioner, server, path, ganeshaConfigPath *string +) + +func init() { + flags.SetFlagsFromEnv(serverCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetLoggingFlags(serverCmd.Flags()) + + provisioner = serverCmd.Flags().String("provisioner", "", "Name of the provisioner. The provisioner will only provision volumes for claims that request a StorageClass with a provisioner field set equal to this name.") + server = serverCmd.Flags().String("server", os.Getenv("ROOK_NFS_SERVICE_HOST"), "Name of the NFS server") + ganeshaConfigPath = serverCmd.Flags().String("ganeshaConfigPath", "", "ConfigPath of nfs ganesha") + + serverCmd.RunE = startServer +} + +func startServer(cmd *cobra.Command, args []string) error { + rook.SetLogLevel() + rook.LogStartupInfo(serverCmd.Flags()) + if len(*provisioner) == 0 { + return errors.New("--provisioner is a required parameter") + } else if len(*ganeshaConfigPath) == 0 { + return errors.New("--ganeshaConfigPath is a required parameter") + } + + clientset, _, _, err := rook.GetClientset() + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to get k8s clients. %+v", err)) + } + + logger.Infof("Setting up NFS server!") + + err = nfs.Setup(*ganeshaConfigPath) + if err != nil { + logger.Fatalf("Error setting up NFS server: %v", err) + } + + logger.Infof("starting NFS server") + go func() { + for { + // This blocks until server exits (presumably due to an error) + err = nfs.Run(*ganeshaConfigPath) + if err != nil { + logger.Errorf("NFS server Exited Unexpectedly with err: %v", err) + } + + // take a moment before trying to restart + time.Sleep(time.Second) + } + }() + + // Wait for NFS server to come up before continuing provisioner process + time.Sleep(5 * time.Second) + + serverVersion, err := clientset.Discovery().ServerVersion() + if err != nil { + logger.Fatalf("Error getting server version: %v", err) + } + + clientNFSProvisioner := nfs.NewNFSProvisioner(clientset, *server) + + pc := controller.NewProvisionController(clientset, *provisioner, clientNFSProvisioner, serverVersion.GitVersion) + pc.Run(wait.NeverStop) + return nil +} diff --git a/images/nfs/Dockerfile b/images/nfs/Dockerfile index 85f2de584..ca6be1510 100644 --- a/images/nfs/Dockerfile +++ b/images/nfs/Dockerfile @@ -15,36 +15,26 @@ #Portions of this file came from https://github.com/kubernetes-incubator/external-storage/blob/master/nfs/deploy/docker/Dockerfile, which uses the same license. FROM NFS_BASEIMAGE - # Build ganesha from source, installing deps and removing them in one line. # Why? # 1. Root_Id_Squash, only present in >= 2.4.0.3 which is not yet packaged # 2. Set NFS_V4_RECOV_ROOT to /export # 3. Use device major/minor as fsid major/minor to work on OverlayFS -RUN dnf install -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel jemalloc-devel libnfsidmap-devel patch libnsl2-devel && dnf clean all \ - && curl -L https://github.com/nfs-ganesha/nfs-ganesha/archive/V2.4.1.tar.gz | tar zx \ - && curl -L https://github.com/nfs-ganesha/nfs-ganesha/commit/c583534b166be198755d905c82a7687d83b458d8.patch -o /nfs-ganesha.patch \ - && curl -L https://github.com/nfs-ganesha/ntirpc/archive/v1.4.4.tar.gz | tar zx \ - && rm -r nfs-ganesha-2.4.1/src/libntirpc \ - && mv ntirpc-1.4.4 nfs-ganesha-2.4.1/src/libntirpc \ - && cd nfs-ganesha-2.4.1 \ - && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_CONFIG=vfs_only src/ \ - && patch -p1 < /nfs-ganesha.patch \ - && make \ - && make install \ - && cp src/scripts/ganeshactl/org.ganesha.nfsd.conf /etc/dbus-1/system.d/ \ - && cd .. \ - && rm -rf nfs-ganesha-2.4.1 /nfs-ganesha.patch \ - && dnf remove -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel jemalloc-devel libnfsidmap-devel patch && dnf clean all - -RUN dnf install -y dbus-x11 rpcbind hostname nfs-utils xfsprogs jemalloc libnfsidmap && dnf clean all - -RUN mkdir -p /var/run/dbus -RUN mkdir -p /export - -COPY start.sh / - +RUN DEBIAN_FRONTEND=noninteractive \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FE869A9 \ + && echo "deb http://ppa.launchpad.net/gluster/nfs-ganesha-2.5/ubuntu xenial main" > /etc/apt/sources.list.d/nfs-ganesha-2.5.list \ + && echo "deb http://ppa.launchpad.net/gluster/libntirpc-1.5/ubuntu xenial main" > /etc/apt/sources.list.d/libntirpc-1.5.list \ + && echo "deb http://ppa.launchpad.net/gluster/glusterfs-3.13/ubuntu xenial main" > /etc/apt/sources.list.d/glusterfs-3.13.list \ + && apt-get update \ + && apt-get install -y netbase nfs-common dbus nfs-ganesha nfs-ganesha-vfs glusterfs-common \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && mkdir -p /run/rpcbind /export /var/run/dbus \ + && touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr \ + && chmod 755 /run/rpcbind/* \ + && chown messagebus:messagebus /var/run/dbus + EXPOSE 2049 38465-38467 662 111/udp 111 COPY rook /usr/local/bin/ diff --git a/images/nfs/Makefile b/images/nfs/Makefile index 440e3e18a..2b9c46555 100755 --- a/images/nfs/Makefile +++ b/images/nfs/Makefile @@ -19,7 +19,7 @@ include ../image.mk NFS_IMAGE = $(BUILD_REGISTRY)/nfs-$(GOARCH) -NFS_BASE ?= fedora:28 +NFS_BASE ?= ubuntu:xenial ifeq ($(GOARCH),amd64) NFS_BASEIMAGE = $(NFS_BASE) @@ -41,7 +41,6 @@ IMAGE_FILENAME := $(IMAGE_OUTPUT_DIR)/nfs.tar.gz do.build: @echo === docker build $(NFS_IMAGE) @cp Dockerfile $(TEMP) - @cp start.sh $(TEMP) @cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP) @cd $(TEMP) && $(SED_CMD) 's|NFS_BASEIMAGE|$(NFS_BASEIMAGE)|g' Dockerfile @$(DOCKERCMD) build $(BUILD_ARGS) \ diff --git a/images/nfs/start.sh b/images/nfs/start.sh deleted file mode 100755 index 0a46f217c..000000000 --- a/images/nfs/start.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Rook Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#Portions of this file came from https://github.com/mitcdh/docker-nfs-ganesha/blob/master/start.sh - -set -e - -# Options for starting Ganesha -: ${GANESHA_LOGFILE:="/dev/stdout"} -: ${GANESHA_CONFIGFILE:="/nfs-ganesha/config/nfs-ganesha-config"} -: ${GANESHA_OPTIONS:="-N NIV_DEBUG"} # NIV_DEBUG -: ${GANESHA_EPOCH:=""} - -function init_rpc { - echo "Starting rpcbind" - rpcbind || return 0 - rpc.statd -L || return 0 - rpc.idmapd || return 0 - sleep 1 -} - -function init_dbus { - echo "Starting dbus" - rm -f /var/run/dbus/system_bus_socket - rm -f /var/run/dbus/pid - dbus-uuidgen --ensure - dbus-daemon --system --fork - sleep 1 -} - -function startup_script { - if [ -f "${STARTUP_SCRIPT}" ]; then - /bin/sh ${STARTUP_SCRIPT} - fi -} - -startup_script - -init_rpc -init_dbus - -echo "Starting Ganesha NFS" -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib -exec /usr/bin/ganesha.nfsd -F -L ${GANESHA_LOGFILE} -f ${GANESHA_CONFIGFILE} ${GANESHA_OPTIONS} diff --git a/pkg/apis/nfs.rook.io/v1alpha1/types.go b/pkg/apis/nfs.rook.io/v1alpha1/types.go index 144311814..7fd8d358d 100644 --- a/pkg/apis/nfs.rook.io/v1alpha1/types.go +++ b/pkg/apis/nfs.rook.io/v1alpha1/types.go @@ -17,7 +17,7 @@ package v1alpha1 import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -53,6 +53,11 @@ type NFSServerSpec struct { // Replicas of the NFS daemon Replicas int `json:"replicas,omitempty"` + // ServiceAccountName is the name of the ServiceAccount to use to run this nfs server pod. + // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + // +optional + ServiceAccountName string `json:"serviceAccountName,omitempty" protobuf:"bytes,8,opt,name=serviceAccountName"` + // The parameters to configure the NFS export Exports []ExportsSpec `json:"exports,omitempty"` } diff --git a/pkg/operator/nfs/controller.go b/pkg/operator/nfs/controller.go index 3a9d75e9d..abad5f068 100644 --- a/pkg/operator/nfs/controller.go +++ b/pkg/operator/nfs/controller.go @@ -28,7 +28,8 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,9 +41,7 @@ import ( const ( customResourceName = "nfsserver" customResourceNamePlural = "nfsservers" - appName = "rook-nfs" - nfsConfigMapName = "nfs-ganesha-config" - nfsConfigMapPath = "/nfs-ganesha/config" + NFSConfigMapPath = "/nfs-ganesha/config" nfsPort = 2049 rpcPort = 111 noneMode = "none" @@ -99,7 +98,7 @@ type nfsServer struct { func newNfsServer(c *nfsv1alpha1.NFSServer, context *clusterd.Context) *nfsServer { return &nfsServer{ - name: appName, + name: c.Name, context: context, namespace: c.Namespace, spec: c.Spec, @@ -134,9 +133,9 @@ func getServerConfig(exports []nfsv1alpha1.ExportsSpec) map[string]map[string]st return claimConfigOpt } -func createAppLabels() map[string]string { +func createAppLabels(nfsServer *nfsServer) map[string]string { return map[string]string{ - k8sutil.AppAttr: appName, + k8sutil.AppAttr: nfsServer.name, } } @@ -155,32 +154,58 @@ func createServicePorts() []v1.ServicePort { } } -func (c *Controller) createNFSService(nfsServer *nfsServer) error { +func (c *Controller) createStorageClass(nfsServer *nfsServer) error { + for _, export := range nfsServer.spec.Exports { + nfsStorageClass := &storage.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: export.Name, + Annotations: map[string]string{ + "rook.io/nfs-path": "/" + export.PersistentVolumeClaim.ClaimName, + }, + OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, + Labels: createAppLabels(nfsServer), + }, + Provisioner: "rook.io/" + nfsServer.name, + } + if _, err := c.context.Clientset.StorageV1().StorageClasses().Create(nfsStorageClass); err != nil { + if !errors.IsAlreadyExists(err) { + return err + } + logger.Infof("storage class %s already exists", nfsStorageClass.GetName()) + } else { + logger.Infof("storage class %s created", nfsStorageClass.GetName()) + } + } + return nil +} + +func (c *Controller) createNFSService(nfsServer *nfsServer) (*v1.Service, error) { // This service is meant to be used by clients to access NFS. nfsService := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: nfsServer.name, Namespace: nfsServer.namespace, OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), }, Spec: v1.ServiceSpec{ - Selector: createAppLabels(), + Selector: createAppLabels(nfsServer), Type: v1.ServiceTypeClusterIP, Ports: createServicePorts(), }, } - if _, err := c.context.Clientset.CoreV1().Services(nfsServer.namespace).Create(nfsService); err != nil { + svc, err := c.context.Clientset.CoreV1().Services(nfsServer.namespace).Create(nfsService) + if err != nil { if !errors.IsAlreadyExists(err) { - return err + return nil, err } logger.Infof("nfs service %s already exists in namespace %s", nfsService.Name, nfsService.Namespace) } else { logger.Infof("nfs service %s started in namespace %s", nfsService.Name, nfsService.Namespace) } - return nil + return svc, nil } func createCephNFSExport(id int, path string, access string, squash string) string { @@ -241,13 +266,13 @@ func (c *Controller) createNFSConfigMap(nfsServer *nfsServer) error { configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: nfsConfigMapName, + Name: nfsServer.name, Namespace: nfsServer.namespace, OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), }, Data: map[string]string{ - nfsConfigMapName: nfsGaneshaConfig, + nfsServer.name: nfsGaneshaConfig, }, } _, err := c.context.Clientset.CoreV1().ConfigMaps(nfsServer.namespace).Create(configMap) @@ -271,9 +296,9 @@ func getPVCNameList(spec *nfsv1alpha1.NFSServerSpec) []string { return pvcNameList } -func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume { +func createPVCSpecList(nfsServer *nfsServer) []v1.Volume { pvcSpecList := make([]v1.Volume, 0) - pvcNameList := getPVCNameList(spec) + pvcNameList := getPVCNameList(&nfsServer.spec) for _, claimName := range pvcNameList { pvcSpecList = append(pvcSpecList, v1.Volume{ Name: claimName, @@ -288,14 +313,14 @@ func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume { configMapSrc := &v1.ConfigMapVolumeSource{ Items: []v1.KeyToPath{ { - Key: nfsConfigMapName, - Path: nfsConfigMapName, + Key: nfsServer.name, + Path: nfsServer.name, }, }, } - configMapSrc.Name = nfsConfigMapName + configMapSrc.Name = nfsServer.name configMapVol := v1.Volume{ - Name: nfsConfigMapName, + Name: nfsServer.name, VolumeSource: v1.VolumeSource{ ConfigMap: configMapSrc, }, @@ -305,9 +330,9 @@ func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume { return pvcSpecList } -func createVolumeMountList(spec *nfsv1alpha1.NFSServerSpec) []v1.VolumeMount { +func createVolumeMountList(nfsServer *nfsServer) []v1.VolumeMount { volumeMountList := make([]v1.VolumeMount, 0) - pvcNameList := getPVCNameList(spec) + pvcNameList := getPVCNameList(&nfsServer.spec) for _, claimName := range pvcNameList { volumeMountList = append(volumeMountList, v1.VolumeMount{ Name: claimName, @@ -316,27 +341,29 @@ func createVolumeMountList(spec *nfsv1alpha1.NFSServerSpec) []v1.VolumeMount { } configMapVolMount := v1.VolumeMount{ - Name: nfsConfigMapName, - MountPath: nfsConfigMapPath, + Name: nfsServer.name, + MountPath: NFSConfigMapPath, } volumeMountList = append(volumeMountList, configMapVolMount) return volumeMountList } -func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { +func (c *Controller) createNfsPodSpec(nfsServer *nfsServer, service *v1.Service) v1.PodTemplateSpec { nfsPodSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: nfsServer.name, Namespace: nfsServer.namespace, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), }, Spec: v1.PodSpec{ + ServiceAccountName: nfsServer.spec.ServiceAccountName, Containers: []v1.Container{ { - Name: nfsServer.name, - Image: c.containerImage, - Command: []string{"/start.sh"}, + ImagePullPolicy: "IfNotPresent", + Name: nfsServer.name, + Image: c.containerImage, + Args: []string{"nfs", "server", "--provisioner=rook.io/" + nfsServer.name, "--ganeshaConfigPath=" + NFSConfigMapPath + "/" + nfsServer.name, "--server=" + service.Spec.ClusterIP}, Ports: []v1.ContainerPort{ { Name: "nfs-port", @@ -347,7 +374,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { ContainerPort: int32(rpcPort), }, }, - VolumeMounts: createVolumeMountList(&nfsServer.spec), + VolumeMounts: createVolumeMountList(nfsServer), SecurityContext: &v1.SecurityContext{ Capabilities: &v1.Capabilities{ Add: []v1.Capability{ @@ -358,7 +385,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { }, }, }, - Volumes: createPVCSpecList(&nfsServer.spec), + Volumes: createPVCSpecList(nfsServer), }, } @@ -368,19 +395,19 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { func (c *Controller) createNfsStatefulSet(nfsServer *nfsServer, replicas int32) error { appsClient := c.context.Clientset.AppsV1() - nfsPodSpec := c.createNfsPodSpec(nfsServer) + nfsPodSpec := c.createNfsPodSpec(nfsServer, service) statefulSet := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: nfsServer.name, Namespace: nfsServer.namespace, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, }, Spec: appsv1.StatefulSetSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ - MatchLabels: createAppLabels(), + MatchLabels: createAppLabels(nfsServer), }, Template: nfsPodSpec, ServiceName: nfsServer.name, @@ -415,7 +442,8 @@ func (c *Controller) onAdd(obj interface{}) { } logger.Infof("creating nfs server service in namespace %s", nfsServer.namespace) - if err := c.createNFSService(nfsServer); err != nil { + svc, err := c.createNFSService(nfsServer) + if err != nil { logger.Errorf("Unable to create NFS service %+v", err) } @@ -425,9 +453,14 @@ func (c *Controller) onAdd(obj interface{}) { } logger.Infof("creating nfs server stateful set in namespace %s", nfsServer.namespace) - if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas)); err != nil { + if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas), svc); err != nil { logger.Errorf("Unable to create NFS stateful set %+v", err) } + + logger.Infof("creating nfs server storageclass") + if err := c.createStorageClass(nfsServer); err != nil { + logger.Errorf("Unable to create NFS storageclass %+v", err) + } } func (c *Controller) onUpdate(oldObj, newObj interface{}) { diff --git a/pkg/operator/nfs/controller_test.go b/pkg/operator/nfs/controller_test.go index c3db1b825..a0394d832 100644 --- a/pkg/operator/nfs/controller_test.go +++ b/pkg/operator/nfs/controller_test.go @@ -25,11 +25,13 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) +const appName = "nfs-server-X" + func TestValidateNFSServerSpec(t *testing.T) { // first, test that a good NFSServerSpec is good @@ -122,13 +124,13 @@ func TestOnAdd(t *testing.T) { controller.onAdd(nfsserver) // verify client service - clientService, err := clientset.CoreV1().Services(namespace).Get(appName, metav1.GetOptions{}) + clientService, err := clientset.CoreV1().Services(namespace).Get(nfsserver.GetName(), metav1.GetOptions{}) assert.Nil(t, err) assert.NotNil(t, clientService) assert.Equal(t, v1.ServiceTypeClusterIP, clientService.Spec.Type) // verify nfs-ganesha config in the configmap - configMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(nfsConfigMapName, metav1.GetOptions{}) + configMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(nfsserver.GetName(), metav1.GetOptions{}) assert.Nil(t, err) assert.NotNil(t, configMap) nfsGaneshaConfig := ` @@ -149,7 +151,7 @@ NFS_Core_Param { fsid_device = true; }` - assert.Equal(t, nfsGaneshaConfig, configMap.Data[nfsConfigMapName]) + assert.Equal(t, nfsGaneshaConfig, configMap.Data[nfsserver.GetName()]) // verify stateful set ss, err := clientset.AppsV1().StatefulSets(namespace).Get(appName, metav1.GetOptions{}) @@ -161,7 +163,7 @@ NFS_Core_Param container := ss.Spec.Template.Spec.Containers[0] assert.Equal(t, 2, len(container.VolumeMounts)) - expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: "nfs-ganesha-config", MountPath: "/nfs-ganesha/config"}} + expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: nfsserver.GetName(), MountPath: "/nfs-ganesha/config"}} assert.Equal(t, expectedVolumeMounts, container.VolumeMounts) } diff --git a/pkg/operator/nfs/provisioner.go b/pkg/operator/nfs/provisioner.go new file mode 100644 index 000000000..51f37e3dd --- /dev/null +++ b/pkg/operator/nfs/provisioner.go @@ -0,0 +1,118 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "fmt" + + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" + v1 "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/apis/core/v1/helper" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type nfsProvisioner struct { + client kubernetes.Interface + server string +} + +const ( + claimPath = "rook.io/nfs-path" +) + +var _ controller.Provisioner = &nfsProvisioner{} + +// NewNFSProvisioner returns an instance of nfsProvisioner +func NewNFSProvisioner(clientset kubernetes.Interface, server string) nfsProvisioner { + return nfsProvisioner{clientset, server} +} + +func (p nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) { + logger.Infof("nfs provisioner: VolumeOptions %v", options) + + // Get the storage class for this volume. + storageClass, err := p.getClassForPVC(options.PVC) + if err != nil { + return nil, err + } + + if options.PVC.Spec.Selector != nil { + return nil, fmt.Errorf("claim Selector is not supported") + } + + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: options.PVName, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy, + AccessModes: options.PVC.Spec.AccessModes, + MountOptions: options.MountOptions, + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)], + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + NFS: &v1.NFSVolumeSource{ + Server: p.server, + Path: storageClass.GetAnnotations()[claimPath], + ReadOnly: false, + }, + }, + }, + } + + return pv, nil +} + +func (p nfsProvisioner) Delete(volume *v1.PersistentVolume) error { + return nil +} + +// getClassForPV returns StorageClass +func (p nfsProvisioner) getClassForPV(pv *v1.PersistentVolume) (*storage.StorageClass, error) { + if p.client == nil { + return nil, fmt.Errorf("Cannot get kube client") + } + className := helper.GetPersistentVolumeClass(pv) + if className == "" { + return nil, fmt.Errorf("Volume has no storage class") + } + class, err := p.client.StorageV1().StorageClasses().Get(className, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return class, nil +} + +// getClassForPVC returns StorageClass +func (p nfsProvisioner) getClassForPVC(pvc *v1.PersistentVolumeClaim) (*storage.StorageClass, error) { + if p.client == nil { + return nil, fmt.Errorf("Cannot get kube client") + } + className := helper.GetPersistentVolumeClaimClass(pvc) + if className == "" { + return nil, fmt.Errorf("Volume has no storage class") + } + class, err := p.client.StorageV1().StorageClasses().Get(className, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return class, nil +} diff --git a/pkg/operator/nfs/server.go b/pkg/operator/nfs/server.go new file mode 100644 index 000000000..4f190fb57 --- /dev/null +++ b/pkg/operator/nfs/server.go @@ -0,0 +1,100 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "fmt" + "os/exec" + "syscall" +) + +const ( + ganeshaLog = "/dev/stdout" + ganeshaOptions = "NIV_DEBUG" +) + +// Setup sets up various prerequisites and settings for the server. If an error +// is encountered at any point it returns it instantly +func Setup(ganeshaConfig string) error { + // Start rpcbind if it is not started yet + cmd := exec.Command("rpcinfo", "127.0.0.1") + if err := cmd.Run(); err != nil { + cmd = exec.Command("rpcbind", "-w") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("Starting rpcbind failed with error: %v, output: %s", err, out) + } + } + + cmd = exec.Command("rpc.statd") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("rpc.statd failed with error: %v, output: %s", err, out) + } + + // Start dbus, needed for ganesha dynamic exports + cmd = exec.Command("dbus-daemon", "--system") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("dbus-daemon failed with error: %v, output: %s", err, out) + } + + err := setRlimitNOFILE() + if err != nil { + logger.Warningf("Error setting RLIMIT_NOFILE, there may be 'Too many open files' errors later: %v", err) + } + return nil +} + +// Run : run the NFS server in the foreground until it exits +// Ideally, it should never exit when run in foreground mode +// We force foreground to allow the provisioner process to restart +// the server if it crashes - daemonization prevents us from using Wait() +// for this purpose +func Run(ganeshaConfig string) error { + // Start ganesha.nfsd + logger.Infof("Running NFS server!") + cmd := exec.Command("ganesha.nfsd", "-F", "-L", ganeshaLog, "-f", ganeshaConfig, "-N", ganeshaOptions) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("ganesha.nfsd failed with error: %v, output: %s", err, out) + } + + return nil +} + +func setRlimitNOFILE() error { + var rlimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + return fmt.Errorf("error getting RLIMIT_NOFILE: %v", err) + } + logger.Infof("starting RLIMIT_NOFILE rlimit.Cur %d, rlimit.Max %d", rlimit.Cur, rlimit.Max) + rlimit.Max = 1024 * 1024 + rlimit.Cur = 1024 * 1024 + err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + return err + } + err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + return fmt.Errorf("error getting RLIMIT_NOFILE: %v", err) + } + logger.Infof("ending RLIMIT_NOFILE rlimit.Cur %d, rlimit.Max %d", rlimit.Cur, rlimit.Max) + return nil +} + +// Stop stops the NFS server. +func Stop() { + // /bin/dbus-send --system --dest=org.ganesha.nfsd --type=method_call /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.shutdown +} diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index 5542a18b8..2d8c110b5 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -113,12 +113,12 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass return fmt.Errorf("Failed to create nfs server: %+v ", err) } - if err := h.k8shelper.WaitForPodCount("app=rook-nfs", namespace, 1); err != nil { + if err := h.k8shelper.WaitForPodCount("app="+namespace, namespace, 1); err != nil { logger.Errorf("nfs server pods in namespace %s not found", namespace) return err } - err := h.k8shelper.WaitForLabeledPodsToRun("app=rook-nfs", namespace) + err := h.k8shelper.WaitForLabeledPodsToRun("app="+namespace, namespace) if err != nil { logger.Errorf("nfs server pods in namespace %s are not running", namespace) return err @@ -132,18 +132,8 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error { logger.Info("creating volume from nfs server in namespace %s", namespace) - clusterIP, err := h.GetNFSServerClusterIP(namespace) - if err != nil { - return err - } - nfsServerPV := h.manifests.GetNFSServerPV(namespace, clusterIP) nfsServerPVC := h.manifests.GetNFSServerPVC() - logger.Info("creating nfs server pv") - if _, err := h.k8shelper.KubectlWithStdin(nfsServerPV, createFromStdinArgs...); err != nil { - return err - } - logger.Info("creating nfs server pvc") if _, err := h.k8shelper.KubectlWithStdin(nfsServerPVC, createFromStdinArgs...); err != nil { return err @@ -193,6 +183,8 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { checkError(h.T(), err, "cannot uninstall hostpath provisioner") h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("run-nfs-client-provisioner", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("nfs-client-provisioner-runner", nil) logger.Infof("done removing the operator from namespace %s", systemNamespace) } @@ -202,15 +194,3 @@ func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testNa h.k8shelper.GetLogs("rook-nfs-operator", Env.HostType, systemNamespace, testName) h.k8shelper.GetLogs("rook-nfs", Env.HostType, namespace, testName) } - -// GetNFSServerClusterIP gets the nfs server cluster ip on which it serves -func (h *NFSInstaller) GetNFSServerClusterIP(namespace string) (string, error) { - clusterIP := "" - service, err := h.k8shelper.GetService("rook-nfs", namespace) - if err != nil { - logger.Errorf("nfs server service in namespace %s is not active", namespace) - return clusterIP, err - } - clusterIP = service.Spec.ClusterIP - return clusterIP, nil -} diff --git a/tests/framework/installer/nfs_manifests.go b/tests/framework/installer/nfs_manifests.go index a66ffd96b..6a61fff69 100644 --- a/tests/framework/installer/nfs_manifests.go +++ b/tests/framework/installer/nfs_manifests.go @@ -43,10 +43,10 @@ spec: // GetNFSServerOperator returns the NFSServer operator definition func (i *NFSManifests) GetNFSServerOperator(namespace string) string { - return `kind: Namespace -apiVersion: v1 + return `apiVersion: v1 +kind: Namespace metadata: - name: ` + namespace + ` + name: ` + namespace + ` --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole @@ -54,12 +54,13 @@ metadata: name: rook-nfs-operator rules: - apiGroups: - - "" + - "*" resources: - namespaces - configmaps - pods - services + - storageclasses verbs: - get - watch @@ -124,6 +125,7 @@ spec: serviceAccountName: rook-nfs-operator containers: - name: rook-nfs-operator + imagePullPolicy: IfNotPresent image: rook/nfs:master args: ["nfs", "operator"] env: @@ -183,7 +185,7 @@ kind: PersistentVolumeClaim metadata: name: nfs-pv-claim spec: - storageClassName: nfs-sc + storageClassName: nfs-share accessModes: - ReadWriteMany resources: @@ -195,7 +197,7 @@ kind: PersistentVolumeClaim metadata: name: nfs-pv-claim-bigger spec: - storageClassName: nfs-sc + storageClassName: nfs-share1 accessModes: - ReadWriteMany resources: @@ -206,7 +208,69 @@ spec: // GetNFSServer returns NFSServer CRD instance definition func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string { - return `apiVersion: v1 + return `kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: ` + namespace + ` + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: ` + namespace + ` +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: ` + namespace + ` +subjects: + - kind: ServiceAccount + name: ` + namespace + ` + # replace with namespace where provisioner is deployed + namespace: ` + namespace + ` +roleRef: + kind: Role + name: leader-locking-nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ` + namespace + ` + namespace: ` + namespace + ` +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-claim @@ -238,6 +302,7 @@ metadata: name: ` + namespace + ` namespace: ` + namespace + ` spec: + serviceAccountName: ` + namespace + ` replicas: ` + strconv.Itoa(count) + ` exports: - name: nfs-share diff --git a/tests/integration/nfs_test.go b/tests/integration/nfs_test.go index 39cfe7acf..ebf198c66 100644 --- a/tests/integration/nfs_test.go +++ b/tests/integration/nfs_test.go @@ -105,7 +105,7 @@ func (suite *NfsSuite) TestNfsServerInstallation() { "1 rook-nfs-operator must be in Running state") // verify nfs server instances are running OK - assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"), + assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"), fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount)) // verify bigger export is running OK