From 23efbf658bc04bca8d5efac1d6f6f811d0a78648 Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Tue, 5 Mar 2019 02:46:52 +0530 Subject: [PATCH 1/5] enhance(NFS): adds dynamic provisoning for nfs Signed-off-by: Ashish Ranjan This commit adds dynamic provisioning for nfs --- cluster/examples/kubernetes/nfs/nfs.yaml | 63 ++++++++++ cluster/examples/kubernetes/nfs/operator.yaml | 3 +- cluster/examples/kubernetes/nfs/pvc.yaml | 5 +- cmd/rook/nfs/nfs.go | 1 + cmd/rook/nfs/server.go | 103 +++++++++++++++ images/nfs/Dockerfile | 38 +++--- images/nfs/Makefile | 3 +- images/nfs/start.sh | 57 --------- pkg/apis/nfs.rook.io/v1alpha1/types.go | 7 +- pkg/operator/nfs/controller.go | 109 ++++++++++------ pkg/operator/nfs/controller_test.go | 12 +- pkg/operator/nfs/provisioner.go | 118 ++++++++++++++++++ pkg/operator/nfs/server.go | 100 +++++++++++++++ tests/framework/installer/nfs_installer.go | 28 +---- tests/framework/installer/nfs_manifests.go | 79 ++++++++++-- tests/integration/nfs_test.go | 2 +- 16 files changed, 566 insertions(+), 162 deletions(-) create mode 100644 cmd/rook/nfs/server.go delete mode 100755 images/nfs/start.sh create mode 100644 pkg/operator/nfs/provisioner.go create mode 100644 pkg/operator/nfs/server.go diff --git a/cluster/examples/kubernetes/nfs/nfs.yaml b/cluster/examples/kubernetes/nfs/nfs.yaml index 29d42de47..8f9e980b5 100644 --- a/cluster/examples/kubernetes/nfs/nfs.yaml +++ b/cluster/examples/kubernetes/nfs/nfs.yaml @@ -3,6 +3,68 @@ kind: Namespace metadata: name: rook-nfs --- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: rook-nfs + namespace: rook-nfs +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: rook-nfs +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: rook-nfs +subjects: + - kind: ServiceAccount + name: rook-nfs + # replace with namespace where provisioner is deployed + namespace: rook-nfs +roleRef: + kind: Role + name: leader-locking-nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-nfs + namespace: rook-nfs +--- # A default storageclass must be present apiVersion: v1 kind: PersistentVolumeClaim @@ -22,6 +84,7 @@ metadata: name: rook-nfs namespace: rook-nfs spec: + serviceAccountName: rook-nfs replicas: 1 exports: - name: nfs-share diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index 95c6d08b2..2464251c5 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -23,12 +23,13 @@ metadata: name: rook-nfs-operator rules: - apiGroups: - - "" + - "*" resources: - namespaces - configmaps - pods - services + - storageclasses verbs: - get - watch diff --git a/cluster/examples/kubernetes/nfs/pvc.yaml b/cluster/examples/kubernetes/nfs/pvc.yaml index 84d70ec8d..721d442bb 100644 --- a/cluster/examples/kubernetes/nfs/pvc.yaml +++ b/cluster/examples/kubernetes/nfs/pvc.yaml @@ -2,10 +2,11 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: rook-nfs-pv-claim + annotations: + volume.beta.kubernetes.io/storage-class: "nfs-share" spec: - storageClassName: rook-nfs-sc accessModes: - ReadWriteMany resources: requests: - storage: 1Mi + storage: 1Mi \ No newline at end of file diff --git a/cmd/rook/nfs/nfs.go b/cmd/rook/nfs/nfs.go index cbd21276d..5740d009d 100644 --- a/cmd/rook/nfs/nfs.go +++ b/cmd/rook/nfs/nfs.go @@ -36,6 +36,7 @@ var ( func init() { Cmd.AddCommand(operatorCmd) + Cmd.AddCommand(serverCmd) } func createContext() *clusterd.Context { diff --git a/cmd/rook/nfs/server.go b/cmd/rook/nfs/server.go new file mode 100644 index 000000000..3dcec60ba --- /dev/null +++ b/cmd/rook/nfs/server.go @@ -0,0 +1,103 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/operator/nfs" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" +) + +var serverCmd = &cobra.Command{ + Use: "server", + Short: "Runs the NFS server to deploy and manage NFS server in kubernetes clusters", + Long: `Runs the NFS operator to deploy and manage NFS server in kubernetes clusters. +https://github.com/rook/rook`, +} + +var ( + provisioner, server, path, ganeshaConfigPath *string +) + +func init() { + flags.SetFlagsFromEnv(serverCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetLoggingFlags(serverCmd.Flags()) + + provisioner = serverCmd.Flags().String("provisioner", "", "Name of the provisioner. The provisioner will only provision volumes for claims that request a StorageClass with a provisioner field set equal to this name.") + server = serverCmd.Flags().String("server", os.Getenv("ROOK_NFS_SERVICE_HOST"), "Name of the NFS server") + ganeshaConfigPath = serverCmd.Flags().String("ganeshaConfigPath", "", "ConfigPath of nfs ganesha") + + serverCmd.RunE = startServer +} + +func startServer(cmd *cobra.Command, args []string) error { + rook.SetLogLevel() + rook.LogStartupInfo(serverCmd.Flags()) + if len(*provisioner) == 0 { + return errors.New("--provisioner is a required parameter") + } else if len(*ganeshaConfigPath) == 0 { + return errors.New("--ganeshaConfigPath is a required parameter") + } + + clientset, _, _, err := rook.GetClientset() + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to get k8s clients. %+v", err)) + } + + logger.Infof("Setting up NFS server!") + + err = nfs.Setup(*ganeshaConfigPath) + if err != nil { + logger.Fatalf("Error setting up NFS server: %v", err) + } + + logger.Infof("starting NFS server") + go func() { + for { + // This blocks until server exits (presumably due to an error) + err = nfs.Run(*ganeshaConfigPath) + if err != nil { + logger.Errorf("NFS server Exited Unexpectedly with err: %v", err) + } + + // take a moment before trying to restart + time.Sleep(time.Second) + } + }() + + // Wait for NFS server to come up before continuing provisioner process + time.Sleep(5 * time.Second) + + serverVersion, err := clientset.Discovery().ServerVersion() + if err != nil { + logger.Fatalf("Error getting server version: %v", err) + } + + clientNFSProvisioner := nfs.NewNFSProvisioner(clientset, *server) + + pc := controller.NewProvisionController(clientset, *provisioner, clientNFSProvisioner, serverVersion.GitVersion) + pc.Run(wait.NeverStop) + return nil +} diff --git a/images/nfs/Dockerfile b/images/nfs/Dockerfile index 85f2de584..ca6be1510 100644 --- a/images/nfs/Dockerfile +++ b/images/nfs/Dockerfile @@ -15,36 +15,26 @@ #Portions of this file came from https://github.com/kubernetes-incubator/external-storage/blob/master/nfs/deploy/docker/Dockerfile, which uses the same license. FROM NFS_BASEIMAGE - # Build ganesha from source, installing deps and removing them in one line. # Why? # 1. Root_Id_Squash, only present in >= 2.4.0.3 which is not yet packaged # 2. Set NFS_V4_RECOV_ROOT to /export # 3. Use device major/minor as fsid major/minor to work on OverlayFS -RUN dnf install -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel jemalloc-devel libnfsidmap-devel patch libnsl2-devel && dnf clean all \ - && curl -L https://github.com/nfs-ganesha/nfs-ganesha/archive/V2.4.1.tar.gz | tar zx \ - && curl -L https://github.com/nfs-ganesha/nfs-ganesha/commit/c583534b166be198755d905c82a7687d83b458d8.patch -o /nfs-ganesha.patch \ - && curl -L https://github.com/nfs-ganesha/ntirpc/archive/v1.4.4.tar.gz | tar zx \ - && rm -r nfs-ganesha-2.4.1/src/libntirpc \ - && mv ntirpc-1.4.4 nfs-ganesha-2.4.1/src/libntirpc \ - && cd nfs-ganesha-2.4.1 \ - && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_CONFIG=vfs_only src/ \ - && patch -p1 < /nfs-ganesha.patch \ - && make \ - && make install \ - && cp src/scripts/ganeshactl/org.ganesha.nfsd.conf /etc/dbus-1/system.d/ \ - && cd .. \ - && rm -rf nfs-ganesha-2.4.1 /nfs-ganesha.patch \ - && dnf remove -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel jemalloc-devel libnfsidmap-devel patch && dnf clean all - -RUN dnf install -y dbus-x11 rpcbind hostname nfs-utils xfsprogs jemalloc libnfsidmap && dnf clean all - -RUN mkdir -p /var/run/dbus -RUN mkdir -p /export - -COPY start.sh / - +RUN DEBIAN_FRONTEND=noninteractive \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FE869A9 \ + && echo "deb http://ppa.launchpad.net/gluster/nfs-ganesha-2.5/ubuntu xenial main" > /etc/apt/sources.list.d/nfs-ganesha-2.5.list \ + && echo "deb http://ppa.launchpad.net/gluster/libntirpc-1.5/ubuntu xenial main" > /etc/apt/sources.list.d/libntirpc-1.5.list \ + && echo "deb http://ppa.launchpad.net/gluster/glusterfs-3.13/ubuntu xenial main" > /etc/apt/sources.list.d/glusterfs-3.13.list \ + && apt-get update \ + && apt-get install -y netbase nfs-common dbus nfs-ganesha nfs-ganesha-vfs glusterfs-common \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && mkdir -p /run/rpcbind /export /var/run/dbus \ + && touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr \ + && chmod 755 /run/rpcbind/* \ + && chown messagebus:messagebus /var/run/dbus + EXPOSE 2049 38465-38467 662 111/udp 111 COPY rook /usr/local/bin/ diff --git a/images/nfs/Makefile b/images/nfs/Makefile index 440e3e18a..2b9c46555 100755 --- a/images/nfs/Makefile +++ b/images/nfs/Makefile @@ -19,7 +19,7 @@ include ../image.mk NFS_IMAGE = $(BUILD_REGISTRY)/nfs-$(GOARCH) -NFS_BASE ?= fedora:28 +NFS_BASE ?= ubuntu:xenial ifeq ($(GOARCH),amd64) NFS_BASEIMAGE = $(NFS_BASE) @@ -41,7 +41,6 @@ IMAGE_FILENAME := $(IMAGE_OUTPUT_DIR)/nfs.tar.gz do.build: @echo === docker build $(NFS_IMAGE) @cp Dockerfile $(TEMP) - @cp start.sh $(TEMP) @cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP) @cd $(TEMP) && $(SED_CMD) 's|NFS_BASEIMAGE|$(NFS_BASEIMAGE)|g' Dockerfile @$(DOCKERCMD) build $(BUILD_ARGS) \ diff --git a/images/nfs/start.sh b/images/nfs/start.sh deleted file mode 100755 index 0a46f217c..000000000 --- a/images/nfs/start.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Copyright 2018 The Rook Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#Portions of this file came from https://github.com/mitcdh/docker-nfs-ganesha/blob/master/start.sh - -set -e - -# Options for starting Ganesha -: ${GANESHA_LOGFILE:="/dev/stdout"} -: ${GANESHA_CONFIGFILE:="/nfs-ganesha/config/nfs-ganesha-config"} -: ${GANESHA_OPTIONS:="-N NIV_DEBUG"} # NIV_DEBUG -: ${GANESHA_EPOCH:=""} - -function init_rpc { - echo "Starting rpcbind" - rpcbind || return 0 - rpc.statd -L || return 0 - rpc.idmapd || return 0 - sleep 1 -} - -function init_dbus { - echo "Starting dbus" - rm -f /var/run/dbus/system_bus_socket - rm -f /var/run/dbus/pid - dbus-uuidgen --ensure - dbus-daemon --system --fork - sleep 1 -} - -function startup_script { - if [ -f "${STARTUP_SCRIPT}" ]; then - /bin/sh ${STARTUP_SCRIPT} - fi -} - -startup_script - -init_rpc -init_dbus - -echo "Starting Ganesha NFS" -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib -exec /usr/bin/ganesha.nfsd -F -L ${GANESHA_LOGFILE} -f ${GANESHA_CONFIGFILE} ${GANESHA_OPTIONS} diff --git a/pkg/apis/nfs.rook.io/v1alpha1/types.go b/pkg/apis/nfs.rook.io/v1alpha1/types.go index 144311814..7fd8d358d 100644 --- a/pkg/apis/nfs.rook.io/v1alpha1/types.go +++ b/pkg/apis/nfs.rook.io/v1alpha1/types.go @@ -17,7 +17,7 @@ package v1alpha1 import ( rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -53,6 +53,11 @@ type NFSServerSpec struct { // Replicas of the NFS daemon Replicas int `json:"replicas,omitempty"` + // ServiceAccountName is the name of the ServiceAccount to use to run this nfs server pod. + // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + // +optional + ServiceAccountName string `json:"serviceAccountName,omitempty" protobuf:"bytes,8,opt,name=serviceAccountName"` + // The parameters to configure the NFS export Exports []ExportsSpec `json:"exports,omitempty"` } diff --git a/pkg/operator/nfs/controller.go b/pkg/operator/nfs/controller.go index 3a9d75e9d..abad5f068 100644 --- a/pkg/operator/nfs/controller.go +++ b/pkg/operator/nfs/controller.go @@ -28,7 +28,8 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,9 +41,7 @@ import ( const ( customResourceName = "nfsserver" customResourceNamePlural = "nfsservers" - appName = "rook-nfs" - nfsConfigMapName = "nfs-ganesha-config" - nfsConfigMapPath = "/nfs-ganesha/config" + NFSConfigMapPath = "/nfs-ganesha/config" nfsPort = 2049 rpcPort = 111 noneMode = "none" @@ -99,7 +98,7 @@ type nfsServer struct { func newNfsServer(c *nfsv1alpha1.NFSServer, context *clusterd.Context) *nfsServer { return &nfsServer{ - name: appName, + name: c.Name, context: context, namespace: c.Namespace, spec: c.Spec, @@ -134,9 +133,9 @@ func getServerConfig(exports []nfsv1alpha1.ExportsSpec) map[string]map[string]st return claimConfigOpt } -func createAppLabels() map[string]string { +func createAppLabels(nfsServer *nfsServer) map[string]string { return map[string]string{ - k8sutil.AppAttr: appName, + k8sutil.AppAttr: nfsServer.name, } } @@ -155,32 +154,58 @@ func createServicePorts() []v1.ServicePort { } } -func (c *Controller) createNFSService(nfsServer *nfsServer) error { +func (c *Controller) createStorageClass(nfsServer *nfsServer) error { + for _, export := range nfsServer.spec.Exports { + nfsStorageClass := &storage.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: export.Name, + Annotations: map[string]string{ + "rook.io/nfs-path": "/" + export.PersistentVolumeClaim.ClaimName, + }, + OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, + Labels: createAppLabels(nfsServer), + }, + Provisioner: "rook.io/" + nfsServer.name, + } + if _, err := c.context.Clientset.StorageV1().StorageClasses().Create(nfsStorageClass); err != nil { + if !errors.IsAlreadyExists(err) { + return err + } + logger.Infof("storage class %s already exists", nfsStorageClass.GetName()) + } else { + logger.Infof("storage class %s created", nfsStorageClass.GetName()) + } + } + return nil +} + +func (c *Controller) createNFSService(nfsServer *nfsServer) (*v1.Service, error) { // This service is meant to be used by clients to access NFS. nfsService := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: nfsServer.name, Namespace: nfsServer.namespace, OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), }, Spec: v1.ServiceSpec{ - Selector: createAppLabels(), + Selector: createAppLabels(nfsServer), Type: v1.ServiceTypeClusterIP, Ports: createServicePorts(), }, } - if _, err := c.context.Clientset.CoreV1().Services(nfsServer.namespace).Create(nfsService); err != nil { + svc, err := c.context.Clientset.CoreV1().Services(nfsServer.namespace).Create(nfsService) + if err != nil { if !errors.IsAlreadyExists(err) { - return err + return nil, err } logger.Infof("nfs service %s already exists in namespace %s", nfsService.Name, nfsService.Namespace) } else { logger.Infof("nfs service %s started in namespace %s", nfsService.Name, nfsService.Namespace) } - return nil + return svc, nil } func createCephNFSExport(id int, path string, access string, squash string) string { @@ -241,13 +266,13 @@ func (c *Controller) createNFSConfigMap(nfsServer *nfsServer) error { configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: nfsConfigMapName, + Name: nfsServer.name, Namespace: nfsServer.namespace, OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), }, Data: map[string]string{ - nfsConfigMapName: nfsGaneshaConfig, + nfsServer.name: nfsGaneshaConfig, }, } _, err := c.context.Clientset.CoreV1().ConfigMaps(nfsServer.namespace).Create(configMap) @@ -271,9 +296,9 @@ func getPVCNameList(spec *nfsv1alpha1.NFSServerSpec) []string { return pvcNameList } -func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume { +func createPVCSpecList(nfsServer *nfsServer) []v1.Volume { pvcSpecList := make([]v1.Volume, 0) - pvcNameList := getPVCNameList(spec) + pvcNameList := getPVCNameList(&nfsServer.spec) for _, claimName := range pvcNameList { pvcSpecList = append(pvcSpecList, v1.Volume{ Name: claimName, @@ -288,14 +313,14 @@ func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume { configMapSrc := &v1.ConfigMapVolumeSource{ Items: []v1.KeyToPath{ { - Key: nfsConfigMapName, - Path: nfsConfigMapName, + Key: nfsServer.name, + Path: nfsServer.name, }, }, } - configMapSrc.Name = nfsConfigMapName + configMapSrc.Name = nfsServer.name configMapVol := v1.Volume{ - Name: nfsConfigMapName, + Name: nfsServer.name, VolumeSource: v1.VolumeSource{ ConfigMap: configMapSrc, }, @@ -305,9 +330,9 @@ func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume { return pvcSpecList } -func createVolumeMountList(spec *nfsv1alpha1.NFSServerSpec) []v1.VolumeMount { +func createVolumeMountList(nfsServer *nfsServer) []v1.VolumeMount { volumeMountList := make([]v1.VolumeMount, 0) - pvcNameList := getPVCNameList(spec) + pvcNameList := getPVCNameList(&nfsServer.spec) for _, claimName := range pvcNameList { volumeMountList = append(volumeMountList, v1.VolumeMount{ Name: claimName, @@ -316,27 +341,29 @@ func createVolumeMountList(spec *nfsv1alpha1.NFSServerSpec) []v1.VolumeMount { } configMapVolMount := v1.VolumeMount{ - Name: nfsConfigMapName, - MountPath: nfsConfigMapPath, + Name: nfsServer.name, + MountPath: NFSConfigMapPath, } volumeMountList = append(volumeMountList, configMapVolMount) return volumeMountList } -func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { +func (c *Controller) createNfsPodSpec(nfsServer *nfsServer, service *v1.Service) v1.PodTemplateSpec { nfsPodSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: nfsServer.name, Namespace: nfsServer.namespace, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), }, Spec: v1.PodSpec{ + ServiceAccountName: nfsServer.spec.ServiceAccountName, Containers: []v1.Container{ { - Name: nfsServer.name, - Image: c.containerImage, - Command: []string{"/start.sh"}, + ImagePullPolicy: "IfNotPresent", + Name: nfsServer.name, + Image: c.containerImage, + Args: []string{"nfs", "server", "--provisioner=rook.io/" + nfsServer.name, "--ganeshaConfigPath=" + NFSConfigMapPath + "/" + nfsServer.name, "--server=" + service.Spec.ClusterIP}, Ports: []v1.ContainerPort{ { Name: "nfs-port", @@ -347,7 +374,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { ContainerPort: int32(rpcPort), }, }, - VolumeMounts: createVolumeMountList(&nfsServer.spec), + VolumeMounts: createVolumeMountList(nfsServer), SecurityContext: &v1.SecurityContext{ Capabilities: &v1.Capabilities{ Add: []v1.Capability{ @@ -358,7 +385,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { }, }, }, - Volumes: createPVCSpecList(&nfsServer.spec), + Volumes: createPVCSpecList(nfsServer), }, } @@ -368,19 +395,19 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec { func (c *Controller) createNfsStatefulSet(nfsServer *nfsServer, replicas int32) error { appsClient := c.context.Clientset.AppsV1() - nfsPodSpec := c.createNfsPodSpec(nfsServer) + nfsPodSpec := c.createNfsPodSpec(nfsServer, service) statefulSet := appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: nfsServer.name, Namespace: nfsServer.namespace, - Labels: createAppLabels(), + Labels: createAppLabels(nfsServer), OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, }, Spec: appsv1.StatefulSetSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ - MatchLabels: createAppLabels(), + MatchLabels: createAppLabels(nfsServer), }, Template: nfsPodSpec, ServiceName: nfsServer.name, @@ -415,7 +442,8 @@ func (c *Controller) onAdd(obj interface{}) { } logger.Infof("creating nfs server service in namespace %s", nfsServer.namespace) - if err := c.createNFSService(nfsServer); err != nil { + svc, err := c.createNFSService(nfsServer) + if err != nil { logger.Errorf("Unable to create NFS service %+v", err) } @@ -425,9 +453,14 @@ func (c *Controller) onAdd(obj interface{}) { } logger.Infof("creating nfs server stateful set in namespace %s", nfsServer.namespace) - if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas)); err != nil { + if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas), svc); err != nil { logger.Errorf("Unable to create NFS stateful set %+v", err) } + + logger.Infof("creating nfs server storageclass") + if err := c.createStorageClass(nfsServer); err != nil { + logger.Errorf("Unable to create NFS storageclass %+v", err) + } } func (c *Controller) onUpdate(oldObj, newObj interface{}) { diff --git a/pkg/operator/nfs/controller_test.go b/pkg/operator/nfs/controller_test.go index c3db1b825..a0394d832 100644 --- a/pkg/operator/nfs/controller_test.go +++ b/pkg/operator/nfs/controller_test.go @@ -25,11 +25,13 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" ) +const appName = "nfs-server-X" + func TestValidateNFSServerSpec(t *testing.T) { // first, test that a good NFSServerSpec is good @@ -122,13 +124,13 @@ func TestOnAdd(t *testing.T) { controller.onAdd(nfsserver) // verify client service - clientService, err := clientset.CoreV1().Services(namespace).Get(appName, metav1.GetOptions{}) + clientService, err := clientset.CoreV1().Services(namespace).Get(nfsserver.GetName(), metav1.GetOptions{}) assert.Nil(t, err) assert.NotNil(t, clientService) assert.Equal(t, v1.ServiceTypeClusterIP, clientService.Spec.Type) // verify nfs-ganesha config in the configmap - configMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(nfsConfigMapName, metav1.GetOptions{}) + configMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(nfsserver.GetName(), metav1.GetOptions{}) assert.Nil(t, err) assert.NotNil(t, configMap) nfsGaneshaConfig := ` @@ -149,7 +151,7 @@ NFS_Core_Param { fsid_device = true; }` - assert.Equal(t, nfsGaneshaConfig, configMap.Data[nfsConfigMapName]) + assert.Equal(t, nfsGaneshaConfig, configMap.Data[nfsserver.GetName()]) // verify stateful set ss, err := clientset.AppsV1().StatefulSets(namespace).Get(appName, metav1.GetOptions{}) @@ -161,7 +163,7 @@ NFS_Core_Param container := ss.Spec.Template.Spec.Containers[0] assert.Equal(t, 2, len(container.VolumeMounts)) - expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: "nfs-ganesha-config", MountPath: "/nfs-ganesha/config"}} + expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: nfsserver.GetName(), MountPath: "/nfs-ganesha/config"}} assert.Equal(t, expectedVolumeMounts, container.VolumeMounts) } diff --git a/pkg/operator/nfs/provisioner.go b/pkg/operator/nfs/provisioner.go new file mode 100644 index 000000000..51f37e3dd --- /dev/null +++ b/pkg/operator/nfs/provisioner.go @@ -0,0 +1,118 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "fmt" + + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" + v1 "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/apis/core/v1/helper" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type nfsProvisioner struct { + client kubernetes.Interface + server string +} + +const ( + claimPath = "rook.io/nfs-path" +) + +var _ controller.Provisioner = &nfsProvisioner{} + +// NewNFSProvisioner returns an instance of nfsProvisioner +func NewNFSProvisioner(clientset kubernetes.Interface, server string) nfsProvisioner { + return nfsProvisioner{clientset, server} +} + +func (p nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) { + logger.Infof("nfs provisioner: VolumeOptions %v", options) + + // Get the storage class for this volume. + storageClass, err := p.getClassForPVC(options.PVC) + if err != nil { + return nil, err + } + + if options.PVC.Spec.Selector != nil { + return nil, fmt.Errorf("claim Selector is not supported") + } + + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: options.PVName, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy, + AccessModes: options.PVC.Spec.AccessModes, + MountOptions: options.MountOptions, + Capacity: v1.ResourceList{ + v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)], + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + NFS: &v1.NFSVolumeSource{ + Server: p.server, + Path: storageClass.GetAnnotations()[claimPath], + ReadOnly: false, + }, + }, + }, + } + + return pv, nil +} + +func (p nfsProvisioner) Delete(volume *v1.PersistentVolume) error { + return nil +} + +// getClassForPV returns StorageClass +func (p nfsProvisioner) getClassForPV(pv *v1.PersistentVolume) (*storage.StorageClass, error) { + if p.client == nil { + return nil, fmt.Errorf("Cannot get kube client") + } + className := helper.GetPersistentVolumeClass(pv) + if className == "" { + return nil, fmt.Errorf("Volume has no storage class") + } + class, err := p.client.StorageV1().StorageClasses().Get(className, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return class, nil +} + +// getClassForPVC returns StorageClass +func (p nfsProvisioner) getClassForPVC(pvc *v1.PersistentVolumeClaim) (*storage.StorageClass, error) { + if p.client == nil { + return nil, fmt.Errorf("Cannot get kube client") + } + className := helper.GetPersistentVolumeClaimClass(pvc) + if className == "" { + return nil, fmt.Errorf("Volume has no storage class") + } + class, err := p.client.StorageV1().StorageClasses().Get(className, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return class, nil +} diff --git a/pkg/operator/nfs/server.go b/pkg/operator/nfs/server.go new file mode 100644 index 000000000..4f190fb57 --- /dev/null +++ b/pkg/operator/nfs/server.go @@ -0,0 +1,100 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "fmt" + "os/exec" + "syscall" +) + +const ( + ganeshaLog = "/dev/stdout" + ganeshaOptions = "NIV_DEBUG" +) + +// Setup sets up various prerequisites and settings for the server. If an error +// is encountered at any point it returns it instantly +func Setup(ganeshaConfig string) error { + // Start rpcbind if it is not started yet + cmd := exec.Command("rpcinfo", "127.0.0.1") + if err := cmd.Run(); err != nil { + cmd = exec.Command("rpcbind", "-w") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("Starting rpcbind failed with error: %v, output: %s", err, out) + } + } + + cmd = exec.Command("rpc.statd") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("rpc.statd failed with error: %v, output: %s", err, out) + } + + // Start dbus, needed for ganesha dynamic exports + cmd = exec.Command("dbus-daemon", "--system") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("dbus-daemon failed with error: %v, output: %s", err, out) + } + + err := setRlimitNOFILE() + if err != nil { + logger.Warningf("Error setting RLIMIT_NOFILE, there may be 'Too many open files' errors later: %v", err) + } + return nil +} + +// Run : run the NFS server in the foreground until it exits +// Ideally, it should never exit when run in foreground mode +// We force foreground to allow the provisioner process to restart +// the server if it crashes - daemonization prevents us from using Wait() +// for this purpose +func Run(ganeshaConfig string) error { + // Start ganesha.nfsd + logger.Infof("Running NFS server!") + cmd := exec.Command("ganesha.nfsd", "-F", "-L", ganeshaLog, "-f", ganeshaConfig, "-N", ganeshaOptions) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("ganesha.nfsd failed with error: %v, output: %s", err, out) + } + + return nil +} + +func setRlimitNOFILE() error { + var rlimit syscall.Rlimit + err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + return fmt.Errorf("error getting RLIMIT_NOFILE: %v", err) + } + logger.Infof("starting RLIMIT_NOFILE rlimit.Cur %d, rlimit.Max %d", rlimit.Cur, rlimit.Max) + rlimit.Max = 1024 * 1024 + rlimit.Cur = 1024 * 1024 + err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + return err + } + err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit) + if err != nil { + return fmt.Errorf("error getting RLIMIT_NOFILE: %v", err) + } + logger.Infof("ending RLIMIT_NOFILE rlimit.Cur %d, rlimit.Max %d", rlimit.Cur, rlimit.Max) + return nil +} + +// Stop stops the NFS server. +func Stop() { + // /bin/dbus-send --system --dest=org.ganesha.nfsd --type=method_call /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.shutdown +} diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index 5542a18b8..2d8c110b5 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -113,12 +113,12 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass return fmt.Errorf("Failed to create nfs server: %+v ", err) } - if err := h.k8shelper.WaitForPodCount("app=rook-nfs", namespace, 1); err != nil { + if err := h.k8shelper.WaitForPodCount("app="+namespace, namespace, 1); err != nil { logger.Errorf("nfs server pods in namespace %s not found", namespace) return err } - err := h.k8shelper.WaitForLabeledPodsToRun("app=rook-nfs", namespace) + err := h.k8shelper.WaitForLabeledPodsToRun("app="+namespace, namespace) if err != nil { logger.Errorf("nfs server pods in namespace %s are not running", namespace) return err @@ -132,18 +132,8 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error { logger.Info("creating volume from nfs server in namespace %s", namespace) - clusterIP, err := h.GetNFSServerClusterIP(namespace) - if err != nil { - return err - } - nfsServerPV := h.manifests.GetNFSServerPV(namespace, clusterIP) nfsServerPVC := h.manifests.GetNFSServerPVC() - logger.Info("creating nfs server pv") - if _, err := h.k8shelper.KubectlWithStdin(nfsServerPV, createFromStdinArgs...); err != nil { - return err - } - logger.Info("creating nfs server pvc") if _, err := h.k8shelper.KubectlWithStdin(nfsServerPVC, createFromStdinArgs...); err != nil { return err @@ -193,6 +183,8 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) { checkError(h.T(), err, "cannot uninstall hostpath provisioner") h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("run-nfs-client-provisioner", nil) + h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("nfs-client-provisioner-runner", nil) logger.Infof("done removing the operator from namespace %s", systemNamespace) } @@ -202,15 +194,3 @@ func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testNa h.k8shelper.GetLogs("rook-nfs-operator", Env.HostType, systemNamespace, testName) h.k8shelper.GetLogs("rook-nfs", Env.HostType, namespace, testName) } - -// GetNFSServerClusterIP gets the nfs server cluster ip on which it serves -func (h *NFSInstaller) GetNFSServerClusterIP(namespace string) (string, error) { - clusterIP := "" - service, err := h.k8shelper.GetService("rook-nfs", namespace) - if err != nil { - logger.Errorf("nfs server service in namespace %s is not active", namespace) - return clusterIP, err - } - clusterIP = service.Spec.ClusterIP - return clusterIP, nil -} diff --git a/tests/framework/installer/nfs_manifests.go b/tests/framework/installer/nfs_manifests.go index a66ffd96b..6a61fff69 100644 --- a/tests/framework/installer/nfs_manifests.go +++ b/tests/framework/installer/nfs_manifests.go @@ -43,10 +43,10 @@ spec: // GetNFSServerOperator returns the NFSServer operator definition func (i *NFSManifests) GetNFSServerOperator(namespace string) string { - return `kind: Namespace -apiVersion: v1 + return `apiVersion: v1 +kind: Namespace metadata: - name: ` + namespace + ` + name: ` + namespace + ` --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole @@ -54,12 +54,13 @@ metadata: name: rook-nfs-operator rules: - apiGroups: - - "" + - "*" resources: - namespaces - configmaps - pods - services + - storageclasses verbs: - get - watch @@ -124,6 +125,7 @@ spec: serviceAccountName: rook-nfs-operator containers: - name: rook-nfs-operator + imagePullPolicy: IfNotPresent image: rook/nfs:master args: ["nfs", "operator"] env: @@ -183,7 +185,7 @@ kind: PersistentVolumeClaim metadata: name: nfs-pv-claim spec: - storageClassName: nfs-sc + storageClassName: nfs-share accessModes: - ReadWriteMany resources: @@ -195,7 +197,7 @@ kind: PersistentVolumeClaim metadata: name: nfs-pv-claim-bigger spec: - storageClassName: nfs-sc + storageClassName: nfs-share1 accessModes: - ReadWriteMany resources: @@ -206,7 +208,69 @@ spec: // GetNFSServer returns NFSServer CRD instance definition func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string { - return `apiVersion: v1 + return `kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: ` + namespace + ` + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: ` + namespace + ` +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: ` + namespace + ` +subjects: + - kind: ServiceAccount + name: ` + namespace + ` + # replace with namespace where provisioner is deployed + namespace: ` + namespace + ` +roleRef: + kind: Role + name: leader-locking-nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ` + namespace + ` + namespace: ` + namespace + ` +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: test-claim @@ -238,6 +302,7 @@ metadata: name: ` + namespace + ` namespace: ` + namespace + ` spec: + serviceAccountName: ` + namespace + ` replicas: ` + strconv.Itoa(count) + ` exports: - name: nfs-share diff --git a/tests/integration/nfs_test.go b/tests/integration/nfs_test.go index 39cfe7acf..ebf198c66 100644 --- a/tests/integration/nfs_test.go +++ b/tests/integration/nfs_test.go @@ -105,7 +105,7 @@ func (suite *NfsSuite) TestNfsServerInstallation() { "1 rook-nfs-operator must be in Running state") // verify nfs server instances are running OK - assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"), + assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"), fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount)) // verify bigger export is running OK From 9f3debc53fd50caa4dc4dda23b0bd43277fcb189 Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Fri, 5 Apr 2019 00:53:37 +0530 Subject: [PATCH 2/5] refactor(nfs,provisioning): move claimPath info from annotation to parameters Signed-off-by: Ashish Ranjan This commit moves claimPath information from annotation to parameters and adds option to enable/disable storageclass creation. --- Gopkg.lock | 26 +++++++++++++++ cluster/examples/kubernetes/nfs/nfs.yaml | 3 +- cluster/examples/kubernetes/nfs/operator.yaml | 1 + cluster/examples/kubernetes/nfs/pvc.yaml | 3 +- pkg/apis/nfs.rook.io/v1alpha1/types.go | 3 ++ pkg/operator/nfs/controller.go | 32 ++++++++++--------- pkg/operator/nfs/controller_test.go | 2 +- pkg/operator/nfs/provisioner.go | 26 ++++++++------- tests/framework/installer/nfs_manifests.go | 6 ++-- 9 files changed, 69 insertions(+), 33 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index cd4d70c2d..4d0ac331a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -438,6 +438,30 @@ revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1" version = "v1.2" +[[projects]] + branch = "master" + digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" + name = "github.com/petar/GoLLRB" + packages = ["llrb"] + pruneopts = "UT" + revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" + +[[projects]] + digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" + name = "github.com/peterbourgon/diskv" + packages = ["."] + pruneopts = "UT" + revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" + version = "v2.0.1" + +[[projects]] + digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "UT" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" + [[projects]] digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" @@ -1367,6 +1391,8 @@ "github.com/google/uuid", "github.com/icrowley/fake", "github.com/jbw976/go-ps", + "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller", + "github.com/pkg/errors", "github.com/rook/operator-kit", "github.com/spf13/cobra", "github.com/spf13/pflag", diff --git a/cluster/examples/kubernetes/nfs/nfs.yaml b/cluster/examples/kubernetes/nfs/nfs.yaml index 8f9e980b5..bd2fcce47 100644 --- a/cluster/examples/kubernetes/nfs/nfs.yaml +++ b/cluster/examples/kubernetes/nfs/nfs.yaml @@ -87,7 +87,8 @@ spec: serviceAccountName: rook-nfs replicas: 1 exports: - - name: nfs-share + - name: share1 + createStorageClass: true server: accessMode: ReadWrite squash: "none" diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index 2464251c5..1a615cdcd 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -95,6 +95,7 @@ spec: containers: - name: rook-nfs-operator image: rook/nfs:master + imagePullPolicy: IfNotPresent args: ["nfs", "operator"] env: - name: POD_NAME diff --git a/cluster/examples/kubernetes/nfs/pvc.yaml b/cluster/examples/kubernetes/nfs/pvc.yaml index 721d442bb..6af4d8e28 100644 --- a/cluster/examples/kubernetes/nfs/pvc.yaml +++ b/cluster/examples/kubernetes/nfs/pvc.yaml @@ -2,9 +2,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: rook-nfs-pv-claim - annotations: - volume.beta.kubernetes.io/storage-class: "nfs-share" spec: + storageClassName: "rook-nfs-share1" accessModes: - ReadWriteMany resources: diff --git a/pkg/apis/nfs.rook.io/v1alpha1/types.go b/pkg/apis/nfs.rook.io/v1alpha1/types.go index 7fd8d358d..4a2c6f0fa 100644 --- a/pkg/apis/nfs.rook.io/v1alpha1/types.go +++ b/pkg/apis/nfs.rook.io/v1alpha1/types.go @@ -70,6 +70,9 @@ type ExportsSpec struct { // The NFS server configuration Server ServerSpec `json:"server,omitempty"` + // Configuration to create storageclass for nfs share + CreateStorageClass bool `json:"createStorageClass,omitempty"` + // PVC from which the NFS daemon gets storage for sharing PersistentVolumeClaim v1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"` } diff --git a/pkg/operator/nfs/controller.go b/pkg/operator/nfs/controller.go index abad5f068..3c9cc2e14 100644 --- a/pkg/operator/nfs/controller.go +++ b/pkg/operator/nfs/controller.go @@ -156,24 +156,26 @@ func createServicePorts() []v1.ServicePort { func (c *Controller) createStorageClass(nfsServer *nfsServer) error { for _, export := range nfsServer.spec.Exports { - nfsStorageClass := &storage.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: export.Name, - Annotations: map[string]string{ - "rook.io/nfs-path": "/" + export.PersistentVolumeClaim.ClaimName, + if export.CreateStorageClass { + nfsStorageClass := &storage.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: nfsServer.name + "-" + export.Name, + OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, + Labels: createAppLabels(nfsServer), }, - OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, - Labels: createAppLabels(nfsServer), - }, - Provisioner: "rook.io/" + nfsServer.name, - } - if _, err := c.context.Clientset.StorageV1().StorageClasses().Create(nfsStorageClass); err != nil { - if !errors.IsAlreadyExists(err) { - return err + Provisioner: "rook.io/" + nfsServer.name, + Parameters: map[string]string{"claimPath": "/" + export.PersistentVolumeClaim.ClaimName}, + } + if _, err := c.context.Clientset.StorageV1().StorageClasses().Create(nfsStorageClass); err != nil { + if !errors.IsAlreadyExists(err) { + return err + } + logger.Infof("storage class %s already exists", nfsStorageClass.GetName()) + } else { + logger.Infof("storage class %s created", nfsStorageClass.GetName()) } - logger.Infof("storage class %s already exists", nfsStorageClass.GetName()) } else { - logger.Infof("storage class %s created", nfsStorageClass.GetName()) + logger.Infof("default storage class creation disabled for share: %v", export.Name) } } return nil diff --git a/pkg/operator/nfs/controller_test.go b/pkg/operator/nfs/controller_test.go index a0394d832..ea4835591 100644 --- a/pkg/operator/nfs/controller_test.go +++ b/pkg/operator/nfs/controller_test.go @@ -92,7 +92,7 @@ func TestOnAdd(t *testing.T) { namespace := "rook-nfs-test" nfsserver := &nfsv1alpha1.NFSServer{ ObjectMeta: metav1.ObjectMeta{ - Name: "nfs-server-X", + Name: appName, Namespace: namespace, }, Spec: nfsv1alpha1.NFSServerSpec{ diff --git a/pkg/operator/nfs/provisioner.go b/pkg/operator/nfs/provisioner.go index 51f37e3dd..b81c28bb5 100644 --- a/pkg/operator/nfs/provisioner.go +++ b/pkg/operator/nfs/provisioner.go @@ -18,8 +18,10 @@ package nfs import ( "fmt" + "strings" "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/client-go/kubernetes" @@ -33,18 +35,14 @@ type nfsProvisioner struct { server string } -const ( - claimPath = "rook.io/nfs-path" -) - var _ controller.Provisioner = &nfsProvisioner{} // NewNFSProvisioner returns an instance of nfsProvisioner -func NewNFSProvisioner(clientset kubernetes.Interface, server string) nfsProvisioner { - return nfsProvisioner{clientset, server} +func NewNFSProvisioner(clientset kubernetes.Interface, server string) *nfsProvisioner { + return &nfsProvisioner{clientset, server} } -func (p nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) { +func (p *nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) { logger.Infof("nfs provisioner: VolumeOptions %v", options) // Get the storage class for this volume. @@ -57,6 +55,11 @@ func (p nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.Persist return nil, fmt.Errorf("claim Selector is not supported") } + claimPath, present := storageClass.Parameters["claimPath"] + if !present { + return nil, errors.Errorf("NFS share Path not found in the storageclass: %v", storageClass.GetName()) + } + pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: options.PVName, @@ -71,22 +74,21 @@ func (p nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.Persist PersistentVolumeSource: v1.PersistentVolumeSource{ NFS: &v1.NFSVolumeSource{ Server: p.server, - Path: storageClass.GetAnnotations()[claimPath], + Path: strings.TrimSpace(claimPath), ReadOnly: false, }, }, }, } - return pv, nil } -func (p nfsProvisioner) Delete(volume *v1.PersistentVolume) error { +func (p *nfsProvisioner) Delete(volume *v1.PersistentVolume) error { return nil } // getClassForPV returns StorageClass -func (p nfsProvisioner) getClassForPV(pv *v1.PersistentVolume) (*storage.StorageClass, error) { +func (p *nfsProvisioner) getClassForPV(pv *v1.PersistentVolume) (*storage.StorageClass, error) { if p.client == nil { return nil, fmt.Errorf("Cannot get kube client") } @@ -102,7 +104,7 @@ func (p nfsProvisioner) getClassForPV(pv *v1.PersistentVolume) (*storage.Storage } // getClassForPVC returns StorageClass -func (p nfsProvisioner) getClassForPVC(pvc *v1.PersistentVolumeClaim) (*storage.StorageClass, error) { +func (p *nfsProvisioner) getClassForPVC(pvc *v1.PersistentVolumeClaim) (*storage.StorageClass, error) { if p.client == nil { return nil, fmt.Errorf("Cannot get kube client") } diff --git a/tests/framework/installer/nfs_manifests.go b/tests/framework/installer/nfs_manifests.go index 6a61fff69..9ed19dd5d 100644 --- a/tests/framework/installer/nfs_manifests.go +++ b/tests/framework/installer/nfs_manifests.go @@ -185,7 +185,7 @@ kind: PersistentVolumeClaim metadata: name: nfs-pv-claim spec: - storageClassName: nfs-share + storageClassName: nfs-ns-nfs-share accessModes: - ReadWriteMany resources: @@ -197,7 +197,7 @@ kind: PersistentVolumeClaim metadata: name: nfs-pv-claim-bigger spec: - storageClassName: nfs-share1 + storageClassName: nfs-ns-nfs-share1 accessModes: - ReadWriteMany resources: @@ -306,12 +306,14 @@ spec: replicas: ` + strconv.Itoa(count) + ` exports: - name: nfs-share + createStorageClass: true server: accessMode: ReadWrite squash: "none" persistentVolumeClaim: claimName: test-claim - name: nfs-share1 + createStorageClass: true server: accessMode: ReadWrite squash: "none" From 858c8eede719dbc6d3b1c8ac973b527664606605 Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Wed, 24 Apr 2019 14:44:50 +0530 Subject: [PATCH 3/5] refactor(operator): removed wild card from the operator Signed-off-by: Ashish Ranjan --- Gopkg.lock | 19 +------------------ Gopkg.toml | 14 +++++++++++++- cluster/examples/kubernetes/nfs/operator.yaml | 9 ++++++++- cmd/rook/nfs/server.go | 2 +- pkg/operator/nfs/controller.go | 2 +- pkg/operator/nfs/provisioner.go | 2 +- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 4d0ac331a..b9dda9af9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -438,22 +438,6 @@ revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1" version = "v1.2" -[[projects]] - branch = "master" - digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" - name = "github.com/petar/GoLLRB" - packages = ["llrb"] - pruneopts = "UT" - revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" - -[[projects]] - digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" - name = "github.com/peterbourgon/diskv" - packages = ["."] - pruneopts = "UT" - revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" - version = "v2.0.1" - [[projects]] digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b" name = "github.com/pkg/errors" @@ -1342,7 +1326,6 @@ revision = "21c4ce38f2a793ec01e925ddc31216500183b773" [[projects]] - branch = "release-1.14" digest = "1:a2147c72c6ac436c7cbd3ffaefb3a8ca26264dd60fed5aec22cf4ed6d6c42690" name = "sigs.k8s.io/sig-storage-lib-external-provisioner" packages = [ @@ -1352,6 +1335,7 @@ ] pruneopts = "UT" revision = "3eea7193b4c5c034f885913c03105178e9e23a4f" + version = "v3.1.0" [[projects]] digest = "1:fef5992ddf85e9ccad899b11338339858936a10fbce96b9d5c43767669d95398" @@ -1391,7 +1375,6 @@ "github.com/google/uuid", "github.com/icrowley/fake", "github.com/jbw976/go-ps", - "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller", "github.com/pkg/errors", "github.com/rook/operator-kit", "github.com/spf13/cobra", diff --git a/Gopkg.toml b/Gopkg.toml index 7268a3bb5..db6f59d5a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -92,7 +92,7 @@ ignored = [ [[constraint]] name = "sigs.k8s.io/sig-storage-lib-external-provisioner" - branch = "release-1.14" + version = "v3.1.0" # Kubernetes 1.14 dependencies on random package versions: [[override]] @@ -113,3 +113,15 @@ ignored = [ [[override]] name = "bitbucket.org/ww/goautoneg" source = "github.com/rancher/goautoneg" + +[[override]] + name = "k8s.io/apiextensions-apiserver" + version = "kubernetes-1.14.0" + +[[override]] + name = "k8s.io/apimachinery" + version = "kubernetes-1.14.0" + +[[override]] + name = "k8s.io/client-go" + version = "v11.0.0" diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index 1a615cdcd..00dab6c5b 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -23,7 +23,7 @@ metadata: name: rook-nfs-operator rules: - apiGroups: - - "*" + - "" resources: - namespaces - configmaps @@ -53,6 +53,13 @@ rules: - "*" verbs: - "*" +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - create --- apiVersion: v1 kind: ServiceAccount diff --git a/cmd/rook/nfs/server.go b/cmd/rook/nfs/server.go index 3dcec60ba..a8d88ea27 100644 --- a/cmd/rook/nfs/server.go +++ b/cmd/rook/nfs/server.go @@ -22,12 +22,12 @@ import ( "os" "time" - "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" "github.com/rook/rook/cmd/rook/rook" "github.com/rook/rook/pkg/operator/nfs" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/sig-storage-lib-external-provisioner/controller" ) var serverCmd = &cobra.Command{ diff --git a/pkg/operator/nfs/controller.go b/pkg/operator/nfs/controller.go index 3c9cc2e14..e5220ed71 100644 --- a/pkg/operator/nfs/controller.go +++ b/pkg/operator/nfs/controller.go @@ -394,7 +394,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer, service *v1.Service) return nfsPodSpec } -func (c *Controller) createNfsStatefulSet(nfsServer *nfsServer, replicas int32) error { +func (c *Controller) createNfsStatefulSet(nfsServer *nfsServer, replicas int32, service *v1.Service) error { appsClient := c.context.Clientset.AppsV1() nfsPodSpec := c.createNfsPodSpec(nfsServer, service) diff --git a/pkg/operator/nfs/provisioner.go b/pkg/operator/nfs/provisioner.go index b81c28bb5..1c34978fb 100644 --- a/pkg/operator/nfs/provisioner.go +++ b/pkg/operator/nfs/provisioner.go @@ -20,12 +20,12 @@ import ( "fmt" "strings" - "github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/apis/core/v1/helper" + "sigs.k8s.io/sig-storage-lib-external-provisioner/controller" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) From edb2b5e76f6949f07f49811bbe52824806f167ec Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Wed, 24 Apr 2019 15:32:21 +0530 Subject: [PATCH 4/5] refactor(nfs): updated nfs documentation with Dynamic provisioning method Signed-off-by: Ashish Ranjan --- Documentation/nfs.md | 129 +++++++++++++++++- .../examples/kubernetes/nfs/pvc-dynamic.yaml | 11 ++ cluster/examples/kubernetes/nfs/pvc.yaml | 4 +- 3 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 cluster/examples/kubernetes/nfs/pvc-dynamic.yaml diff --git a/Documentation/nfs.md b/Documentation/nfs.md index 8526e1f0c..2cf903b1d 100644 --- a/Documentation/nfs.md +++ b/Documentation/nfs.md @@ -61,7 +61,69 @@ kind: Namespace metadata: name: rook-nfs --- -# A default StorageClass must be present +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-client-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-client-provisioner +subjects: + - kind: ServiceAccount + name: rook-nfs + namespace: rook-nfs +roleRef: + kind: ClusterRole + name: nfs-client-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: rook-nfs +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: leader-locking-nfs-client-provisioner + namespace: rook-nfs +subjects: + - kind: ServiceAccount + name: rook-nfs + # replace with namespace where provisioner is deployed + namespace: rook-nfs +roleRef: + kind: Role + name: leader-locking-nfs-client-provisioner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-nfs + namespace: rook-nfs +--- +# A default storageclass must be present apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -80,15 +142,20 @@ metadata: name: rook-nfs namespace: rook-nfs spec: + serviceAccountName: rook-nfs replicas: 1 exports: - - name: nfs-share + - name: share1 + createStorageClass: true server: accessMode: ReadWrite squash: "none" # A Persistent Volume Claim must be created before creating NFS CRD instance. persistentVolumeClaim: claimName: nfs-default-claim + # A key/value list of annotations + annotations: + # key: value ``` With the `nfs.yaml` file saved, now create the NFS server as shown: @@ -279,6 +346,64 @@ After the NFS server pod is running, follow the same instructions from the previ After that, follow the rest of the instructions in the [Accessing the Export](nfs.md#accessing-the-export) section and then the [Consuming the Export](nfs.md#consuming-the-export) section to consume the NFS volume. +## Dynamic provisioning with NFS +With PR https://github.com/rook/rook/pull/2758 rook starts supporting dynamic provisioning with NFS. This example will be showing how dynamic provisioning feature can be used for nfs. + +Once the NFS Operator and an instance of NFSServer is deployed. A storageclass with below example can be created. + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + app: rook-nfs + name: rook-nfs-share1 +parameters: + claimPath: / +provisioner: rook.io/ +reclaimPolicy: Delete +volumeBindingMode: Immediate +``` +and can be used for provisioning volumes using PVC similar to example given below. + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rook-nfs-pv-claim +spec: + storageClassName: "rook-nfs-share1" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Mi +``` + +There is an option for automatically creating a storageclass for nfs which creates a storageclass for each export along with the nfs-server instance. The name of the storage class is generated in -. This option can be enabled by making `exports[].createStorageClass` field to true. For example: +```yaml +apiVersion: nfs.rook.io/v1alpha1 +kind: NFSServer +metadata: + name: rook-nfs + namespace: rook-nfs +spec: + serviceAccountName: rook-nfs + replicas: 1 + exports: + - name: share1 + createStorageClass: true + server: + accessMode: ReadWrite + squash: "none" + # A Persistent Volume Claim must be created before creating NFS CRD instance. + persistentVolumeClaim: + claimName: nfs-default-claim + # A key/value list of annotations + annotations: + # key: value +``` + ## Teardown To clean up all resources associated with this walk-through, you can run the commands below. diff --git a/cluster/examples/kubernetes/nfs/pvc-dynamic.yaml b/cluster/examples/kubernetes/nfs/pvc-dynamic.yaml new file mode 100644 index 000000000..389eda80b --- /dev/null +++ b/cluster/examples/kubernetes/nfs/pvc-dynamic.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rook-nfs-pv-claim +spec: + storageClassName: "rook-nfs-share1" + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Mi diff --git a/cluster/examples/kubernetes/nfs/pvc.yaml b/cluster/examples/kubernetes/nfs/pvc.yaml index 6af4d8e28..f376ac192 100644 --- a/cluster/examples/kubernetes/nfs/pvc.yaml +++ b/cluster/examples/kubernetes/nfs/pvc.yaml @@ -3,9 +3,9 @@ kind: PersistentVolumeClaim metadata: name: rook-nfs-pv-claim spec: - storageClassName: "rook-nfs-share1" accessModes: - ReadWriteMany + storageClassName: "" resources: requests: - storage: 1Mi \ No newline at end of file + storage: 1Mi From dac028ab2491172146f641dc889c42a6f8f53551 Mon Sep 17 00:00:00 2001 From: Ashish Ranjan Date: Fri, 26 Apr 2019 04:24:17 +0530 Subject: [PATCH 5/5] enhance(nfs): remove one to one mapping for nfs provisioner Signed-off-by: Ashish Ranjan This commit removes one to one mapping of NFS provisioner and adds a dedicated provisioner for nfs. --- Documentation/nfs.md | 229 ++----------- cluster/examples/kubernetes/nfs/nfs-ceph.yaml | 82 ++--- cluster/examples/kubernetes/nfs/nfs.yaml | 63 ---- cluster/examples/kubernetes/nfs/operator.yaml | 321 +++++++++++------- cluster/examples/kubernetes/nfs/pv.yaml | 19 -- .../examples/kubernetes/nfs/pvc-dynamic.yaml | 11 - cluster/examples/kubernetes/nfs/pvc.yaml | 3 +- cluster/examples/kubernetes/nfs/sc.yaml | 13 + cmd/rook/nfs/nfs.go | 1 + cmd/rook/nfs/provisioner.go | 70 ++++ cmd/rook/nfs/server.go | 45 +-- images/nfs/Dockerfile | 2 +- pkg/apis/nfs.rook.io/v1alpha1/types.go | 10 +- pkg/operator/nfs/controller.go | 60 +--- pkg/operator/nfs/controller_test.go | 2 +- pkg/operator/nfs/provisioner.go | 54 ++- pkg/operator/nfs/server.go | 1 + tests/framework/installer/nfs_installer.go | 2 +- tests/framework/installer/nfs_manifests.go | 186 ++++++---- 19 files changed, 543 insertions(+), 631 deletions(-) delete mode 100644 cluster/examples/kubernetes/nfs/pv.yaml delete mode 100644 cluster/examples/kubernetes/nfs/pvc-dynamic.yaml create mode 100644 cluster/examples/kubernetes/nfs/sc.yaml create mode 100644 cmd/rook/nfs/provisioner.go diff --git a/Documentation/nfs.md b/Documentation/nfs.md index 2cf903b1d..c8bcf8091 100644 --- a/Documentation/nfs.md +++ b/Documentation/nfs.md @@ -32,8 +32,9 @@ You can check if the operator is up and running with: ```console kubectl -n rook-nfs-system get pod -NAME READY STATUS RESTARTS AGE -rook-nfs-operator-b8d6d955d-cdq2v 1/1 Running 0 1m +NAME READY STATUS RESTARTS AGE +rook-nfs-operator-879f5bf8b-gnwht 1/1 Running 0 29m +rook-nfs-provisioner-65f4874c8f-kkz6b 1/1 Running 0 29m ``` ## Create and Initialize NFS Server @@ -61,68 +62,6 @@ kind: Namespace metadata: name: rook-nfs --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nfs-client-provisioner-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: run-nfs-client-provisioner -subjects: - - kind: ServiceAccount - name: rook-nfs - namespace: rook-nfs -roleRef: - kind: ClusterRole - name: nfs-client-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: leader-locking-nfs-client-provisioner - namespace: rook-nfs -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: leader-locking-nfs-client-provisioner - namespace: rook-nfs -subjects: - - kind: ServiceAccount - name: rook-nfs - # replace with namespace where provisioner is deployed - namespace: rook-nfs -roleRef: - kind: Role - name: leader-locking-nfs-client-provisioner - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-nfs - namespace: rook-nfs ---- # A default storageclass must be present apiVersion: v1 kind: PersistentVolumeClaim @@ -146,7 +85,6 @@ spec: replicas: 1 exports: - name: share1 - createStorageClass: true server: accessMode: ReadWrite squash: "none" @@ -185,54 +123,33 @@ rook-nfs-0 1/1 Running 0 2m If the NFS server pod is in the `Running` state, then we have successfully created an exported NFS share that clients can start to access over the network. ### Accessing the Export +With PR https://github.com/rook/rook/pull/2758 rook starts supporting dynamic provisioning with NFS. This example will be showing how dynamic provisioning feature can be used for nfs. -To access the export from another pod, you must first manually create a `PersistentVolume` with the connection information. -This experience is a bit cumbersome, but will be improved in the future with dynamic provisioning support. -First, find the current IP address of your NFS server pod using the following command: - -```console -kubectl -n rook-nfs get service -l app=rook-nfs -o jsonpath='{.items[0].spec.clusterIP}' -``` - -Copy this IP address and insert it into the following content in place of the `` placeholder. -Then also replace the `/` value with the `claimName` that was configured in the `nfs.yaml`. -In this case, it will be `nfs-default-claim` and it will look like `path: "/nfs-default-claim"`. -Finally, save the updated content to a file called `pv.yaml`. +Once the NFS Operator and an instance of NFSServer is deployed. A storageclass similar to below example has to be created to dynamically provisioning volumes. ```yaml -apiVersion: v1 -kind: PersistentVolume +apiVersion: storage.k8s.io/v1 +kind: StorageClass metadata: - name: rook-nfs-pv - namespace: rook-nfs -spec: - capacity: - storage: 1Mi - accessModes: - - ReadWriteMany - nfs: - # FIXME: replace with the current IP of the NFS server pod. - # Run the below command to get the current IP address: - # kubectl -n rook-nfs get service -l app=rook-nfs -o jsonpath='{.items[0].spec.clusterIP}' - server: - path: "/" + labels: + app: rook-nfs + name: rook-nfs-share1 +parameters: + exportName: share1 + nfsServerName: rook-nfs + nfsServerNamespace: rook-nfs +provisioner: rook.io/nfs-provisioner +reclaimPolicy: Delete +volumeBindingMode: Immediate ``` -With the `pv.yaml` file saved, we can create the PV object: +#### Note: The storageclass need to have the following 3 parameters passed. +1. `exportName`: It tells the provisioner which export to use for provisioning the volumes. +2. `nfsServerName`: It is the name of the NFSServer instance. +3. `nfsServerNamespace`: It namespace where the NFSServer instance is running. -```console -kubectl create -f pv.yaml -``` -### Consuming the Export - -Now we can consume the PV that we just created by creating an example web server app that uses a `PersistentVolumeClaim` to claim the exported volume. -There are 2 pods that comprise this example: - -1. A web server pod that will read and display the contents of the NFS share -1. A writer pod that will write random data to the NFS share so the website will continually update - -First, save the PVC definition in a file called `pvc.yaml`: +Once the above storageclass has been created create a PV claim referencing the storageclass as shown in the example given below. ```yaml apiVersion: v1 @@ -240,50 +157,14 @@ kind: PersistentVolumeClaim metadata: name: rook-nfs-pv-claim spec: + storageClassName: "rook-nfs-share1" accessModes: - ReadWriteMany - storageClassName: "" resources: requests: storage: 1Mi ``` -Then create the PVC: - -```console -kubectl create -f pvc.yaml -``` - -Start both the busybox pod (writer) and the web server from the `cluster/examples/kubernetes/nfs` folder: - -```console -kubectl create -f busybox-rc.yaml -kubectl create -f web-rc.yaml -``` - -Let's confirm that the expected busybox writer pods and web server pods are **all** up and in the `Running` state: - -```console -kubectl get pod -l app=nfs-demo -``` - -In order to be able to reach the web server over the network, let's create a service for it: - -```console -kubectl create -f web-service.yaml -``` - -We can then use the busybox writer pod we launched before to check that nginx is serving the data appropriately. -In the below 1-liner command, we use `kubectl exec` to run a command in the busybox writer pod that uses `wget` to retrieve the web page that the web server pod is hosting. As the busybox writer pod continues to write a new timestamp, we should see the returned output also update every ~10 seconds or so. - -```console -> echo; kubectl exec $(kubectl get pod -l name=nfs-busybox -o jsonpath='{.items[0].metadata.name}') -- wget -qO- http://$(kubectl get services nfs-web -o jsonpath='{.spec.clusterIP}'); echo - -Thu Oct 22 19:28:55 UTC 2015 -nfs-busybox-w3s4t - -``` - ## Rook Ceph volume example In this alternative example, we will use a different underlying volume as an export for the NFS server. @@ -340,69 +221,7 @@ Create the NFS server instance that you saved in `nfs-ceph.yaml`: kubectl create -f nfs-ceph.yaml ``` -After the NFS server pod is running, follow the same instructions from the previous example to access and consume the NFS share, with the following exception: - -* Replace the `/` value in the PV definition in the [Accessing the Export](nfs.md#accessing-the-export) section with the `claimName` that was configured in the `nfs-ceph.yaml`. In this case it will be `nfs-ceph-claim` and will look like `path: "/nfs-ceph-claim"`. - -After that, follow the rest of the instructions in the [Accessing the Export](nfs.md#accessing-the-export) section and then the [Consuming the Export](nfs.md#consuming-the-export) section to consume the NFS volume. - -## Dynamic provisioning with NFS -With PR https://github.com/rook/rook/pull/2758 rook starts supporting dynamic provisioning with NFS. This example will be showing how dynamic provisioning feature can be used for nfs. - -Once the NFS Operator and an instance of NFSServer is deployed. A storageclass with below example can be created. - -```yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - labels: - app: rook-nfs - name: rook-nfs-share1 -parameters: - claimPath: / -provisioner: rook.io/ -reclaimPolicy: Delete -volumeBindingMode: Immediate -``` -and can be used for provisioning volumes using PVC similar to example given below. - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: rook-nfs-pv-claim -spec: - storageClassName: "rook-nfs-share1" - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Mi -``` - -There is an option for automatically creating a storageclass for nfs which creates a storageclass for each export along with the nfs-server instance. The name of the storage class is generated in -. This option can be enabled by making `exports[].createStorageClass` field to true. For example: -```yaml -apiVersion: nfs.rook.io/v1alpha1 -kind: NFSServer -metadata: - name: rook-nfs - namespace: rook-nfs -spec: - serviceAccountName: rook-nfs - replicas: 1 - exports: - - name: share1 - createStorageClass: true - server: - accessMode: ReadWrite - squash: "none" - # A Persistent Volume Claim must be created before creating NFS CRD instance. - persistentVolumeClaim: - claimName: nfs-default-claim - # A key/value list of annotations - annotations: - # key: value -``` +After the NFS server pod is running, follow the same instructions from the previous example to access and consume the NFS share. ## Teardown diff --git a/cluster/examples/kubernetes/nfs/nfs-ceph.yaml b/cluster/examples/kubernetes/nfs/nfs-ceph.yaml index 9c621b896..1a259b2f4 100644 --- a/cluster/examples/kubernetes/nfs/nfs-ceph.yaml +++ b/cluster/examples/kubernetes/nfs/nfs-ceph.yaml @@ -1,41 +1,41 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: rook-nfs ---- -# A rook ceph cluster must be running -# Create a rook ceph cluster using examples in rook/cluster/examples/kubernetes/ceph -# Refer to https://rook.io/docs/rook/master/ceph-quickstart.html for a quick rook cluster setup -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: nfs-ceph-claim - namespace: rook-nfs -spec: - storageClassName: rook-ceph-block - accessModes: - - ReadWriteMany - resources: - requests: - storage: 2Gi ---- -apiVersion: nfs.rook.io/v1alpha1 -kind: NFSServer -metadata: - name: rook-nfs - namespace: rook-nfs -spec: - replicas: 1 - exports: - - name: nfs-share - server: - accessMode: ReadWrite - squash: "none" - # A Persistent Volume Claim must be created before creating NFS CRD instance. - # Create a Ceph cluster for using this example - # Create a ceph PVC after creating the rook ceph cluster using ceph-pvc.yaml - persistentVolumeClaim: - claimName: nfs-ceph-claim - # A key value list of annotations - annotations: - # key: value +apiVersion: v1 +kind: Namespace +metadata: + name: rook-nfs +--- +# A rook ceph cluster must be running +# Create a rook ceph cluster using examples in rook/cluster/examples/kubernetes/ceph +# Refer to https://rook.io/docs/rook/master/ceph-quickstart.html for a quick rook cluster setup +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nfs-ceph-claim + namespace: rook-nfs +spec: + storageClassName: rook-ceph-block + accessModes: + - ReadWriteMany + resources: + requests: + storage: 2Gi +--- +apiVersion: nfs.rook.io/v1alpha1 +kind: NFSServer +metadata: + name: rook-nfs + namespace: rook-nfs +spec: + replicas: 1 + exports: + - name: share1 + server: + accessMode: ReadWrite + squash: "none" + # A Persistent Volume Claim must be created before creating NFS CRD instance. + # Create a Ceph cluster for using this example + # Create a ceph PVC after creating the rook ceph cluster using ceph-pvc.yaml + persistentVolumeClaim: + claimName: nfs-ceph-claim + # A key value list of annotations + annotations: + # key: value diff --git a/cluster/examples/kubernetes/nfs/nfs.yaml b/cluster/examples/kubernetes/nfs/nfs.yaml index bd2fcce47..aa6a07967 100644 --- a/cluster/examples/kubernetes/nfs/nfs.yaml +++ b/cluster/examples/kubernetes/nfs/nfs.yaml @@ -3,68 +3,6 @@ kind: Namespace metadata: name: rook-nfs --- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nfs-client-provisioner-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: run-nfs-client-provisioner -subjects: - - kind: ServiceAccount - name: rook-nfs - namespace: rook-nfs -roleRef: - kind: ClusterRole - name: nfs-client-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: leader-locking-nfs-client-provisioner - namespace: rook-nfs -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: leader-locking-nfs-client-provisioner - namespace: rook-nfs -subjects: - - kind: ServiceAccount - name: rook-nfs - # replace with namespace where provisioner is deployed - namespace: rook-nfs -roleRef: - kind: Role - name: leader-locking-nfs-client-provisioner - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-nfs - namespace: rook-nfs ---- # A default storageclass must be present apiVersion: v1 kind: PersistentVolumeClaim @@ -88,7 +26,6 @@ spec: replicas: 1 exports: - name: share1 - createStorageClass: true server: accessMode: ReadWrite squash: "none" diff --git a/cluster/examples/kubernetes/nfs/operator.yaml b/cluster/examples/kubernetes/nfs/operator.yaml index 00dab6c5b..d470320b1 100644 --- a/cluster/examples/kubernetes/nfs/operator.yaml +++ b/cluster/examples/kubernetes/nfs/operator.yaml @@ -1,115 +1,206 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: rook-nfs-system ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: nfsservers.nfs.rook.io -spec: - group: nfs.rook.io - names: - kind: NFSServer - listKind: NFSServerList - plural: nfsservers - singular: nfsserver - scope: Namespaced - version: v1alpha1 ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: rook-nfs-operator -rules: -- apiGroups: - - "" - resources: - - namespaces - - configmaps - - pods - - services - - storageclasses - verbs: - - get - - watch - - create -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - get - - create -- apiGroups: - - nfs.rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - rook.io - resources: - - "*" - verbs: - - "*" -- apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - create ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-nfs-operator - namespace: rook-nfs-system ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-nfs-operator - namespace: rook-nfs-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-nfs-operator -subjects: -- kind: ServiceAccount - name: rook-nfs-operator - namespace: rook-nfs-system ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: rook-nfs-operator - namespace: rook-nfs-system - labels: - app: rook-nfs-operator -spec: - replicas: 1 - selector: - matchLabels: - app: rook-nfs-operator - template: - metadata: - labels: - app: rook-nfs-operator - spec: - serviceAccountName: rook-nfs-operator - containers: - - name: rook-nfs-operator - image: rook/nfs:master - imagePullPolicy: IfNotPresent - args: ["nfs", "operator"] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace +apiVersion: v1 +kind: Namespace +metadata: + name: rook-nfs-system +--- +# NFS operator deployment specifications +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: nfsservers.nfs.rook.io +spec: + group: nfs.rook.io + names: + kind: NFSServer + listKind: NFSServerList + plural: nfsservers + singular: nfsserver + scope: Namespaced + version: v1alpha1 +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-nfs-operator +rules: +- apiGroups: + - "" + resources: + - namespaces + - configmaps + - pods + - services + - storageclasses + verbs: + - get + - watch + - create +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - create +- apiGroups: + - nfs.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - create +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-nfs-operator + namespace: rook-nfs-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-nfs-operator + namespace: rook-nfs-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-nfs-operator +subjects: +- kind: ServiceAccount + name: rook-nfs-operator + namespace: rook-nfs-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rook-nfs-operator + namespace: rook-nfs-system + labels: + app: rook-nfs-operator +spec: + replicas: 1 + selector: + matchLabels: + app: rook-nfs-operator + template: + metadata: + labels: + app: rook-nfs-operator + spec: + serviceAccountName: rook-nfs-operator + containers: + - name: rook-nfs-operator + image: rook/nfs:master + imagePullPolicy: IfNotPresent + args: ["nfs", "operator"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +--- +# NFS provisioner deployment specification +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-nfs-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["podsecuritypolicies"] + resourceNames: ["nfs-provisioner"] + verbs: ["use"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: + - nfs.rook.io + resources: + - "*" + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-provisioner +subjects: + - kind: ServiceAccount + name: rook-nfs-provisioner + # replace with namespace where provisioner is deployed + namespace: rook-nfs-system +roleRef: + kind: ClusterRole + name: rook-nfs-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-nfs-provisioner + namespace: rook-nfs-system +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: rook-nfs-provisioner + namespace: rook-nfs-system +spec: + selector: + matchLabels: + app: rook-nfs-provisioner + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: rook-nfs-provisioner + spec: + serviceAccount: rook-nfs-provisioner + containers: + - name: rook-nfs-provisioner + image: rook/nfs:master + imagePullPolicy: IfNotPresent + args: ["nfs", "provisioner","--provisioner=rook.io/nfs-provisioner"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/cluster/examples/kubernetes/nfs/pv.yaml b/cluster/examples/kubernetes/nfs/pv.yaml deleted file mode 100644 index e44da6ea1..000000000 --- a/cluster/examples/kubernetes/nfs/pv.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: rook-nfs-pv - namespace: rook-nfs - annotations: - volume.beta.kubernetes.io/mount-options: "vers=4.1" -spec: - storageClassName: rook-nfs-sc - capacity: - storage: 1Mi - accessModes: - - ReadWriteMany - nfs: - # FIXME: replace with the current IP of the NFS server pod. - # Run the below command to get the current IP address: - # kubectl -n rook-nfs get service -l app=rook-nfs -o jsonpath='{.items[0].spec.clusterIP}' - server: - path: "/" diff --git a/cluster/examples/kubernetes/nfs/pvc-dynamic.yaml b/cluster/examples/kubernetes/nfs/pvc-dynamic.yaml deleted file mode 100644 index 389eda80b..000000000 --- a/cluster/examples/kubernetes/nfs/pvc-dynamic.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: rook-nfs-pv-claim -spec: - storageClassName: "rook-nfs-share1" - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Mi diff --git a/cluster/examples/kubernetes/nfs/pvc.yaml b/cluster/examples/kubernetes/nfs/pvc.yaml index f376ac192..789de6b86 100644 --- a/cluster/examples/kubernetes/nfs/pvc.yaml +++ b/cluster/examples/kubernetes/nfs/pvc.yaml @@ -1,11 +1,12 @@ +--- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: rook-nfs-pv-claim spec: + storageClassName: "rook-nfs-share1" accessModes: - ReadWriteMany - storageClassName: "" resources: requests: storage: 1Mi diff --git a/cluster/examples/kubernetes/nfs/sc.yaml b/cluster/examples/kubernetes/nfs/sc.yaml new file mode 100644 index 000000000..2f25f3c7c --- /dev/null +++ b/cluster/examples/kubernetes/nfs/sc.yaml @@ -0,0 +1,13 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + app: rook-nfs + name: rook-nfs-share1 +parameters: + exportName: share1 + nfsServerName: rook-nfs + nfsServerNamespace: rook-nfs +provisioner: rook.io/nfs-provisioner +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/cmd/rook/nfs/nfs.go b/cmd/rook/nfs/nfs.go index 5740d009d..773ca62b8 100644 --- a/cmd/rook/nfs/nfs.go +++ b/cmd/rook/nfs/nfs.go @@ -36,6 +36,7 @@ var ( func init() { Cmd.AddCommand(operatorCmd) + Cmd.AddCommand(provisonerCmd) Cmd.AddCommand(serverCmd) } diff --git a/cmd/rook/nfs/provisioner.go b/cmd/rook/nfs/provisioner.go new file mode 100644 index 000000000..b7bc8f57a --- /dev/null +++ b/cmd/rook/nfs/provisioner.go @@ -0,0 +1,70 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "errors" + "fmt" + + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/operator/nfs" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/sig-storage-lib-external-provisioner/controller" +) + +var provisonerCmd = &cobra.Command{ + Use: "provisioner", + Short: "Runs the NFS provisioner for provisioning volumes", + Long: "Runs the NFS provisioner for provisioning volumes from the rook provisioned nfs servers", +} + +var ( + provisioner *string +) + +func init() { + flags.SetFlagsFromEnv(provisonerCmd.Flags(), rook.RookEnvVarPrefix) + flags.SetLoggingFlags(provisonerCmd.Flags()) + + provisioner = provisonerCmd.Flags().String("provisioner", "", "Name of the provisioner. The provisioner will only provision volumes for claims that request a StorageClass with a provisioner field set equal to this name.") + provisonerCmd.RunE = startProvisioner +} + +func startProvisioner(cmd *cobra.Command, args []string) error { + rook.SetLogLevel() + rook.LogStartupInfo(serverCmd.Flags()) + if len(*provisioner) == 0 { + return errors.New("--provisioner is a required parameter") + } + + clientset, _, rookClientset, err := rook.GetClientset() + if err != nil { + rook.TerminateFatal(fmt.Errorf("failed to get k8s clients. %+v", err)) + } + + serverVersion, err := clientset.Discovery().ServerVersion() + if err != nil { + logger.Fatalf("Error getting server version: %v", err) + } + + clientNFSProvisioner := nfs.NewNFSProvisioner(clientset, rookClientset) + pc := controller.NewProvisionController(clientset, *provisioner, clientNFSProvisioner, serverVersion.GitVersion) + pc.Run(wait.NeverStop) + return nil +} diff --git a/cmd/rook/nfs/server.go b/cmd/rook/nfs/server.go index a8d88ea27..2e39e4c1a 100644 --- a/cmd/rook/nfs/server.go +++ b/cmd/rook/nfs/server.go @@ -18,16 +18,11 @@ package nfs import ( "errors" - "fmt" - "os" - "time" "github.com/rook/rook/cmd/rook/rook" "github.com/rook/rook/pkg/operator/nfs" "github.com/rook/rook/pkg/util/flags" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/sig-storage-lib-external-provisioner/controller" ) var serverCmd = &cobra.Command{ @@ -38,15 +33,13 @@ https://github.com/rook/rook`, } var ( - provisioner, server, path, ganeshaConfigPath *string + ganeshaConfigPath *string ) func init() { flags.SetFlagsFromEnv(serverCmd.Flags(), rook.RookEnvVarPrefix) flags.SetLoggingFlags(serverCmd.Flags()) - provisioner = serverCmd.Flags().String("provisioner", "", "Name of the provisioner. The provisioner will only provision volumes for claims that request a StorageClass with a provisioner field set equal to this name.") - server = serverCmd.Flags().String("server", os.Getenv("ROOK_NFS_SERVICE_HOST"), "Name of the NFS server") ganeshaConfigPath = serverCmd.Flags().String("ganeshaConfigPath", "", "ConfigPath of nfs ganesha") serverCmd.RunE = startServer @@ -55,49 +48,23 @@ func init() { func startServer(cmd *cobra.Command, args []string) error { rook.SetLogLevel() rook.LogStartupInfo(serverCmd.Flags()) - if len(*provisioner) == 0 { - return errors.New("--provisioner is a required parameter") - } else if len(*ganeshaConfigPath) == 0 { + if len(*ganeshaConfigPath) == 0 { return errors.New("--ganeshaConfigPath is a required parameter") } - clientset, _, _, err := rook.GetClientset() - if err != nil { - rook.TerminateFatal(fmt.Errorf("failed to get k8s clients. %+v", err)) - } - logger.Infof("Setting up NFS server!") - err = nfs.Setup(*ganeshaConfigPath) + err := nfs.Setup(*ganeshaConfigPath) if err != nil { logger.Fatalf("Error setting up NFS server: %v", err) } logger.Infof("starting NFS server") - go func() { - for { - // This blocks until server exits (presumably due to an error) - err = nfs.Run(*ganeshaConfigPath) - if err != nil { - logger.Errorf("NFS server Exited Unexpectedly with err: %v", err) - } - - // take a moment before trying to restart - time.Sleep(time.Second) - } - }() - - // Wait for NFS server to come up before continuing provisioner process - time.Sleep(5 * time.Second) - - serverVersion, err := clientset.Discovery().ServerVersion() + // This blocks until server exits (presumably due to an error) + err = nfs.Run(*ganeshaConfigPath) if err != nil { - logger.Fatalf("Error getting server version: %v", err) + logger.Errorf("NFS server Exited Unexpectedly with err: %v", err) } - clientNFSProvisioner := nfs.NewNFSProvisioner(clientset, *server) - - pc := controller.NewProvisionController(clientset, *provisioner, clientNFSProvisioner, serverVersion.GitVersion) - pc.Run(wait.NeverStop) return nil } diff --git a/images/nfs/Dockerfile b/images/nfs/Dockerfile index ca6be1510..5319ec5c5 100644 --- a/images/nfs/Dockerfile +++ b/images/nfs/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Portions of this file came from https://github.com/kubernetes-incubator/external-storage/blob/master/nfs/deploy/docker/Dockerfile, which uses the same license. +#Portions of this file came from https://github.com/mitcdh/docker-nfs-ganesha/blob/master/Dockerfile, which uses the same license. FROM NFS_BASEIMAGE # Build ganesha from source, installing deps and removing them in one line. diff --git a/pkg/apis/nfs.rook.io/v1alpha1/types.go b/pkg/apis/nfs.rook.io/v1alpha1/types.go index 4a2c6f0fa..6febf79a8 100644 --- a/pkg/apis/nfs.rook.io/v1alpha1/types.go +++ b/pkg/apis/nfs.rook.io/v1alpha1/types.go @@ -45,7 +45,7 @@ type NFSServerList struct { Items []NFSServer `json:"items"` } -// NFSSpec represents the spec of NFS daemon +// NFSServerSpec represents the spec of NFS daemon type NFSServerSpec struct { // The annotations-related configuration to add/set on each Pod related object. Annotations rookalpha.Annotations `json:"annotations,omitempty"` @@ -53,11 +53,6 @@ type NFSServerSpec struct { // Replicas of the NFS daemon Replicas int `json:"replicas,omitempty"` - // ServiceAccountName is the name of the ServiceAccount to use to run this nfs server pod. - // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ - // +optional - ServiceAccountName string `json:"serviceAccountName,omitempty" protobuf:"bytes,8,opt,name=serviceAccountName"` - // The parameters to configure the NFS export Exports []ExportsSpec `json:"exports,omitempty"` } @@ -70,9 +65,6 @@ type ExportsSpec struct { // The NFS server configuration Server ServerSpec `json:"server,omitempty"` - // Configuration to create storageclass for nfs share - CreateStorageClass bool `json:"createStorageClass,omitempty"` - // PVC from which the NFS daemon gets storage for sharing PersistentVolumeClaim v1.PersistentVolumeClaimVolumeSource `json:"persistentVolumeClaim,omitempty"` } diff --git a/pkg/operator/nfs/controller.go b/pkg/operator/nfs/controller.go index e5220ed71..3d6e6609e 100644 --- a/pkg/operator/nfs/controller.go +++ b/pkg/operator/nfs/controller.go @@ -29,7 +29,6 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - storage "k8s.io/api/storage/v1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -154,33 +153,6 @@ func createServicePorts() []v1.ServicePort { } } -func (c *Controller) createStorageClass(nfsServer *nfsServer) error { - for _, export := range nfsServer.spec.Exports { - if export.CreateStorageClass { - nfsStorageClass := &storage.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: nfsServer.name + "-" + export.Name, - OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef}, - Labels: createAppLabels(nfsServer), - }, - Provisioner: "rook.io/" + nfsServer.name, - Parameters: map[string]string{"claimPath": "/" + export.PersistentVolumeClaim.ClaimName}, - } - if _, err := c.context.Clientset.StorageV1().StorageClasses().Create(nfsStorageClass); err != nil { - if !errors.IsAlreadyExists(err) { - return err - } - logger.Infof("storage class %s already exists", nfsStorageClass.GetName()) - } else { - logger.Infof("storage class %s created", nfsStorageClass.GetName()) - } - } else { - logger.Infof("default storage class creation disabled for share: %v", export.Name) - } - } - return nil -} - func (c *Controller) createNFSService(nfsServer *nfsServer) (*v1.Service, error) { // This service is meant to be used by clients to access NFS. nfsService := &v1.Service{ @@ -285,25 +257,23 @@ func (c *Controller) createNFSConfigMap(nfsServer *nfsServer) error { return nil } -func getPVCNameList(spec *nfsv1alpha1.NFSServerSpec) []string { - exports := spec.Exports - pvcNameList := make([]string, 0) - for _, export := range exports { +func getPVCMount(spec *nfsv1alpha1.NFSServerSpec) map[string]string { + pvcMounts := map[string]string{} + for _, export := range spec.Exports { claimName := export.PersistentVolumeClaim.ClaimName if claimName != "" { - pvcNameList = append(pvcNameList, claimName) + pvcMounts[export.Name] = export.PersistentVolumeClaim.ClaimName } } - - return pvcNameList + return pvcMounts } func createPVCSpecList(nfsServer *nfsServer) []v1.Volume { pvcSpecList := make([]v1.Volume, 0) - pvcNameList := getPVCNameList(&nfsServer.spec) - for _, claimName := range pvcNameList { + pvcNameList := getPVCMount(&nfsServer.spec) + for shareName, claimName := range pvcNameList { pvcSpecList = append(pvcSpecList, v1.Volume{ - Name: claimName, + Name: shareName, VolumeSource: v1.VolumeSource{ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ ClaimName: claimName, @@ -334,10 +304,10 @@ func createPVCSpecList(nfsServer *nfsServer) []v1.Volume { func createVolumeMountList(nfsServer *nfsServer) []v1.VolumeMount { volumeMountList := make([]v1.VolumeMount, 0) - pvcNameList := getPVCNameList(&nfsServer.spec) - for _, claimName := range pvcNameList { + pvcMount := getPVCMount(&nfsServer.spec) + for shareName, claimName := range pvcMount { volumeMountList = append(volumeMountList, v1.VolumeMount{ - Name: claimName, + Name: shareName, MountPath: "/" + claimName, }) } @@ -359,13 +329,12 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer, service *v1.Service) Labels: createAppLabels(nfsServer), }, Spec: v1.PodSpec{ - ServiceAccountName: nfsServer.spec.ServiceAccountName, Containers: []v1.Container{ { ImagePullPolicy: "IfNotPresent", Name: nfsServer.name, Image: c.containerImage, - Args: []string{"nfs", "server", "--provisioner=rook.io/" + nfsServer.name, "--ganeshaConfigPath=" + NFSConfigMapPath + "/" + nfsServer.name, "--server=" + service.Spec.ClusterIP}, + Args: []string{"nfs", "server", "--ganeshaConfigPath=" + NFSConfigMapPath + "/" + nfsServer.name}, Ports: []v1.ContainerPort{ { Name: "nfs-port", @@ -458,11 +427,6 @@ func (c *Controller) onAdd(obj interface{}) { if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas), svc); err != nil { logger.Errorf("Unable to create NFS stateful set %+v", err) } - - logger.Infof("creating nfs server storageclass") - if err := c.createStorageClass(nfsServer); err != nil { - logger.Errorf("Unable to create NFS storageclass %+v", err) - } } func (c *Controller) onUpdate(oldObj, newObj interface{}) { diff --git a/pkg/operator/nfs/controller_test.go b/pkg/operator/nfs/controller_test.go index ea4835591..cdb1868ef 100644 --- a/pkg/operator/nfs/controller_test.go +++ b/pkg/operator/nfs/controller_test.go @@ -163,7 +163,7 @@ NFS_Core_Param container := ss.Spec.Template.Spec.Containers[0] assert.Equal(t, 2, len(container.VolumeMounts)) - expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: nfsserver.GetName(), MountPath: "/nfs-ganesha/config"}} + expectedVolumeMounts := []v1.VolumeMount{{Name: "export-test", MountPath: "/test-claim"}, {Name: nfsserver.GetName(), MountPath: "/nfs-ganesha/config"}} assert.Equal(t, expectedVolumeMounts, container.VolumeMounts) } diff --git a/pkg/operator/nfs/provisioner.go b/pkg/operator/nfs/provisioner.go index 1c34978fb..b741f3199 100644 --- a/pkg/operator/nfs/provisioner.go +++ b/pkg/operator/nfs/provisioner.go @@ -18,9 +18,9 @@ package nfs import ( "fmt" - "strings" "github.com/pkg/errors" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/client-go/kubernetes" @@ -31,15 +31,15 @@ import ( ) type nfsProvisioner struct { - client kubernetes.Interface - server string + client kubernetes.Interface + rookClient rookclient.Interface } var _ controller.Provisioner = &nfsProvisioner{} // NewNFSProvisioner returns an instance of nfsProvisioner -func NewNFSProvisioner(clientset kubernetes.Interface, server string) *nfsProvisioner { - return &nfsProvisioner{clientset, server} +func NewNFSProvisioner(clientset kubernetes.Interface, rookClientset rookclient.Interface) *nfsProvisioner { + return &nfsProvisioner{clientset, rookClientset} } func (p *nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) { @@ -55,11 +55,25 @@ func (p *nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis return nil, fmt.Errorf("claim Selector is not supported") } - claimPath, present := storageClass.Parameters["claimPath"] + nfsServerName, present := storageClass.Parameters["nfsServerName"] if !present { return nil, errors.Errorf("NFS share Path not found in the storageclass: %v", storageClass.GetName()) } + nfsNamespace, present := storageClass.Parameters["nfsServerNamespace"] + if !present { + return nil, errors.Errorf("NFS share Path not found in the storageclass: %v", storageClass.GetName()) + } + + exportName, present := storageClass.Parameters["exportName"] + if !present { + return nil, errors.Errorf("NFS share Path not found in the storageclass: %v", storageClass.GetName()) + } + + nfsVolumeSource, err := p.getNFSVolumeSource(nfsServerName, nfsNamespace, exportName) + if err != nil { + return nil, err + } pv := &v1.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: options.PVName, @@ -72,11 +86,7 @@ func (p *nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)], }, PersistentVolumeSource: v1.PersistentVolumeSource{ - NFS: &v1.NFSVolumeSource{ - Server: p.server, - Path: strings.TrimSpace(claimPath), - ReadOnly: false, - }, + NFS: nfsVolumeSource, }, }, } @@ -118,3 +128,25 @@ func (p *nfsProvisioner) getClassForPVC(pvc *v1.PersistentVolumeClaim) (*storage } return class, nil } + +// getNFSVolumeSource returns the nfs source configuration for the pv +func (p *nfsProvisioner) getNFSVolumeSource(nfsServerName, namespace, exportName string) (*v1.NFSVolumeSource, error) { + nfsServer, err := p.rookClient.NfsV1alpha1().NFSServers(namespace).Get(nfsServerName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + nfsService, err := p.client.CoreV1().Services(namespace).Get(nfsServerName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + for _, export := range nfsServer.Spec.Exports { + if export.Name == exportName { + return &v1.NFSVolumeSource{ + ReadOnly: false, + Server: nfsService.Spec.ClusterIP, + Path: "/" + export.PersistentVolumeClaim.ClaimName, + }, nil + } + } + return nil, errors.Errorf("exportName not Found") +} diff --git a/pkg/operator/nfs/server.go b/pkg/operator/nfs/server.go index 4f190fb57..b271d11cc 100644 --- a/pkg/operator/nfs/server.go +++ b/pkg/operator/nfs/server.go @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Portion of this file is comming from https://github.com/kubernetes-incubator/external-storage/blob/master/nfs/pkg/server/server.go package nfs import ( diff --git a/tests/framework/installer/nfs_installer.go b/tests/framework/installer/nfs_installer.go index 2d8c110b5..e8537ebfb 100644 --- a/tests/framework/installer/nfs_installer.go +++ b/tests/framework/installer/nfs_installer.go @@ -132,7 +132,7 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error { logger.Info("creating volume from nfs server in namespace %s", namespace) - nfsServerPVC := h.manifests.GetNFSServerPVC() + nfsServerPVC := h.manifests.GetNFSServerPVC(namespace) logger.Info("creating nfs server pvc") if _, err := h.k8shelper.KubectlWithStdin(nfsServerPVC, createFromStdinArgs...); err != nil { diff --git a/tests/framework/installer/nfs_manifests.go b/tests/framework/installer/nfs_manifests.go index 9ed19dd5d..5f8f713b2 100644 --- a/tests/framework/installer/nfs_manifests.go +++ b/tests/framework/installer/nfs_manifests.go @@ -137,6 +137,93 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-nfs-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["podsecuritypolicies"] + resourceNames: ["nfs-provisioner"] + verbs: ["use"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: + - nfs.rook.io + resources: + - "*" + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: run-nfs-provisioner +subjects: + - kind: ServiceAccount + name: rook-nfs-provisioner + # replace with namespace where provisioner is deployed + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rook-nfs-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-nfs-provisioner + namespace: ` + namespace + ` +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: rook-nfs-provisioner + namespace: ` + namespace + ` +spec: + selector: + matchLabels: + app: rook-nfs-provisioner + replicas: 1 + strategy: + type: Recreate + template: + metadata: + labels: + app: rook-nfs-provisioner + spec: + serviceAccount: rook-nfs-provisioner + containers: + - name: rook-nfs-provisioner + image: rook/nfs:master + imagePullPolicy: IfNotPresent + args: ["nfs", "provisioner","--provisioner=rook.io/nfs-provisioner"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace ` } @@ -179,8 +266,38 @@ spec: } // GetNFSServerPVC returns NFSServer PVC definition -func (i *NFSManifests) GetNFSServerPVC() string { - return `apiVersion: v1 +func (i *NFSManifests) GetNFSServerPVC(namespace string) string { + return ` +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + app: rook-nfs + name: nfs-ns-nfs-share +parameters: + exportName: nfs-share + nfsServerName: ` + namespace + ` + nfsServerNamespace: ` + namespace + ` +provisioner: rook.io/nfs-provisioner +reclaimPolicy: Delete +volumeBindingMode: Immediate +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + app: rook-nfs + name: nfs-ns-nfs-share1 +parameters: + exportName: nfs-share1 + nfsServerName: ` + namespace + ` + nfsServerNamespace: ` + namespace + ` +provisioner: rook.io/nfs-provisioner +reclaimPolicy: Delete +volumeBindingMode: Immediate +--- +apiVersion: v1 kind: PersistentVolumeClaim metadata: name: nfs-pv-claim @@ -208,68 +325,7 @@ spec: // GetNFSServer returns NFSServer CRD instance definition func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string { - return `kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nfs-client-provisioner-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: run-nfs-client-provisioner -subjects: - - kind: ServiceAccount - name: ` + namespace + ` - namespace: ` + namespace + ` -roleRef: - kind: ClusterRole - name: nfs-client-provisioner-runner - apiGroup: rbac.authorization.k8s.io ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: leader-locking-nfs-client-provisioner - namespace: ` + namespace + ` -rules: - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: leader-locking-nfs-client-provisioner - namespace: ` + namespace + ` -subjects: - - kind: ServiceAccount - name: ` + namespace + ` - # replace with namespace where provisioner is deployed - namespace: ` + namespace + ` -roleRef: - kind: Role - name: leader-locking-nfs-client-provisioner - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ` + namespace + ` - namespace: ` + namespace + ` ---- + return ` apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -306,14 +362,12 @@ spec: replicas: ` + strconv.Itoa(count) + ` exports: - name: nfs-share - createStorageClass: true server: accessMode: ReadWrite squash: "none" persistentVolumeClaim: claimName: test-claim - name: nfs-share1 - createStorageClass: true server: accessMode: ReadWrite squash: "none"