enhance(NFS): adds dynamic provisoning for nfs

Signed-off-by: Ashish Ranjan <ashishranjan738@gmail.com>

This commit adds dynamic provisioning for nfs
This commit is contained in:
Ashish Ranjan committed 2019-04-26 04:25:49 +05:30
1 parent e749783a21
commit 23efbf658b
16 files changed
+566 -162

No files matched your search

+63
View File
@@ -3,6 +3,68 @@ kind: Namespace
metadata:
name: rook-nfs
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: rook-nfs
namespace: rook-nfs
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: rook-nfs
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: rook-nfs
subjects:
- kind: ServiceAccount
name: rook-nfs
# replace with namespace where provisioner is deployed
namespace: rook-nfs
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-nfs
namespace: rook-nfs
---
# A default storageclass must be present
apiVersion: v1
kind: PersistentVolumeClaim
@@ -22,6 +84,7 @@ metadata:
name: rook-nfs
namespace: rook-nfs
spec:
serviceAccountName: rook-nfs
replicas: 1
exports:
- name: nfs-share
@@ -23,12 +23,13 @@ metadata:
name: rook-nfs-operator
rules:
- apiGroups:
- ""
- "*"
resources:
- namespaces
- configmaps
- pods
- services
- storageclasses
verbs:
- get
- watch
+3 -2
View File
@@ -2,10 +2,11 @@ apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rook-nfs-pv-claim
annotations:
volume.beta.kubernetes.io/storage-class: "nfs-share"
spec:
storageClassName: rook-nfs-sc
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Mi
storage: 1Mi
+1
View File
@@ -36,6 +36,7 @@ var (
func init() {
Cmd.AddCommand(operatorCmd)
Cmd.AddCommand(serverCmd)
}
func createContext() *clusterd.Context {
+103
View File
@@ -0,0 +1,103 @@
/*
Copyright 2018 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfs
import (
"errors"
"fmt"
"os"
"time"
"github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller"
"github.com/rook/rook/cmd/rook/rook"
"github.com/rook/rook/pkg/operator/nfs"
"github.com/rook/rook/pkg/util/flags"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/util/wait"
)
var serverCmd = &cobra.Command{
Use: "server",
Short: "Runs the NFS server to deploy and manage NFS server in kubernetes clusters",
Long: `Runs the NFS operator to deploy and manage NFS server in kubernetes clusters.
https://github.com/rook/rook`,
}
var (
provisioner, server, path, ganeshaConfigPath *string
)
func init() {
flags.SetFlagsFromEnv(serverCmd.Flags(), rook.RookEnvVarPrefix)
flags.SetLoggingFlags(serverCmd.Flags())
provisioner = serverCmd.Flags().String("provisioner", "", "Name of the provisioner. The provisioner will only provision volumes for claims that request a StorageClass with a provisioner field set equal to this name.")
server = serverCmd.Flags().String("server", os.Getenv("ROOK_NFS_SERVICE_HOST"), "Name of the NFS server")
ganeshaConfigPath = serverCmd.Flags().String("ganeshaConfigPath", "", "ConfigPath of nfs ganesha")
serverCmd.RunE = startServer
}
func startServer(cmd *cobra.Command, args []string) error {
rook.SetLogLevel()
rook.LogStartupInfo(serverCmd.Flags())
if len(*provisioner) == 0 {
return errors.New("--provisioner is a required parameter")
} else if len(*ganeshaConfigPath) == 0 {
return errors.New("--ganeshaConfigPath is a required parameter")
}
clientset, _, _, err := rook.GetClientset()
if err != nil {
rook.TerminateFatal(fmt.Errorf("failed to get k8s clients. %+v", err))
}
logger.Infof("Setting up NFS server!")
err = nfs.Setup(*ganeshaConfigPath)
if err != nil {
logger.Fatalf("Error setting up NFS server: %v", err)
}
logger.Infof("starting NFS server")
go func() {
for {
// This blocks until server exits (presumably due to an error)
err = nfs.Run(*ganeshaConfigPath)
if err != nil {
logger.Errorf("NFS server Exited Unexpectedly with err: %v", err)
}
// take a moment before trying to restart
time.Sleep(time.Second)
}
}()
// Wait for NFS server to come up before continuing provisioner process
time.Sleep(5 * time.Second)
serverVersion, err := clientset.Discovery().ServerVersion()
if err != nil {
logger.Fatalf("Error getting server version: %v", err)
}
clientNFSProvisioner := nfs.NewNFSProvisioner(clientset, *server)
pc := controller.NewProvisionController(clientset, *provisioner, clientNFSProvisioner, serverVersion.GitVersion)
pc.Run(wait.NeverStop)
return nil
}
+14 -24
View File
@@ -15,36 +15,26 @@
#Portions of this file came from https://github.com/kubernetes-incubator/external-storage/blob/master/nfs/deploy/docker/Dockerfile, which uses the same license.
FROM NFS_BASEIMAGE
# Build ganesha from source, installing deps and removing them in one line.
# Why?
# 1. Root_Id_Squash, only present in >= 2.4.0.3 which is not yet packaged
# 2. Set NFS_V4_RECOV_ROOT to /export
# 3. Use device major/minor as fsid major/minor to work on OverlayFS
RUN dnf install -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel jemalloc-devel libnfsidmap-devel patch libnsl2-devel && dnf clean all \
&& curl -L https://github.com/nfs-ganesha/nfs-ganesha/archive/V2.4.1.tar.gz | tar zx \
&& curl -L https://github.com/nfs-ganesha/nfs-ganesha/commit/c583534b166be198755d905c82a7687d83b458d8.patch -o /nfs-ganesha.patch \
&& curl -L https://github.com/nfs-ganesha/ntirpc/archive/v1.4.4.tar.gz | tar zx \
&& rm -r nfs-ganesha-2.4.1/src/libntirpc \
&& mv ntirpc-1.4.4 nfs-ganesha-2.4.1/src/libntirpc \
&& cd nfs-ganesha-2.4.1 \
&& cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_CONFIG=vfs_only src/ \
&& patch -p1 < /nfs-ganesha.patch \
&& make \
&& make install \
&& cp src/scripts/ganeshactl/org.ganesha.nfsd.conf /etc/dbus-1/system.d/ \
&& cd .. \
&& rm -rf nfs-ganesha-2.4.1 /nfs-ganesha.patch \
&& dnf remove -y tar gcc cmake autoconf libtool bison flex make gcc-c++ krb5-devel dbus-devel jemalloc-devel libnfsidmap-devel patch && dnf clean all
RUN dnf install -y dbus-x11 rpcbind hostname nfs-utils xfsprogs jemalloc libnfsidmap && dnf clean all
RUN mkdir -p /var/run/dbus
RUN mkdir -p /export
COPY start.sh /
RUN DEBIAN_FRONTEND=noninteractive \
&& apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FE869A9 \
&& echo "deb http://ppa.launchpad.net/gluster/nfs-ganesha-2.5/ubuntu xenial main" > /etc/apt/sources.list.d/nfs-ganesha-2.5.list \
&& echo "deb http://ppa.launchpad.net/gluster/libntirpc-1.5/ubuntu xenial main" > /etc/apt/sources.list.d/libntirpc-1.5.list \
&& echo "deb http://ppa.launchpad.net/gluster/glusterfs-3.13/ubuntu xenial main" > /etc/apt/sources.list.d/glusterfs-3.13.list \
&& apt-get update \
&& apt-get install -y netbase nfs-common dbus nfs-ganesha nfs-ganesha-vfs glusterfs-common \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
&& mkdir -p /run/rpcbind /export /var/run/dbus \
&& touch /run/rpcbind/rpcbind.xdr /run/rpcbind/portmap.xdr \
&& chmod 755 /run/rpcbind/* \
&& chown messagebus:messagebus /var/run/dbus
EXPOSE 2049 38465-38467 662 111/udp 111
COPY rook /usr/local/bin/
+1 -2
View File
@@ -19,7 +19,7 @@ include ../image.mk
NFS_IMAGE = $(BUILD_REGISTRY)/nfs-$(GOARCH)
NFS_BASE ?= fedora:28
NFS_BASE ?= ubuntu:xenial
ifeq ($(GOARCH),amd64)
NFS_BASEIMAGE = $(NFS_BASE)
@@ -41,7 +41,6 @@ IMAGE_FILENAME := $(IMAGE_OUTPUT_DIR)/nfs.tar.gz
do.build:
@echo === docker build $(NFS_IMAGE)
@cp Dockerfile $(TEMP)
@cp start.sh $(TEMP)
@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
@cd $(TEMP) && $(SED_CMD) 's|NFS_BASEIMAGE|$(NFS_BASEIMAGE)|g' Dockerfile
@$(DOCKERCMD) build $(BUILD_ARGS) \
-57
View File
@@ -1,57 +0,0 @@
#!/bin/bash
# Copyright 2018 The Rook Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#Portions of this file came from https://github.com/mitcdh/docker-nfs-ganesha/blob/master/start.sh
set -e
# Options for starting Ganesha
: ${GANESHA_LOGFILE:="/dev/stdout"}
: ${GANESHA_CONFIGFILE:="/nfs-ganesha/config/nfs-ganesha-config"}
: ${GANESHA_OPTIONS:="-N NIV_DEBUG"} # NIV_DEBUG
: ${GANESHA_EPOCH:=""}
function init_rpc {
echo "Starting rpcbind"
rpcbind || return 0
rpc.statd -L || return 0
rpc.idmapd || return 0
sleep 1
}
function init_dbus {
echo "Starting dbus"
rm -f /var/run/dbus/system_bus_socket
rm -f /var/run/dbus/pid
dbus-uuidgen --ensure
dbus-daemon --system --fork
sleep 1
}
function startup_script {
if [ -f "${STARTUP_SCRIPT}" ]; then
/bin/sh ${STARTUP_SCRIPT}
fi
}
startup_script
init_rpc
init_dbus
echo "Starting Ganesha NFS"
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
exec /usr/bin/ganesha.nfsd -F -L ${GANESHA_LOGFILE} -f ${GANESHA_CONFIGFILE} ${GANESHA_OPTIONS}
+6 -1
View File
@@ -17,7 +17,7 @@ package v1alpha1
import (
rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -53,6 +53,11 @@ type NFSServerSpec struct {
// Replicas of the NFS daemon
Replicas int `json:"replicas,omitempty"`
// ServiceAccountName is the name of the ServiceAccount to use to run this nfs server pod.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
// +optional
ServiceAccountName string `json:"serviceAccountName,omitempty" protobuf:"bytes,8,opt,name=serviceAccountName"`
// The parameters to configure the NFS export
Exports []ExportsSpec `json:"exports,omitempty"`
}
+71 -38
View File
@@ -28,7 +28,8 @@ import (
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -40,9 +41,7 @@ import (
const (
customResourceName = "nfsserver"
customResourceNamePlural = "nfsservers"
appName = "rook-nfs"
nfsConfigMapName = "nfs-ganesha-config"
nfsConfigMapPath = "/nfs-ganesha/config"
NFSConfigMapPath = "/nfs-ganesha/config"
nfsPort = 2049
rpcPort = 111
noneMode = "none"
@@ -99,7 +98,7 @@ type nfsServer struct {
func newNfsServer(c *nfsv1alpha1.NFSServer, context *clusterd.Context) *nfsServer {
return &nfsServer{
name: appName,
name: c.Name,
context: context,
namespace: c.Namespace,
spec: c.Spec,
@@ -134,9 +133,9 @@ func getServerConfig(exports []nfsv1alpha1.ExportsSpec) map[string]map[string]st
return claimConfigOpt
}
func createAppLabels() map[string]string {
func createAppLabels(nfsServer *nfsServer) map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.AppAttr: nfsServer.name,
}
}
@@ -155,32 +154,58 @@ func createServicePorts() []v1.ServicePort {
}
}
func (c *Controller) createNFSService(nfsServer *nfsServer) error {
func (c *Controller) createStorageClass(nfsServer *nfsServer) error {
for _, export := range nfsServer.spec.Exports {
nfsStorageClass := &storage.StorageClass{
ObjectMeta: metav1.ObjectMeta{
Name: export.Name,
Annotations: map[string]string{
"rook.io/nfs-path": "/" + export.PersistentVolumeClaim.ClaimName,
},
OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef},
Labels: createAppLabels(nfsServer),
},
Provisioner: "rook.io/" + nfsServer.name,
}
if _, err := c.context.Clientset.StorageV1().StorageClasses().Create(nfsStorageClass); err != nil {
if !errors.IsAlreadyExists(err) {
return err
}
logger.Infof("storage class %s already exists", nfsStorageClass.GetName())
} else {
logger.Infof("storage class %s created", nfsStorageClass.GetName())
}
}
return nil
}
func (c *Controller) createNFSService(nfsServer *nfsServer) (*v1.Service, error) {
// This service is meant to be used by clients to access NFS.
nfsService := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: nfsServer.name,
Namespace: nfsServer.namespace,
OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef},
Labels: createAppLabels(),
Labels: createAppLabels(nfsServer),
},
Spec: v1.ServiceSpec{
Selector: createAppLabels(),
Selector: createAppLabels(nfsServer),
Type: v1.ServiceTypeClusterIP,
Ports: createServicePorts(),
},
}
if _, err := c.context.Clientset.CoreV1().Services(nfsServer.namespace).Create(nfsService); err != nil {
svc, err := c.context.Clientset.CoreV1().Services(nfsServer.namespace).Create(nfsService)
if err != nil {
if !errors.IsAlreadyExists(err) {
return err
return nil, err
}
logger.Infof("nfs service %s already exists in namespace %s", nfsService.Name, nfsService.Namespace)
} else {
logger.Infof("nfs service %s started in namespace %s", nfsService.Name, nfsService.Namespace)
}
return nil
return svc, nil
}
func createCephNFSExport(id int, path string, access string, squash string) string {
@@ -241,13 +266,13 @@ func (c *Controller) createNFSConfigMap(nfsServer *nfsServer) error {
configMap := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: nfsConfigMapName,
Name: nfsServer.name,
Namespace: nfsServer.namespace,
OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef},
Labels: createAppLabels(),
Labels: createAppLabels(nfsServer),
},
Data: map[string]string{
nfsConfigMapName: nfsGaneshaConfig,
nfsServer.name: nfsGaneshaConfig,
},
}
_, err := c.context.Clientset.CoreV1().ConfigMaps(nfsServer.namespace).Create(configMap)
@@ -271,9 +296,9 @@ func getPVCNameList(spec *nfsv1alpha1.NFSServerSpec) []string {
return pvcNameList
}
func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume {
func createPVCSpecList(nfsServer *nfsServer) []v1.Volume {
pvcSpecList := make([]v1.Volume, 0)
pvcNameList := getPVCNameList(spec)
pvcNameList := getPVCNameList(&nfsServer.spec)
for _, claimName := range pvcNameList {
pvcSpecList = append(pvcSpecList, v1.Volume{
Name: claimName,
@@ -288,14 +313,14 @@ func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume {
configMapSrc := &v1.ConfigMapVolumeSource{
Items: []v1.KeyToPath{
{
Key: nfsConfigMapName,
Path: nfsConfigMapName,
Key: nfsServer.name,
Path: nfsServer.name,
},
},
}
configMapSrc.Name = nfsConfigMapName
configMapSrc.Name = nfsServer.name
configMapVol := v1.Volume{
Name: nfsConfigMapName,
Name: nfsServer.name,
VolumeSource: v1.VolumeSource{
ConfigMap: configMapSrc,
},
@@ -305,9 +330,9 @@ func createPVCSpecList(spec *nfsv1alpha1.NFSServerSpec) []v1.Volume {
return pvcSpecList
}
func createVolumeMountList(spec *nfsv1alpha1.NFSServerSpec) []v1.VolumeMount {
func createVolumeMountList(nfsServer *nfsServer) []v1.VolumeMount {
volumeMountList := make([]v1.VolumeMount, 0)
pvcNameList := getPVCNameList(spec)
pvcNameList := getPVCNameList(&nfsServer.spec)
for _, claimName := range pvcNameList {
volumeMountList = append(volumeMountList, v1.VolumeMount{
Name: claimName,
@@ -316,27 +341,29 @@ func createVolumeMountList(spec *nfsv1alpha1.NFSServerSpec) []v1.VolumeMount {
}
configMapVolMount := v1.VolumeMount{
Name: nfsConfigMapName,
MountPath: nfsConfigMapPath,
Name: nfsServer.name,
MountPath: NFSConfigMapPath,
}
volumeMountList = append(volumeMountList, configMapVolMount)
return volumeMountList
}
func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec {
func (c *Controller) createNfsPodSpec(nfsServer *nfsServer, service *v1.Service) v1.PodTemplateSpec {
nfsPodSpec := v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: nfsServer.name,
Namespace: nfsServer.namespace,
Labels: createAppLabels(),
Labels: createAppLabels(nfsServer),
},
Spec: v1.PodSpec{
ServiceAccountName: nfsServer.spec.ServiceAccountName,
Containers: []v1.Container{
{
Name: nfsServer.name,
Image: c.containerImage,
Command: []string{"/start.sh"},
ImagePullPolicy: "IfNotPresent",
Name: nfsServer.name,
Image: c.containerImage,
Args: []string{"nfs", "server", "--provisioner=rook.io/" + nfsServer.name, "--ganeshaConfigPath=" + NFSConfigMapPath + "/" + nfsServer.name, "--server=" + service.Spec.ClusterIP},
Ports: []v1.ContainerPort{
{
Name: "nfs-port",
@@ -347,7 +374,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec {
ContainerPort: int32(rpcPort),
},
},
VolumeMounts: createVolumeMountList(&nfsServer.spec),
VolumeMounts: createVolumeMountList(nfsServer),
SecurityContext: &v1.SecurityContext{
Capabilities: &v1.Capabilities{
Add: []v1.Capability{
@@ -358,7 +385,7 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec {
},
},
},
Volumes: createPVCSpecList(&nfsServer.spec),
Volumes: createPVCSpecList(nfsServer),
},
}
@@ -368,19 +395,19 @@ func (c *Controller) createNfsPodSpec(nfsServer *nfsServer) v1.PodTemplateSpec {
func (c *Controller) createNfsStatefulSet(nfsServer *nfsServer, replicas int32) error {
appsClient := c.context.Clientset.AppsV1()
nfsPodSpec := c.createNfsPodSpec(nfsServer)
nfsPodSpec := c.createNfsPodSpec(nfsServer, service)
statefulSet := appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: nfsServer.name,
Namespace: nfsServer.namespace,
Labels: createAppLabels(),
Labels: createAppLabels(nfsServer),
OwnerReferences: []metav1.OwnerReference{nfsServer.ownerRef},
},
Spec: appsv1.StatefulSetSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: createAppLabels(),
MatchLabels: createAppLabels(nfsServer),
},
Template: nfsPodSpec,
ServiceName: nfsServer.name,
@@ -415,7 +442,8 @@ func (c *Controller) onAdd(obj interface{}) {
}
logger.Infof("creating nfs server service in namespace %s", nfsServer.namespace)
if err := c.createNFSService(nfsServer); err != nil {
svc, err := c.createNFSService(nfsServer)
if err != nil {
logger.Errorf("Unable to create NFS service %+v", err)
}
@@ -425,9 +453,14 @@ func (c *Controller) onAdd(obj interface{}) {
}
logger.Infof("creating nfs server stateful set in namespace %s", nfsServer.namespace)
if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas)); err != nil {
if err := c.createNfsStatefulSet(nfsServer, int32(nfsServer.spec.Replicas), svc); err != nil {
logger.Errorf("Unable to create NFS stateful set %+v", err)
}
logger.Infof("creating nfs server storageclass")
if err := c.createStorageClass(nfsServer); err != nil {
logger.Errorf("Unable to create NFS storageclass %+v", err)
}
}
func (c *Controller) onUpdate(oldObj, newObj interface{}) {
+7 -5
View File
@@ -25,11 +25,13 @@ import (
"github.com/rook/rook/pkg/operator/k8sutil"
testop "github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
const appName = "nfs-server-X"
func TestValidateNFSServerSpec(t *testing.T) {
// first, test that a good NFSServerSpec is good
@@ -122,13 +124,13 @@ func TestOnAdd(t *testing.T) {
controller.onAdd(nfsserver)
// verify client service
clientService, err := clientset.CoreV1().Services(namespace).Get(appName, metav1.GetOptions{})
clientService, err := clientset.CoreV1().Services(namespace).Get(nfsserver.GetName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.NotNil(t, clientService)
assert.Equal(t, v1.ServiceTypeClusterIP, clientService.Spec.Type)
// verify nfs-ganesha config in the configmap
configMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(nfsConfigMapName, metav1.GetOptions{})
configMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(nfsserver.GetName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.NotNil(t, configMap)
nfsGaneshaConfig := `
@@ -149,7 +151,7 @@ NFS_Core_Param
{
fsid_device = true;
}`
assert.Equal(t, nfsGaneshaConfig, configMap.Data[nfsConfigMapName])
assert.Equal(t, nfsGaneshaConfig, configMap.Data[nfsserver.GetName()])
// verify stateful set
ss, err := clientset.AppsV1().StatefulSets(namespace).Get(appName, metav1.GetOptions{})
@@ -161,7 +163,7 @@ NFS_Core_Param
container := ss.Spec.Template.Spec.Containers[0]
assert.Equal(t, 2, len(container.VolumeMounts))
expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: "nfs-ganesha-config", MountPath: "/nfs-ganesha/config"}}
expectedVolumeMounts := []v1.VolumeMount{{Name: "test-claim", MountPath: "/test-claim"}, {Name: nfsserver.GetName(), MountPath: "/nfs-ganesha/config"}}
assert.Equal(t, expectedVolumeMounts, container.VolumeMounts)
}
+118
View File
@@ -0,0 +1,118 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfs
import (
"fmt"
"github.com/kubernetes-sigs/sig-storage-lib-external-provisioner/controller"
v1 "k8s.io/api/core/v1"
storage "k8s.io/api/storage/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/apis/core/v1/helper"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type nfsProvisioner struct {
client kubernetes.Interface
server string
}
const (
claimPath = "rook.io/nfs-path"
)
var _ controller.Provisioner = &nfsProvisioner{}
// NewNFSProvisioner returns an instance of nfsProvisioner
func NewNFSProvisioner(clientset kubernetes.Interface, server string) nfsProvisioner {
return nfsProvisioner{clientset, server}
}
func (p nfsProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
logger.Infof("nfs provisioner: VolumeOptions %v", options)
// Get the storage class for this volume.
storageClass, err := p.getClassForPVC(options.PVC)
if err != nil {
return nil, err
}
if options.PVC.Spec.Selector != nil {
return nil, fmt.Errorf("claim Selector is not supported")
}
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: options.PVName,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
AccessModes: options.PVC.Spec.AccessModes,
MountOptions: options.MountOptions,
Capacity: v1.ResourceList{
v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
},
PersistentVolumeSource: v1.PersistentVolumeSource{
NFS: &v1.NFSVolumeSource{
Server: p.server,
Path: storageClass.GetAnnotations()[claimPath],
ReadOnly: false,
},
},
},
}
return pv, nil
}
func (p nfsProvisioner) Delete(volume *v1.PersistentVolume) error {
return nil
}
// getClassForPV returns StorageClass
func (p nfsProvisioner) getClassForPV(pv *v1.PersistentVolume) (*storage.StorageClass, error) {
if p.client == nil {
return nil, fmt.Errorf("Cannot get kube client")
}
className := helper.GetPersistentVolumeClass(pv)
if className == "" {
return nil, fmt.Errorf("Volume has no storage class")
}
class, err := p.client.StorageV1().StorageClasses().Get(className, metav1.GetOptions{})
if err != nil {
return nil, err
}
return class, nil
}
// getClassForPVC returns StorageClass
func (p nfsProvisioner) getClassForPVC(pvc *v1.PersistentVolumeClaim) (*storage.StorageClass, error) {
if p.client == nil {
return nil, fmt.Errorf("Cannot get kube client")
}
className := helper.GetPersistentVolumeClaimClass(pvc)
if className == "" {
return nil, fmt.Errorf("Volume has no storage class")
}
class, err := p.client.StorageV1().StorageClasses().Get(className, metav1.GetOptions{})
if err != nil {
return nil, err
}
return class, nil
}
+100
View File
@@ -0,0 +1,100 @@
/*
Copyright 2019 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nfs
import (
"fmt"
"os/exec"
"syscall"
)
const (
ganeshaLog = "/dev/stdout"
ganeshaOptions = "NIV_DEBUG"
)
// Setup sets up various prerequisites and settings for the server. If an error
// is encountered at any point it returns it instantly
func Setup(ganeshaConfig string) error {
// Start rpcbind if it is not started yet
cmd := exec.Command("rpcinfo", "127.0.0.1")
if err := cmd.Run(); err != nil {
cmd = exec.Command("rpcbind", "-w")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("Starting rpcbind failed with error: %v, output: %s", err, out)
}
}
cmd = exec.Command("rpc.statd")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("rpc.statd failed with error: %v, output: %s", err, out)
}
// Start dbus, needed for ganesha dynamic exports
cmd = exec.Command("dbus-daemon", "--system")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("dbus-daemon failed with error: %v, output: %s", err, out)
}
err := setRlimitNOFILE()
if err != nil {
logger.Warningf("Error setting RLIMIT_NOFILE, there may be 'Too many open files' errors later: %v", err)
}
return nil
}
// Run : run the NFS server in the foreground until it exits
// Ideally, it should never exit when run in foreground mode
// We force foreground to allow the provisioner process to restart
// the server if it crashes - daemonization prevents us from using Wait()
// for this purpose
func Run(ganeshaConfig string) error {
// Start ganesha.nfsd
logger.Infof("Running NFS server!")
cmd := exec.Command("ganesha.nfsd", "-F", "-L", ganeshaLog, "-f", ganeshaConfig, "-N", ganeshaOptions)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("ganesha.nfsd failed with error: %v, output: %s", err, out)
}
return nil
}
func setRlimitNOFILE() error {
var rlimit syscall.Rlimit
err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)
if err != nil {
return fmt.Errorf("error getting RLIMIT_NOFILE: %v", err)
}
logger.Infof("starting RLIMIT_NOFILE rlimit.Cur %d, rlimit.Max %d", rlimit.Cur, rlimit.Max)
rlimit.Max = 1024 * 1024
rlimit.Cur = 1024 * 1024
err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
if err != nil {
return err
}
err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)
if err != nil {
return fmt.Errorf("error getting RLIMIT_NOFILE: %v", err)
}
logger.Infof("ending RLIMIT_NOFILE rlimit.Cur %d, rlimit.Max %d", rlimit.Cur, rlimit.Max)
return nil
}
// Stop stops the NFS server.
func Stop() {
// /bin/dbus-send --system --dest=org.ganesha.nfsd --type=method_call /org/ganesha/nfsd/admin org.ganesha.nfsd.admin.shutdown
}
+4 -24
View File
@@ -113,12 +113,12 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass
return fmt.Errorf("Failed to create nfs server: %+v ", err)
}
if err := h.k8shelper.WaitForPodCount("app=rook-nfs", namespace, 1); err != nil {
if err := h.k8shelper.WaitForPodCount("app="+namespace, namespace, 1); err != nil {
logger.Errorf("nfs server pods in namespace %s not found", namespace)
return err
}
err := h.k8shelper.WaitForLabeledPodsToRun("app=rook-nfs", namespace)
err := h.k8shelper.WaitForLabeledPodsToRun("app="+namespace, namespace)
if err != nil {
logger.Errorf("nfs server pods in namespace %s are not running", namespace)
return err
@@ -132,18 +132,8 @@ func (h *NFSInstaller) CreateNFSServer(namespace string, count int, storageClass
func (h *NFSInstaller) CreateNFSServerVolume(namespace string) error {
logger.Info("creating volume from nfs server in namespace %s", namespace)
clusterIP, err := h.GetNFSServerClusterIP(namespace)
if err != nil {
return err
}
nfsServerPV := h.manifests.GetNFSServerPV(namespace, clusterIP)
nfsServerPVC := h.manifests.GetNFSServerPVC()
logger.Info("creating nfs server pv")
if _, err := h.k8shelper.KubectlWithStdin(nfsServerPV, createFromStdinArgs...); err != nil {
return err
}
logger.Info("creating nfs server pvc")
if _, err := h.k8shelper.KubectlWithStdin(nfsServerPVC, createFromStdinArgs...); err != nil {
return err
@@ -193,6 +183,8 @@ func (h *NFSInstaller) UninstallNFSServer(systemNamespace, namespace string) {
checkError(h.T(), err, "cannot uninstall hostpath provisioner")
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("anon-user-access", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("run-nfs-client-provisioner", nil)
h.k8shelper.Clientset.RbacV1beta1().ClusterRoles().Delete("nfs-client-provisioner-runner", nil)
logger.Infof("done removing the operator from namespace %s", systemNamespace)
}
@@ -202,15 +194,3 @@ func (h *NFSInstaller) GatherAllNFSServerLogs(systemNamespace, namespace, testNa
h.k8shelper.GetLogs("rook-nfs-operator", Env.HostType, systemNamespace, testName)
h.k8shelper.GetLogs("rook-nfs", Env.HostType, namespace, testName)
}
// GetNFSServerClusterIP gets the nfs server cluster ip on which it serves
func (h *NFSInstaller) GetNFSServerClusterIP(namespace string) (string, error) {
clusterIP := ""
service, err := h.k8shelper.GetService("rook-nfs", namespace)
if err != nil {
logger.Errorf("nfs server service in namespace %s is not active", namespace)
return clusterIP, err
}
clusterIP = service.Spec.ClusterIP
return clusterIP, nil
}
+72 -7
View File
@@ -43,10 +43,10 @@ spec:
// GetNFSServerOperator returns the NFSServer operator definition
func (i *NFSManifests) GetNFSServerOperator(namespace string) string {
return `kind: Namespace
apiVersion: v1
return `apiVersion: v1
kind: Namespace
metadata:
name: ` + namespace + `
name: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
@@ -54,12 +54,13 @@ metadata:
name: rook-nfs-operator
rules:
- apiGroups:
- ""
- "*"
resources:
- namespaces
- configmaps
- pods
- services
- storageclasses
verbs:
- get
- watch
@@ -124,6 +125,7 @@ spec:
serviceAccountName: rook-nfs-operator
containers:
- name: rook-nfs-operator
imagePullPolicy: IfNotPresent
image: rook/nfs:master
args: ["nfs", "operator"]
env:
@@ -183,7 +185,7 @@ kind: PersistentVolumeClaim
metadata:
name: nfs-pv-claim
spec:
storageClassName: nfs-sc
storageClassName: nfs-share
accessModes:
- ReadWriteMany
resources:
@@ -195,7 +197,7 @@ kind: PersistentVolumeClaim
metadata:
name: nfs-pv-claim-bigger
spec:
storageClassName: nfs-sc
storageClassName: nfs-share1
accessModes:
- ReadWriteMany
resources:
@@ -206,7 +208,69 @@ spec:
// GetNFSServer returns NFSServer CRD instance definition
func (i *NFSManifests) GetNFSServer(namespace string, count int, storageClassName string) string {
return `apiVersion: v1
return `kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: ` + namespace + `
namespace: ` + namespace + `
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: ` + namespace + `
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: ` + namespace + `
subjects:
- kind: ServiceAccount
name: ` + namespace + `
# replace with namespace where provisioner is deployed
namespace: ` + namespace + `
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: ` + namespace + `
namespace: ` + namespace + `
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: test-claim
@@ -238,6 +302,7 @@ metadata:
name: ` + namespace + `
namespace: ` + namespace + `
spec:
serviceAccountName: ` + namespace + `
replicas: ` + strconv.Itoa(count) + `
exports:
- name: nfs-share
+1 -1
View File
@@ -105,7 +105,7 @@ func (suite *NfsSuite) TestNfsServerInstallation() {
"1 rook-nfs-operator must be in Running state")
// verify nfs server instances are running OK
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState("rook-nfs", suite.namespace, suite.instanceCount, "Running"),
assert.True(suite.T(), suite.k8shelper.CheckPodCountAndState(suite.namespace, suite.namespace, suite.instanceCount, "Running"),
fmt.Sprintf("%d rook-nfs pods must be in Running state", suite.instanceCount))
// verify bigger export is running OK