Files
my-rook-config/pkg/operator/nfs/quota.go
T

266 lines
7.1 KiB
Go

package nfs
import (
"fmt"
"io/ioutil"
"math"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"sigs.k8s.io/sig-storage-lib-external-provisioner/v6/mount"
)
type Quotaer interface {
CreateProjectQuota(projectsFile, directory, limit string) (string, error)
RemoveProjectQuota(projectID uint16, projectsFile, block string) error
RestoreProjectQuota() error
}
type Quota struct {
mutex *sync.Mutex
projectsIDs map[string]map[uint16]bool
}
func NewProjectQuota() (Quotaer, error) {
projectsIDs := map[string]map[uint16]bool{}
mountEntries, err := findProjectQuotaMount()
if err != nil {
return nil, err
}
for _, entry := range mountEntries {
exportName := filepath.Base(entry.Mountpoint)
projectsIDs[exportName] = map[uint16]bool{}
projectsFile := filepath.Join(entry.Mountpoint, "projects")
_, err := os.Stat(projectsFile)
if os.IsNotExist(err) {
logger.Infof("creating new project file %s", projectsFile)
file, cerr := os.Create(projectsFile)
if cerr != nil {
return nil, fmt.Errorf("error creating xfs projects file %s: %v", projectsFile, cerr)
}
if err := file.Close(); err != nil {
return nil, err
}
} else {
logger.Infof("found project file %s, restoring project ids", projectsFile)
re := regexp.MustCompile("(?m:^([0-9]+):/.+$)")
projectIDs, err := restoreProjectIDs(projectsFile, re)
if err != nil {
logger.Errorf("error while populating projectIDs map, there may be errors setting quotas later if projectIDs are reused: %v", err)
}
projectsIDs[exportName] = projectIDs
}
}
quota := &Quota{
mutex: &sync.Mutex{},
projectsIDs: projectsIDs,
}
if err := quota.RestoreProjectQuota(); err != nil {
return nil, err
}
return quota, nil
}
func findProjectQuotaMount() ([]*mount.Info, error) {
var entries []*mount.Info
allEntries, err := mount.GetMounts()
if err != nil {
return nil, err
}
for _, entry := range allEntries {
// currently we only support xfs
if entry.Fstype != "xfs" {
continue
}
if filepath.Dir(entry.Mountpoint) == mountPath && (strings.Contains(entry.VfsOpts, "pquota") || strings.Contains(entry.VfsOpts, "prjquota")) {
entries = append(entries, entry)
}
}
return entries, nil
}
func restoreProjectIDs(projectsFile string, re *regexp.Regexp) (map[uint16]bool, error) {
ids := map[uint16]bool{}
digitsRe := "([0-9]+)"
if !strings.Contains(re.String(), digitsRe) {
return ids, fmt.Errorf("regexp %s doesn't contain digits submatch %s", re.String(), digitsRe)
}
read, err := ioutil.ReadFile(projectsFile) // #nosec
if err != nil {
return ids, err
}
allMatches := re.FindAllSubmatch(read, -1)
for _, match := range allMatches {
digits := match[1]
if id, err := strconv.ParseUint(string(digits), 10, 16); err == nil {
ids[uint16(id)] = true
}
}
return ids, nil
}
func (q *Quota) CreateProjectQuota(projectsFile, directory, limit string) (string, error) {
exportName := filepath.Base(filepath.Dir(projectsFile))
q.mutex.Lock()
projectID := uint16(1)
for ; projectID < math.MaxUint16; projectID++ {
if _, ok := q.projectsIDs[exportName][projectID]; !ok {
break
}
}
q.projectsIDs[exportName][projectID] = true
block := strconv.FormatUint(uint64(projectID), 10) + ":" + directory + ":" + limit + "\n"
file, err := os.OpenFile(projectsFile, os.O_APPEND|os.O_WRONLY, 0600) // #nosec
if err != nil {
q.mutex.Unlock()
return "", err
}
defer func() {
if err := file.Close(); err != nil {
logger.Errorf("Error closing file: %s\n", err)
}
}()
if _, err = file.WriteString(block); err != nil {
q.mutex.Unlock()
return "", err
}
if err := file.Sync(); err != nil {
q.mutex.Unlock()
return "", err
}
logger.Infof("set project to %s for directory %s with limit %s", projectsFile, directory, limit)
if err := q.setProject(projectID, projectsFile, directory, block); err != nil {
q.mutex.Unlock()
return "", err
}
logger.Infof("set quota for project id %d with limit %s", projectID, limit)
if err := q.setQuota(projectID, projectsFile, directory, limit); err != nil {
q.mutex.Unlock()
_ = q.removeProject(projectID, projectsFile, block)
}
q.mutex.Unlock()
return block, nil
}
func (q *Quota) RemoveProjectQuota(projectID uint16, projectsFile, block string) error {
return q.removeProject(projectID, projectsFile, block)
}
func (q *Quota) RestoreProjectQuota() error {
mountEntries, err := findProjectQuotaMount()
if err != nil {
return err
}
for _, entry := range mountEntries {
projectsFile := filepath.Join(entry.Mountpoint, "projects")
if _, err := os.Stat(projectsFile); err != nil {
if os.IsNotExist(err) {
continue
}
return err
}
read, err := ioutil.ReadFile(projectsFile) // #nosec
if err != nil {
return err
}
re := regexp.MustCompile("(?m:^([0-9]+):(.+):(.+)$\n)")
matches := re.FindAllSubmatch(read, -1)
for _, match := range matches {
projectID, _ := strconv.ParseUint(string(match[1]), 10, 16)
directory := string(match[2])
bhard := string(match[3])
if _, err := os.Stat(directory); os.IsNotExist(err) {
_ = q.removeProject(uint16(projectID), projectsFile, string(match[0]))
continue
}
block := strconv.FormatUint(uint64(projectID), 10) + ":" + directory + ":" + bhard + "\n"
if err := q.setProject(uint16(projectID), projectsFile, directory, block); err != nil {
return err
}
logger.Infof("restoring quotas from project file %s for project id %s", string(match[1]), projectsFile)
if err := q.setQuota(uint16(projectID), projectsFile, directory, bhard); err != nil {
return fmt.Errorf("error restoring quota for directory %s: %v", directory, err)
}
}
}
return nil
}
func (q *Quota) setProject(projectID uint16, projectsFile, directory, block string) error {
cmd := exec.Command("xfs_quota", "-x", "-c", fmt.Sprintf("project -s -p %s %s", directory, strconv.FormatUint(uint64(projectID), 10)), filepath.Dir(projectsFile)) // #nosec
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("xfs_quota failed with error: %v, output: %s", err, out)
}
return nil
}
func (q *Quota) setQuota(projectID uint16, projectsFile, directory, bhard string) error {
exportName := filepath.Base(filepath.Dir(projectsFile))
if !q.projectsIDs[exportName][projectID] {
return fmt.Errorf("project with id %v has not been added", projectID)
}
cmd := exec.Command("xfs_quota", "-x", "-c", fmt.Sprintf("limit -p bhard=%s %s", bhard, strconv.FormatUint(uint64(projectID), 10)), filepath.Dir(projectsFile)) // #nosec
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("xfs_quota failed with error: %v, output: %s", err, out)
}
return nil
}
func (q *Quota) removeProject(projectID uint16, projectsFile, block string) error {
exportName := filepath.Base(filepath.Dir(projectsFile))
q.mutex.Lock()
delete(q.projectsIDs[exportName], projectID)
read, err := ioutil.ReadFile(projectsFile) // #nosec
if err != nil {
q.mutex.Unlock()
return err
}
removed := strings.Replace(string(read), block, "", -1)
err = ioutil.WriteFile(projectsFile, []byte(removed), 0)
if err != nil {
q.mutex.Unlock()
return err
}
q.mutex.Unlock()
return nil
}