From 466053bfc063bf1f3473c4e189ccb7d4a889f0b2 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Fri, 28 Jul 2017 16:19:34 -0700 Subject: [PATCH] operator: remove provisioner dependence on rest api --- pkg/operator/mon/mon.go | 18 +++-- pkg/operator/mon/mon_test.go | 12 +-- pkg/operator/mon/spec.go | 2 +- pkg/operator/operator.go | 2 +- pkg/operator/provisioner/provisioner.go | 80 ++++++++++---------- pkg/operator/provisioner/provisioner_test.go | 37 +++++++-- 6 files changed, 90 insertions(+), 61 deletions(-) diff --git a/pkg/operator/mon/mon.go b/pkg/operator/mon/mon.go index cb85d18ab..13b22ff79 100644 --- a/pkg/operator/mon/mon.go +++ b/pkg/operator/mon/mon.go @@ -41,6 +41,12 @@ import ( var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mon") const ( + // EndpointConfigMapName is the name of the configmap with mon endpoints + EndpointConfigMapName = "rook-ceph-mon-endpoints" + + // EndpointDataKey is the name of the key inside the mon configmap to get the endpoints + EndpointDataKey = "data" + appName = "rook-ceph-mon" monNodeAttr = "mon_node" monClusterAttr = "mon_cluster" @@ -49,8 +55,6 @@ const ( monSecretName = "mon-secret" adminSecretName = "admin-secret" clusterSecretName = "cluster-name" - monConfigMapName = "mon-config" - monEndpointKey = "endpoints" maxMonIDKey = "maxMonId" ) @@ -342,14 +346,14 @@ func (c *Cluster) waitForPodToStart(name string) (string, error) { func (c *Cluster) saveMonConfig() error { configMap := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: monConfigMapName, + Name: EndpointConfigMapName, Namespace: c.Namespace, Annotations: map[string]string{}, }, } configMap.Data = map[string]string{ - monEndpointKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors), - maxMonIDKey: strconv.Itoa(c.maxMonID), + EndpointDataKey: mon.FlattenMonEndpoints(c.clusterInfo.Monitors), + maxMonIDKey: strconv.Itoa(c.maxMonID), } _, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Create(configMap) @@ -384,7 +388,7 @@ func (c *Cluster) writeConnectionConfig() error { } func (c *Cluster) loadMonConfig() error { - cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(monConfigMapName, metav1.GetOptions{}) + cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) if err != nil { if !errors.IsNotFound(err) { return err @@ -396,7 +400,7 @@ func (c *Cluster) loadMonConfig() error { } // Parse the monitor List - if info, ok := cm.Data[monEndpointKey]; ok { + if info, ok := cm.Data[EndpointDataKey]; ok { c.clusterInfo.Monitors = mon.ParseMonEndpoints(info) } else { c.clusterInfo.Monitors = map[string]*mon.CephMonitorConfig{} diff --git a/pkg/operator/mon/mon_test.go b/pkg/operator/mon/mon_test.go index 66278cb6b..647170418 100644 --- a/pkg/operator/mon/mon_test.go +++ b/pkg/operator/mon/mon_test.go @@ -107,18 +107,18 @@ func TestSaveMonEndpoints(t *testing.T) { err := c.saveMonConfig() assert.Nil(t, err) - cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config", metav1.GetOptions{}) + cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, "mon1=1.2.3.1:6790", cm.Data["endpoints"]) + assert.Equal(t, "mon1=1.2.3.1:6790", cm.Data[EndpointDataKey]) // update the config map c.clusterInfo.Monitors["mon1"].Endpoint = "2.3.4.5:6790" err = c.saveMonConfig() assert.Nil(t, err) - cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config", metav1.GetOptions{}) + cm, err = c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, "mon1=2.3.4.5:6790", cm.Data["endpoints"]) + assert.Equal(t, "mon1=2.3.4.5:6790", cm.Data[EndpointDataKey]) } func TestCheckHealth(t *testing.T) { @@ -147,9 +147,9 @@ func TestCheckHealth(t *testing.T) { err = c.failoverMon("mon1") assert.Nil(t, err) - cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get("mon-config", metav1.GetOptions{}) + cm, err := c.context.Clientset.CoreV1().ConfigMaps(c.Namespace).Get(EndpointConfigMapName, metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, "rook-ceph-mon11=:6790", cm.Data["endpoints"]) + assert.Equal(t, "rook-ceph-mon11=:6790", cm.Data[EndpointDataKey]) } func TestMonInQuourm(t *testing.T) { diff --git a/pkg/operator/mon/spec.go b/pkg/operator/mon/spec.go index bf513b72f..feea98ea1 100644 --- a/pkg/operator/mon/spec.go +++ b/pkg/operator/mon/spec.go @@ -34,7 +34,7 @@ func ClusterNameEnvVar(name string) v1.EnvVar { // EndpointEnvVar is the mon endpoint environment var func EndpointEnvVar() v1.EnvVar { - ref := &v1.ConfigMapKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: monConfigMapName}, Key: monEndpointKey} + ref := &v1.ConfigMapKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: EndpointConfigMapName}, Key: EndpointDataKey} return v1.EnvVar{Name: "ROOKD_MON_ENDPOINTS", ValueFrom: &v1.EnvVarSource{ConfigMapKeyRef: ref}} } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index cadb45033..c438f02f9 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -65,7 +65,7 @@ func New(context *clusterd.Context) *Operator { logger.Errorf("failed to create Operator. %+v.", err) return nil } - volumeProvisioner := provisioner.New(context.Clientset) + volumeProvisioner := provisioner.New(context) schemes := []kit.CustomResource{cluster.ClusterResource, pool.PoolResource} return &Operator{ diff --git a/pkg/operator/provisioner/provisioner.go b/pkg/operator/provisioner/provisioner.go index cc2374edb..6ece3f68d 100644 --- a/pkg/operator/provisioner/provisioner.go +++ b/pkg/operator/provisioner/provisioner.go @@ -23,14 +23,14 @@ import ( "github.com/coreos/pkg/capnslog" "github.com/kubernetes-incubator/external-storage/lib/controller" - "github.com/rook/rook/pkg/model" - "github.com/rook/rook/pkg/operator/api" + ceph "github.com/rook/rook/pkg/ceph/client" + cephmon "github.com/rook/rook/pkg/ceph/mon" + "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" - rookclient "github.com/rook/rook/pkg/rook/client" + "github.com/rook/rook/pkg/operator/mon" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/kubernetes" ) const ( @@ -43,7 +43,7 @@ var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-provisioner") // RookVolumeProvisioner is used to provision Rook volumes on Kubernetes type RookVolumeProvisioner struct { - client kubernetes.Interface + context *clusterd.Context // Configuration of rook volume provisioner provConfig provisionerConfig @@ -64,9 +64,9 @@ type provisionerConfig struct { } // New creates RookVolumeProvisioner -func New(client kubernetes.Interface) controller.Provisioner { +func New(context *clusterd.Context) controller.Provisioner { return &RookVolumeProvisioner{ - client: client, + context: context, } } @@ -93,22 +93,16 @@ func (p *RookVolumeProvisioner) Provision(options controller.VolumeOptions) (*v1 imageName := createImageName(options.PVName) - rookClient, err := api.GetRookClient(p.provConfig.clusterNamespace, p.client) - if err != nil { - return nil, fmt.Errorf("Failed to get rook client: %v", err) - } - - res, err := createVolume(imageName, p.provConfig.pool, requestBytes, rookClient) + err = p.createVolume(imageName, p.provConfig.pool, requestBytes) if err != nil { return nil, err } - logger.Infof("Rook block image created: %s", res) - rookClientInfo, err := rookClient.GetClientAccessInfo() + monitors, err := p.getMonitorEndpoints() if err != nil { - return nil, fmt.Errorf("Failed to get rook client information: %v", err) + return nil, err } - monitors := processMonAddresses(rookClientInfo.MonAddresses) + radosUser := fmt.Sprintf("%s-rook-user", p.provConfig.clusterName) secretRef := new(v1.LocalObjectReference) secretRef.Name = fmt.Sprintf("%s-rook-user", p.provConfig.clusterName) @@ -141,38 +135,31 @@ func (p *RookVolumeProvisioner) Provision(options controller.VolumeOptions) (*v1 } // createVolume creates a rook block volume. -func createVolume(image, pool string, size int64, client rookclient.RookRestClient) (string, error) { - newImage := model.BlockImage{ - Name: image, - PoolName: pool, - Size: uint64(size), +func (p *RookVolumeProvisioner) createVolume(image, pool string, size int64) error { + if image == "" || pool == "" || size == 0 { + return fmt.Errorf("image missing required fields (image=%s, pool=%s, size=%d)", image, pool, size) } - res, err := client.CreateBlockImage(newImage) + createdImage, err := ceph.CreateImage(p.context, p.provConfig.clusterName, image, pool, uint64(size)) if err != nil { - return "", fmt.Errorf("Failed to create rook block image %s/%s: %v", pool, image, err) + return fmt.Errorf("Failed to create rook block image %s/%s: %v", pool, image, err) } + logger.Infof("Rook block image created: %s", createdImage.Name) - return res, nil + return nil } // Delete removes the storage asset that was created by Provision represented // by the given PV. func (p *RookVolumeProvisioner) Delete(volume *v1.PersistentVolume) error { - rookClient, err := api.GetRookClient(p.provConfig.clusterNamespace, p.client) - if err != nil { - return fmt.Errorf("Failed to get rook client: %v", err) - } + logger.Infof("Deleting volume %s", volume.Name) - image := model.BlockImage{ - Name: volume.Spec.PersistentVolumeSource.RBD.RBDImage, - PoolName: p.provConfig.pool, - } - - _, err = rookClient.DeleteBlockImage(image) + name := volume.Spec.PersistentVolumeSource.RBD.RBDImage + err := ceph.DeleteImage(p.context, p.provConfig.clusterName, name, p.provConfig.pool) if err != nil { return fmt.Errorf("Failed to delete rook block image %s/%s: %v", p.provConfig.pool, volume.Name, err) } + logger.Infof("succeeded deleting volume %+v", volume) return nil } @@ -212,13 +199,24 @@ func parseClassParameters(params map[string]string) (*provisionerConfig, error) return &cfg, nil } -func processMonAddresses(monAddresses []string) []string { - monAddrs := make([]string, len(monAddresses)) - for i, addr := range monAddresses { - mon := strings.Split(addr, "/") - monAddrs[i] = mon[0] +func (p *RookVolumeProvisioner) getMonitorEndpoints() ([]string, error) { + cm, err := p.context.Clientset.CoreV1().ConfigMaps(p.provConfig.clusterName).Get(mon.EndpointConfigMapName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get mon endpoints. %+v", err) } - return monAddrs + + // Parse the monitor List + info, ok := cm.Data[mon.EndpointDataKey] + if !ok { + return nil, fmt.Errorf("failed to find mon endpoints in config map: %+v", cm.Data) + } + + mons := cephmon.ParseMonEndpoints(info) + var endpoints []string + for _, mon := range mons { + endpoints = append(endpoints, mon.Endpoint) + } + return endpoints, nil } func createImageName(pvName string) string { diff --git a/pkg/operator/provisioner/provisioner_test.go b/pkg/operator/provisioner/provisioner_test.go index 2a0ff837b..9e5412825 100644 --- a/pkg/operator/provisioner/provisioner_test.go +++ b/pkg/operator/provisioner/provisioner_test.go @@ -20,18 +20,45 @@ import ( "fmt" "testing" + "k8s.io/api/core/v1" + "strings" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/mon" + "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestProcessMonAddresses(t *testing.T) { - unformattedMons := []string{"10.0.0.1:8124/0", "10.0.0.2:8125", "10.0.0.3/24"} - formattedMons := processMonAddresses(unformattedMons) + clientset := test.New(3) + context := &clusterd.Context{Clientset: clientset} + ns := "myns" + config := provisionerConfig{clusterName: ns} + p := &RookVolumeProvisioner{context: context, provConfig: config} - assert.Equal(t, "10.0.0.1:8124", formattedMons[0]) - assert.Equal(t, "10.0.0.2:8125", formattedMons[1]) - assert.Equal(t, "10.0.0.3", formattedMons[2]) + // create the test set of mons + testConfigMap := &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: mon.EndpointConfigMapName}, + Data: map[string]string{mon.EndpointDataKey: "mon1=10.0.0.1:6790,mon2=10.0.0.2:8125,mon3=10.0.0.3:6790"}} + _, err := clientset.CoreV1().ConfigMaps(ns).Create(testConfigMap) + require.Nil(t, err) + + formattedMons, err := p.getMonitorEndpoints() + require.Nil(t, err) + findMon(t, "10.0.0.1:6790", formattedMons) + findMon(t, "10.0.0.2:8125", formattedMons) + findMon(t, "10.0.0.3:6790", formattedMons) +} + +func findMon(t *testing.T, expected string, mons []string) { + for _, mon := range mons { + if expected == mon { + return + } + } + assert.Fail(t, fmt.Sprintf("expected: %s in %v", expected, mons)) } func TestParseClassParameters(t *testing.T) {