diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index f1c62b40c..82c21aa7a 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -80,6 +80,7 @@ func (c *CephNFSController) makeDeployment(nfs cephv1.CephNFS, cfg daemonConfig) ObjectMeta: metav1.ObjectMeta{ Name: instanceName(nfs, cfg.ID), Namespace: nfs.Namespace, + Labels: getLabels(nfs, cfg.ID), }, } k8sutil.AddRookVersionLabelToDeployment(deployment) diff --git a/pkg/operator/ceph/nfs/spec_test.go b/pkg/operator/ceph/nfs/spec_test.go new file mode 100644 index 000000000..2cc6456d4 --- /dev/null +++ b/pkg/operator/ceph/nfs/spec_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nfs + +import ( + "testing" + + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "github.com/rook/rook/pkg/clusterd" + cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" + "github.com/rook/rook/pkg/operator/ceph/config" + optest "github.com/rook/rook/pkg/operator/test" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestDeploymentSpec(t *testing.T) { + nfs := cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NFSGaneshaSpec{ + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "myfs-data0", + Namespace: "nfs-test-ns", + }, + Server: cephv1.GaneshaServerSpec{ + Active: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + }, + }, + } + + clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} + c := NewCephNFSController( + clusterInfo, + &clusterd.Context{Clientset: optest.New(1)}, + "/var/lib/rook", + "rook-ceph-test-ns", + "rook/rook:testimage", + cephv1.CephVersionSpec{Image: "ceph/ceph:myceph"}, + false, + metav1.OwnerReference{}, + ) + + id := "i" + configName := "rook-ceph-nfs-my-nfs-i" + cfg := daemonConfig{ + ID: id, + ConfigConfigMap: configName, + DataPathMap: &config.DataPathMap{ + HostDataDir: "", // nfs daemon does not store data on host, ... + ContainerDataDir: cephconfig.DefaultConfigDir, // does share data in containers using emptyDir, ... + HostLogDir: "", // and does not log to /var/log/ceph dir + }, + } + + d := c.makeDeployment(nfs, cfg) + + // Deployment should have Ceph labels + optest.AssertLabelsContainRookRequirements(t, d.ObjectMeta.Labels, appName) + + podTemplate := optest.NewPodTemplateSpecTester(t, &d.Spec.Template) + podTemplate.RunFullSuite( + appName, + optest.ResourceLimitExpectations{ + CPUResourceLimit: "500", + MemoryResourceLimit: "1Ki", + CPUResourceRequest: "200", + MemoryResourceRequest: "512", + }, + ) +} diff --git a/pkg/operator/ceph/test/containers.go b/pkg/operator/ceph/test/containers.go index 0b2844254..e8b909055 100644 --- a/pkg/operator/ceph/test/containers.go +++ b/pkg/operator/ceph/test/containers.go @@ -163,24 +163,12 @@ func (ct *ContainersTester) AssertCephImagesMatch(image string) { } } -// AssertResourceSpec asserts that the container under test's resource limits/requests match the -// given (in string format) resource limits/requests. -func (ct *ContainersTester) AssertResourceSpec(cpuResourceLimit, cpuResourceRequest, memoryResourceLimit, memoryResourceRequest string) { - for _, c := range ct.containers { - assert.Equal(ct.t, cpuResourceLimit, c.Resources.Limits.Cpu().String()) - assert.Equal(ct.t, cpuResourceRequest, c.Resources.Requests.Cpu().String()) - assert.Equal(ct.t, memoryResourceLimit, c.Resources.Limits.Memory().String()) - assert.Equal(ct.t, memoryResourceRequest, c.Resources.Requests.Memory().String()) - } -} - // RunFullSuite runs all assertion tests for the Containers under test. func (ct *ContainersTester) RunFullSuite(cephImage, cpuResourceLimit, cpuResourceRequest, memoryResourceLimit, memoryResourceRequest string) { ct.AssertEnvVarsContainCephRequirements() ct.AssertArgReferencesMatchEnvVars() ct.AssertArgsContainCephRequirements() ct.AssertCephImagesMatch(cephImage) - ct.AssertResourceSpec(cpuResourceLimit, cpuResourceRequest, memoryResourceLimit, memoryResourceRequest) } func isCephCommand(command []string) bool { diff --git a/pkg/operator/ceph/test/podspec.go b/pkg/operator/ceph/test/podspec.go index 2fadf8f77..32318796c 100644 --- a/pkg/operator/ceph/test/podspec.go +++ b/pkg/operator/ceph/test/podspec.go @@ -103,23 +103,6 @@ func (ps *PodSpecTester) AssertVolumesMeetCephRequirements( "required volumes don't exist in pod spec's volume list:", ps.spec.Volumes) } -// AssertVolumesAndMountsMatch asserts that all of the volume mounts in the pod spec under test's -// containers have a volume which sources them. It also asserts that each volume is used at least -// once by any of the mounts in containers. -func (ps *PodSpecTester) AssertVolumesAndMountsMatch() { - pod := optest.VolumesAndMountsTestDefinition{ - VolumesSpec: &optest.VolumesSpec{Moniker: "volumes", Volumes: ps.spec.Volumes}, - MountsSpecItems: []*optest.MountsSpec{}, - } - for _, c := range allContainers(ps.spec) { - pod.MountsSpecItems = append( - pod.MountsSpecItems, - &optest.MountsSpec{Moniker: "mounts of container " + c.Name, Mounts: c.VolumeMounts}, - ) - } - pod.TestMountsMatchVolumes(ps.t) -} - // AssertRestartPolicyAlways asserts that the pod spec is set to always restart on failure. func (ps *PodSpecTester) AssertRestartPolicyAlways() { assert.Equal(ps.t, v1.RestartPolicyAlways, ps.spec.RestartPolicy) @@ -130,7 +113,15 @@ func (ps *PodSpecTester) RunFullSuite( daemonType config.DaemonType, resourceName, cephImage, cpuResourceLimit, cpuResourceRequest, memoryResourceLimit, memoryResourceRequest string, ) { - ps.AssertVolumesAndMountsMatch() + resourceExpectations := optest.ResourceLimitExpectations{ + CPUResourceLimit: cpuResourceLimit, + MemoryResourceLimit: memoryResourceLimit, + CPUResourceRequest: cpuResourceRequest, + MemoryResourceRequest: memoryResourceRequest, + } + ops := optest.NewPodSpecTester(ps.t, ps.spec) + ops.RunFullSuite(resourceExpectations) + ps.AssertVolumesMeetCephRequirements(daemonType, resourceName) ps.AssertRestartPolicyAlways() ps.Containers().RunFullSuite(cephImage, cpuResourceLimit, cpuResourceRequest, memoryResourceLimit, memoryResourceRequest) diff --git a/pkg/operator/ceph/test/spec.go b/pkg/operator/ceph/test/spec.go index 8bc181335..401001503 100644 --- a/pkg/operator/ceph/test/spec.go +++ b/pkg/operator/ceph/test/spec.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/rook/rook/pkg/operator/ceph/config" + optest "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" ) @@ -78,12 +79,13 @@ func AssertLabelsContainCephRequirements( t *testing.T, labels map[string]string, daemonType config.DaemonType, daemonID, appName, namespace string, ) { + optest.AssertLabelsContainRookRequirements(t, labels, appName) + resourceLabels := []string{} for k, v := range labels { resourceLabels = append(resourceLabels, fmt.Sprintf("%s=%s", k, v)) } expectedLabels := []string{ - "app=" + appName, "ceph_daemon_id=" + daemonID, string(daemonType) + "=" + daemonID, "rook_cluster" + "=" + namespace, diff --git a/pkg/operator/test/containers.go b/pkg/operator/test/containers.go new file mode 100644 index 000000000..f7254787e --- /dev/null +++ b/pkg/operator/test/containers.go @@ -0,0 +1,115 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" +) + +// A ContainersTester is a helper exposing methods for testing required Rook specifications common +// for all Rook containers. +type ContainersTester struct { + t *testing.T + containers []v1.Container +} + +// ResourceLimitExpectations allows a test to define expectations for resource limits on containers. +// If any field is left as an empty string, that field will not be tested. +type ResourceLimitExpectations struct { + CPUResourceLimit string + CPUResourceRequest string + MemoryResourceLimit string + MemoryResourceRequest string +} + +// Containers creates a ContainersTester from a parent PodSpecTester. Because ContainersTester is +// intended to test the full list of containers (both init and run containers) in a PodSpec, this +// method is the only way of creating a ContainersTester. +func (ps *PodSpecTester) Containers() *ContainersTester { + return &ContainersTester{ + t: ps.t, + containers: allContainers(ps.spec), + } +} + +// AssertArgReferencesMatchEnvVars asserts that for each container under test, any references to +// Kubernetes environment variables (e.g., $(POD_NAME)), have an environment variable set to source +// the value. +func (ct *ContainersTester) AssertArgReferencesMatchEnvVars() { + for _, c := range ct.containers { + assert.Subset(ct.t, varNames(&c), argEnvReferences(&c), + "container: "+c.Name, + "references to env vars in args do not match env vars", + "args:", c.Args, "envs:", c.Env) + } +} + +// AssertResourceSpec asserts that the container under test's resource limits/requests match the +// given (in string format) resource limits/requests. +func (ct *ContainersTester) AssertResourceSpec(expectations ResourceLimitExpectations) { + for _, c := range ct.containers { + if expectations.CPUResourceLimit != "" { + assert.Equal(ct.t, expectations.CPUResourceLimit, c.Resources.Limits.Cpu().String()) + } + if expectations.CPUResourceRequest != "" { + assert.Equal(ct.t, expectations.CPUResourceRequest, c.Resources.Requests.Cpu().String()) + } + if expectations.MemoryResourceLimit != "" { + assert.Equal(ct.t, expectations.MemoryResourceLimit, c.Resources.Limits.Memory().String()) + } + if expectations.MemoryResourceRequest != "" { + assert.Equal(ct.t, expectations.MemoryResourceRequest, c.Resources.Requests.Memory().String()) + } + } +} + +// RunFullSuite runs all assertion tests for the Containers under test. +func (ct *ContainersTester) RunFullSuite(resourceExpectations ResourceLimitExpectations) { + ct.AssertArgReferencesMatchEnvVars() + ct.AssertResourceSpec(resourceExpectations) +} + +func argEnvReferences(c *v1.Container) []string { + argRefSet := map[string]bool{} + for _, a := range c.Args { + argRefRegex, e := regexp.Compile("\\$\\(([a-zA-Z][a-zA-Z0-9_]*)\\)") + if e != nil { + panic("could not compile argument reference regexp") + } + matches := argRefRegex.FindAllStringSubmatch(a, -1) + for _, m := range matches { + argRefSet[m[1]] = true + } + } + refs := []string{} + for r := range argRefSet { + refs = append(refs, r) + } + return refs +} + +func varNames(c *v1.Container) []string { + vars := []string{} + for _, v := range c.Env { + vars = append(vars, v.Name) + } + return vars +} diff --git a/pkg/operator/test/podspec.go b/pkg/operator/test/podspec.go new file mode 100644 index 000000000..7adee974b --- /dev/null +++ b/pkg/operator/test/podspec.go @@ -0,0 +1,67 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "testing" + + v1 "k8s.io/api/core/v1" +) + +// A PodSpecTester is a helper exposing methods for testing required Ceph specifications common for +// all Ceph PodSpecs. +type PodSpecTester struct { + t *testing.T + spec *v1.PodSpec +} + +// Spec creates a PodSpecTester from a parent PodTemplateSpecTester. +func (pt *PodTemplateSpecTester) Spec() *PodSpecTester { + return NewPodSpecTester(pt.t, &pt.template.Spec) +} + +// NewPodSpecTester creates a new tester to test the given PodSpec. +func NewPodSpecTester(t *testing.T, spec *v1.PodSpec) *PodSpecTester { + return &PodSpecTester{t: t, spec: spec} +} + +// AssertVolumesAndMountsMatch asserts that all of the volume mounts in the pod spec under test's +// containers have a volume which sources them. It also asserts that each volume is used at least +// once by any of the mounts in containers. +func (ps *PodSpecTester) AssertVolumesAndMountsMatch() { + pod := VolumesAndMountsTestDefinition{ + VolumesSpec: &VolumesSpec{Moniker: "volumes", Volumes: ps.spec.Volumes}, + MountsSpecItems: []*MountsSpec{}, + } + for _, c := range allContainers(ps.spec) { + pod.MountsSpecItems = append( + pod.MountsSpecItems, + &MountsSpec{Moniker: "mounts of container " + c.Name, Mounts: c.VolumeMounts}, + ) + } + pod.TestMountsMatchVolumes(ps.t) +} + +// RunFullSuite runs all assertion tests for the PodSpec under test and its sub-resources. +func (ps *PodSpecTester) RunFullSuite(resourceExpectations ResourceLimitExpectations) { + ps.AssertVolumesAndMountsMatch() + ps.Containers().RunFullSuite(resourceExpectations) +} + +func allContainers(p *v1.PodSpec) []v1.Container { + return append(p.InitContainers, p.Containers...) +} diff --git a/pkg/operator/test/podtemplatespec.go b/pkg/operator/test/podtemplatespec.go new file mode 100644 index 000000000..5a5e2f361 --- /dev/null +++ b/pkg/operator/test/podtemplatespec.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "testing" + + v1 "k8s.io/api/core/v1" +) + +// A PodTemplateSpecTester is a helper exposing methods for testing required Rook specifications +// common for all Rook PodTemplateSpecs. +type PodTemplateSpecTester struct { + t *testing.T + template *v1.PodTemplateSpec +} + +// NewPodTemplateSpecTester creates a new tester to test the given PodTemplateSpec +func NewPodTemplateSpecTester(t *testing.T, template *v1.PodTemplateSpec) *PodTemplateSpecTester { + return &PodTemplateSpecTester{t: t, template: template} +} + +// AssertLabelsContainRookRequirements asserts that the PodTemplateSpec under test contains labels +// which all Rook pods should have. +func (pt *PodTemplateSpecTester) AssertLabelsContainRookRequirements(appName string) { + AssertLabelsContainRookRequirements(pt.t, pt.template.ObjectMeta.Labels, appName) +} + +// RunFullSuite runs all assertion tests for the PodTemplateSpec under test and its sub-resources. +func (pt *PodTemplateSpecTester) RunFullSuite( + appName string, + resourceExpectations ResourceLimitExpectations, +) { + pt.AssertLabelsContainRookRequirements(appName) + pt.Spec().RunFullSuite(resourceExpectations) +} diff --git a/pkg/operator/test/spec.go b/pkg/operator/test/spec.go index 41c6dd882..956e19c4b 100644 --- a/pkg/operator/test/spec.go +++ b/pkg/operator/test/spec.go @@ -19,8 +19,10 @@ package test import ( "fmt" "strings" + "testing" "github.com/coreos/pkg/capnslog" + "github.com/stretchr/testify/assert" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "ceph-op-testlib") @@ -68,3 +70,18 @@ func ArgumentsMatchExpected(actualArgs []string, expectedArgs [][]string) error } return nil } + +// AssertLabelsContainRookRequirements asserts that the the labels under test contain the labels +// which all Rook pods should have. This can be used with labels for Kubernetes Deployments, +// DaemonSets, etc. +func AssertLabelsContainRookRequirements(t *testing.T, labels map[string]string, appName string) { + resourceLabels := []string{} + for k, v := range labels { + resourceLabels = append(resourceLabels, fmt.Sprintf("%s=%s", k, v)) + } + expectedLabels := []string{ + "app=" + appName, + } + assert.Subset(t, resourceLabels, expectedLabels, + "labels on resource do not match Rook requirements", labels) +}