Blaine Gardner
ccafc2b715
When IPv6-only network mode is enabled and when mons are configured to
listen only on msgrV2 port, Rook has to add the port to the mon
--public-bind-addr in order to force mons to stop listening on the
msgrV1 port. This can cause problems for IPv6 clusters where adding
':<port>' to the end of an IP addr looks like a valid IPv6 address.
In IPv6 mode, Rook needs to set this to '[<ip>]:<port>' with the
brackets to be a non-ambiguous IPv6 address.
In IPv4 mode, Rook needs to keep its existing behavior. IPv4 mode should
also continue to be the defualt assumption if the network config isn't
specified.
If 'dualStack: true' is specified, Rook is informed that the environment
is dual stack, but Rook doesn't know which IP family to expect from the
pod IP. In this scenario, it is safest for Rook not to apply the port,
since assuming either IPv4 or IPv6 format could be incorrect. In this
scenario, mons may still listen on msgrV1 ports, but this is preferable
to an environment where mons perpetually crash.
Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
(cherry picked from commit 573a47e736)
259 lines
8.9 KiB
Go
259 lines
8.9 KiB
Go
/*
|
|
Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package mon
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/clusterd"
|
|
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
|
"github.com/rook/rook/pkg/operator/ceph/config"
|
|
"github.com/rook/rook/pkg/operator/ceph/controller"
|
|
"github.com/rook/rook/pkg/operator/ceph/test"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
testop "github.com/rook/rook/pkg/operator/test"
|
|
"github.com/stretchr/testify/assert"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
)
|
|
|
|
func TestPodSpecs(t *testing.T) {
|
|
testPodSpec(t, "a", true)
|
|
testPodSpec(t, "mon0", true)
|
|
testPodSpec(t, "a", false)
|
|
testPodSpec(t, "mon0", false)
|
|
}
|
|
|
|
func testPodSpec(t *testing.T, monID string, pvc bool) {
|
|
clientset := testop.New(t, 1)
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
c := New(
|
|
context.TODO(),
|
|
&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"},
|
|
"ns",
|
|
cephv1.ClusterSpec{},
|
|
ownerInfo,
|
|
)
|
|
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "rook/rook:myversion")
|
|
c.spec.CephVersion = cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:myceph"}
|
|
c.spec.Resources = map[string]v1.ResourceRequirements{}
|
|
c.spec.DataDirHostPath = "/var/lib/rook"
|
|
c.spec.Resources["mon"] = v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
|
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
|
|
},
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
|
|
v1.ResourceMemory: *resource.NewQuantity(500.0, resource.BinarySI),
|
|
},
|
|
}
|
|
c.spec.PriorityClassNames = map[cephv1.KeyType]string{
|
|
cephv1.KeyMon: "my-priority-class",
|
|
}
|
|
monConfig := testGenMonConfig(monID)
|
|
|
|
d, err := c.makeDeployment(monConfig, false)
|
|
assert.NoError(t, err)
|
|
assert.NotNil(t, d)
|
|
assert.Equal(t, k8sutil.DefaultServiceAccount, d.Spec.Template.Spec.ServiceAccountName)
|
|
|
|
if pvc {
|
|
d.Spec.Template.Spec.Volumes = append(
|
|
d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataPVC("i-am-pvc"))
|
|
} else {
|
|
d.Spec.Template.Spec.Volumes = append(
|
|
d.Spec.Template.Spec.Volumes, controller.DaemonVolumesDataHostPath(monConfig.DataPathMap)...)
|
|
}
|
|
|
|
// Deployment should have Ceph labels
|
|
test.AssertLabelsContainCephRequirements(t, d.ObjectMeta.Labels,
|
|
config.MonType, monID, AppName, "ns", "default", "cephclusters.ceph.rook.io", "ceph-mon")
|
|
|
|
podTemplate := test.NewPodTemplateSpecTester(t, &d.Spec.Template)
|
|
podTemplate.RunFullSuite(config.MonType, monID, AppName, "ns", "quay.io/ceph/ceph:myceph",
|
|
"200", "100", "1337", "500", /* resources */
|
|
"my-priority-class", "default", "cephclusters.ceph.rook.io", "ceph-mon")
|
|
|
|
t.Run(("check mon ConfigureProbe"), func(t *testing.T) {
|
|
c.spec.HealthCheck.StartupProbe = make(map[cephv1.KeyType]*cephv1.ProbeSpec)
|
|
c.spec.HealthCheck.StartupProbe[cephv1.KeyMon] = &cephv1.ProbeSpec{Disabled: false, Probe: &v1.Probe{InitialDelaySeconds: 1000}}
|
|
c.spec.HealthCheck.LivenessProbe = make(map[cephv1.KeyType]*cephv1.ProbeSpec)
|
|
c.spec.HealthCheck.LivenessProbe[cephv1.KeyMon] = &cephv1.ProbeSpec{Disabled: false, Probe: &v1.Probe{InitialDelaySeconds: 900}}
|
|
container := c.makeMonDaemonContainer(monConfig)
|
|
assert.NotNil(t, container.LivenessProbe)
|
|
assert.NotNil(t, container.StartupProbe)
|
|
assert.Equal(t, int32(900), container.LivenessProbe.InitialDelaySeconds)
|
|
assert.Equal(t, int32(1000), container.StartupProbe.InitialDelaySeconds)
|
|
})
|
|
|
|
t.Run(("msgr2 not required"), func(t *testing.T) {
|
|
container := c.makeMonDaemonContainer(monConfig)
|
|
checkMsgr2Required(t, container, false, false, false)
|
|
})
|
|
|
|
t.Run(("require msgr2"), func(t *testing.T) {
|
|
monConfig.Port = DefaultMsgr2Port
|
|
container := c.makeMonDaemonContainer(monConfig)
|
|
checkMsgr2Required(t, container, true, true, false)
|
|
})
|
|
|
|
t.Run(("require msgr2 -- dual stack"), func(t *testing.T) {
|
|
monConfig.Port = DefaultMsgr2Port
|
|
c.spec.Network = cephv1.NetworkSpec{
|
|
DualStack: true,
|
|
}
|
|
monConfig.Port = DefaultMsgr2Port
|
|
container := c.makeMonDaemonContainer(monConfig)
|
|
checkMsgr2Required(t, container, true, false, false)
|
|
})
|
|
|
|
t.Run(("require msgr2 -- IPv4"), func(t *testing.T) {
|
|
monConfig.Port = DefaultMsgr2Port
|
|
c.spec.Network = cephv1.NetworkSpec{
|
|
DualStack: false,
|
|
IPFamily: cephv1.IPv4,
|
|
}
|
|
monConfig.Port = DefaultMsgr2Port
|
|
container := c.makeMonDaemonContainer(monConfig)
|
|
checkMsgr2Required(t, container, true, true, false)
|
|
})
|
|
|
|
t.Run(("require msgr2 -- IPv6"), func(t *testing.T) {
|
|
monConfig.Port = DefaultMsgr2Port
|
|
c.spec.Network = cephv1.NetworkSpec{
|
|
DualStack: false,
|
|
IPFamily: cephv1.IPv6,
|
|
}
|
|
monConfig.Port = DefaultMsgr2Port
|
|
container := c.makeMonDaemonContainer(monConfig)
|
|
checkMsgr2Required(t, container, true, true, true)
|
|
})
|
|
}
|
|
|
|
func checkMsgr2Required(t *testing.T, container v1.Container, expectedRequireMsgr2, expectedPort, expectedBrackets bool) {
|
|
foundDisabledMsgr1 := false
|
|
foundMsgr2Port := false
|
|
foundBrackets := false
|
|
|
|
for _, arg := range container.Args {
|
|
if arg == "--ms-bind-msgr1=false" {
|
|
foundDisabledMsgr1 = true
|
|
}
|
|
if strings.HasPrefix(arg, "--public-bind-addr=") {
|
|
// flag should always refer to the env var, no matter what
|
|
assert.Contains(t, arg, "$(ROOK_POD_IP)")
|
|
if strings.HasSuffix(arg, ":3300") {
|
|
foundMsgr2Port = true
|
|
}
|
|
// brackets are expected for IPv6 addrs
|
|
if strings.Contains(arg, "[$(ROOK_POD_IP)]") {
|
|
foundBrackets = true
|
|
}
|
|
}
|
|
}
|
|
assert.Equal(t, expectedRequireMsgr2, foundDisabledMsgr1)
|
|
assert.Equal(t, expectedPort, foundMsgr2Port)
|
|
assert.Equal(t, expectedBrackets, foundBrackets)
|
|
}
|
|
|
|
func TestDeploymentPVCSpec(t *testing.T) {
|
|
clientset := testop.New(t, 1)
|
|
ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef()
|
|
c := New(
|
|
context.TODO(),
|
|
&clusterd.Context{Clientset: clientset, ConfigDir: "/var/lib/rook"},
|
|
"ns",
|
|
cephv1.ClusterSpec{},
|
|
ownerInfo,
|
|
)
|
|
setCommonMonProperties(c, 0, cephv1.MonSpec{Count: 3, AllowMultiplePerNode: true}, "rook/rook:myversion")
|
|
c.spec.CephVersion = cephv1.CephVersionSpec{Image: "quay.io/ceph/ceph:myceph"}
|
|
c.spec.Resources = map[string]v1.ResourceRequirements{}
|
|
c.spec.Resources["mon"] = v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
|
v1.ResourceMemory: *resource.NewQuantity(1337.0, resource.BinarySI),
|
|
},
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: *resource.NewQuantity(100.0, resource.BinarySI),
|
|
v1.ResourceMemory: *resource.NewQuantity(500.0, resource.BinarySI),
|
|
},
|
|
}
|
|
monConfig := testGenMonConfig("a")
|
|
|
|
// configured with default storage request
|
|
c.spec.Mon.VolumeClaimTemplate = &cephv1.VolumeClaimTemplate{}
|
|
pvc, err := c.makeDeploymentPVC(monConfig, false)
|
|
assert.NoError(t, err)
|
|
defaultReq, err := resource.ParseQuantity(cephMonDefaultStorageRequest)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pvc.Spec.Resources.Requests[v1.ResourceStorage], defaultReq)
|
|
|
|
// limit is preserved
|
|
req, err := resource.ParseQuantity("22Gi")
|
|
assert.NoError(t, err)
|
|
c.spec.Mon.VolumeClaimTemplate = &cephv1.VolumeClaimTemplate{
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
Resources: v1.VolumeResourceRequirements{
|
|
Limits: v1.ResourceList{v1.ResourceStorage: req},
|
|
},
|
|
},
|
|
}
|
|
pvc, err = c.makeDeploymentPVC(monConfig, false)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pvc.Spec.Resources.Limits[v1.ResourceStorage], req)
|
|
|
|
// request is preserved
|
|
req, err = resource.ParseQuantity("23Gi")
|
|
assert.NoError(t, err)
|
|
c.spec.Mon.VolumeClaimTemplate = &cephv1.VolumeClaimTemplate{
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
Resources: v1.VolumeResourceRequirements{
|
|
Requests: v1.ResourceList{v1.ResourceStorage: req},
|
|
},
|
|
},
|
|
}
|
|
pvc, err = c.makeDeploymentPVC(monConfig, false)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, pvc.Spec.Resources.Requests[v1.ResourceStorage], req)
|
|
}
|
|
|
|
func testRequiredDuringScheduling(t *testing.T, hostNetwork, allowMultiplePerNode, required bool) {
|
|
c := New(
|
|
context.TODO(),
|
|
&clusterd.Context{},
|
|
"ns",
|
|
cephv1.ClusterSpec{},
|
|
&k8sutil.OwnerInfo{},
|
|
)
|
|
|
|
c.spec.Network.HostNetwork = hostNetwork
|
|
c.spec.Mon.AllowMultiplePerNode = allowMultiplePerNode
|
|
assert.Equal(t, required, requiredDuringScheduling(&c.spec))
|
|
}
|
|
|
|
func TestRequiredDuringScheduling(t *testing.T) {
|
|
testRequiredDuringScheduling(t, false, false, true)
|
|
testRequiredDuringScheduling(t, true, false, true)
|
|
testRequiredDuringScheduling(t, true, true, true)
|
|
testRequiredDuringScheduling(t, false, true, false)
|
|
}
|