From f489f99e13de2b335f2ea007738e35fd948cbf60 Mon Sep 17 00:00:00 2001 From: Travis Nielsen Date: Wed, 30 Oct 2024 13:44:11 -0600 Subject: [PATCH] core: set resources on the cmd reporter jobs If the cmd-reporter key is set under the resources element in the CephCluster CR, the memory and cpu request and limits will be set on the ceph detect version job and network job accordingly. Signed-off-by: Travis Nielsen --- .../CRDs/Cluster/ceph-cluster-crd.md | 1 + deploy/examples/cluster.yaml | 1 + pkg/apis/ceph.rook.io/v1/resources.go | 25 ++++++++++++------- pkg/operator/ceph/controller/network.go | 1 + pkg/operator/ceph/controller/network_test.go | 7 +++--- pkg/operator/ceph/controller/version.go | 1 + .../k8sutil/cmdreporter/cmdreporter.go | 7 ++++++ 7 files changed, 30 insertions(+), 13 deletions(-) diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index fce660364..d76a649d3 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -485,6 +485,7 @@ You can set resource requests/limits for Rook components through the [Resource R It scrapes for Ceph daemon core dumps and sends them to the Ceph manager crash module so that core dumps are centralized and can be easily listed/accessed. You can read more about the [Ceph Crash module](https://docs.ceph.com/docs/master/mgr/crash/). * `logcollector`: Set resource requests/limits for the log collector. When enabled, this container runs as side-car to each Ceph daemons. +* `cmd-reporter`: Set resource requests/limits for the jobs that detect the ceph version and collect network info. * `cleanup`: Set resource requests/limits for cleanup job, responsible for wiping cluster's data after uninstall * `exporter`: Set resource requests/limits for Ceph exporter. diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 345897fb5..68612419b 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -251,6 +251,7 @@ spec: # logcollector: # cleanup: # exporter: + # cmd-reporter: # The option to automatically remove OSDs that are out and are safe to destroy. removeOSDsIfOutAndSafeToRemove: false priorityClassNames: diff --git a/pkg/apis/ceph.rook.io/v1/resources.go b/pkg/apis/ceph.rook.io/v1/resources.go index b87089fde..daebe4aaf 100644 --- a/pkg/apis/ceph.rook.io/v1/resources.go +++ b/pkg/apis/ceph.rook.io/v1/resources.go @@ -31,6 +31,8 @@ const ( ResourcesKeyOSD = "osd" // ResourcesKeyPrepareOSD represents the name of resource in the CR for the osd prepare job ResourcesKeyPrepareOSD = "prepareosd" + // ResourcesKeyCmdReporter represents the name of resource in the CR for the detect version and network jobs + ResourcesKeyCmdReporter = "cmd-reporter" // ResourcesKeyMDS represents the name of resource in the CR for the mds ResourcesKeyMDS = "mds" // ResourcesKeyCrashCollector represents the name of resource in the CR for the crash @@ -47,22 +49,22 @@ const ( ResourcesKeyCephExporter = "exporter" ) -// GetMgrResources returns the placement for the MGR service +// GetMgrResources returns the resources for the MGR service func GetMgrResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyMgr] } -// GetMgrSidecarResources returns the placement for the MGR sidecar container +// GetMgrSidecarResources returns the resources for the MGR sidecar container func GetMgrSidecarResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyMgrSidecar] } -// GetMonResources returns the placement for the monitors +// GetMonResources returns the resources for the monitors func GetMonResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyMon] } -// GetOSDResources returns the placement for all OSDs or for OSDs of specified device class (hdd, nvme, ssd) +// GetOSDResources returns the resources for all OSDs or for OSDs of specified device class (hdd, nvme, ssd) func GetOSDResources(p ResourceSpec, deviceClass string) v1.ResourceRequirements { if deviceClass == "" { return p[ResourcesKeyOSD] @@ -80,27 +82,32 @@ func getOSDResourceKeyForDeviceClass(deviceClass string) string { return ResourcesKeyOSD + "-" + deviceClass } -// GetPrepareOSDResources returns the placement for the OSDs prepare job +// GetPrepareOSDResources returns the resources for the OSDs prepare job func GetPrepareOSDResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyPrepareOSD] } -// GetCrashCollectorResources returns the placement for the crash daemon +// GetCmdReporterResources returns the resources for the detect version job +func GetCmdReporterResources(p ResourceSpec) v1.ResourceRequirements { + return p[ResourcesKeyCmdReporter] +} + +// GetCrashCollectorResources returns the resources for the crash daemon func GetCrashCollectorResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyCrashCollector] } -// GetLogCollectorResources returns the placement for the crash daemon +// GetLogCollectorResources returns the resources for the logo collector func GetLogCollectorResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyLogCollector] } -// GetCleanupResources returns the placement for the cleanup job +// GetCleanupResources returns the resources for the cleanup job func GetCleanupResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyCleanup] } -// GetCephExporterResources returns the placement for the cleanup job +// GetCephExporterResources returns the resources for the cleanup job func GetCephExporterResources(p ResourceSpec) v1.ResourceRequirements { return p[ResourcesKeyCephExporter] } diff --git a/pkg/operator/ceph/controller/network.go b/pkg/operator/ceph/controller/network.go index b97cb454a..3b96c9623 100644 --- a/pkg/operator/ceph/controller/network.go +++ b/pkg/operator/ceph/controller/network.go @@ -252,6 +252,7 @@ func discoverAddressRanges( rookImage, rookImage, clusterSpec.CephVersion.ImagePullPolicy, + clusterSpec.Resources, ) if err != nil { return ranges, errors.Wrapf(err, "failed to set up ceph %q network canary", cephNetwork) diff --git a/pkg/operator/ceph/controller/network_test.go b/pkg/operator/ceph/controller/network_test.go index 8cc5beab1..afb967b19 100644 --- a/pkg/operator/ceph/controller/network_test.go +++ b/pkg/operator/ceph/controller/network_test.go @@ -54,9 +54,9 @@ func (m *mockCmdReporter) Run(ctx context.Context, timeout time.Duration) (stdou } // returns a newCmdReporter function that returns the given mockCmdReporter and error -func mockNewCmdReporter(m *mockCmdReporter, err error) func(clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo, appName string, jobName string, jobNamespace string, cmd []string, args []string, rookImage string, runImage string, imagePullPolicy v1.PullPolicy) (cmdreporter.CmdReporterInterface, error) { - return func(clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo, appName, jobName, jobNamespace string, cmd, args []string, rookImage, runImage string, imagePullPolicy v1.PullPolicy) (cmdreporter.CmdReporterInterface, error) { - job, err := cmdreporter.MockCmdReporterJob(clientset, ownerInfo, appName, jobName, jobNamespace, cmd, args, rookImage, runImage, imagePullPolicy) +func mockNewCmdReporter(m *mockCmdReporter, err error) func(clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo, appName string, jobName string, jobNamespace string, cmd []string, args []string, rookImage string, runImage string, imagePullPolicy v1.PullPolicy, resources cephv1.ResourceSpec) (cmdreporter.CmdReporterInterface, error) { + return func(clientset kubernetes.Interface, ownerInfo *k8sutil.OwnerInfo, appName, jobName, jobNamespace string, cmd, args []string, rookImage, runImage string, imagePullPolicy v1.PullPolicy, resources cephv1.ResourceSpec) (cmdreporter.CmdReporterInterface, error) { + job, err := cmdreporter.MockCmdReporterJob(clientset, ownerInfo, appName, jobName, jobNamespace, cmd, args, rookImage, runImage, imagePullPolicy, resources) if err != nil { // okay to panic here because this is a unit test setup failure, not part of code testing panic(fmt.Sprintf("error setting up mock CmdReporter job: %v", err)) @@ -82,7 +82,6 @@ func newTestConfigsWithNetworkSpec(n cephv1.NetworkSpec) (*clusterd.Context, *ce func Test_discoverAddressRanges(t *testing.T) { oldNewCmdReporter := newCmdReporter defer func() { newCmdReporter = oldNewCmdReporter }() - t.Run("public network selected", func(t *testing.T) { clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec( cephv1.NetworkSpec{ diff --git a/pkg/operator/ceph/controller/version.go b/pkg/operator/ceph/controller/version.go index 915672fda..fa3d91f01 100644 --- a/pkg/operator/ceph/controller/version.go +++ b/pkg/operator/ceph/controller/version.go @@ -74,6 +74,7 @@ func DetectCephVersion(ctx context.Context, rookImage, namespace, jobName string rookImage, cephImage, cephClusterSpec.CephVersion.ImagePullPolicy, + cephClusterSpec.Resources, ) if err != nil { return nil, errors.Wrap(err, "failed to set up ceph version job") diff --git a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go index 3de895d3a..6a5eea450 100644 --- a/pkg/operator/k8sutil/cmdreporter/cmdreporter.go +++ b/pkg/operator/k8sutil/cmdreporter/cmdreporter.go @@ -80,6 +80,7 @@ type cmdReporterCfg struct { rookImage string runImage string imagePullPolicy v1.PullPolicy + resources cephv1.ResourceSpec } // New creates a new CmdReporter. @@ -101,6 +102,7 @@ func New( cmd, args []string, rookImage, runImage string, imagePullPolicy v1.PullPolicy, + resources cephv1.ResourceSpec, ) (CmdReporterInterface, error) { cfg := &cmdReporterCfg{ clientset: clientset, @@ -113,6 +115,7 @@ func New( rookImage: rookImage, runImage: runImage, imagePullPolicy: imagePullPolicy, + resources: resources, } // Validate contents of config struct, not inputs to function to catch any developer errors @@ -352,6 +355,7 @@ func (cr *cmdReporterCfg) initContainers() []v1.Container { }, Image: cr.rookImage, ImagePullPolicy: cr.imagePullPolicy, + Resources: cephv1.GetCmdReporterResources(cr.resources), } _, copyBinsMount := copyBinariesVolAndMount() c.VolumeMounts = []v1.VolumeMount{copyBinsMount} @@ -382,6 +386,7 @@ func (cr *cmdReporterCfg) container() (*v1.Container, error) { }, Image: cr.runImage, ImagePullPolicy: cr.imagePullPolicy, + Resources: cephv1.GetCmdReporterResources(cr.resources), } if cr.needToCopyBinaries() { _, copyBinsMount := copyBinariesVolAndMount() @@ -419,6 +424,7 @@ func MockCmdReporterJob( rookImage string, runImage string, imagePullPolicy v1.PullPolicy, + resources cephv1.ResourceSpec, ) (*batch.Job, error) { cfg := &cmdReporterCfg{ clientset: clientset, @@ -431,6 +437,7 @@ func MockCmdReporterJob( rookImage: rookImage, runImage: runImage, imagePullPolicy: imagePullPolicy, + resources: resources, } return cfg.initJobSpec() }