feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Merge pull request #12169 from rook/mergify/bp/release-1.11/pr-12069

Browse Source 
test: add multus validation test routine to rook binary (backport #12069)
This commit is contained in:
Travis Nielsen authored and GitHub committed 2023-05-02 13:55:45 -06:00
2 parents bd9874d408 8bb189c5c8
commit d8bd4fa6d3
23 files changed
+1600 -158
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/CRDs/Cluster/ceph-cluster-crd.md
+19 -4
View File
@@ -221,7 +221,7 @@ Based on the configuration, the operator will do the following:
public: rook-ceph/rook-public-nw
```
2. If only the `cluster` selector is specified, the internal cluster traffic* will happen on that network. All other traffic to mons, OSDs, and other daemons will be on the default network.
2. If only the `cluster` selector is specified, the internal cluster traffic\* will happen on that network. All other traffic to mons, OSDs, and other daemons will be on the default network.
```yaml
network:
@@ -230,7 +230,7 @@ Based on the configuration, the operator will do the following:
cluster: rook-ceph/rook-cluster-nw
```
3. If both `public` and `cluster` selectors are specified the first one will run all the communication network and the second the internal cluster network*
3. If both `public` and `cluster` selectors are specified the first one will run all the communication network and the second the internal cluster network\*
```yaml
network:
@@ -274,7 +274,7 @@ spec:
```
* Ensure that `master` matches the network interface of the host that you want to use.
* Ipam type `whereabouts` is required because it makes sure that all the pods get a unique IP address from the multus network.
* IPAM type `whereabouts` is required because it makes sure that all the pods get a unique IP address from the multus network.
* The NetworkAttachmentDefinition should be referenced along with the namespace in which it is present like `public: <namespace>/<name of NAD>`.
e.g., the network attachment definition are in `default` namespace:
@@ -286,7 +286,22 @@ spec:
* This format is required in order to use the NetworkAttachmentDefinition across namespaces.
* In Openshift, to use a NetworkAttachmentDefinition (NAD) across namespaces, the NAD must be deployed in the `default` namespace. The NAD is then referenced with the namespace: `default/rook-public-nw`
#### Known limitations with Multus
##### Validating Multus configuration
We **highly** recommend validating your Multus configuration before you install Rook. A tool exists
to facilitate validating the Multus configuration. After installing the Rook operator and before
installing any Custom Resources, run the tool from the operator pod.
The tool's CLI is designed to be as helpful as possible. Get help text for the multus validation
tool like so:
```console
kubectl --namespace rook-ceph exec -it deploy/rook-ceph-operator -- rook ctl multus validation run --help
```
If the tool fails, it will suggest what things may be preventing Multus networks from working
properly, and it will request the logs and outputs that will help debug issues.
##### Known limitations with Multus
Daemons leveraging Kubernetes service IPs (Monitors, Managers, Rados Gateways) are not listening on the NAD specified in the `selectors`.
Instead the daemon listens on the default network, however the NAD is attached to the container,
cmd/rook/ceph/ceph.go
+3 -2
View File
@@ -31,8 +31,9 @@ import (
// Cmd is the main command for operator and daemons.
var Cmd = &cobra.Command{
Use: "ceph",
Short: "Main command for Ceph operator and daemons.",
Use: "ceph",
Short: "Main command for Ceph operator and daemons.",
Hidden: true, // do not advertise to end users
}
var (
cmd/rook/discover.go
+3 -2
View File
@@ -26,8 +26,9 @@ import (
var (
discoverCmd = &cobra.Command{
Use: "discover",
Short: "Discover devices",
Use: "discover",
Short: "Discover devices",
Hidden: true, // do not advertise to end users
}
// interval between discovering devices
cmd/rook/main.go
+8
View File
@@ -20,6 +20,7 @@ import (
"github.com/rook/rook/cmd/rook/ceph"
rook "github.com/rook/rook/cmd/rook/rook"
"github.com/rook/rook/cmd/rook/userfacing"
"github.com/rook/rook/cmd/rook/util"
"github.com/rook/rook/cmd/rook/version"
)
@@ -43,4 +44,11 @@ func addCommands() {
util.CmdReporterCmd,
util.CopyBinsCmd,
)
// do double diligence to ensure non-user-facing commands are hidden from end users
for _, cmd := range rook.RootCmd.Commands() {
cmd.Hidden = true
}
// all user-facing commands
rook.RootCmd.AddCommand(userfacing.Commands...)
}
cmd/rook/rook/rook.go
+9 -59
View File
@@ -36,10 +36,8 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
const (
@@ -48,14 +46,14 @@ const (
)
var RootCmd = &cobra.Command{
Use: "rook",
Use: "rook",
Short: "Rook (rook.io) Kubernetes operator and user tools",
Hidden: false,
}
var (
logLevelRaw string
operatorImage string
serviceAccountName string
logger = capnslog.NewPackageLogger("github.com/rook/rook", "rookcmd")
logLevelRaw string
logger = capnslog.NewPackageLogger("github.com/rook/rook", "multus-validation")
)
// Initialize the configuration parameters. The precedence from lowest to highest is:
@@ -64,8 +62,9 @@ var (
// 3. command line parameter
func init() {
RootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: ERROR,WARNING,INFO,DEBUG)")
RootCmd.PersistentFlags().StringVar(&operatorImage, "operator-image", "", "Override the image url that the operator uses. The default is read from the operator pod.")
RootCmd.PersistentFlags().StringVar(&serviceAccountName, "service-account", "", "Override the service account that the operator uses. The default is read from the operator pod.")
RootCmd.InitDefaultHelpCmd()
RootCmd.InitDefaultHelpFlag()
RootCmd.InitDefaultCompletionCmd()
// load the environment variables
flags.SetFlagsFromEnv(RootCmd.Flags(), RookEnvVarPrefix)
@@ -97,44 +96,7 @@ func NewContext() *clusterd.Context {
// Try to read config from in-cluster env
context.KubeConfig, err = rest.InClusterConfig()
if err != nil {
// **Not** running inside a cluster - running the operator outside of the cluster.
// This mode is for developers running the operator on their dev machines
// for faster development, or to run operator cli tools manually to a remote cluster.
// We setup the API server config from default user file locations (most notably ~/.kube/config),
// and also change the executor to work remotely and run kubernetes jobs.
logger.Info("setting up the context to outside of the cluster")
// Try to read config from user config files
context.KubeConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
clientcmd.NewDefaultClientConfigLoadingRules(),
&clientcmd.ConfigOverrides{}).ClientConfig()
TerminateOnError(err, "failed to get k8s config")
// When running outside, we need to setup an executor that runs the commands as kubernetes jobs.
// This allows the operator code to execute tools that are available in the operator image
// or just have to be run inside the cluster in order to talk to the pods and services directly.
context.Executor = &exec.TranslateCommandExecutor{
Executor: context.Executor,
Translator: func(
command string,
arg ...string,
) (string, []string) {
jobName := "rook-exec-job-" + string(uuid.NewUUID())
transCommand := "kubectl"
transArgs := append([]string{
"run", jobName,
"--image=" + operatorImage,
"--serviceaccount=" + serviceAccountName,
"--restart=Never",
"--attach",
"--rm",
"--quiet",
"--command", "--",
command}, arg...)
return transCommand, transArgs
},
}
TerminateOnError(err, "failed to get k8s cluster config")
}
context.Clientset, err = kubernetes.NewForConfig(context.KubeConfig)
@@ -153,12 +115,6 @@ func NewContext() *clusterd.Context {
}
func GetOperatorImage(ctx context.Context, clientset kubernetes.Interface, containerName string) string {
// If provided as a flag then use that value
if operatorImage != "" {
return operatorImage
}
// Getting the info of the operator pod
pod, err := k8sutil.GetRunningPod(ctx, clientset)
TerminateOnError(err, "failed to get pod")
@@ -171,12 +127,6 @@ func GetOperatorImage(ctx context.Context, clientset kubernetes.Interface, conta
}
func GetOperatorServiceAccount(ctx context.Context, clientset kubernetes.Interface) string {
// If provided as a flag then use that value
if serviceAccountName != "" {
return serviceAccountName
}
// Getting the info of the operator pod
pod, err := k8sutil.GetRunningPod(ctx, clientset)
TerminateOnError(err, "failed to get pod")
cmd/rook/secret.go
+1
View File
@@ -41,6 +41,7 @@ var KeyManagementCmd = &cobra.Command{
Long: `The secret sub-command helps interacting with Key Management System.
It can perform various actions such as retrieving the content of a Key Encryption Key and
rotating Key Encryption Key.`,
Hidden: true, // do not advertise to end users
}
func init() {
cmd/rook/userfacing/multus/multus.go
+41
View File
@@ -0,0 +1,41 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package multus
import (
"github.com/rook/rook/cmd/rook/userfacing/multus/validation"
"github.com/spf13/cobra"
)
func init() {
Cmd.AddCommand(
validation.Cmd,
)
}
// Cmd is the 'multus' CLI command
var Cmd = &cobra.Command{
Use: "multus",
Short: "Get help configuring Multus for compatibility with Rook",
Long: `
Get help configuring Multus for compatibility with Rook.
Though intended for Multus, these utilities should support any multi-net
provider that implements the Kubernetes Network Plumbing Working Group's
Multi-Network Spec: https://github.com/k8snetworkplumbingwg/multi-net-spec
`,
}
cmd/rook/userfacing/multus/validation/validation.go
+216
View File
@@ -0,0 +1,216 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validation
import (
"context"
"fmt"
"os"
"strconv"
"time"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/cmd/rook/rook"
"github.com/rook/rook/pkg/daemon/multus"
"github.com/rook/rook/pkg/operator/k8sutil"
"github.com/spf13/cobra"
)
// defaults
var (
DefaultValidationNamespace = "rook-ceph"
// 1 mon, 3 osds, 1 mgr, 1 mds, 1 nfs, 1 rgw, 1 rbdmirror, 1 cephfsmirror,
// (1 csi provisioner, 1 csi plugin) x3 for rbd, cephfs, and nfs CSI drivers
DefaultDaemonsPerNode = 16
DefaultTimeoutMinutes = 3
)
// config
var (
validationConfig = multus.ValidationTest{
Logger: capnslog.NewPackageLogger("github.com/rook/rook", "multus-validation"),
}
)
// commands
var (
// parent 'validation' command
Cmd = &cobra.Command{
Use: "validation",
Short: "Run and manage Multus validation tests for Rook",
}
// 'validation run' command
runCmd = &cobra.Command{
Use: "run [--public-network=<nad-name>] [--cluster-network=<nad-name>]",
Short: "Run a Multus validation test for Rook",
Long: `
Run a validation test that determines whether the current Multus and system
configurations will support Rook with Multus.
This is a fairly long-running test. It starts up a web server and many
clients to verify that Multus network communication works properly.
It does *not* perform any load testing. Networks that cannot support high
volumes of Ceph traffic may still encounter runtime issues. This may be
particularly noticeable with high I/O load or during OSD rebalancing
(see: https://docs.ceph.com/en/latest/architecture/#rebalancing).
`,
Run: func(cmd *cobra.Command, args []string) {
runValidation(cmd.Context())
},
Args: cobra.NoArgs,
}
// 'validation cleanup' command
cleanupCmd = &cobra.Command{
Use: "cleanup",
Short: "Clean up Multus validation test resources",
Run: func(cmd *cobra.Command, args []string) {
runCleanup(cmd.Context())
},
Args: cobra.NoArgs,
}
)
func init() {
validationConfig.Clientset = rook.NewContext().Clientset
Cmd.AddCommand(runCmd)
Cmd.AddCommand(cleanupCmd)
// flags on ALL subcommands - makes output more straightforward than using PersistentFlags() global flags on parent
for _, subCommand := range Cmd.Commands() {
// the default namespace is the current namespace the operator pod is running in if possible
defaultNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar)
if defaultNamespace == "" {
defaultNamespace = DefaultValidationNamespace
}
subCommand.Flags().StringVarP(&validationConfig.Namespace, "namespace", "n", defaultNamespace,
"The namespace for validation test resources. "+
"It is recommended to set this to the namespace in which Rook's Ceph cluster will be installed.")
// VarPF() keeps the the specific var passed to it for setting at runtime, and the current
// val of that var when VarPF() is called is used as the default
validationConfig.ResourceTimeout = time.Duration(DefaultTimeoutMinutes) * time.Minute
t := (*timeoutMinutes)(&validationConfig.ResourceTimeout)
subCommand.Flags().VarPF(t, "timeout-minutes", "", /* no shorthand */
"The time to wait for resources to change to the expected state. For example, for the "+
"test web server to start, for test clients to become ready, or for test resources to be deleted. "+
"At longest, this may need to reflect the time it takes for client pods to to pull images, "+
"get address assignments, and then for each client to determine that its network connection is stable. "+
"Minimum: 1 minute. Recommended: 2 minutes or more.")
}
// flags for 'validation run'
runCmd.Flags().StringVar(&validationConfig.PublicNetwork, "public-network", "",
"The name of the Network Attachment Definition (NAD) that will be used for Ceph's public network. "+
"This should be a namespaced name in the form <namespace>/<name> if the NAD is defined in a different namespace from the cluster namespace.")
runCmd.Flags().StringVar(&validationConfig.ClusterNetwork, "cluster-network", "",
"The name of the Network Attachment Definition (NAD) that will be used for Ceph's cluster network. "+
"This should be a namespaced name in the form <namespace>/<name> if the NAD is defined in a different namespace from the cluster namespace.")
runCmd.Flags().IntVar(&validationConfig.DaemonsPerNode, "daemons-per-node", DefaultDaemonsPerNode,
"The number of validation test daemons to run per node. "+
"It is recommended to set this to the maximum number of Ceph daemons that can run on any node in the worst case of node failure(s). "+
"The default value is set to the worst-case value for a Rook Ceph cluster with 3 portable OSDs, 3 portable monitors, "+
"and where all optional child resources have been created with 1 daemon such that they all might run on a single node in a failure scenario. "+
"If you aren't sure what to choose for this value, add 1 for each additional OSD beyond 3.")
// flags for 'validation cleanup'
// none
}
func runValidation(ctx context.Context) {
results, err := validationConfig.Run(ctx)
report := results.SuggestedDebuggingReport()
// success/failure message
fmt.Print("\n")
switch {
case err != nil:
fmt.Printf("RESULT: multus validation test failed: %v\n\n", err)
case report == "":
fmt.Print("RESULT: multus validation test succeeded!\n\n")
runCleanup(ctx)
os.Exit(0) // success!
case report != "":
// suggestions are bad
fmt.Print("RESULT: multus validation test succeeded, but there are suggestions\n\n")
}
// output report suggestions
fmt.Print(report + "\n")
// help users help us help them
fmt.Println("leaving multus validation test resources running for manual debugging")
fmt.Print(`
For assistance debugging, collect the following into an archive file:
- Output of this utility
- Network Attachment Definitions (NADs) used by this test
- A write-up describing the network configuration you are trying to achieve including the
intended network for Ceph public/client traffic, intended network for Ceph cluster traffic,
interface names and CIDRs for both networks, and any other details that are relevant.
- 'ifconfig' output from at least one Kubernetes worker node
- 'kubectl get pods -o wide' output from the test namespace
- 'kubectl describe pods' output from the test namespace
- 'kubectl get pods -o yaml' output from the test namespace
- 'kubectl get daemonsets' output from the test namespace
- 'kubectl describe daemonsets' output from the test namespace
- 'kubectl get daemonsets -o yaml' output from the test namespace
- 'kubectl logs multus-validation-test-web-server' output from the test namespace
- 'kubectl get nodes -o wide' output
`)
// tell them how to cleanup
fmt.Printf("\nTo clean up resources when you are done debugging: %s", cleanupCmd.CommandPath())
os.Exit(1)
}
func runCleanup(ctx context.Context) {
fmt.Printf("cleaning up multus validation test resources in namespace %q\n", validationConfig.Namespace)
results, err := validationConfig.CleanUp(ctx)
if err != nil {
fmt.Printf("multus validation test cleanup failed: %v\n\n", err)
fmt.Println(results.SuggestedDebuggingReport())
return
}
fmt.Print("multus validation test resources were successfully cleaned up\n")
}
// custom flag types
// implements pflag.Value interface to validate and set resource timeout and enforce nonzero
type timeoutMinutes time.Duration
func (t *timeoutMinutes) String() string { return time.Duration(*t).String() }
func (t *timeoutMinutes) Set(v string) error {
i, err := strconv.Atoi(v)
if err != nil {
return err
}
if i < 1 {
return fmt.Errorf("timeout must be greater than 0")
}
*t = timeoutMinutes(time.Duration(i) * time.Minute)
return nil
}
func (t timeoutMinutes) Type() string {
return "timeoutMinutes"
}
cmd/rook/userfacing/userfacing.go
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// contains user facing commands, separated to a subdir for convenience
package userfacing
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/rook/rook/cmd/rook/userfacing/multus"
"github.com/spf13/cobra"
)
var Commands = []*cobra.Command{
multus.Cmd,
}
var stopSignalCapture context.CancelFunc
func init() {
// ensure all user-facing commands have some basic assumptions covered
for _, cmd := range Commands {
// they should not be hidden
cmd.Hidden = false
// they should have a context that captures ctrl-c interrupts
cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
ctx := context.Background()
ctx, stopSignalCapture = signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
cmd.SetContext(ctx)
return cmd.ValidateArgs(args)
}
// and should stop capturing when exiting
cmd.PersistentPostRun = func(cmd *cobra.Command, args []string) {
stopSignalCapture()
}
// they should always provide help
cmd.InitDefaultHelpCmd()
cmd.InitDefaultHelpFlag()
}
}
cmd/rook/util/cmdreporter.go
+3 -2
View File
@@ -46,8 +46,9 @@ error storing the command results into the ConfigMap. An application label
is applied to the ConfigMap. Run will also terminate if the label already
exists and has a different application's name name; this may indicate that
it is not safe for cmd-reporter to edit the ConfigMap.`,
Args: cobra.NoArgs,
Run: runCmdReporter,
Args: cobra.NoArgs,
Run: runCmdReporter,
Hidden: true, // do not advertise to end users
}
var (
cmd/rook/util/copybins.go
+3 -2
View File
@@ -37,8 +37,9 @@ binary to a Kubernetes EmptyDir volume mounted at the given directory in a pod's
init container. From the pod's main container, mount the volume which now
contains the 'rook' binary, and call 'rook cmd-reporter run' from
'rook' in order to run the desired command from a non-Rook container.`,
Args: cobra.NoArgs,
RunE: runCopyBins,
Args: cobra.NoArgs,
RunE: runCopyBins,
Hidden: true, // do not advertise to end users
}
func init() {
design/common/multi-homed-cluster.md
-87
View File
@@ -1,87 +0,0 @@
# Rook Operators Multi-homed Cluster Network Spec
## Overview
This project aims to create new API to enable multiple network interface for
Rook storage providers. Currently, Rook providers only choice is to use
`hostNetwork` or not. The new API will be used to define networks resource for
Rook clusters. Rook operators will be able to consume those definitions and
manage them. Therefore, it enables more fine-grained control over storage
providers network access.
## Current Implementation
To achieve non-flat networking model, Rook can choose to enable `hostNetwork`
and expose host network interfaces to Storage Provider pods.
Ceph Rook cluster network definition example:
```yaml
network:
hostNetwork: true
```
The Ceph operator without specifying this configuration will always
default to pod networking.
## Proposed Design
Rook operators can define storage cluster's network using network provider.
Network provider example includes host, and multus. To configure the cluster
network, cluster CRD needs to tell the network provider the appropriate
`NetworkInterfaceSelector`. `NetworkInterfaceSelector` will be provided as list
of `interfaces` key-value.
```yaml
network:
provider: <network-provider>
interfaces:
<key>: <network-interface-selector>
<key>: <network-interface-selector>
```
### Network Provider
Network provider determines multi-homing method and the network interface
selector. Using host network provider, pod can use host's network namespace
and its interfaces to use as cluster network. Using popular multi-plugin CNI
plugin such as [multus][multus-cni], Rook operators can attach multiple network
interfaces to pods.
In the future, works can be added to support other multi network CNI plugins
such as [knitter][knitter-cni] or [genie][genie-cni].
One thing to remember, leaving the network configuration empty will default to
kubernetes cluster default networking.
### Keys
The key for each interface key-value pairs are left to each storage providers to
decide. The network interface selectors with key values `public` and
`cluster` are specified for Ceph, while other storage providers can define their
own key names if desired.
### Network Interface Selector
Network interface selector is fed to network provider to connect pod to cluster
network. This selector may vary from one network provider to another. For
example, host network provider only needs to know the interface name.
On the other hand, multi-plugin CNI plugin needs to know the network attachment
definition's name and vice versa. Multi-plugin such as multus may seem to follow
_Network Attachment Selection Annotation_ documented at [Kubernetes Network
Custom Resource Definition De-facto Standard][network-crd]. However, their
implementation has extra features not covered by the standard such as
`@<interface-name>` notation or `interfaceRequest` property documented
[here][multus-annotation].
## Example Configurations
### Ceph Rook Cluster Network definition example
<!--TODO-->
[multus-cni]: https://github.com/intel/multus-cni
[knitter-cni]: https://github.com/ZTE/Knitter
[genie-cni]: https://github.com/cni-genie/CNI-Genie/
[network-crd]: https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ/edit
[multus-annotation]: https://github.com/intel/multus-cni/blob/master/doc/how-to-use.md#run-pod-with-network-annotation
design/common/multi-net-multus.md
+109
View File
@@ -0,0 +1,109 @@
# Multi-homed Network (Multus)
## Overview
This project aims to create an API to enable multiple network interface for
Rook storage providers. Currently, Rook providers only choice is to use
`hostNetwork` or not. The API will be used to define networks resource for
Rook clusters. It enables more fine-grained control over network access.
## Legacy Implementation
To achieve non-flat networking model, Rook can choose to enable `hostNetwork`
and expose host network interfaces to Storage Provider pods.
Ceph Rook cluster network definition example:
```yaml
network:
hostNetwork: true
```
The Ceph operator without specifying this configuration will always
default to pod networking.
## Multi-homed Network Design
Rook operators can define storage cluster's network using network provider.
Network provider example includes host, and multus. To configure the cluster
network, cluster CRD needs to tell the network provider the appropriate
`NetworkInterfaceSelector`. `NetworkInterfaceSelector` will be provided as list
of `interfaces` key-value.
```yaml
network:
provider: <network-provider>
interfaces:
<key>: <network-interface-selector>
<key>: <network-interface-selector>
```
### Network Provider
Network provider determines multi-homing method and the network interface
selector. Using host network provider, pod can use host's network namespace
and its interfaces to use as cluster network. Using popular multi-plugin CNI
plugin such as [multus][multus-cni], Rook operators can attach multiple network
interfaces to pods.
One thing to remember, leaving the network configuration empty will default to
kubernetes cluster default networking.
### Keys
The network interface selectors with key values `public` and
`cluster` are specified for Ceph.
### Network Interface Selector
Network interface selector is fed to network provider to connect pod to cluster
network. This selector may vary from one network provider to another. For
example, host network provider only needs to know the interface name.
On the other hand, multi-plugin CNI plugin needs to know the network attachment
definition's name and vice versa. Multi-plugin such as multus may seem to follow
_Network Attachment Selection Annotation_ documented by Kubernetes Network Plumbing Working Group's
[Multi-Net Spec].
Any future multi-homed network provider that implements the Network Plumbing WG's Multi-net spec may
use the `multus` network. Previously-identified providers [CNI-Genie] and [Knitter] have since gone
dormant (upd. 2023), and Multus continues to be the most prominent known provider.
## Multi-Net validation tester
Rook will implement a user-runnable routine that validates a multi-net configuration based on the
[Multi-Net Spec]. Users can input one or both of the Network Attachment Definitions (NADs) for
Ceph's `multus` network, and the routine will perform cursory validation of the ability of the
specified networks to support Ceph.
The routine will start up **a single** web server with the specified networks attached (via NADs).
The routine will also start up a number of clients on each node that will test the network(s)'s
connections by HTTP(S) requests to the web server.
A client represents a simplified view of a Ceph daemon from the networking perspective. The number
of clients is intended to represent the number of Ceph daemons that could run on a node in the worst
possible failure case where all daemons are rescheduled to a single node. This helps verify that the
network provider is able to successfully assign IP addresses to Ceph daemon pods in the worst case.
Some network interface software/hardware may limit the number of addresses that can be assigned to
an interface on a given node, and this helps verify that issue is not present.
It is important that the web server be only **a single** instance. If clients collocated on the same
node with the web server can make successful HTTP(S) requests but clients on other nodes are not,
that is an important indicator of inter-node traffic being blocked.
The validation test routine will run in the Rook operator container. Rook's kubectl [Krew plugin]
may facilitate running the test, but it is useful to have the option of executing a long-running
validation routine running in the Kubernetes cluster instead of on an administrator's log-in
session. Additionally, the results of the validation tester may become important for users wanting
to get help with Multus configuration, and having a tool that is present in the Rook container image
will allow users to run the routine for bug reports more readily than if they needed to install the
Krew plugin.
<!--
LINKS
-->
[multus-cni]: https://github.com/intel/multus-cni
[multi-net spec]: https://github.com/k8snetworkplumbingwg/multi-net-spec
[cni-genie]: https://github.com/cni-genie/CNI-Genie/
[knitter]: https://github.com/ZTE/Knitter
[krew plugin]: https://github.com/rook/kubectl-rook-ceph
go.mod
+1
View File
@@ -82,6 +82,7 @@ require (
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible // indirect
github.com/circonus-labs/circonusllhist v0.1.3 // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/containernetworking/cni v1.0.1 // indirect
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba // indirect
go.sum
+6
View File
@@ -304,6 +304,8 @@ github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBd
github.com/containerd/containerd v1.5.17 h1:NLDEI//zhMZpR3DS/AP0qiN+dzYKNAwJaNXCnCmYcgY=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg=
github.com/containernetworking/cni v1.0.1 h1:9OIL/sZmMYDBe+G8svzILAlulUpaDTUjeAbtH/JNLBo=
github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -1295,6 +1297,7 @@ github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
@@ -1310,6 +1313,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc=
@@ -1477,6 +1481,7 @@ github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0
github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
github.com/sean-/conswriter v0.0.0-20180208195008-f5ae3917a627/go.mod h1:7zjs06qF79/FKAJpBvFx3P8Ww4UTIMAe+lpNXDHziac=
github.com/sean-/pager v0.0.0-20180208200047-666be9bf53b5/go.mod h1:BeybITEsBEg6qbIiqJ6/Bqeq25bCLbL7YFmpaFfJDuM=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@@ -2226,6 +2231,7 @@ gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76
gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
pkg/daemon/multus/client-daemonset.yaml
+63
View File
@@ -0,0 +1,63 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: multus-validation-test-client-{{ .ClientID }}
labels:
app: multus-validation-test-client
app.kubernetes.io/name: "client"
app.kubernetes.io/instance: "client-{{ .ClientID }}"
app.kubernetes.io/component: "client"
app.kubernetes.io/part-of: "multus-validation-test"
app.kubernetes.io/managed-by: "rook-cli"
spec:
selector:
matchLabels:
app: multus-validation-test-client
clientID: "{{ .ClientID }}"
template:
metadata:
labels:
app: multus-validation-test-client
clientID: "{{ .ClientID }}"
annotations:
k8s.v1.cni.cncf.io/networks: "{{ .NetworksAnnotationValue }}"
spec:
# TODO: selectors, affinities, tolerations
securityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
containers:
{{ range $name, $address := .NetworkNamesAndAddresses }}
- name: readiness-check-web-server-{{ $name }}-addr
# use nginx image because it's already pulled for the nginx pod and has a non-root user
image: nginx:stable-alpine
command:
- sleep
- infinity
resources: {}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
# A readiness probe makes validation testing easier than investigate container logs.
# Additionally, readiness probe failures don't result in CrashLoopBackoff -- ideal here,
# where ever-longer back-offs would cause tests to run for much longer than necessary.
readinessProbe:
# Low failure threshold and high success threshold. Intended to be very sensitive to
# failures. If probe fails with any regularity, Ceph OSDs likely won't be stable.
failureThreshold: 1
successThreshold: 12
periodSeconds: 5
# Assumption: a network with a latency more than 4 seconds for this validation test's
# simple client-server response likely won't support acceptable performance for any
# production Ceph cluster.
timeoutSeconds: 4
# TODO: exec:curl works but httpGet fails. Why? need custom header?
exec:
command:
- "curl"
- "--insecure"
- "{{ $address }}:8080"
{{ end }}
pkg/daemon/multus/nginx-config.yaml
+18
View File
@@ -0,0 +1,18 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: multus-validation-test-web-server-conf
labels:
app: multus-validation-test-web-server
data:
server.conf: |
server {
listen 8080;
server_name localhost;
# return the client ip upon connect
location / {
default_type text/plain;
return 200 "$remote_addr\n";
}
}
pkg/daemon/multus/nginx-pod.yaml
+52
View File
@@ -0,0 +1,52 @@
# This is a Pod and not a Deployment because there is intended to only be one running
apiVersion: apps/v1
kind: Pod
metadata:
name: multus-validation-test-web-server
labels:
app: multus-validation-test-web-server
app.kubernetes.io/name: "nginx"
app.kubernetes.io/instance: "nginx" # there is only one instance
app.kubernetes.io/component: "server"
app.kubernetes.io/part-of: "multus-validation-test"
app.kubernetes.io/managed-by: "rook-cli"
annotations:
k8s.v1.cni.cncf.io/networks: "{{ .NetworksAnnotationValue }}"
spec:
# TODO: selectors, affinities, tolerations
securityContext:
runAsNonRoot: true
runAsUser: 101
runAsGroup: 101
seccompProfile:
type: RuntimeDefault
containers:
- name: multus-validation-test-web-server
image: nginx:stable-alpine
resources: {}
ports:
- containerPort: 8080
readinessProbe:
httpGet:
port: 8080
scheme: HTTP
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- "ALL"
volumeMounts:
- name: var-cache-nginx
mountPath: /var/cache/nginx
- name: server-conf
mountPath: /etc/nginx/conf.d
- name: var-run
mountPath: /var/run
volumes:
- name: var-cache-nginx
emptyDir: {}
- name: server-conf
configMap:
name: multus-validation-test-web-server-conf
- name: var-run
emptyDir: {}
pkg/daemon/multus/resources.go
+280
View File
@@ -0,0 +1,280 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package multus
import (
"context"
"fmt"
"time"
core "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
)
type podNetworkInfo struct {
// node the pod is running on
nodeName string
// multus public addr attached (if any)
publicAddr string
// multus cluster addr attached (if any)
clusterAddr string
}
var manualCleanupInstructions = fmt.Sprintf(
"manually delete owner configmap %q, and wait for all multus-validation-test resources to be deleted", ownerConfigMapName)
var previousTestSuggestion = "there could be a past test preventing this one from proceeding; " + manualCleanupInstructions
var unableToProvideAddressSuggestions = []string{
"multus may be unable to provide addresses for pods",
"check networking events on the pod and multus logs",
"macvlan: NIC or switch hardware/software may block the association of some number of additional MAC addresses on an interface",
"macvlan: interfaces and network switching must enable promiscuous mode to allow receiving packets for unknown (Multus) MACs",
"macvlan/ipvlan: switch hardware/software may block an interface from receiving packets to an unknown (Multus) IP",
}
// create a validation test config object that stores the configuration of the running validation
// test. this object serves as the owner of all associated test objects. when this object is
// deleted, all validation test objects should also be deleted, effectively cleaning up all
// components of this test.
func (vt *ValidationTest) createOwningConfigMap(ctx context.Context) ([]meta.OwnerReference, error) {
c := core.ConfigMap{
ObjectMeta: meta.ObjectMeta{
Name: ownerConfigMapName,
},
}
configObject, err := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Create(ctx, &c, meta.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("failed to create validation test config object [%+v]: %w", c, err)
}
// for cleanup, we want to make sure all children are deleted
BlockOwnerDeletion := true
refToConfigObject := meta.OwnerReference{
APIVersion: "v1",
Kind: "ConfigMap",
Name: configObject.GetName(),
UID: configObject.GetUID(),
BlockOwnerDeletion: &BlockOwnerDeletion,
}
return []meta.OwnerReference{refToConfigObject}, nil
}
func (vt *ValidationTest) startWebServer(ctx context.Context, owners []meta.OwnerReference) error {
pod, err := vt.generateWebServerPod()
if err != nil {
return fmt.Errorf("failed to generate web server pod: %w", err)
}
pod.SetOwnerReferences(owners) // set owner refs so cleanup is easier
configMap, err := vt.generateWebServerConfigMap()
if err != nil {
return fmt.Errorf("failed to generate web server config: %w", err)
}
configMap.SetOwnerReferences(owners) // set owner refs so cleanup is easier
// create configmap before pod so pod doesn't crashloopbackoff on first creation
_, err = vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Create(ctx, configMap, meta.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create web server config: %w", err)
}
_, err = vt.Clientset.CoreV1().Pods(vt.Namespace).Create(ctx, pod, meta.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create web server pod: %w", err)
}
return nil
}
func (vt *ValidationTest) getWebServerInfo(
ctx context.Context,
desiredPublicNet, desiredClusterNet *types.NamespacedName,
) (podNetworkInfo, []string, error) {
podInfo := podNetworkInfo{}
pod, err := vt.Clientset.CoreV1().Pods(vt.Namespace).Get(ctx, webServerPodName(), meta.GetOptions{})
if err != nil {
return podInfo, []string{}, fmt.Errorf("unexpected error when getting web server pod: %w", err)
}
var publicAddr, clusterAddr string
publicAddr, clusterAddr, networkSuggestions, err := getNetworksFromPod(pod, desiredPublicNet, desiredClusterNet)
if err != nil {
return podInfo, networkSuggestions, fmt.Errorf("no web server network info: %w", err)
}
if !podIsReady(*pod) {
return podInfo, []string{}, fmt.Errorf("web server pod is not ready yet")
}
podInfo.nodeName = pod.Spec.NodeName
podInfo.publicAddr = publicAddr
podInfo.clusterAddr = clusterAddr
return podInfo, []string{}, nil // no suggestions if successful
}
func (vt *ValidationTest) startClients(
ctx context.Context,
owners []meta.OwnerReference,
serverPublicAddr, serverClusterAddr string,
) error {
for i := 0; i < vt.DaemonsPerNode; i++ {
ds, err := vt.generateClientDaemonSet(i, serverPublicAddr, serverClusterAddr)
if err != nil {
return fmt.Errorf("failed to generate client daemonset for client #%d: %w", i, err)
}
ds.SetOwnerReferences(owners) // set owner refs so cleanup is easier
_, err = vt.Clientset.AppsV1().DaemonSets(vt.Namespace).Create(ctx, ds, meta.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create client daemonset for client #%d: %w", i, err)
}
}
return nil
}
// if this returns success, the return value is guaranteed to be greater than 0
func (vt *ValidationTest) getExpectedNumberOfClientPods(ctx context.Context) (int, error) {
var agreedNumberScheduled int32
listOpts := meta.ListOptions{
LabelSelector: clientAppLabel(),
}
dsets, err := vt.Clientset.AppsV1().DaemonSets(vt.Namespace).List(ctx, listOpts)
if err != nil {
return 0, fmt.Errorf("unexpected error listing client daemonsets: %w", err)
}
if len(dsets.Items) < vt.DaemonsPerNode {
return 0, fmt.Errorf("got fewer client daemonsets [%d] than should exist [%d]", len(dsets.Items), vt.DaemonsPerNode)
}
for _, d := range dsets.Items {
numScheduled := d.Status.CurrentNumberScheduled
if numScheduled == 0 {
return 0, fmt.Errorf("a daemonset expects zero scheduled client pods; cannot validate with no running clients")
}
if agreedNumberScheduled == 0 {
agreedNumberScheduled = numScheduled
} else if numScheduled != agreedNumberScheduled {
return 0, fmt.Errorf("daemonsets do not all agree on the number of expected client pods")
}
}
return int(agreedNumberScheduled) * vt.DaemonsPerNode, nil
}
func (vt *ValidationTest) allClientsAreRunning(ctx context.Context, expectedNumPods int) (bool, error) {
pods, err := vt.getClientPods(ctx, expectedNumPods)
if err != nil {
return false, fmt.Errorf("unexpected error getting client pods: %w", err)
}
numRunning := 0
for _, p := range pods.Items {
if podIsRunning(p) {
numRunning++
}
}
if numRunning != expectedNumPods {
return false, fmt.Errorf("number of running clients [%d] is not the number expected [%d]", numRunning, expectedNumPods)
}
return true, nil
}
func (vt *ValidationTest) numClientsReady(ctx context.Context, expectedNumPods int) (int, error) {
pods, err := vt.getClientPods(ctx, expectedNumPods)
if err != nil {
return 0, fmt.Errorf("unexpected error getting client pods: %w", err)
}
numReady := 0
for _, p := range pods.Items {
if podIsReady(p) {
numReady++
}
}
return numReady, nil
}
func (vt *ValidationTest) getClientPods(ctx context.Context, expectedNumPods int) (*core.PodList, error) {
listOpts := meta.ListOptions{
LabelSelector: clientAppLabel(),
}
pods, err := vt.Clientset.CoreV1().Pods(vt.Namespace).List(ctx, listOpts)
if err != nil {
return nil, fmt.Errorf("failed to list client pods: %w", err)
}
if len(pods.Items) != expectedNumPods {
return nil, fmt.Errorf("the number of pods listed [%d] does not match the number expected [%d]", len(pods.Items), expectedNumPods)
}
return pods, err
}
func (vt *ValidationTest) cleanUpTestResources() (string, error) {
// need a clean, non-canceled context in case the test is canceled by ctrl-c
ctx := context.Background()
// delete the config object in the foreground so we wait until all validation test resources are
// gone before stopping, and do it now because there's no need to wait for just a test
var gracePeriodZero int64 = 0
deleteForeground := meta.DeletePropagationForeground
delOpts := meta.DeleteOptions{
PropagationPolicy: &deleteForeground,
GracePeriodSeconds: &gracePeriodZero,
}
err := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Delete(ctx, ownerConfigMapName, delOpts)
if err != nil {
if !kerrors.IsNotFound(err) {
return manualCleanupInstructions, fmt.Errorf("failed to clean up multus validation test resources: %w", err)
}
return "", nil
}
// clients take a long time to terminate, and the 0 grace period set on the configmap doesn't
// propagate to dependents. make a best-effort attempt to delete client pods with 0 grace period
listOpts := meta.ListOptions{
LabelSelector: clientAppLabel(),
}
// ignore errors for a best-effort attempt, they will delete eventually
_ = vt.Clientset.CoreV1().Pods(vt.Namespace).DeleteCollection(ctx, delOpts, listOpts)
// wait for resources to be cleaned up
ctx, cancel := context.WithTimeout(ctx, vt.ResourceTimeout)
defer cancel()
lastSuggestion := ""
err = wait.PollUntilWithContext(ctx, 2*time.Second, func(ctx context.Context) (done bool, err error) {
_, getErr := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Get(ctx, ownerConfigMapName, meta.GetOptions{})
if getErr != nil {
if kerrors.IsNotFound(getErr) {
return true, nil
}
lastSuggestion = fmt.Sprintf("unexpected error when cleaning up multus validation test resources; attempting to continue: %v", err)
}
return false, nil
})
if err != nil {
return lastSuggestion + "; " + manualCleanupInstructions,
fmt.Errorf("failed waiting for multus validation test resources to be deleted: %w", err)
}
return "", nil
}
pkg/daemon/multus/statemachine.go
+96
View File
@@ -0,0 +1,96 @@
package multus
import (
"context"
"fmt"
"time"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type validationState interface {
Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error)
}
type validationStateMachine struct {
vt *ValidationTest
state validationState
timer *time.Timer
stateWasChanged bool
resourceOwnerRefs []meta.OwnerReference
testResults *ValidationTestResults
lastSuggestions []string
lastErr error
done bool
}
func (vsm *validationStateMachine) SetNextState(nextState validationState) {
vsm.stateWasChanged = true
vsm.state = nextState
}
func (vsm *validationStateMachine) Exit() {
vsm.done = true
}
func (vsm *validationStateMachine) Run(ctx context.Context) (*ValidationTestResults, error) {
vsm.timer = time.NewTimer(vsm.vt.ResourceTimeout)
defer func() { vsm.timer.Stop() }()
for {
select {
case <-ctx.Done():
return vsm.exitContextCanceled(ctx)
case <-vsm.timer.C:
vsm.testResults.addSuggestions(vsm.lastSuggestions...)
return vsm.testResults, fmt.Errorf("multus validation test timed out: %w", vsm.lastErr)
default:
// give each state the full resource timeout to run successfully
if vsm.stateWasChanged {
vsm.resetTimer()
vsm.stateWasChanged = false
}
// run the state
suggestions, err := vsm.state.Run(ctx, vsm)
// if the context was canceled, the error message won't be as useful as gathered from
// the last context, so exit before updating the latest suggestions and error
if ctx.Err() != nil {
return vsm.exitContextCanceled(ctx)
}
// record the latest suggestions and error
vsm.lastErr = err
vsm.lastSuggestions = suggestions
// exit the state machine, error or not
if vsm.done {
vsm.testResults.addSuggestions(vsm.lastSuggestions...)
if vsm.lastErr != nil {
return vsm.testResults, fmt.Errorf("multus validation test failed: %w", vsm.lastErr)
}
return vsm.testResults, nil
}
if err != nil {
vsm.vt.Logger.Infof("continuing: %s", err)
}
time.Sleep(2 * time.Second)
}
}
}
func (vsm *validationStateMachine) resetTimer() {
if !vsm.timer.Stop() {
<-vsm.timer.C
}
vsm.timer.Reset(vsm.vt.ResourceTimeout)
}
func (vsm *validationStateMachine) exitContextCanceled(ctx context.Context) (*ValidationTestResults, error) {
vsm.testResults.addSuggestions(vsm.lastSuggestions...)
return vsm.testResults, fmt.Errorf("context canceled before multus validation test could complete: %s: %w", ctx.Err().Error(), vsm.lastErr)
}
pkg/daemon/multus/templates.go
+149
View File
@@ -0,0 +1,149 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package multus
import (
"bytes"
_ "embed"
"fmt"
"strings"
"text/template"
apps "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/yaml"
)
var (
//go:embed nginx-pod.yaml
nginxPodTemplate string
//go:embed nginx-config.yaml
nginxConfigTemplate string
//go:embed client-daemonset.yaml
clientDaemonSet string
)
type webServerTemplateConfig struct {
NetworksAnnotationValue string
}
type clientTemplateConfig struct {
ClientID int
NetworksAnnotationValue string
NetworkNamesAndAddresses map[string]string
}
func webServerPodName() string {
return "multus-validation-test-web-server"
}
func clientAppLabel() string {
return "app=multus-validation-test-client"
}
func (vt *ValidationTest) generateWebServerTemplateConfig() webServerTemplateConfig {
return webServerTemplateConfig{
NetworksAnnotationValue: vt.generateNetworksAnnotationValue(),
}
}
func (vt *ValidationTest) generateClientTemplateConfig(clientID int, serverPublicAddr, serverClusterAddr string) clientTemplateConfig {
netNamesAndAddresses := map[string]string{}
if serverPublicAddr != "" {
netNamesAndAddresses["public"] = serverPublicAddr
}
if serverClusterAddr != "" {
netNamesAndAddresses["cluster"] = serverClusterAddr
}
return clientTemplateConfig{
ClientID: clientID,
NetworksAnnotationValue: vt.generateNetworksAnnotationValue(),
NetworkNamesAndAddresses: netNamesAndAddresses,
}
}
func (vt *ValidationTest) generateWebServerPod() (*core.Pod, error) {
t, err := loadTemplate("webServerPod", nginxPodTemplate, vt.generateWebServerTemplateConfig())
if err != nil {
return nil, fmt.Errorf("failed to load web server pod template: %w", err)
}
var p core.Pod
err = yaml.Unmarshal(t, &p)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal web server pod template: %w", err)
}
return &p, nil
}
func (vt *ValidationTest) generateWebServerConfigMap() (*core.ConfigMap, error) {
t, err := loadTemplate("webServerConfigMap", nginxConfigTemplate, vt.generateWebServerTemplateConfig())
if err != nil {
return nil, fmt.Errorf("failed to load web server configmap template: %w", err)
}
var cm core.ConfigMap
err = yaml.Unmarshal(t, &cm)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal web server configmap template: %w", err)
}
return &cm, nil
}
func (vt *ValidationTest) generateClientDaemonSet(
clientID int,
serverPublicAddr, serverClusterAddr string,
) (*apps.DaemonSet, error) {
t, err := loadTemplate("clientDaemonSet", clientDaemonSet, vt.generateClientTemplateConfig(clientID, serverPublicAddr, serverClusterAddr))
if err != nil {
return nil, fmt.Errorf("failed to load client daemonset template: %w", err)
}
var d apps.DaemonSet
err = yaml.Unmarshal(t, &d)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal client daemonset template: %w", err)
}
return &d, nil
}
func (vt *ValidationTest) generateNetworksAnnotationValue() string {
nets := []string{}
if vt.PublicNetwork != "" {
nets = append(nets, vt.PublicNetwork)
}
if vt.ClusterNetwork != "" {
nets = append(nets, vt.ClusterNetwork)
}
return strings.Join(nets, ",")
}
func loadTemplate(name, templateFileText string, config interface{}) ([]byte, error) {
var writer bytes.Buffer
t := template.New(name)
t, err := t.Parse(templateFileText)
if err != nil {
return nil, fmt.Errorf("failed to parse template %q: %w", name, err)
}
err = t.Execute(&writer, config)
return writer.Bytes(), err
}
pkg/daemon/multus/util.go
+129
View File
@@ -0,0 +1,129 @@
package multus
import (
"fmt"
"os"
nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
// Logger defines the log types that this library will use for outputting status information to the
// user. Because this library may be called interactively by a user or programmatically by another
// application, we allow the calling application to make a very simple Logger type that will be
// compatible with this library.
type Logger interface {
Infof(format string, args ...interface{})
Debugf(format string, args ...interface{})
Warningf(format string, args ...interface{})
}
type SimpleStderrLogger struct{}
func (*SimpleStderrLogger) Infof(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, " INFO: "+format+"\n", args...)
}
func (*SimpleStderrLogger) Debugf(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, "DEBUG: "+format+"\n", args...)
}
func (*SimpleStderrLogger) Warningf(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, " WARN: "+format+"\n", args...)
}
func getNetworksFromPod(
pod *core.Pod,
desiredPublicNet, desiredClusterNet *types.NamespacedName,
) (
publicAddr, clusterAddr string, suggestions []string, err error,
) {
nets, err := nadutils.GetNetworkStatus(pod)
if err != nil {
return "", "", unableToProvideAddressSuggestions,
fmt.Errorf("pod has no network status yet: %w", err)
}
if len(nets) == 0 {
return "", "", unableToProvideAddressSuggestions,
fmt.Errorf("pod has no attached networks yet")
}
// if the pod has networks attached, parse all of them, and report any debugging suggestions
// associated for each one
publicAddr, clusterAddr = "", ""
suggestions = []string{}
for _, net := range nets {
nsName, err := networkNamespacedName(net.Name, pod.Namespace)
if err != nil {
suggestions = append(suggestions,
fmt.Sprintf("pod has an attached network with an un-parse-able network name [%s]; "+
"not sure what to do; this is unlikely to resolve", net.Name),
)
}
if len(net.IPs) == 0 {
suggestions = append(suggestions,
fmt.Sprintf("pod has an attached network [%s] with no IP; "+
"could this be an IPAM issue?", net.Name))
}
ip := net.IPs[0] // any IP should work
if desiredPublicNet != nil && nsName == *desiredPublicNet {
// FOUND DESIRED PUBLIC ADDR
publicAddr = ip
}
if desiredClusterNet != nil && nsName == *desiredClusterNet {
// FOUND DESIRED CLUSTER ADDR
clusterAddr = ip
}
}
if desiredPublicNet != nil && publicAddr == "" {
suggestions = append(suggestions, "pod does not have a public network attachment")
}
if desiredClusterNet != nil && clusterAddr == "" {
suggestions = append(suggestions, "pod does not have a cluster network attachment")
}
// if there are any suggestions, something isn't right
if len(suggestions) > 0 {
suggestions = append(suggestions, unableToProvideAddressSuggestions...)
return publicAddr, clusterAddr, suggestions, fmt.Errorf("did not find all desired networks")
}
// found what we were looking for
return publicAddr, clusterAddr, []string{}, nil
}
func podIsRunning(pod core.Pod) bool {
return pod.Status.Phase == core.PodRunning
}
func podIsReady(pod core.Pod) bool {
for _, c := range pod.Status.Conditions {
if c.Type == core.PodReady {
return c.Status == core.ConditionTrue
}
}
return false
}
func networkNamespacedName(netName, podNamespace string) (types.NamespacedName, error) {
nsName := types.NamespacedName{}
// what we really want is the result of parsePodNetworkObjectText(), a private method.
// because the network name is in the same format as network annotations, we can at
// at least use ParseNetworkAnnotation() to parse the network name
netSelections, err := nadutils.ParseNetworkAnnotation(netName, podNamespace)
if err != nil {
return nsName, fmt.Errorf("failed to parse network name %q into namespaced name: %w", netName, err)
}
if len(netSelections) != 1 {
return nsName, fmt.Errorf("failed to parse network name %q into a single namespaced network name, instead: %v", netName, netSelections)
}
net := netSelections[0]
nsName.Namespace = net.Namespace
nsName.Name = net.Name
return nsName, nil
}
pkg/daemon/multus/validation.go
+332
View File
@@ -0,0 +1,332 @@
/*
Copyright 2023 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package multus
import (
"context"
"fmt"
"time"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
)
const (
// the name of the config object that "owns" all other resources created for this test
// this does not need to be unique per test. per-namespace is fine
ownerConfigMapName = "multus-validation-test-owner"
flakyNetworkSuggestion = "the underlying network may be flaky or not have the bandwidth to support a production ceph cluster; " +
"even if the validation test passes, this could still be an issue"
)
// A Multus ValidationTest runs a number of Multus-connected pods to validate that a Kubernetes
// environment is suitable for Rook to run Ceph in.
type ValidationTest struct {
Clientset kubernetes.Interface
// Namespace is the intended namespace where the validation test will be run. This is
// recommended to be where the Rook-Ceph cluster will be installed.
Namespace string
// PublicNetwork is the name of the Network Attachment Definition (NAD) which will
// be used for the Ceph cluster's public network. This should be a namespaced name in the form
// <namespace>/<name> if the NAD is defined in a different namespace from the cluster namespace.
PublicNetwork string
// ClusterNetwork is the name of the Network Attachment Definition (NAD) which will
// be used for the Ceph cluster's cluster network. This should be a namespaced name in the form
// <namespace>/<name> if the NAD is defined in a different namespace from the cluster namespace.
ClusterNetwork string
// DaemonsPerNode is the number of Multus-connected validation daemons that will run as part of
// the test. This should be set to the largest number of Ceph daemons that may run on a node in
// the worst case. Remember to consider failure cases where nodes or availability zones are
// offline and their workloads must be rescheduled.
DaemonsPerNode int
// ResourceTimeout is the time to wait for resources to change to the expected state. For
// example, for the test web server to start, for test clients to become ready, or for test
// resources to be deleted. At longest, this may need to reflect the time it takes for client
// pods to to pull images, get address assignments, and then for each client to determine that
// its network connection is stable.
//
// This should be at least 1 minute. 2 minutes or more is recommended.
ResourceTimeout time.Duration
// Logger an instance of the basic log implementation used by this library.
Logger Logger
// TODO: allow overriding the nginx server image from CLI, and set from default var that can be
// overridden at build time
}
// ValidationTestResults contains results from a validation test.
type ValidationTestResults struct {
suggestedDebugging []string
}
func (vtr *ValidationTestResults) SuggestedDebuggingReport() string {
if vtr == nil || len(vtr.suggestedDebugging) == 0 {
return ""
}
out := fmt.Sprintln("Suggested things to investigate before installing with Multus:")
for _, s := range vtr.suggestedDebugging {
out = out + " - " + fmt.Sprintln(s)
}
return out
}
func (vtr *ValidationTestResults) addSuggestions(s ...string) {
for _, sug := range s {
if sug == "" {
continue
}
vtr.suggestedDebugging = append(vtr.suggestedDebugging, sug)
}
}
/*
* Validation state machine state definitions
* state machine is linear and very simple, with steps numbered 1-N
*/
// 1. Get web server info
type getWebServerInfoState struct {
desiredPublicNet, desiredClusterNet *types.NamespacedName
}
func (s *getWebServerInfoState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) {
info, suggestions, err := vsm.vt.getWebServerInfo(ctx, s.desiredPublicNet, s.desiredClusterNet)
if err != nil {
return suggestions, err
}
vsm.SetNextState(&startClientsState{
webServerInfo: info,
})
return []string{}, nil
}
// 2. Start clients
type startClientsState struct {
webServerInfo podNetworkInfo
}
func (s *startClientsState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) {
vsm.vt.Logger.Infof("starting %d client DaemonSets", vsm.vt.DaemonsPerNode)
err = vsm.vt.startClients(ctx, vsm.resourceOwnerRefs, s.webServerInfo.publicAddr, s.webServerInfo.clusterAddr)
if err != nil {
eErr := fmt.Errorf("failed to start clients: %w", err)
vsm.Exit() // this is a whole validation test failure if we can't start clients
return []string{}, eErr
}
vsm.SetNextState(&getNumExpectedClientsState{})
return []string{}, nil
}
// 3. Get expected number of clients
type getNumExpectedClientsState struct {
expectedNumClients int
expectedNumClientsValueChanged time.Time
}
var podSchedulerDebounceTime = 30 * time.Second
func (s *getNumExpectedClientsState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) {
expectedNumClientPods, err := vsm.vt.getExpectedNumberOfClientPods(ctx)
if err != nil {
return []string{"inability to schedule DaemonSets is likely an issue with the Kubernetes cluster itself"},
fmt.Errorf("expected number of client not yet ready: %w", err)
}
if s.expectedNumClients < expectedNumClientPods {
s.expectedNumClients = expectedNumClientPods
s.expectedNumClientsValueChanged = time.Now()
}
if time.Since(s.expectedNumClientsValueChanged) < podSchedulerDebounceTime {
vsm.vt.Logger.Infof("waiting to ensure num expected clients to stabilizes at %d", s.expectedNumClients)
return []string{}, nil
}
vsm.vt.Logger.Infof("expecting %d clients", expectedNumClientPods)
vsm.SetNextState(&verifyAllClientsRunningState{
expectedNumClients: expectedNumClientPods,
})
return []string{}, nil
}
// 4. All client pods should be in running state, even if they aren't "Ready"
type verifyAllClientsRunningState struct {
expectedNumClients int
}
func (s *verifyAllClientsRunningState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) {
running, err := vsm.vt.allClientsAreRunning(ctx, s.expectedNumClients)
suggestions = append([]string{
"clients not being able to run can mean multus is unable to provide them with addresses"},
unableToProvideAddressSuggestions...)
if err != nil {
return suggestions, err
} else if !running {
return suggestions, nil // not running, but no specific error
}
vsm.vt.Logger.Infof("all %d clients are running - but may not be ready", s.expectedNumClients)
vsm.SetNextState(&verifyAllClientsReadyState{
expectedNumClients: s.expectedNumClients,
})
return []string{}, nil
}
// 5. All client pods should be in "Ready" state
type verifyAllClientsReadyState struct {
expectedNumClients int
// keep some info to heuristically determine if the network might be flaky/overloaded
prevNumReady int
timeClientsStartedBecomingReady time.Time
suggestFlaky bool
}
func (s *verifyAllClientsReadyState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) {
numReady, err := vsm.vt.numClientsReady(ctx, s.expectedNumClients)
collocationSuggestion := "if clients on the same node as the web server become ready but not others, " +
"there may be a network firewall or security policy blocking inter-node traffic on multus networks"
defaultSuggestions := append([]string{collocationSuggestion, flakyNetworkSuggestion}, unableToProvideAddressSuggestions...)
if err != nil {
return defaultSuggestions, err
}
s.checkIfFlaky(vsm, numReady)
if numReady != s.expectedNumClients {
return defaultSuggestions, fmt.Errorf("number of ready clients [%d] is not the number expected [%d]", numReady, s.expectedNumClients)
}
vsm.vt.Logger.Infof("all %d clients are ready", s.expectedNumClients)
suggestionsOnSuccess := []string{}
if s.suggestFlaky {
suggestionsOnSuccess = append(suggestionsOnSuccess,
fmt.Sprintf("not all clients became ready within %s; %s", flakyThreshold.String(), flakyNetworkSuggestion))
}
vsm.Exit() // DONE!
return suggestionsOnSuccess, nil
}
// clients should all become ready within a pretty short amount of time since they all should start
// pretty simultaneously
// TODO: allow tuning this
// TODO: pull the image first on all nodes to ensure flakiness isn't affected by image pull time
var flakyThreshold = 20 * time.Second
func (s *verifyAllClientsReadyState) checkIfFlaky(vsm *validationStateMachine, numReady int) {
if s.suggestFlaky {
return // no need to do any checks if network is already found flaky
}
if numReady > s.prevNumReady && s.timeClientsStartedBecomingReady.IsZero() {
vsm.vt.Logger.Debugf("clients started becoming ready")
s.timeClientsStartedBecomingReady = time.Now()
return
}
if !s.timeClientsStartedBecomingReady.IsZero() {
// check to see how long it took since clients first started becoming ready. if the time is
// longer than the flaky threshold, warn the user, and record that the network is flaky
if time.Since(s.timeClientsStartedBecomingReady) > flakyThreshold {
vsm.vt.Logger.Warningf(
"network seems flaky; the time since clients started becoming ready until now is greater than %s", flakyThreshold.String())
s.suggestFlaky = true
}
}
}
// Run the Multus validation test.
func (vt *ValidationTest) Run(ctx context.Context) (*ValidationTestResults, error) {
if vt.Logger == nil {
vt.Logger = &SimpleStderrLogger{}
vt.Logger.Infof("no logger was specified; using a simple stderr logger")
}
vt.Logger.Infof("starting multus validation test with the following config:")
vt.Logger.Infof(" namespace: %q", vt.Namespace)
vt.Logger.Infof(" public network: %q", vt.PublicNetwork)
vt.Logger.Infof(" cluster network: %q", vt.ClusterNetwork)
vt.Logger.Infof(" daemons per node: %d", vt.DaemonsPerNode)
vt.Logger.Infof(" resource timeout: %v", vt.ResourceTimeout)
if vt.PublicNetwork == "" && vt.ClusterNetwork == "" {
return nil, fmt.Errorf("at least one of 'public network' and 'cluster network' must be specified")
}
var desiredPublicNet *types.NamespacedName = nil
var desiredClusterNet *types.NamespacedName = nil
if vt.PublicNetwork != "" {
n, err := networkNamespacedName(vt.PublicNetwork, vt.Namespace)
if err != nil {
return nil, fmt.Errorf("public network is an invalid NAD name: %w", err)
}
desiredPublicNet = &n
}
if vt.ClusterNetwork != "" {
n, err := networkNamespacedName(vt.ClusterNetwork, vt.Namespace)
if err != nil {
return nil, fmt.Errorf("cluster network is an invalid NAD name: %w", err)
}
desiredClusterNet = &n
}
testResults := &ValidationTestResults{
suggestedDebugging: []string{},
}
// configmap's purpose is to serve as the owner resource object for all other test resources.
// this allows users to clean up a botched test easily just by deleting this configmap
owningConfigMap, err := vt.createOwningConfigMap(ctx)
if err != nil {
testResults.addSuggestions(previousTestSuggestion)
return testResults, fmt.Errorf("failed to create validation test config object: %w", err)
}
err = vt.startWebServer(ctx, owningConfigMap)
if err != nil {
testResults.addSuggestions(previousTestSuggestion)
return testResults, fmt.Errorf("failed to start web server: %w", err)
}
// start the state machine
vsm := &validationStateMachine{
vt: vt,
resourceOwnerRefs: owningConfigMap,
testResults: testResults,
lastSuggestions: []string{},
}
startingState := &getWebServerInfoState{
desiredPublicNet: desiredPublicNet,
desiredClusterNet: desiredClusterNet,
}
vsm.SetNextState(startingState)
return vsm.Run(ctx)
}
// CleanUp cleans up Multus validation test resources. It returns a suggestion for manual action if
// clean up was unsuccessful.
func (vt *ValidationTest) CleanUp(ctx context.Context) (*ValidationTestResults, error) {
var err error
res := ValidationTestResults{
suggestedDebugging: []string{},
}
suggestions, err := vt.cleanUpTestResources()
res.addSuggestions(suggestions)
return &res, err
}