diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index a078d3c36..d910acf0b 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -221,7 +221,7 @@ Based on the configuration, the operator will do the following: public: rook-ceph/rook-public-nw ``` -2. If only the `cluster` selector is specified, the internal cluster traffic* will happen on that network. All other traffic to mons, OSDs, and other daemons will be on the default network. +2. If only the `cluster` selector is specified, the internal cluster traffic\* will happen on that network. All other traffic to mons, OSDs, and other daemons will be on the default network. ```yaml network: @@ -230,7 +230,7 @@ Based on the configuration, the operator will do the following: cluster: rook-ceph/rook-cluster-nw ``` -3. If both `public` and `cluster` selectors are specified the first one will run all the communication network and the second the internal cluster network* +3. If both `public` and `cluster` selectors are specified the first one will run all the communication network and the second the internal cluster network\* ```yaml network: @@ -274,7 +274,7 @@ spec: ``` * Ensure that `master` matches the network interface of the host that you want to use. -* Ipam type `whereabouts` is required because it makes sure that all the pods get a unique IP address from the multus network. +* IPAM type `whereabouts` is required because it makes sure that all the pods get a unique IP address from the multus network. * The NetworkAttachmentDefinition should be referenced along with the namespace in which it is present like `public: /`. e.g., the network attachment definition are in `default` namespace: @@ -286,7 +286,22 @@ spec: * This format is required in order to use the NetworkAttachmentDefinition across namespaces. * In Openshift, to use a NetworkAttachmentDefinition (NAD) across namespaces, the NAD must be deployed in the `default` namespace. The NAD is then referenced with the namespace: `default/rook-public-nw` -#### Known limitations with Multus +##### Validating Multus configuration + +We **highly** recommend validating your Multus configuration before you install Rook. A tool exists +to facilitate validating the Multus configuration. After installing the Rook operator and before +installing any Custom Resources, run the tool from the operator pod. + +The tool's CLI is designed to be as helpful as possible. Get help text for the multus validation +tool like so: +```console +kubectl --namespace rook-ceph exec -it deploy/rook-ceph-operator -- rook ctl multus validation run --help +``` + +If the tool fails, it will suggest what things may be preventing Multus networks from working +properly, and it will request the logs and outputs that will help debug issues. + +##### Known limitations with Multus Daemons leveraging Kubernetes service IPs (Monitors, Managers, Rados Gateways) are not listening on the NAD specified in the `selectors`. Instead the daemon listens on the default network, however the NAD is attached to the container, diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index 31889758a..756fd9f50 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -31,8 +31,9 @@ import ( // Cmd is the main command for operator and daemons. var Cmd = &cobra.Command{ - Use: "ceph", - Short: "Main command for Ceph operator and daemons.", + Use: "ceph", + Short: "Main command for Ceph operator and daemons.", + Hidden: true, // do not advertise to end users } var ( diff --git a/cmd/rook/discover.go b/cmd/rook/discover.go index d33eeb658..712b07dc7 100644 --- a/cmd/rook/discover.go +++ b/cmd/rook/discover.go @@ -26,8 +26,9 @@ import ( var ( discoverCmd = &cobra.Command{ - Use: "discover", - Short: "Discover devices", + Use: "discover", + Short: "Discover devices", + Hidden: true, // do not advertise to end users } // interval between discovering devices diff --git a/cmd/rook/main.go b/cmd/rook/main.go index 73872729f..ba6ef8cfd 100644 --- a/cmd/rook/main.go +++ b/cmd/rook/main.go @@ -20,6 +20,7 @@ import ( "github.com/rook/rook/cmd/rook/ceph" rook "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/cmd/rook/userfacing" "github.com/rook/rook/cmd/rook/util" "github.com/rook/rook/cmd/rook/version" ) @@ -43,4 +44,11 @@ func addCommands() { util.CmdReporterCmd, util.CopyBinsCmd, ) + // do double diligence to ensure non-user-facing commands are hidden from end users + for _, cmd := range rook.RootCmd.Commands() { + cmd.Hidden = true + } + + // all user-facing commands + rook.RootCmd.AddCommand(userfacing.Commands...) } diff --git a/cmd/rook/rook/rook.go b/cmd/rook/rook/rook.go index 723a52dcf..7480e4429 100644 --- a/cmd/rook/rook/rook.go +++ b/cmd/rook/rook/rook.go @@ -36,10 +36,8 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" ) const ( @@ -48,14 +46,14 @@ const ( ) var RootCmd = &cobra.Command{ - Use: "rook", + Use: "rook", + Short: "Rook (rook.io) Kubernetes operator and user tools", + Hidden: false, } var ( - logLevelRaw string - operatorImage string - serviceAccountName string - logger = capnslog.NewPackageLogger("github.com/rook/rook", "rookcmd") + logLevelRaw string + logger = capnslog.NewPackageLogger("github.com/rook/rook", "multus-validation") ) // Initialize the configuration parameters. The precedence from lowest to highest is: @@ -64,8 +62,9 @@ var ( // 3. command line parameter func init() { RootCmd.PersistentFlags().StringVar(&logLevelRaw, "log-level", "INFO", "logging level for logging/tracing output (valid values: ERROR,WARNING,INFO,DEBUG)") - RootCmd.PersistentFlags().StringVar(&operatorImage, "operator-image", "", "Override the image url that the operator uses. The default is read from the operator pod.") - RootCmd.PersistentFlags().StringVar(&serviceAccountName, "service-account", "", "Override the service account that the operator uses. The default is read from the operator pod.") + RootCmd.InitDefaultHelpCmd() + RootCmd.InitDefaultHelpFlag() + RootCmd.InitDefaultCompletionCmd() // load the environment variables flags.SetFlagsFromEnv(RootCmd.Flags(), RookEnvVarPrefix) @@ -97,44 +96,7 @@ func NewContext() *clusterd.Context { // Try to read config from in-cluster env context.KubeConfig, err = rest.InClusterConfig() if err != nil { - - // **Not** running inside a cluster - running the operator outside of the cluster. - // This mode is for developers running the operator on their dev machines - // for faster development, or to run operator cli tools manually to a remote cluster. - // We setup the API server config from default user file locations (most notably ~/.kube/config), - // and also change the executor to work remotely and run kubernetes jobs. - logger.Info("setting up the context to outside of the cluster") - - // Try to read config from user config files - context.KubeConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - &clientcmd.ConfigOverrides{}).ClientConfig() - TerminateOnError(err, "failed to get k8s config") - - // When running outside, we need to setup an executor that runs the commands as kubernetes jobs. - // This allows the operator code to execute tools that are available in the operator image - // or just have to be run inside the cluster in order to talk to the pods and services directly. - context.Executor = &exec.TranslateCommandExecutor{ - Executor: context.Executor, - Translator: func( - command string, - arg ...string, - ) (string, []string) { - jobName := "rook-exec-job-" + string(uuid.NewUUID()) - transCommand := "kubectl" - transArgs := append([]string{ - "run", jobName, - "--image=" + operatorImage, - "--serviceaccount=" + serviceAccountName, - "--restart=Never", - "--attach", - "--rm", - "--quiet", - "--command", "--", - command}, arg...) - return transCommand, transArgs - }, - } + TerminateOnError(err, "failed to get k8s cluster config") } context.Clientset, err = kubernetes.NewForConfig(context.KubeConfig) @@ -153,12 +115,6 @@ func NewContext() *clusterd.Context { } func GetOperatorImage(ctx context.Context, clientset kubernetes.Interface, containerName string) string { - - // If provided as a flag then use that value - if operatorImage != "" { - return operatorImage - } - // Getting the info of the operator pod pod, err := k8sutil.GetRunningPod(ctx, clientset) TerminateOnError(err, "failed to get pod") @@ -171,12 +127,6 @@ func GetOperatorImage(ctx context.Context, clientset kubernetes.Interface, conta } func GetOperatorServiceAccount(ctx context.Context, clientset kubernetes.Interface) string { - - // If provided as a flag then use that value - if serviceAccountName != "" { - return serviceAccountName - } - // Getting the info of the operator pod pod, err := k8sutil.GetRunningPod(ctx, clientset) TerminateOnError(err, "failed to get pod") diff --git a/cmd/rook/secret.go b/cmd/rook/secret.go index e3eaf6b46..9291a96c3 100644 --- a/cmd/rook/secret.go +++ b/cmd/rook/secret.go @@ -41,6 +41,7 @@ var KeyManagementCmd = &cobra.Command{ Long: `The secret sub-command helps interacting with Key Management System. It can perform various actions such as retrieving the content of a Key Encryption Key and rotating Key Encryption Key.`, + Hidden: true, // do not advertise to end users } func init() { diff --git a/cmd/rook/userfacing/multus/multus.go b/cmd/rook/userfacing/multus/multus.go new file mode 100644 index 000000000..ab654b5ff --- /dev/null +++ b/cmd/rook/userfacing/multus/multus.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multus + +import ( + "github.com/rook/rook/cmd/rook/userfacing/multus/validation" + "github.com/spf13/cobra" +) + +func init() { + Cmd.AddCommand( + validation.Cmd, + ) +} + +// Cmd is the 'multus' CLI command +var Cmd = &cobra.Command{ + Use: "multus", + Short: "Get help configuring Multus for compatibility with Rook", + Long: ` +Get help configuring Multus for compatibility with Rook. + +Though intended for Multus, these utilities should support any multi-net +provider that implements the Kubernetes Network Plumbing Working Group's +Multi-Network Spec: https://github.com/k8snetworkplumbingwg/multi-net-spec +`, +} diff --git a/cmd/rook/userfacing/multus/validation/validation.go b/cmd/rook/userfacing/multus/validation/validation.go new file mode 100644 index 000000000..8a6c66faf --- /dev/null +++ b/cmd/rook/userfacing/multus/validation/validation.go @@ -0,0 +1,216 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validation + +import ( + "context" + "fmt" + "os" + "strconv" + "time" + + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/daemon/multus" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/spf13/cobra" +) + +// defaults +var ( + DefaultValidationNamespace = "rook-ceph" + + // 1 mon, 3 osds, 1 mgr, 1 mds, 1 nfs, 1 rgw, 1 rbdmirror, 1 cephfsmirror, + // (1 csi provisioner, 1 csi plugin) x3 for rbd, cephfs, and nfs CSI drivers + DefaultDaemonsPerNode = 16 + + DefaultTimeoutMinutes = 3 +) + +// config +var ( + validationConfig = multus.ValidationTest{ + Logger: capnslog.NewPackageLogger("github.com/rook/rook", "multus-validation"), + } +) + +// commands +var ( + // parent 'validation' command + Cmd = &cobra.Command{ + Use: "validation", + Short: "Run and manage Multus validation tests for Rook", + } + + // 'validation run' command + runCmd = &cobra.Command{ + Use: "run [--public-network=] [--cluster-network=]", + Short: "Run a Multus validation test for Rook", + Long: ` +Run a validation test that determines whether the current Multus and system +configurations will support Rook with Multus. + +This is a fairly long-running test. It starts up a web server and many +clients to verify that Multus network communication works properly. + +It does *not* perform any load testing. Networks that cannot support high +volumes of Ceph traffic may still encounter runtime issues. This may be +particularly noticeable with high I/O load or during OSD rebalancing +(see: https://docs.ceph.com/en/latest/architecture/#rebalancing). +`, + Run: func(cmd *cobra.Command, args []string) { + runValidation(cmd.Context()) + }, + Args: cobra.NoArgs, + } + + // 'validation cleanup' command + cleanupCmd = &cobra.Command{ + Use: "cleanup", + Short: "Clean up Multus validation test resources", + Run: func(cmd *cobra.Command, args []string) { + runCleanup(cmd.Context()) + }, + Args: cobra.NoArgs, + } +) + +func init() { + validationConfig.Clientset = rook.NewContext().Clientset + + Cmd.AddCommand(runCmd) + Cmd.AddCommand(cleanupCmd) + + // flags on ALL subcommands - makes output more straightforward than using PersistentFlags() global flags on parent + for _, subCommand := range Cmd.Commands() { + // the default namespace is the current namespace the operator pod is running in if possible + defaultNamespace := os.Getenv(k8sutil.PodNamespaceEnvVar) + if defaultNamespace == "" { + defaultNamespace = DefaultValidationNamespace + } + subCommand.Flags().StringVarP(&validationConfig.Namespace, "namespace", "n", defaultNamespace, + "The namespace for validation test resources. "+ + "It is recommended to set this to the namespace in which Rook's Ceph cluster will be installed.") + + // VarPF() keeps the the specific var passed to it for setting at runtime, and the current + // val of that var when VarPF() is called is used as the default + validationConfig.ResourceTimeout = time.Duration(DefaultTimeoutMinutes) * time.Minute + t := (*timeoutMinutes)(&validationConfig.ResourceTimeout) + subCommand.Flags().VarPF(t, "timeout-minutes", "", /* no shorthand */ + "The time to wait for resources to change to the expected state. For example, for the "+ + "test web server to start, for test clients to become ready, or for test resources to be deleted. "+ + "At longest, this may need to reflect the time it takes for client pods to to pull images, "+ + "get address assignments, and then for each client to determine that its network connection is stable. "+ + "Minimum: 1 minute. Recommended: 2 minutes or more.") + } + + // flags for 'validation run' + runCmd.Flags().StringVar(&validationConfig.PublicNetwork, "public-network", "", + "The name of the Network Attachment Definition (NAD) that will be used for Ceph's public network. "+ + "This should be a namespaced name in the form / if the NAD is defined in a different namespace from the cluster namespace.") + runCmd.Flags().StringVar(&validationConfig.ClusterNetwork, "cluster-network", "", + "The name of the Network Attachment Definition (NAD) that will be used for Ceph's cluster network. "+ + "This should be a namespaced name in the form / if the NAD is defined in a different namespace from the cluster namespace.") + runCmd.Flags().IntVar(&validationConfig.DaemonsPerNode, "daemons-per-node", DefaultDaemonsPerNode, + "The number of validation test daemons to run per node. "+ + "It is recommended to set this to the maximum number of Ceph daemons that can run on any node in the worst case of node failure(s). "+ + "The default value is set to the worst-case value for a Rook Ceph cluster with 3 portable OSDs, 3 portable monitors, "+ + "and where all optional child resources have been created with 1 daemon such that they all might run on a single node in a failure scenario. "+ + "If you aren't sure what to choose for this value, add 1 for each additional OSD beyond 3.") + + // flags for 'validation cleanup' + // none +} + +func runValidation(ctx context.Context) { + results, err := validationConfig.Run(ctx) + report := results.SuggestedDebuggingReport() + + // success/failure message + fmt.Print("\n") + switch { + case err != nil: + fmt.Printf("RESULT: multus validation test failed: %v\n\n", err) + case report == "": + fmt.Print("RESULT: multus validation test succeeded!\n\n") + runCleanup(ctx) + os.Exit(0) // success! + case report != "": + // suggestions are bad + fmt.Print("RESULT: multus validation test succeeded, but there are suggestions\n\n") + } + + // output report suggestions + fmt.Print(report + "\n") + + // help users help us help them + fmt.Println("leaving multus validation test resources running for manual debugging") + fmt.Print(` +For assistance debugging, collect the following into an archive file: + - Output of this utility + - Network Attachment Definitions (NADs) used by this test + - A write-up describing the network configuration you are trying to achieve including the + intended network for Ceph public/client traffic, intended network for Ceph cluster traffic, + interface names and CIDRs for both networks, and any other details that are relevant. + - 'ifconfig' output from at least one Kubernetes worker node + - 'kubectl get pods -o wide' output from the test namespace + - 'kubectl describe pods' output from the test namespace + - 'kubectl get pods -o yaml' output from the test namespace + - 'kubectl get daemonsets' output from the test namespace + - 'kubectl describe daemonsets' output from the test namespace + - 'kubectl get daemonsets -o yaml' output from the test namespace + - 'kubectl logs multus-validation-test-web-server' output from the test namespace + - 'kubectl get nodes -o wide' output +`) + + // tell them how to cleanup + fmt.Printf("\nTo clean up resources when you are done debugging: %s", cleanupCmd.CommandPath()) + + os.Exit(1) +} + +func runCleanup(ctx context.Context) { + fmt.Printf("cleaning up multus validation test resources in namespace %q\n", validationConfig.Namespace) + results, err := validationConfig.CleanUp(ctx) + if err != nil { + fmt.Printf("multus validation test cleanup failed: %v\n\n", err) + fmt.Println(results.SuggestedDebuggingReport()) + return + } + fmt.Print("multus validation test resources were successfully cleaned up\n") +} + +// custom flag types + +// implements pflag.Value interface to validate and set resource timeout and enforce nonzero +type timeoutMinutes time.Duration + +func (t *timeoutMinutes) String() string { return time.Duration(*t).String() } +func (t *timeoutMinutes) Set(v string) error { + i, err := strconv.Atoi(v) + if err != nil { + return err + } + if i < 1 { + return fmt.Errorf("timeout must be greater than 0") + } + *t = timeoutMinutes(time.Duration(i) * time.Minute) + return nil +} +func (t timeoutMinutes) Type() string { + return "timeoutMinutes" +} diff --git a/cmd/rook/userfacing/userfacing.go b/cmd/rook/userfacing/userfacing.go new file mode 100644 index 000000000..4d2348f8b --- /dev/null +++ b/cmd/rook/userfacing/userfacing.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// contains user facing commands, separated to a subdir for convenience +package userfacing + +import ( + "context" + "os" + "os/signal" + "syscall" + + "github.com/rook/rook/cmd/rook/userfacing/multus" + "github.com/spf13/cobra" +) + +var Commands = []*cobra.Command{ + multus.Cmd, +} + +var stopSignalCapture context.CancelFunc + +func init() { + // ensure all user-facing commands have some basic assumptions covered + for _, cmd := range Commands { + // they should not be hidden + cmd.Hidden = false + + // they should have a context that captures ctrl-c interrupts + cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + ctx, stopSignalCapture = signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + cmd.SetContext(ctx) + + return cmd.ValidateArgs(args) + } + // and should stop capturing when exiting + cmd.PersistentPostRun = func(cmd *cobra.Command, args []string) { + stopSignalCapture() + } + + // they should always provide help + cmd.InitDefaultHelpCmd() + cmd.InitDefaultHelpFlag() + } +} diff --git a/cmd/rook/util/cmdreporter.go b/cmd/rook/util/cmdreporter.go index 1ae7af3fd..0d9aac130 100644 --- a/cmd/rook/util/cmdreporter.go +++ b/cmd/rook/util/cmdreporter.go @@ -46,8 +46,9 @@ error storing the command results into the ConfigMap. An application label is applied to the ConfigMap. Run will also terminate if the label already exists and has a different application's name name; this may indicate that it is not safe for cmd-reporter to edit the ConfigMap.`, - Args: cobra.NoArgs, - Run: runCmdReporter, + Args: cobra.NoArgs, + Run: runCmdReporter, + Hidden: true, // do not advertise to end users } var ( diff --git a/cmd/rook/util/copybins.go b/cmd/rook/util/copybins.go index 19707bf52..cf3cb7f5b 100644 --- a/cmd/rook/util/copybins.go +++ b/cmd/rook/util/copybins.go @@ -37,8 +37,9 @@ binary to a Kubernetes EmptyDir volume mounted at the given directory in a pod's init container. From the pod's main container, mount the volume which now contains the 'rook' binary, and call 'rook cmd-reporter run' from 'rook' in order to run the desired command from a non-Rook container.`, - Args: cobra.NoArgs, - RunE: runCopyBins, + Args: cobra.NoArgs, + RunE: runCopyBins, + Hidden: true, // do not advertise to end users } func init() { diff --git a/design/common/multi-homed-cluster.md b/design/common/multi-homed-cluster.md deleted file mode 100644 index 54ded7107..000000000 --- a/design/common/multi-homed-cluster.md +++ /dev/null @@ -1,87 +0,0 @@ -# Rook Operators Multi-homed Cluster Network Spec - -## Overview - -This project aims to create new API to enable multiple network interface for -Rook storage providers. Currently, Rook providers only choice is to use -`hostNetwork` or not. The new API will be used to define networks resource for -Rook clusters. Rook operators will be able to consume those definitions and -manage them. Therefore, it enables more fine-grained control over storage -providers network access. - -## Current Implementation - -To achieve non-flat networking model, Rook can choose to enable `hostNetwork` -and expose host network interfaces to Storage Provider pods. - -Ceph Rook cluster network definition example: -```yaml -network: - hostNetwork: true -``` - -The Ceph operator without specifying this configuration will always -default to pod networking. - -## Proposed Design - -Rook operators can define storage cluster's network using network provider. -Network provider example includes host, and multus. To configure the cluster -network, cluster CRD needs to tell the network provider the appropriate -`NetworkInterfaceSelector`. `NetworkInterfaceSelector` will be provided as list -of `interfaces` key-value. - -```yaml -network: - provider: - interfaces: - : - : -``` - -### Network Provider - -Network provider determines multi-homing method and the network interface -selector. Using host network provider, pod can use host's network namespace -and its interfaces to use as cluster network. Using popular multi-plugin CNI -plugin such as [multus][multus-cni], Rook operators can attach multiple network -interfaces to pods. - -In the future, works can be added to support other multi network CNI plugins -such as [knitter][knitter-cni] or [genie][genie-cni]. - -One thing to remember, leaving the network configuration empty will default to -kubernetes cluster default networking. - -### Keys - -The key for each interface key-value pairs are left to each storage providers to -decide. The network interface selectors with key values `public` and -`cluster` are specified for Ceph, while other storage providers can define their -own key names if desired. - -### Network Interface Selector - -Network interface selector is fed to network provider to connect pod to cluster -network. This selector may vary from one network provider to another. For -example, host network provider only needs to know the interface name. - -On the other hand, multi-plugin CNI plugin needs to know the network attachment -definition's name and vice versa. Multi-plugin such as multus may seem to follow -_Network Attachment Selection Annotation_ documented at [Kubernetes Network -Custom Resource Definition De-facto Standard][network-crd]. However, their -implementation has extra features not covered by the standard such as -`@` notation or `interfaceRequest` property documented -[here][multus-annotation]. - -## Example Configurations - -### Ceph Rook Cluster Network definition example - - - -[multus-cni]: https://github.com/intel/multus-cni -[knitter-cni]: https://github.com/ZTE/Knitter -[genie-cni]: https://github.com/cni-genie/CNI-Genie/ -[network-crd]: https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ/edit -[multus-annotation]: https://github.com/intel/multus-cni/blob/master/doc/how-to-use.md#run-pod-with-network-annotation diff --git a/design/common/multi-net-multus.md b/design/common/multi-net-multus.md new file mode 100644 index 000000000..3bb05cb68 --- /dev/null +++ b/design/common/multi-net-multus.md @@ -0,0 +1,109 @@ +# Multi-homed Network (Multus) + +## Overview + +This project aims to create an API to enable multiple network interface for +Rook storage providers. Currently, Rook providers only choice is to use +`hostNetwork` or not. The API will be used to define networks resource for +Rook clusters. It enables more fine-grained control over network access. + +## Legacy Implementation + +To achieve non-flat networking model, Rook can choose to enable `hostNetwork` +and expose host network interfaces to Storage Provider pods. + +Ceph Rook cluster network definition example: +```yaml +network: + hostNetwork: true +``` + +The Ceph operator without specifying this configuration will always +default to pod networking. + +## Multi-homed Network Design + +Rook operators can define storage cluster's network using network provider. +Network provider example includes host, and multus. To configure the cluster +network, cluster CRD needs to tell the network provider the appropriate +`NetworkInterfaceSelector`. `NetworkInterfaceSelector` will be provided as list +of `interfaces` key-value. + +```yaml +network: + provider: + interfaces: + : + : +``` + +### Network Provider + +Network provider determines multi-homing method and the network interface +selector. Using host network provider, pod can use host's network namespace +and its interfaces to use as cluster network. Using popular multi-plugin CNI +plugin such as [multus][multus-cni], Rook operators can attach multiple network +interfaces to pods. + +One thing to remember, leaving the network configuration empty will default to +kubernetes cluster default networking. + +### Keys + +The network interface selectors with key values `public` and +`cluster` are specified for Ceph. + +### Network Interface Selector + +Network interface selector is fed to network provider to connect pod to cluster +network. This selector may vary from one network provider to another. For +example, host network provider only needs to know the interface name. + +On the other hand, multi-plugin CNI plugin needs to know the network attachment +definition's name and vice versa. Multi-plugin such as multus may seem to follow +_Network Attachment Selection Annotation_ documented by Kubernetes Network Plumbing Working Group's +[Multi-Net Spec]. + +Any future multi-homed network provider that implements the Network Plumbing WG's Multi-net spec may +use the `multus` network. Previously-identified providers [CNI-Genie] and [Knitter] have since gone +dormant (upd. 2023), and Multus continues to be the most prominent known provider. + +## Multi-Net validation tester + +Rook will implement a user-runnable routine that validates a multi-net configuration based on the +[Multi-Net Spec]. Users can input one or both of the Network Attachment Definitions (NADs) for +Ceph's `multus` network, and the routine will perform cursory validation of the ability of the +specified networks to support Ceph. + +The routine will start up **a single** web server with the specified networks attached (via NADs). +The routine will also start up a number of clients on each node that will test the network(s)'s +connections by HTTP(S) requests to the web server. + +A client represents a simplified view of a Ceph daemon from the networking perspective. The number +of clients is intended to represent the number of Ceph daemons that could run on a node in the worst +possible failure case where all daemons are rescheduled to a single node. This helps verify that the +network provider is able to successfully assign IP addresses to Ceph daemon pods in the worst case. +Some network interface software/hardware may limit the number of addresses that can be assigned to +an interface on a given node, and this helps verify that issue is not present. + +It is important that the web server be only **a single** instance. If clients collocated on the same +node with the web server can make successful HTTP(S) requests but clients on other nodes are not, +that is an important indicator of inter-node traffic being blocked. + +The validation test routine will run in the Rook operator container. Rook's kubectl [Krew plugin] +may facilitate running the test, but it is useful to have the option of executing a long-running +validation routine running in the Kubernetes cluster instead of on an administrator's log-in +session. Additionally, the results of the validation tester may become important for users wanting +to get help with Multus configuration, and having a tool that is present in the Rook container image +will allow users to run the routine for bug reports more readily than if they needed to install the +Krew plugin. + + + +[multus-cni]: https://github.com/intel/multus-cni +[multi-net spec]: https://github.com/k8snetworkplumbingwg/multi-net-spec +[cni-genie]: https://github.com/cni-genie/CNI-Genie/ +[knitter]: https://github.com/ZTE/Knitter +[krew plugin]: https://github.com/rook/kubectl-rook-ceph diff --git a/go.mod b/go.mod index 64338e7db..412ea1969 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible // indirect github.com/circonus-labs/circonusllhist v0.1.3 // indirect github.com/cloudflare/circl v1.1.0 // indirect + github.com/containernetworking/cni v1.0.1 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba // indirect diff --git a/go.sum b/go.sum index a8eb0c335..4fcdced0b 100644 --- a/go.sum +++ b/go.sum @@ -304,6 +304,8 @@ github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBd github.com/containerd/containerd v1.5.17 h1:NLDEI//zhMZpR3DS/AP0qiN+dzYKNAwJaNXCnCmYcgY= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containernetworking/cni v1.0.1 h1:9OIL/sZmMYDBe+G8svzILAlulUpaDTUjeAbtH/JNLBo= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -1295,6 +1297,7 @@ github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= @@ -1310,6 +1313,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= @@ -1477,6 +1481,7 @@ github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0 github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/conswriter v0.0.0-20180208195008-f5ae3917a627/go.mod h1:7zjs06qF79/FKAJpBvFx3P8Ww4UTIMAe+lpNXDHziac= github.com/sean-/pager v0.0.0-20180208200047-666be9bf53b5/go.mod h1:BeybITEsBEg6qbIiqJ6/Bqeq25bCLbL7YFmpaFfJDuM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= @@ -2226,6 +2231,7 @@ gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/daemon/multus/client-daemonset.yaml b/pkg/daemon/multus/client-daemonset.yaml new file mode 100644 index 000000000..18fabc2f0 --- /dev/null +++ b/pkg/daemon/multus/client-daemonset.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: multus-validation-test-client-{{ .ClientID }} + labels: + app: multus-validation-test-client + app.kubernetes.io/name: "client" + app.kubernetes.io/instance: "client-{{ .ClientID }}" + app.kubernetes.io/component: "client" + app.kubernetes.io/part-of: "multus-validation-test" + app.kubernetes.io/managed-by: "rook-cli" +spec: + selector: + matchLabels: + app: multus-validation-test-client + clientID: "{{ .ClientID }}" + template: + metadata: + labels: + app: multus-validation-test-client + clientID: "{{ .ClientID }}" + annotations: + k8s.v1.cni.cncf.io/networks: "{{ .NetworksAnnotationValue }}" + spec: + # TODO: selectors, affinities, tolerations + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + {{ range $name, $address := .NetworkNamesAndAddresses }} + - name: readiness-check-web-server-{{ $name }}-addr + # use nginx image because it's already pulled for the nginx pod and has a non-root user + image: nginx:stable-alpine + command: + - sleep + - infinity + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + # A readiness probe makes validation testing easier than investigate container logs. + # Additionally, readiness probe failures don't result in CrashLoopBackoff -- ideal here, + # where ever-longer back-offs would cause tests to run for much longer than necessary. + readinessProbe: + # Low failure threshold and high success threshold. Intended to be very sensitive to + # failures. If probe fails with any regularity, Ceph OSDs likely won't be stable. + failureThreshold: 1 + successThreshold: 12 + periodSeconds: 5 + # Assumption: a network with a latency more than 4 seconds for this validation test's + # simple client-server response likely won't support acceptable performance for any + # production Ceph cluster. + timeoutSeconds: 4 + # TODO: exec:curl works but httpGet fails. Why? need custom header? + exec: + command: + - "curl" + - "--insecure" + - "{{ $address }}:8080" + {{ end }} diff --git a/pkg/daemon/multus/nginx-config.yaml b/pkg/daemon/multus/nginx-config.yaml new file mode 100644 index 000000000..57793c1a7 --- /dev/null +++ b/pkg/daemon/multus/nginx-config.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: multus-validation-test-web-server-conf + labels: + app: multus-validation-test-web-server +data: + server.conf: | + server { + listen 8080; + server_name localhost; + + # return the client ip upon connect + location / { + default_type text/plain; + return 200 "$remote_addr\n"; + } + } diff --git a/pkg/daemon/multus/nginx-pod.yaml b/pkg/daemon/multus/nginx-pod.yaml new file mode 100644 index 000000000..40134c233 --- /dev/null +++ b/pkg/daemon/multus/nginx-pod.yaml @@ -0,0 +1,52 @@ +# This is a Pod and not a Deployment because there is intended to only be one running +apiVersion: apps/v1 +kind: Pod +metadata: + name: multus-validation-test-web-server + labels: + app: multus-validation-test-web-server + app.kubernetes.io/name: "nginx" + app.kubernetes.io/instance: "nginx" # there is only one instance + app.kubernetes.io/component: "server" + app.kubernetes.io/part-of: "multus-validation-test" + app.kubernetes.io/managed-by: "rook-cli" + annotations: + k8s.v1.cni.cncf.io/networks: "{{ .NetworksAnnotationValue }}" +spec: + # TODO: selectors, affinities, tolerations + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: multus-validation-test-web-server + image: nginx:stable-alpine + resources: {} + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + port: 8080 + scheme: HTTP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + volumeMounts: + - name: var-cache-nginx + mountPath: /var/cache/nginx + - name: server-conf + mountPath: /etc/nginx/conf.d + - name: var-run + mountPath: /var/run + volumes: + - name: var-cache-nginx + emptyDir: {} + - name: server-conf + configMap: + name: multus-validation-test-web-server-conf + - name: var-run + emptyDir: {} diff --git a/pkg/daemon/multus/resources.go b/pkg/daemon/multus/resources.go new file mode 100644 index 000000000..f9ae7de86 --- /dev/null +++ b/pkg/daemon/multus/resources.go @@ -0,0 +1,280 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multus + +import ( + "context" + "fmt" + "time" + + core "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" +) + +type podNetworkInfo struct { + // node the pod is running on + nodeName string + + // multus public addr attached (if any) + publicAddr string + + // multus cluster addr attached (if any) + clusterAddr string +} + +var manualCleanupInstructions = fmt.Sprintf( + "manually delete owner configmap %q, and wait for all multus-validation-test resources to be deleted", ownerConfigMapName) + +var previousTestSuggestion = "there could be a past test preventing this one from proceeding; " + manualCleanupInstructions + +var unableToProvideAddressSuggestions = []string{ + "multus may be unable to provide addresses for pods", + "check networking events on the pod and multus logs", + "macvlan: NIC or switch hardware/software may block the association of some number of additional MAC addresses on an interface", + "macvlan: interfaces and network switching must enable promiscuous mode to allow receiving packets for unknown (Multus) MACs", + "macvlan/ipvlan: switch hardware/software may block an interface from receiving packets to an unknown (Multus) IP", +} + +// create a validation test config object that stores the configuration of the running validation +// test. this object serves as the owner of all associated test objects. when this object is +// deleted, all validation test objects should also be deleted, effectively cleaning up all +// components of this test. +func (vt *ValidationTest) createOwningConfigMap(ctx context.Context) ([]meta.OwnerReference, error) { + c := core.ConfigMap{ + ObjectMeta: meta.ObjectMeta{ + Name: ownerConfigMapName, + }, + } + + configObject, err := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Create(ctx, &c, meta.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create validation test config object [%+v]: %w", c, err) + } + + // for cleanup, we want to make sure all children are deleted + BlockOwnerDeletion := true + refToConfigObject := meta.OwnerReference{ + APIVersion: "v1", + Kind: "ConfigMap", + Name: configObject.GetName(), + UID: configObject.GetUID(), + BlockOwnerDeletion: &BlockOwnerDeletion, + } + return []meta.OwnerReference{refToConfigObject}, nil +} + +func (vt *ValidationTest) startWebServer(ctx context.Context, owners []meta.OwnerReference) error { + pod, err := vt.generateWebServerPod() + if err != nil { + return fmt.Errorf("failed to generate web server pod: %w", err) + } + pod.SetOwnerReferences(owners) // set owner refs so cleanup is easier + + configMap, err := vt.generateWebServerConfigMap() + if err != nil { + return fmt.Errorf("failed to generate web server config: %w", err) + } + configMap.SetOwnerReferences(owners) // set owner refs so cleanup is easier + + // create configmap before pod so pod doesn't crashloopbackoff on first creation + _, err = vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Create(ctx, configMap, meta.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create web server config: %w", err) + } + + _, err = vt.Clientset.CoreV1().Pods(vt.Namespace).Create(ctx, pod, meta.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create web server pod: %w", err) + } + + return nil +} + +func (vt *ValidationTest) getWebServerInfo( + ctx context.Context, + desiredPublicNet, desiredClusterNet *types.NamespacedName, +) (podNetworkInfo, []string, error) { + podInfo := podNetworkInfo{} + + pod, err := vt.Clientset.CoreV1().Pods(vt.Namespace).Get(ctx, webServerPodName(), meta.GetOptions{}) + if err != nil { + return podInfo, []string{}, fmt.Errorf("unexpected error when getting web server pod: %w", err) + } + + var publicAddr, clusterAddr string + publicAddr, clusterAddr, networkSuggestions, err := getNetworksFromPod(pod, desiredPublicNet, desiredClusterNet) + if err != nil { + return podInfo, networkSuggestions, fmt.Errorf("no web server network info: %w", err) + } + + if !podIsReady(*pod) { + return podInfo, []string{}, fmt.Errorf("web server pod is not ready yet") + } + + podInfo.nodeName = pod.Spec.NodeName + podInfo.publicAddr = publicAddr + podInfo.clusterAddr = clusterAddr + return podInfo, []string{}, nil // no suggestions if successful +} + +func (vt *ValidationTest) startClients( + ctx context.Context, + owners []meta.OwnerReference, + serverPublicAddr, serverClusterAddr string, +) error { + for i := 0; i < vt.DaemonsPerNode; i++ { + ds, err := vt.generateClientDaemonSet(i, serverPublicAddr, serverClusterAddr) + if err != nil { + return fmt.Errorf("failed to generate client daemonset for client #%d: %w", i, err) + } + ds.SetOwnerReferences(owners) // set owner refs so cleanup is easier + + _, err = vt.Clientset.AppsV1().DaemonSets(vt.Namespace).Create(ctx, ds, meta.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create client daemonset for client #%d: %w", i, err) + } + } + + return nil +} + +// if this returns success, the return value is guaranteed to be greater than 0 +func (vt *ValidationTest) getExpectedNumberOfClientPods(ctx context.Context) (int, error) { + var agreedNumberScheduled int32 + listOpts := meta.ListOptions{ + LabelSelector: clientAppLabel(), + } + dsets, err := vt.Clientset.AppsV1().DaemonSets(vt.Namespace).List(ctx, listOpts) + if err != nil { + return 0, fmt.Errorf("unexpected error listing client daemonsets: %w", err) + } + if len(dsets.Items) < vt.DaemonsPerNode { + return 0, fmt.Errorf("got fewer client daemonsets [%d] than should exist [%d]", len(dsets.Items), vt.DaemonsPerNode) + } + + for _, d := range dsets.Items { + numScheduled := d.Status.CurrentNumberScheduled + if numScheduled == 0 { + return 0, fmt.Errorf("a daemonset expects zero scheduled client pods; cannot validate with no running clients") + } + if agreedNumberScheduled == 0 { + agreedNumberScheduled = numScheduled + } else if numScheduled != agreedNumberScheduled { + return 0, fmt.Errorf("daemonsets do not all agree on the number of expected client pods") + } + } + + return int(agreedNumberScheduled) * vt.DaemonsPerNode, nil +} + +func (vt *ValidationTest) allClientsAreRunning(ctx context.Context, expectedNumPods int) (bool, error) { + pods, err := vt.getClientPods(ctx, expectedNumPods) + if err != nil { + return false, fmt.Errorf("unexpected error getting client pods: %w", err) + } + numRunning := 0 + for _, p := range pods.Items { + if podIsRunning(p) { + numRunning++ + } + } + if numRunning != expectedNumPods { + return false, fmt.Errorf("number of running clients [%d] is not the number expected [%d]", numRunning, expectedNumPods) + } + return true, nil +} + +func (vt *ValidationTest) numClientsReady(ctx context.Context, expectedNumPods int) (int, error) { + pods, err := vt.getClientPods(ctx, expectedNumPods) + if err != nil { + return 0, fmt.Errorf("unexpected error getting client pods: %w", err) + } + numReady := 0 + for _, p := range pods.Items { + if podIsReady(p) { + numReady++ + } + } + return numReady, nil +} + +func (vt *ValidationTest) getClientPods(ctx context.Context, expectedNumPods int) (*core.PodList, error) { + listOpts := meta.ListOptions{ + LabelSelector: clientAppLabel(), + } + pods, err := vt.Clientset.CoreV1().Pods(vt.Namespace).List(ctx, listOpts) + if err != nil { + return nil, fmt.Errorf("failed to list client pods: %w", err) + } + if len(pods.Items) != expectedNumPods { + return nil, fmt.Errorf("the number of pods listed [%d] does not match the number expected [%d]", len(pods.Items), expectedNumPods) + } + return pods, err +} + +func (vt *ValidationTest) cleanUpTestResources() (string, error) { + // need a clean, non-canceled context in case the test is canceled by ctrl-c + ctx := context.Background() + + // delete the config object in the foreground so we wait until all validation test resources are + // gone before stopping, and do it now because there's no need to wait for just a test + var gracePeriodZero int64 = 0 + deleteForeground := meta.DeletePropagationForeground + delOpts := meta.DeleteOptions{ + PropagationPolicy: &deleteForeground, + GracePeriodSeconds: &gracePeriodZero, + } + err := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Delete(ctx, ownerConfigMapName, delOpts) + if err != nil { + if !kerrors.IsNotFound(err) { + return manualCleanupInstructions, fmt.Errorf("failed to clean up multus validation test resources: %w", err) + } + return "", nil + } + + // clients take a long time to terminate, and the 0 grace period set on the configmap doesn't + // propagate to dependents. make a best-effort attempt to delete client pods with 0 grace period + listOpts := meta.ListOptions{ + LabelSelector: clientAppLabel(), + } + // ignore errors for a best-effort attempt, they will delete eventually + _ = vt.Clientset.CoreV1().Pods(vt.Namespace).DeleteCollection(ctx, delOpts, listOpts) + + // wait for resources to be cleaned up + ctx, cancel := context.WithTimeout(ctx, vt.ResourceTimeout) + defer cancel() + lastSuggestion := "" + err = wait.PollUntilWithContext(ctx, 2*time.Second, func(ctx context.Context) (done bool, err error) { + _, getErr := vt.Clientset.CoreV1().ConfigMaps(vt.Namespace).Get(ctx, ownerConfigMapName, meta.GetOptions{}) + if getErr != nil { + if kerrors.IsNotFound(getErr) { + return true, nil + } + lastSuggestion = fmt.Sprintf("unexpected error when cleaning up multus validation test resources; attempting to continue: %v", err) + } + return false, nil + }) + if err != nil { + return lastSuggestion + "; " + manualCleanupInstructions, + fmt.Errorf("failed waiting for multus validation test resources to be deleted: %w", err) + } + + return "", nil +} diff --git a/pkg/daemon/multus/statemachine.go b/pkg/daemon/multus/statemachine.go new file mode 100644 index 000000000..b807ea540 --- /dev/null +++ b/pkg/daemon/multus/statemachine.go @@ -0,0 +1,96 @@ +package multus + +import ( + "context" + "fmt" + "time" + + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type validationState interface { + Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) +} + +type validationStateMachine struct { + vt *ValidationTest + state validationState + timer *time.Timer + stateWasChanged bool + resourceOwnerRefs []meta.OwnerReference + testResults *ValidationTestResults + lastSuggestions []string + lastErr error + done bool +} + +func (vsm *validationStateMachine) SetNextState(nextState validationState) { + vsm.stateWasChanged = true + vsm.state = nextState +} + +func (vsm *validationStateMachine) Exit() { + vsm.done = true +} + +func (vsm *validationStateMachine) Run(ctx context.Context) (*ValidationTestResults, error) { + vsm.timer = time.NewTimer(vsm.vt.ResourceTimeout) + defer func() { vsm.timer.Stop() }() + + for { + select { + case <-ctx.Done(): + return vsm.exitContextCanceled(ctx) + + case <-vsm.timer.C: + vsm.testResults.addSuggestions(vsm.lastSuggestions...) + return vsm.testResults, fmt.Errorf("multus validation test timed out: %w", vsm.lastErr) + + default: + // give each state the full resource timeout to run successfully + if vsm.stateWasChanged { + vsm.resetTimer() + vsm.stateWasChanged = false + } + + // run the state + suggestions, err := vsm.state.Run(ctx, vsm) + + // if the context was canceled, the error message won't be as useful as gathered from + // the last context, so exit before updating the latest suggestions and error + if ctx.Err() != nil { + return vsm.exitContextCanceled(ctx) + } + + // record the latest suggestions and error + vsm.lastErr = err + vsm.lastSuggestions = suggestions + + // exit the state machine, error or not + if vsm.done { + vsm.testResults.addSuggestions(vsm.lastSuggestions...) + if vsm.lastErr != nil { + return vsm.testResults, fmt.Errorf("multus validation test failed: %w", vsm.lastErr) + } + return vsm.testResults, nil + } + if err != nil { + vsm.vt.Logger.Infof("continuing: %s", err) + } + + time.Sleep(2 * time.Second) + } + } +} + +func (vsm *validationStateMachine) resetTimer() { + if !vsm.timer.Stop() { + <-vsm.timer.C + } + vsm.timer.Reset(vsm.vt.ResourceTimeout) +} + +func (vsm *validationStateMachine) exitContextCanceled(ctx context.Context) (*ValidationTestResults, error) { + vsm.testResults.addSuggestions(vsm.lastSuggestions...) + return vsm.testResults, fmt.Errorf("context canceled before multus validation test could complete: %s: %w", ctx.Err().Error(), vsm.lastErr) +} diff --git a/pkg/daemon/multus/templates.go b/pkg/daemon/multus/templates.go new file mode 100644 index 000000000..593ed6c11 --- /dev/null +++ b/pkg/daemon/multus/templates.go @@ -0,0 +1,149 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multus + +import ( + "bytes" + _ "embed" + "fmt" + "strings" + "text/template" + + apps "k8s.io/api/apps/v1" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +var ( + //go:embed nginx-pod.yaml + nginxPodTemplate string + + //go:embed nginx-config.yaml + nginxConfigTemplate string + + //go:embed client-daemonset.yaml + clientDaemonSet string +) + +type webServerTemplateConfig struct { + NetworksAnnotationValue string +} + +type clientTemplateConfig struct { + ClientID int + NetworksAnnotationValue string + NetworkNamesAndAddresses map[string]string +} + +func webServerPodName() string { + return "multus-validation-test-web-server" +} + +func clientAppLabel() string { + return "app=multus-validation-test-client" +} + +func (vt *ValidationTest) generateWebServerTemplateConfig() webServerTemplateConfig { + return webServerTemplateConfig{ + NetworksAnnotationValue: vt.generateNetworksAnnotationValue(), + } +} + +func (vt *ValidationTest) generateClientTemplateConfig(clientID int, serverPublicAddr, serverClusterAddr string) clientTemplateConfig { + netNamesAndAddresses := map[string]string{} + if serverPublicAddr != "" { + netNamesAndAddresses["public"] = serverPublicAddr + } + if serverClusterAddr != "" { + netNamesAndAddresses["cluster"] = serverClusterAddr + } + return clientTemplateConfig{ + ClientID: clientID, + NetworksAnnotationValue: vt.generateNetworksAnnotationValue(), + NetworkNamesAndAddresses: netNamesAndAddresses, + } +} + +func (vt *ValidationTest) generateWebServerPod() (*core.Pod, error) { + t, err := loadTemplate("webServerPod", nginxPodTemplate, vt.generateWebServerTemplateConfig()) + if err != nil { + return nil, fmt.Errorf("failed to load web server pod template: %w", err) + } + + var p core.Pod + err = yaml.Unmarshal(t, &p) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal web server pod template: %w", err) + } + + return &p, nil +} + +func (vt *ValidationTest) generateWebServerConfigMap() (*core.ConfigMap, error) { + t, err := loadTemplate("webServerConfigMap", nginxConfigTemplate, vt.generateWebServerTemplateConfig()) + if err != nil { + return nil, fmt.Errorf("failed to load web server configmap template: %w", err) + } + + var cm core.ConfigMap + err = yaml.Unmarshal(t, &cm) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal web server configmap template: %w", err) + } + + return &cm, nil +} + +func (vt *ValidationTest) generateClientDaemonSet( + clientID int, + serverPublicAddr, serverClusterAddr string, +) (*apps.DaemonSet, error) { + t, err := loadTemplate("clientDaemonSet", clientDaemonSet, vt.generateClientTemplateConfig(clientID, serverPublicAddr, serverClusterAddr)) + if err != nil { + return nil, fmt.Errorf("failed to load client daemonset template: %w", err) + } + + var d apps.DaemonSet + err = yaml.Unmarshal(t, &d) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal client daemonset template: %w", err) + } + + return &d, nil +} + +func (vt *ValidationTest) generateNetworksAnnotationValue() string { + nets := []string{} + if vt.PublicNetwork != "" { + nets = append(nets, vt.PublicNetwork) + } + if vt.ClusterNetwork != "" { + nets = append(nets, vt.ClusterNetwork) + } + return strings.Join(nets, ",") +} + +func loadTemplate(name, templateFileText string, config interface{}) ([]byte, error) { + var writer bytes.Buffer + t := template.New(name) + t, err := t.Parse(templateFileText) + if err != nil { + return nil, fmt.Errorf("failed to parse template %q: %w", name, err) + } + err = t.Execute(&writer, config) + return writer.Bytes(), err +} diff --git a/pkg/daemon/multus/util.go b/pkg/daemon/multus/util.go new file mode 100644 index 000000000..392c7696f --- /dev/null +++ b/pkg/daemon/multus/util.go @@ -0,0 +1,129 @@ +package multus + +import ( + "fmt" + "os" + + nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Logger defines the log types that this library will use for outputting status information to the +// user. Because this library may be called interactively by a user or programmatically by another +// application, we allow the calling application to make a very simple Logger type that will be +// compatible with this library. +type Logger interface { + Infof(format string, args ...interface{}) + Debugf(format string, args ...interface{}) + Warningf(format string, args ...interface{}) +} + +type SimpleStderrLogger struct{} + +func (*SimpleStderrLogger) Infof(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, " INFO: "+format+"\n", args...) +} + +func (*SimpleStderrLogger) Debugf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "DEBUG: "+format+"\n", args...) +} + +func (*SimpleStderrLogger) Warningf(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, " WARN: "+format+"\n", args...) +} + +func getNetworksFromPod( + pod *core.Pod, + desiredPublicNet, desiredClusterNet *types.NamespacedName, +) ( + publicAddr, clusterAddr string, suggestions []string, err error, +) { + nets, err := nadutils.GetNetworkStatus(pod) + if err != nil { + return "", "", unableToProvideAddressSuggestions, + fmt.Errorf("pod has no network status yet: %w", err) + } + if len(nets) == 0 { + return "", "", unableToProvideAddressSuggestions, + fmt.Errorf("pod has no attached networks yet") + } + + // if the pod has networks attached, parse all of them, and report any debugging suggestions + // associated for each one + publicAddr, clusterAddr = "", "" + suggestions = []string{} + for _, net := range nets { + nsName, err := networkNamespacedName(net.Name, pod.Namespace) + if err != nil { + suggestions = append(suggestions, + fmt.Sprintf("pod has an attached network with an un-parse-able network name [%s]; "+ + "not sure what to do; this is unlikely to resolve", net.Name), + ) + } + if len(net.IPs) == 0 { + suggestions = append(suggestions, + fmt.Sprintf("pod has an attached network [%s] with no IP; "+ + "could this be an IPAM issue?", net.Name)) + } + + ip := net.IPs[0] // any IP should work + if desiredPublicNet != nil && nsName == *desiredPublicNet { + // FOUND DESIRED PUBLIC ADDR + publicAddr = ip + } + if desiredClusterNet != nil && nsName == *desiredClusterNet { + // FOUND DESIRED CLUSTER ADDR + clusterAddr = ip + } + } + + if desiredPublicNet != nil && publicAddr == "" { + suggestions = append(suggestions, "pod does not have a public network attachment") + } + if desiredClusterNet != nil && clusterAddr == "" { + suggestions = append(suggestions, "pod does not have a cluster network attachment") + } + + // if there are any suggestions, something isn't right + if len(suggestions) > 0 { + suggestions = append(suggestions, unableToProvideAddressSuggestions...) + return publicAddr, clusterAddr, suggestions, fmt.Errorf("did not find all desired networks") + } + + // found what we were looking for + return publicAddr, clusterAddr, []string{}, nil +} + +func podIsRunning(pod core.Pod) bool { + return pod.Status.Phase == core.PodRunning +} + +func podIsReady(pod core.Pod) bool { + for _, c := range pod.Status.Conditions { + if c.Type == core.PodReady { + return c.Status == core.ConditionTrue + } + } + return false +} + +func networkNamespacedName(netName, podNamespace string) (types.NamespacedName, error) { + nsName := types.NamespacedName{} + // what we really want is the result of parsePodNetworkObjectText(), a private method. + // because the network name is in the same format as network annotations, we can at + // at least use ParseNetworkAnnotation() to parse the network name + netSelections, err := nadutils.ParseNetworkAnnotation(netName, podNamespace) + if err != nil { + return nsName, fmt.Errorf("failed to parse network name %q into namespaced name: %w", netName, err) + } + + if len(netSelections) != 1 { + return nsName, fmt.Errorf("failed to parse network name %q into a single namespaced network name, instead: %v", netName, netSelections) + } + + net := netSelections[0] + nsName.Namespace = net.Namespace + nsName.Name = net.Name + return nsName, nil +} diff --git a/pkg/daemon/multus/validation.go b/pkg/daemon/multus/validation.go new file mode 100644 index 000000000..5180f754d --- /dev/null +++ b/pkg/daemon/multus/validation.go @@ -0,0 +1,332 @@ +/* +Copyright 2023 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multus + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +const ( + // the name of the config object that "owns" all other resources created for this test + // this does not need to be unique per test. per-namespace is fine + ownerConfigMapName = "multus-validation-test-owner" + + flakyNetworkSuggestion = "the underlying network may be flaky or not have the bandwidth to support a production ceph cluster; " + + "even if the validation test passes, this could still be an issue" +) + +// A Multus ValidationTest runs a number of Multus-connected pods to validate that a Kubernetes +// environment is suitable for Rook to run Ceph in. +type ValidationTest struct { + Clientset kubernetes.Interface + + // Namespace is the intended namespace where the validation test will be run. This is + // recommended to be where the Rook-Ceph cluster will be installed. + Namespace string + + // PublicNetwork is the name of the Network Attachment Definition (NAD) which will + // be used for the Ceph cluster's public network. This should be a namespaced name in the form + // / if the NAD is defined in a different namespace from the cluster namespace. + PublicNetwork string + + // ClusterNetwork is the name of the Network Attachment Definition (NAD) which will + // be used for the Ceph cluster's cluster network. This should be a namespaced name in the form + // / if the NAD is defined in a different namespace from the cluster namespace. + ClusterNetwork string + + // DaemonsPerNode is the number of Multus-connected validation daemons that will run as part of + // the test. This should be set to the largest number of Ceph daemons that may run on a node in + // the worst case. Remember to consider failure cases where nodes or availability zones are + // offline and their workloads must be rescheduled. + DaemonsPerNode int + + // ResourceTimeout is the time to wait for resources to change to the expected state. For + // example, for the test web server to start, for test clients to become ready, or for test + // resources to be deleted. At longest, this may need to reflect the time it takes for client + // pods to to pull images, get address assignments, and then for each client to determine that + // its network connection is stable. + // + // This should be at least 1 minute. 2 minutes or more is recommended. + ResourceTimeout time.Duration + + // Logger an instance of the basic log implementation used by this library. + Logger Logger + + // TODO: allow overriding the nginx server image from CLI, and set from default var that can be + // overridden at build time +} + +// ValidationTestResults contains results from a validation test. +type ValidationTestResults struct { + suggestedDebugging []string +} + +func (vtr *ValidationTestResults) SuggestedDebuggingReport() string { + if vtr == nil || len(vtr.suggestedDebugging) == 0 { + return "" + } + out := fmt.Sprintln("Suggested things to investigate before installing with Multus:") + for _, s := range vtr.suggestedDebugging { + out = out + " - " + fmt.Sprintln(s) + } + return out +} + +func (vtr *ValidationTestResults) addSuggestions(s ...string) { + for _, sug := range s { + if sug == "" { + continue + } + vtr.suggestedDebugging = append(vtr.suggestedDebugging, sug) + } +} + +/* + * Validation state machine state definitions + * state machine is linear and very simple, with steps numbered 1-N + */ + +// 1. Get web server info +type getWebServerInfoState struct { + desiredPublicNet, desiredClusterNet *types.NamespacedName +} + +func (s *getWebServerInfoState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) { + info, suggestions, err := vsm.vt.getWebServerInfo(ctx, s.desiredPublicNet, s.desiredClusterNet) + if err != nil { + return suggestions, err + } + vsm.SetNextState(&startClientsState{ + webServerInfo: info, + }) + return []string{}, nil +} + +// 2. Start clients +type startClientsState struct { + webServerInfo podNetworkInfo +} + +func (s *startClientsState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) { + vsm.vt.Logger.Infof("starting %d client DaemonSets", vsm.vt.DaemonsPerNode) + err = vsm.vt.startClients(ctx, vsm.resourceOwnerRefs, s.webServerInfo.publicAddr, s.webServerInfo.clusterAddr) + if err != nil { + eErr := fmt.Errorf("failed to start clients: %w", err) + vsm.Exit() // this is a whole validation test failure if we can't start clients + return []string{}, eErr + } + vsm.SetNextState(&getNumExpectedClientsState{}) + return []string{}, nil +} + +// 3. Get expected number of clients +type getNumExpectedClientsState struct { + expectedNumClients int + expectedNumClientsValueChanged time.Time +} + +var podSchedulerDebounceTime = 30 * time.Second + +func (s *getNumExpectedClientsState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) { + expectedNumClientPods, err := vsm.vt.getExpectedNumberOfClientPods(ctx) + if err != nil { + return []string{"inability to schedule DaemonSets is likely an issue with the Kubernetes cluster itself"}, + fmt.Errorf("expected number of client not yet ready: %w", err) + } + if s.expectedNumClients < expectedNumClientPods { + s.expectedNumClients = expectedNumClientPods + s.expectedNumClientsValueChanged = time.Now() + } + if time.Since(s.expectedNumClientsValueChanged) < podSchedulerDebounceTime { + vsm.vt.Logger.Infof("waiting to ensure num expected clients to stabilizes at %d", s.expectedNumClients) + return []string{}, nil + } + vsm.vt.Logger.Infof("expecting %d clients", expectedNumClientPods) + vsm.SetNextState(&verifyAllClientsRunningState{ + expectedNumClients: expectedNumClientPods, + }) + return []string{}, nil +} + +// 4. All client pods should be in running state, even if they aren't "Ready" +type verifyAllClientsRunningState struct { + expectedNumClients int +} + +func (s *verifyAllClientsRunningState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) { + running, err := vsm.vt.allClientsAreRunning(ctx, s.expectedNumClients) + suggestions = append([]string{ + "clients not being able to run can mean multus is unable to provide them with addresses"}, + unableToProvideAddressSuggestions...) + if err != nil { + return suggestions, err + } else if !running { + return suggestions, nil // not running, but no specific error + } + vsm.vt.Logger.Infof("all %d clients are running - but may not be ready", s.expectedNumClients) + vsm.SetNextState(&verifyAllClientsReadyState{ + expectedNumClients: s.expectedNumClients, + }) + return []string{}, nil +} + +// 5. All client pods should be in "Ready" state +type verifyAllClientsReadyState struct { + expectedNumClients int + + // keep some info to heuristically determine if the network might be flaky/overloaded + prevNumReady int + timeClientsStartedBecomingReady time.Time + suggestFlaky bool +} + +func (s *verifyAllClientsReadyState) Run(ctx context.Context, vsm *validationStateMachine) (suggestions []string, err error) { + numReady, err := vsm.vt.numClientsReady(ctx, s.expectedNumClients) + collocationSuggestion := "if clients on the same node as the web server become ready but not others, " + + "there may be a network firewall or security policy blocking inter-node traffic on multus networks" + defaultSuggestions := append([]string{collocationSuggestion, flakyNetworkSuggestion}, unableToProvideAddressSuggestions...) + if err != nil { + return defaultSuggestions, err + } + + s.checkIfFlaky(vsm, numReady) + + if numReady != s.expectedNumClients { + return defaultSuggestions, fmt.Errorf("number of ready clients [%d] is not the number expected [%d]", numReady, s.expectedNumClients) + } + + vsm.vt.Logger.Infof("all %d clients are ready", s.expectedNumClients) + suggestionsOnSuccess := []string{} + if s.suggestFlaky { + suggestionsOnSuccess = append(suggestionsOnSuccess, + fmt.Sprintf("not all clients became ready within %s; %s", flakyThreshold.String(), flakyNetworkSuggestion)) + } + vsm.Exit() // DONE! + return suggestionsOnSuccess, nil +} + +// clients should all become ready within a pretty short amount of time since they all should start +// pretty simultaneously +// TODO: allow tuning this +// TODO: pull the image first on all nodes to ensure flakiness isn't affected by image pull time +var flakyThreshold = 20 * time.Second + +func (s *verifyAllClientsReadyState) checkIfFlaky(vsm *validationStateMachine, numReady int) { + if s.suggestFlaky { + return // no need to do any checks if network is already found flaky + } + + if numReady > s.prevNumReady && s.timeClientsStartedBecomingReady.IsZero() { + vsm.vt.Logger.Debugf("clients started becoming ready") + s.timeClientsStartedBecomingReady = time.Now() + return + } + + if !s.timeClientsStartedBecomingReady.IsZero() { + // check to see how long it took since clients first started becoming ready. if the time is + // longer than the flaky threshold, warn the user, and record that the network is flaky + if time.Since(s.timeClientsStartedBecomingReady) > flakyThreshold { + vsm.vt.Logger.Warningf( + "network seems flaky; the time since clients started becoming ready until now is greater than %s", flakyThreshold.String()) + s.suggestFlaky = true + } + } +} + +// Run the Multus validation test. +func (vt *ValidationTest) Run(ctx context.Context) (*ValidationTestResults, error) { + if vt.Logger == nil { + vt.Logger = &SimpleStderrLogger{} + vt.Logger.Infof("no logger was specified; using a simple stderr logger") + } + vt.Logger.Infof("starting multus validation test with the following config:") + vt.Logger.Infof(" namespace: %q", vt.Namespace) + vt.Logger.Infof(" public network: %q", vt.PublicNetwork) + vt.Logger.Infof(" cluster network: %q", vt.ClusterNetwork) + vt.Logger.Infof(" daemons per node: %d", vt.DaemonsPerNode) + vt.Logger.Infof(" resource timeout: %v", vt.ResourceTimeout) + + if vt.PublicNetwork == "" && vt.ClusterNetwork == "" { + return nil, fmt.Errorf("at least one of 'public network' and 'cluster network' must be specified") + } + + var desiredPublicNet *types.NamespacedName = nil + var desiredClusterNet *types.NamespacedName = nil + if vt.PublicNetwork != "" { + n, err := networkNamespacedName(vt.PublicNetwork, vt.Namespace) + if err != nil { + return nil, fmt.Errorf("public network is an invalid NAD name: %w", err) + } + desiredPublicNet = &n + } + if vt.ClusterNetwork != "" { + n, err := networkNamespacedName(vt.ClusterNetwork, vt.Namespace) + if err != nil { + return nil, fmt.Errorf("cluster network is an invalid NAD name: %w", err) + } + desiredClusterNet = &n + } + + testResults := &ValidationTestResults{ + suggestedDebugging: []string{}, + } + + // configmap's purpose is to serve as the owner resource object for all other test resources. + // this allows users to clean up a botched test easily just by deleting this configmap + owningConfigMap, err := vt.createOwningConfigMap(ctx) + if err != nil { + testResults.addSuggestions(previousTestSuggestion) + return testResults, fmt.Errorf("failed to create validation test config object: %w", err) + } + + err = vt.startWebServer(ctx, owningConfigMap) + if err != nil { + testResults.addSuggestions(previousTestSuggestion) + return testResults, fmt.Errorf("failed to start web server: %w", err) + } + + // start the state machine + vsm := &validationStateMachine{ + vt: vt, + resourceOwnerRefs: owningConfigMap, + testResults: testResults, + lastSuggestions: []string{}, + } + startingState := &getWebServerInfoState{ + desiredPublicNet: desiredPublicNet, + desiredClusterNet: desiredClusterNet, + } + vsm.SetNextState(startingState) + return vsm.Run(ctx) +} + +// CleanUp cleans up Multus validation test resources. It returns a suggestion for manual action if +// clean up was unsuccessful. +func (vt *ValidationTest) CleanUp(ctx context.Context) (*ValidationTestResults, error) { + var err error + res := ValidationTestResults{ + suggestedDebugging: []string{}, + } + suggestions, err := vt.cleanUpTestResources() + res.addSuggestions(suggestions) + return &res, err +}