From ce1003aef8d7677062ff7bf9fb447d368e35cd2e Mon Sep 17 00:00:00 2001 From: Vineet Badrinath Date: Tue, 16 Jun 2020 18:23:28 +0530 Subject: [PATCH] ceph: adds scripts and components to support admission controllers adds deploy.sh script to deploy validatingwebhookconfiguration and create secrets. adds new command ceph admission-controller to start webhook servers. adds validation for various rook custom resources Signed-off-by: Vineet Badrinath --- Documentation/admission-controller-usage.md | 51 +++++ PendingReleaseNotes.md | 3 + .../rook-ceph/templates/clusterrole.yaml | 9 + .../templates/clusterrolebinding.yaml | 15 ++ .../rook-ceph/templates/serviceaccount.yaml | 7 + cluster/examples/kubernetes/ceph/common.yaml | 29 +++ .../ceph/config-admission-controller.sh | 11 + cmd/rook/ceph/admission.go | 46 ++++ cmd/rook/ceph/ceph.go | 1 + cmd/rook/main.go | 1 - pkg/daemon/admission/controller.go | 119 ++++++++++ pkg/daemon/admission/server.go | 77 +++++++ pkg/daemon/admission/tlsutil.go | 80 +++++++ pkg/operator/ceph/admission.go | 100 +++++++++ pkg/operator/ceph/csi/spec.go | 4 +- pkg/operator/ceph/csi/util.go | 3 +- pkg/operator/ceph/operator.go | 5 + pkg/operator/ceph/webhook.go | 208 ++++++++++++++++++ tests/framework/installer/ceph_installer.go | 80 ++++++- tests/framework/installer/ceph_manifests.go | 29 +++ tests/framework/utils/k8s_helper.go | 6 +- tests/integration/ceph_base_deploy_test.go | 3 +- tests/scripts/deploy_admission_controller.sh | 30 +++ .../deploy_admission_controller_test.sh | 19 ++ tests/scripts/webhook-config.yaml | 22 ++ tests/scripts/webhook-create-signed-cert.sh | 135 ++++++++++++ tests/scripts/webhook-patch-ca-bundle.sh | 14 ++ 27 files changed, 1088 insertions(+), 19 deletions(-) create mode 100644 Documentation/admission-controller-usage.md create mode 100755 cluster/examples/kubernetes/ceph/config-admission-controller.sh create mode 100644 cmd/rook/ceph/admission.go create mode 100644 pkg/daemon/admission/controller.go create mode 100644 pkg/daemon/admission/server.go create mode 100644 pkg/daemon/admission/tlsutil.go create mode 100644 pkg/operator/ceph/admission.go create mode 100644 pkg/operator/ceph/webhook.go create mode 100755 tests/scripts/deploy_admission_controller.sh create mode 100755 tests/scripts/deploy_admission_controller_test.sh create mode 100644 tests/scripts/webhook-config.yaml create mode 100755 tests/scripts/webhook-create-signed-cert.sh create mode 100755 tests/scripts/webhook-patch-ca-bundle.sh diff --git a/Documentation/admission-controller-usage.md b/Documentation/admission-controller-usage.md new file mode 100644 index 000000000..d24080254 --- /dev/null +++ b/Documentation/admission-controller-usage.md @@ -0,0 +1,51 @@ +--- +title: Admission Controller +weight: 2030 +indent: true +--- + +# Admission Controller + +An admission controller intercepts requests to the Kubernetes API server prior to persistence of the object, but after the request is authenticated and authorized. + +Enabling the Rook admission controller is recommended to provide an additional level of validation that Rook is configured correctly with the custom resource (CR) settings. + +## Quick Start + +To deploy the Rook admission controllers we have a helper script that will automate the configuration. + +This script will help us achieve the following tasks +1. Creates self-signed certificate. +1. Creates a Certificate Signing Request(CSR) for the certificate and gets it approved from the Kubernetes cluster. +1. Stores these certificates as a Kubernetes Secret. +1. Creates a Service Account, ClusterRole and ClusterRoleBindings for running the webhook server with minimal priviledges. +1. Creates ValidatingWebhookConfig and fills the CA bundle with the appropriate value from the cluster. + +Run the following commands: +```console +kubectl create -f examples/kubernetes/ceph/common.yaml +cluster/examples/kubernetes/ceph/config-admission-controller.sh +``` +Now that the Secrets have been deployed, we can deploy the operator: +```console +kubectl create -f operator.yaml +``` + +At this point the operator will start the admission controller Deployment automatically and the Webhook will start intercepting requests for Rook resources. + +## Certificate Management + + The script file creates a self-signed Kubernetes approved certificate and deploys it as a secret onto the cluster. It is mandatory that the Secret is named "rook-ceph-admission-controller" because Rook will look for the secret with such name before starting the admission controller servers. + +In the case of deploying from scratch, the script needs to be executed once without any modification, and certificates will be automatically created and deployed as a Secret. + +The above approach of using self-signed certificates is discouraged as it would be the job of the owner to maintain the certificates. The recommended approach would be to use a proper certificate manager and get signed certificates from a known certificate authority. Once these are available, create the secrets using the following command: + +```console +kubectl create secret generic rook-ceph-admission-controller \ + --from-file=key.pem=${PRIVATE_KEY_NAME}.pem \ + --from-file=cert.pem=${PUBLIC_KEY_NAME}.pem +``` + +Once the Secrets are in the cluster, we can modify the parameter `INSTALL_SELF_SIGNED_CERT` to `false` and execute these scripts to deploy the components. This modification is required only when Secrets are created but the components (ValidatingWebhookConfig, RBAC) are yet to be deployed. + diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 921391d2c..bf7e5dda5 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -12,6 +12,9 @@ - CephBlockPool CRD has a new field called `parameters` which allows to set any property on a given [pool](Documentation/ceph-pool-crd.html#add-specific-pool-properties) - OSD changes: - OSD on PVC now supports multipath device. +- Added [admission controller](Documentation/admission-controller-usage.md) support for CRD validations. + - Support for Ceph CRDs is provided. Some validations for CephClusters are included and additional validations can be added for other CRDs + - Can be extended to add support for other providers ### EdgeFS diff --git a/cluster/charts/rook-ceph/templates/clusterrole.yaml b/cluster/charts/rook-ceph/templates/clusterrole.yaml index 7de6c0465..2c7533082 100644 --- a/cluster/charts/rook-ceph/templates/clusterrole.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrole.yaml @@ -516,3 +516,12 @@ rules: verbs: - use {{- end }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-role +rules: + - apiGroups: ["ceph.rook.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] \ No newline at end of file diff --git a/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml b/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml index d05b1292f..49784fb13 100644 --- a/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml +++ b/cluster/charts/rook-ceph/templates/clusterrolebinding.yaml @@ -244,6 +244,20 @@ subjects: name: rook-ceph-mgr namespace: {{ .Release.Namespace }} --- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-rolebinding +subjects: + - kind: ServiceAccount + name: rook-ceph-admission-controller + apiGroup: "" + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: rook-ceph-admission-controller-role + apiGroup: rbac.authorization.k8s.io +--- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -259,3 +273,4 @@ subjects: namespace: {{ .Release.Namespace }} {{- end }} {{- end }} + diff --git a/cluster/charts/rook-ceph/templates/serviceaccount.yaml b/cluster/charts/rook-ceph/templates/serviceaccount.yaml index b22ce66d3..3c4f92427 100644 --- a/cluster/charts/rook-ceph/templates/serviceaccount.yaml +++ b/cluster/charts/rook-ceph/templates/serviceaccount.yaml @@ -76,3 +76,10 @@ metadata: name: rook-csi-rbd-provisioner-sa namespace: {{ .Release.Namespace }} {{ template "imagePullSecrets" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-admission-controller + namespace: {{ .Release.Namespace }} +--- \ No newline at end of file diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index 211a94c46..d04e44665 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -716,6 +716,35 @@ aggregationRule: rbac.ceph.rook.io/aggregate-to-rook-ceph-cluster-mgmt: "true" rules: [] --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-admission-controller + namespace: rook-ceph +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-role +rules: + - apiGroups: ["ceph.rook.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-rolebinding +subjects: + - kind: ServiceAccount + name: rook-ceph-admission-controller + apiGroup: "" + namespace: rook-ceph +roleRef: + kind: ClusterRole + name: rook-ceph-admission-controller-role + apiGroup: rbac.authorization.k8s.io +--- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: diff --git a/cluster/examples/kubernetes/ceph/config-admission-controller.sh b/cluster/examples/kubernetes/ceph/config-admission-controller.sh new file mode 100755 index 000000000..31b84837a --- /dev/null +++ b/cluster/examples/kubernetes/ceph/config-admission-controller.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +BASE_DIR=$(cd "$(dirname "$0")"/../../../..; pwd) + +echo "$BASE_DIR" + +if [ -f "$BASE_DIR"/tests/scripts/deploy_admission_controller.sh ];then + bash "$BASE_DIR"/tests/scripts/deploy_admission_controller.sh +else + echo ""${BASE_DIR}"/deploy_admission_controller.sh not found!" +fi diff --git a/cmd/rook/ceph/admission.go b/cmd/rook/ceph/admission.go new file mode 100644 index 000000000..89870b144 --- /dev/null +++ b/cmd/rook/ceph/admission.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ceph + +import ( + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/daemon/admission" + operator "github.com/rook/rook/pkg/operator/ceph" + "github.com/spf13/cobra" +) + +var ( + admissionCmd = &cobra.Command{ + Use: "admission-controller", + Short: "Starts admission controller", + } +) + +func init() { + admissionCmd.Run = startAdmissionController +} + +func startAdmissionController(cmd *cobra.Command, args []string) { + rook.SetLogLevel() + + rook.LogStartupInfo(admissionCmd.Flags()) + + context := rook.NewContext() + + a := admission.New(context, "ceph", operator.ValidateCephResource) + a.StartServer() +} diff --git a/cmd/rook/ceph/ceph.go b/cmd/rook/ceph/ceph.go index c0904b27a..1b09771ef 100644 --- a/cmd/rook/ceph/ceph.go +++ b/cmd/rook/ceph/ceph.go @@ -56,6 +56,7 @@ func init() { Cmd.AddCommand(cleanUpCmd, operatorCmd, agentCmd, + admissionCmd, osdCmd, configCmd) } diff --git a/cmd/rook/main.go b/cmd/rook/main.go index 291a792cd..32098a63f 100644 --- a/cmd/rook/main.go +++ b/cmd/rook/main.go @@ -40,7 +40,6 @@ func addCommands() { rook.RootCmd.AddCommand( version.VersionCmd, discoverCmd, - // backend commands ceph.Cmd, cockroachdb.Cmd, diff --git a/pkg/daemon/admission/controller.go b/pkg/daemon/admission/controller.go new file mode 100644 index 000000000..ddbcebd89 --- /dev/null +++ b/pkg/daemon/admission/controller.go @@ -0,0 +1,119 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "encoding/json" + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + "io/ioutil" + "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "net/http" +) + +const ( + jsonContentType = `application/json` +) + +var ( + universalDeserializer = serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer() +) + +// admitFunc is a callback for admission controller logic. Given an AdmissionRequest, it returns an error that will be shown when the operation is rejected. +type admitFunc func(*v1beta1.AdmissionRequest, *clusterd.Context) error + +// doServeAdmitFunc parses the HTTP request for an admission controller webhook, and -- in case of a well-formed +// request -- delegates the admission control logic to the given admitFunc. The response body is then returned as raw +// bytes. +func doServeAdmitFunc(w http.ResponseWriter, r *http.Request, a *AdmissionController) ([]byte, error) { + // Step 1: Request validation. Only handle POST requests with a body and json content type. + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return nil, errors.Errorf("invalid method %q, only POST requess are allowed", r.Method) + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return nil, errors.Wrap(err, "could not read request body") + } + if contentType := r.Header.Get("Content-Type"); contentType != jsonContentType { + w.WriteHeader(http.StatusBadRequest) + return nil, errors.Errorf("unsupported content type %q, only %q is supported", contentType, jsonContentType) + } + // Step 2: Parse the AdmissionReview request. + var admissionReviewReq v1beta1.AdmissionReview + if _, _, err := universalDeserializer.Decode(body, nil, &admissionReviewReq); err != nil { + w.WriteHeader(http.StatusBadRequest) + return nil, errors.Wrap(err, "failed to deserialize request") + } else if admissionReviewReq.Request == nil { + w.WriteHeader(http.StatusBadRequest) + return nil, errors.New("malformed admission review. request is nil") + } + logger.Infof("processing webhook request for resource type %s", admissionReviewReq.Request.Kind.Kind) + // Step 3: Construct the AdmissionReview response. + admissionReviewResponse := v1beta1.AdmissionReview{ + Response: &v1beta1.AdmissionResponse{ + UID: admissionReviewReq.Request.UID, + }, + } + err = a.validator(admissionReviewReq.Request, a.context) + if err != nil { + // If the handler returned an error, incorporate the error message into the response and deny the object + // creation. + admissionReviewResponse.Response.Allowed = false + admissionReviewResponse.Response.Result = &metav1.Status{ + Message: err.Error(), + } + } else { + admissionReviewResponse.Response.Allowed = true + } + // Return the AdmissionReview with a response as JSON. + bytes, err := json.Marshal(&admissionReviewResponse) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal response to AdmissionReview") + } + return bytes, nil +} + +// serveAdmitFunc is a wrapper around doServeAdmitFunc that adds error handling and logging. +func serveAdmitFunc(w http.ResponseWriter, r *http.Request, a *AdmissionController) { + logger.Info("handling webhook request") + + var writeErr error + if bytes, err := doServeAdmitFunc(w, r, a); err != nil { + logger.Errorf("failed to handle webhook request. %v", err) + w.WriteHeader(http.StatusInternalServerError) + _, writeErr = w.Write([]byte(err.Error())) + } else { + logger.Info("webhook request handled successfully") + _, writeErr = w.Write(bytes) + } + + if writeErr != nil { + logger.Errorf("failed to write response to webhook request. %v", writeErr) + } +} + +// admitFuncHandler takes an admitFunc and wraps it into a http.Handler by means of calling serveAdmitFunc. +func admitFuncHandler(a *AdmissionController) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serveAdmitFunc(w, r, a) + }) +} diff --git a/pkg/daemon/admission/server.go b/pkg/daemon/admission/server.go new file mode 100644 index 000000000..86cf75737 --- /dev/null +++ b/pkg/daemon/admission/server.go @@ -0,0 +1,77 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "crypto/tls" + "fmt" + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/clusterd" + "net/http" + "path/filepath" +) + +const ( + tlsDir = "/etc/webhook" + tlsCertFile = "tls.crt" + tlsKeyFile = "tls.key" + serverPort int32 = 8079 +) + +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "admission") + validatePath = "/validate" +) + +type AdmissionController struct { + context *clusterd.Context + providerName string + validator admitFunc +} + +func New(context *clusterd.Context, providerName string, validator admitFunc) *AdmissionController { + return &AdmissionController{ + context: context, + providerName: providerName, + validator: validator, + } +} + +// StartServer will start the server +func (a *AdmissionController) StartServer() { + logger.Infof("starting the webhook for backend %q", a.providerName) + certPath := filepath.Join(tlsDir, tlsCertFile) + keyPath := filepath.Join(tlsDir, tlsKeyFile) + keyPair, err := NewTlsKeypairReloader(certPath, keyPath) + if err != nil { + logger.Errorf("failed to load certificate. %v", err) + } + + mux := http.NewServeMux() + mux.Handle(validatePath, admitFuncHandler(a)) + + var httpServer *http.Server + httpServer = &http.Server{ + Addr: fmt.Sprintf(":%d", serverPort), + TLSConfig: &tls.Config{ + GetCertificate: keyPair.GetCertificateFunc(), + }, + Handler: mux, + } + + httpServer.ListenAndServeTLS("", "") +} diff --git a/pkg/daemon/admission/tlsutil.go b/pkg/daemon/admission/tlsutil.go new file mode 100644 index 000000000..e10879253 --- /dev/null +++ b/pkg/daemon/admission/tlsutil.go @@ -0,0 +1,80 @@ +/* + Copyright (c) 2019 Multus Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Original code can be found at https://github.com/openshift/multus-admission-controller +*/ + +package admission + +import ( + "crypto/tls" + "github.com/pkg/errors" + "os" + "os/signal" + "sync" + "syscall" +) + +type tlsKeypairReloader struct { + certMutex sync.RWMutex + cert *tls.Certificate + certPath string + keyPath string +} + +func (keyPair *tlsKeypairReloader) maybeReload() error { + newCert, err := tls.LoadX509KeyPair(keyPair.certPath, keyPair.keyPath) + if err != nil { + return errors.Wrap(err, "failed to load the certificate key pair") + } + logger.Info("certificate reloaded") + keyPair.certMutex.Lock() + defer keyPair.certMutex.Unlock() + keyPair.cert = &newCert + return nil +} + +// GetCertificateFunc will fetch the tls certificate +func (keyPair *tlsKeypairReloader) GetCertificateFunc() func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + keyPair.certMutex.RLock() + defer keyPair.certMutex.RUnlock() + return keyPair.cert, nil + } +} + +// NewTlsKeyPairReloader will load the public and private key pair from given paths +func NewTlsKeypairReloader(certPath, keyPath string) (*tlsKeypairReloader, error) { + result := &tlsKeypairReloader{ + certPath: certPath, + keyPath: keyPath, + } + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, errors.Wrap(err, "failed to load the certificate key pair") + } + result.cert = &cert + + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGHUP) + for range c { + if err := result.maybeReload(); err != nil { + logger.Errorf("failed to reload certificate. %v", err) + } + } + }() + return result, nil +} diff --git a/pkg/operator/ceph/admission.go b/pkg/operator/ceph/admission.go new file mode 100644 index 000000000..eb73f5c73 --- /dev/null +++ b/pkg/operator/ceph/admission.go @@ -0,0 +1,100 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + rookclient "github.com/rook/rook/pkg/client/clientset/versioned" + "github.com/rook/rook/pkg/clusterd" + opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" + "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "strconv" +) + +var ( + cephclusterResource = metav1.GroupVersionResource{Group: cephv1.CustomResourceGroup, Version: cephv1.Version, Resource: opcontroller.ClusterResource.Plural} + rookClientSet rookclient.Interface + universalDeserializer = serializer.NewCodecFactory(runtime.NewScheme()).UniversalDeserializer() +) + +func validateUpdatedCephCluster(updatedCephCluster cephv1.CephCluster, found *cephv1.CephCluster) error { + if updatedCephCluster.Spec.DataDirHostPath != found.Spec.DataDirHostPath { + return errors.Errorf("invalid update : DataDirHostPath change from %q to %q is not allowed", found.Spec.DataDirHostPath, updatedCephCluster.Spec.DataDirHostPath) + } + + if updatedCephCluster.Spec.Network.HostNetwork != found.Spec.Network.HostNetwork { + return errors.Errorf("invalid update : HostNetwork change from %q to %q is not allowed", strconv.FormatBool(found.Spec.Network.HostNetwork), strconv.FormatBool(updatedCephCluster.Spec.Network.HostNetwork)) + } + + if updatedCephCluster.Spec.Network.Provider != found.Spec.Network.Provider { + return errors.Errorf("invalid update : Provider change from %q to %q is not allowed", found.Spec.Network.Provider, updatedCephCluster.Spec.Network.Provider) + } + + return nil +} + +func ValidateCephResource(request *v1beta1.AdmissionRequest, context *clusterd.Context) error { + rookClientSet = context.RookClientset + raw := request.Object.Raw + reqCephCluster := cephv1.CephCluster{} + // Deserializing the incoming CephCluster object + if _, _, err := universalDeserializer.Decode(raw, nil, &reqCephCluster); err != nil { + return errors.Wrap(err, "failed to deserialize pod object") + } + // Checks if CephCluster is being updated and rejects the request if the dataDirHostPath is changed from initial value + if isCephClusterUpdate(request) { + // Fetch the existing ceph cluster object + found, err := fetchExistingCephCluster(request) + if err != nil { + return errors.Wrap(err, "failed to fetch existing cephcluster") + } + err = validateUpdatedCephCluster(reqCephCluster, found) + if err != nil { + return errors.Wrap(err, "failed to validate updated cephcluster") + } + } + if isCephClusterCreate(request) { + //If external mode enabled, then check if other fields are empty + if reqCephCluster.Spec.External.Enable { + if reqCephCluster.Spec.Mon != (cephv1.MonSpec{}) || reqCephCluster.Spec.Dashboard != (cephv1.DashboardSpec{}) || reqCephCluster.Spec.Monitoring != (cephv1.MonitoringSpec{}) || reqCephCluster.Spec.DisruptionManagement != (cephv1.DisruptionManagementSpec{}) || len(reqCephCluster.Spec.Mgr.Modules) > 0 || len(reqCephCluster.Spec.Network.Provider) > 0 || len(reqCephCluster.Spec.Network.Selectors) > 0 { + return errors.New("invalid create : external mode enabled cannot have mon,dashboard,monitoring,network,disruptionManagement,storage fields in CR") + } + + } + } + return nil +} + +func isCephClusterCreate(request *v1beta1.AdmissionRequest) bool { + return request.Resource == cephclusterResource && request.Operation == v1beta1.Create +} + +func fetchExistingCephCluster(request *v1beta1.AdmissionRequest) (*cephv1.CephCluster, error) { + found, err := rookClientSet.CephV1().CephClusters(request.Namespace).Get(request.Name, metav1.GetOptions{}) + if err != nil { + return &cephv1.CephCluster{}, errors.Wrap(err, "failed to find existing cephcluster object") + } + return found, nil +} + +func isCephClusterUpdate(request *v1beta1.AdmissionRequest) bool { + return request.Resource == cephclusterResource && request.Operation == v1beta1.Update +} diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index a39c667e7..6e12fc473 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -400,7 +400,7 @@ func startDrivers(namespace string, clientset kubernetes.Interface, ver *version // apply resource request and limit to rbd provisioner containers applyResourcesToContainers(clientset, rbdProvisionerResource, &rbdProvisionerDeployment.Spec.Template.Spec) k8sutil.SetOwnerRef(&rbdProvisionerDeployment.ObjectMeta, ownerRef) - antiAffinity := getPodAntiAffinity("app", "csi-rbdplugin-provisioner") + antiAffinity := GetPodAntiAffinity("app", "csi-rbdplugin-provisioner") rbdProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity rbdProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ Type: apps.RecreateDeploymentStrategyType, @@ -450,7 +450,7 @@ func startDrivers(namespace string, clientset kubernetes.Interface, ver *version // apply resource request and limit to cephfs provisioner containers applyResourcesToContainers(clientset, cephFSProvisionerResource, &cephfsProvisionerDeployment.Spec.Template.Spec) k8sutil.SetOwnerRef(&cephfsProvisionerDeployment.ObjectMeta, ownerRef) - antiAffinity := getPodAntiAffinity("app", "csi-cephfsplugin-provisioner") + antiAffinity := GetPodAntiAffinity("app", "csi-cephfsplugin-provisioner") cephfsProvisionerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity = &antiAffinity cephfsProvisionerDeployment.Spec.Strategy = apps.DeploymentStrategy{ Type: apps.RecreateDeploymentStrategyType, diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go index 48c152250..4ce5ddd0e 100644 --- a/pkg/operator/ceph/csi/util.go +++ b/pkg/operator/ceph/csi/util.go @@ -221,7 +221,8 @@ func getPortFromConfig(clientset kubernetes.Interface, env string, defaultPort u return uint16(p), nil } -func getPodAntiAffinity(key, value string) corev1.PodAntiAffinity { +// Get PodAntiAffinity from a key and value pair +func GetPodAntiAffinity(key, value string) corev1.PodAntiAffinity { return corev1.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ { diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index 364bc7454..43193573c 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -120,6 +120,11 @@ func (o *Operator) Run() error { } } + logger.Debug("checking for admission controller secrets") + err := StartControllerIfSecretPresent(o.context, o.rookImage) + if err != nil { + return errors.Wrap(err, "failed to start webhook") + } serverVersion, err := o.context.Clientset.Discovery().ServerVersion() if err != nil { return errors.Wrap(err, "failed to get server version") diff --git a/pkg/operator/ceph/webhook.go b/pkg/operator/ceph/webhook.go new file mode 100644 index 000000000..b259d99c4 --- /dev/null +++ b/pkg/operator/ceph/webhook.go @@ -0,0 +1,208 @@ +/* +Copyright 2020 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "github.com/pkg/errors" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/ceph/csi" + "github.com/rook/rook/pkg/operator/k8sutil" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "os" +) + +const ( + appName = "rook-ceph-admission-controller" + secretVolumeName = "webhook-certificates" + serviceAccountName = "rook-ceph-admission-controller" + portName = "webhook-api" + servicePort int32 = 443 + serverPort int32 = 8079 + tlsDir = "/etc/webhook" +) + +var ( + namespace = os.Getenv(k8sutil.PodNamespaceEnvVar) +) + +func isSecretPresent(context *clusterd.Context) (bool, error) { + logger.Infof("looking for secret %q", appName) + _, err := context.Clientset.CoreV1().Secrets(namespace).Get(appName, metav1.GetOptions{}) + if err != nil { + // If secret is not found. All good ! Proceed with rook without admission controllers + if apierrors.IsNotFound(err) { + logger.Infof("secret %q not found. proceeding without the admission controller", appName) + return false, nil + } + return false, err + } + return true, nil +} + +func createWebhookService(context *clusterd.Context) error { + webhookService := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Port: servicePort, + TargetPort: intstr.IntOrString{ + IntVal: serverPort, + }, + }, + }, + + Selector: map[string]string{ + k8sutil.AppAttr: appName, + }, + }, + } + + _, err := k8sutil.CreateOrUpdateService(context.Clientset, namespace, &webhookService) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +// StartControllerIfSecretPresent will initialize the webhook if secret is detected +func StartControllerIfSecretPresent(context *clusterd.Context, admissionImage string) error { + isPresent, err := isSecretPresent(context) + if err != nil { + return errors.Wrap(err, "failed to retrieve secret") + } + if isPresent { + err = initWebhook(context, admissionImage) + if err != nil { + return errors.Wrap(err, "failed to initialize webhook") + } + } + return nil +} + +func initWebhook(context *clusterd.Context, admissionImage string) error { + // At this point volume should be mounted, so proceed with creating the service and validatingwebhookconfig + err := createWebhookService(context) + if err != nil { + return errors.Wrap(err, "failed to create service") + } + err = createWebhookDeployment(context, admissionImage) + if err != nil { + return errors.Wrap(err, "failed to create deployment") + } + return nil +} + +func createWebhookDeployment(context *clusterd.Context, admissionImage string) error { + logger.Info("creating admission controller pods") + admission_parameters := []string{"ceph", + "admission-controller"} + secretVolume := getSecretVolume() + secretVolumeMount := getSecretVolumeMount() + + antiAffinity := csi.GetPodAntiAffinity(k8sutil.AppAttr, appName) + admissionControllerDeployment := getDeployment(secretVolume, antiAffinity, admissionImage, admission_parameters, secretVolumeMount) + + err := k8sutil.CreateDeployment(context.Clientset, appName, namespace, &admissionControllerDeployment) + if err != nil { + return errors.Wrap(err, "failed to create admission-controller deployment") + } + + return nil +} + +func getDeployment(secretVolume corev1.Volume, antiAffinity corev1.PodAntiAffinity, admissionImage string, admission_parameters []string, secretVolumeMount corev1.VolumeMount) v1.Deployment { + var replicas int32 = 2 + + admissionControllerDeployment := v1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + }, + Spec: v1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + k8sutil.AppAttr: appName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: appName, + Namespace: namespace, + Labels: map[string]string{ + k8sutil.AppAttr: appName, + }, + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + secretVolume, + }, + Containers: []corev1.Container{ + { + Name: appName, + Image: admissionImage, + Args: admission_parameters, + Ports: []corev1.ContainerPort{ + { + Name: portName, + ContainerPort: serverPort, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + secretVolumeMount, + }, + }, + }, + ServiceAccountName: serviceAccountName, + Affinity: &corev1.Affinity{ + PodAntiAffinity: &antiAffinity, + }, + }, + }, + }, + } + return admissionControllerDeployment +} + +func getSecretVolumeMount() corev1.VolumeMount { + secretVolumeMount := corev1.VolumeMount{ + Name: secretVolumeName, + ReadOnly: true, + MountPath: tlsDir, + } + return secretVolumeMount +} + +func getSecretVolume() corev1.Volume { + secretVolume := corev1.Volume{ + Name: secretVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: appName, + }, + }, + } + return secretVolume +} diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index ed34159f8..1c9e3dcbe 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -23,6 +23,7 @@ import ( "math/rand" "os" "path" + "path/filepath" "testing" "time" @@ -95,19 +96,13 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { h.k8shelper.ChangeHostnames() } - err = h.k8shelper.CreateNamespace(namespace) + err = h.startAdmissionController(namespace) if err != nil { - if errors.IsAlreadyExists(err) { - logger.Warningf("Namespace %q already exists!!!", namespace) - } else { - return fmt.Errorf("failed to create namespace %q. %v", namespace, err) - } + return fmt.Errorf("Failed to start admission controllers: %v", err) } - if err = h.WipeDisks(namespace, 2); err != nil { return err } - rookOperator := h.Manifests.GetRookOperator(namespace) _, err = h.k8shelper.KubectlWithStdin(rookOperator, createFromStdinArgs...) if err != nil { @@ -119,6 +114,58 @@ func (h *CephInstaller) CreateCephOperator(namespace string) (err error) { return nil } +func (h *CephInstaller) startAdmissionController(namespace string) error { + err := h.k8shelper.CreateNamespace(namespace) + if err != nil { + if errors.IsAlreadyExists(err) { + logger.Warningf("Namespace %q already exists!!!", namespace) + } else { + return fmt.Errorf("failed to create namespace %q. %v", namespace, err) + } + } + currDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to find current working directory. %v", err) + } + rootPath, err := findRookRoot(currDir) + if err != nil { + return fmt.Errorf("failed to find rook root. %v", err) + } + userHome, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to find user home directory. %v", err) + } + err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", namespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", fmt.Sprintf("%s/tests/scripts/deploy_admission_controller_test.sh", rootPath)) + if err != nil { + return err + } + return nil +} + +func findRookRoot(workingDir string) (string, error) { + const folderToFind = "tests" + workingDirectory, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to find current working directory. %v", err) + } + parentPath := workingDirectory + userHome, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to find user home directory. %v", err) + } + for parentPath != userHome { + fmt.Printf("parent path = %s\n", parentPath) + _, err := os.Stat(path.Join(parentPath, folderToFind)) + if os.IsNotExist(err) { + parentPath = filepath.Dir(parentPath) + continue + } + return parentPath, nil + } + + return "", fmt.Errorf("rook root not found above directory %s", workingDir) +} + // CreateRookOperatorViaHelm creates rook operator via Helm chart named local/rook present in local repo func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings string) error { // creating clusterrolebinding for kubeadm env. @@ -130,6 +177,11 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin return fmt.Errorf("Failed to get Version of helm chart %v, err : %v", helmChartName, err) } + err = h.startAdmissionController(namespace) + if err != nil { + return fmt.Errorf("Failed to start admission controllers: %v", err) + } + err = h.helmHelper.InstallLocalRookHelmChart(helmChartName, helmDeployName, helmTag, namespace, chartSettings) if err != nil { return fmt.Errorf("failed to install rook operator via helm, err : %v", err) @@ -139,7 +191,6 @@ func (h *CephInstaller) CreateRookOperatorViaHelm(namespace, chartSettings strin if err = h.WipeDisks(namespace, 2); err != nil { return err } - return nil } @@ -388,7 +439,7 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) { // InstallRook installs rook on k8s func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, storageClassName string, - mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int, skipOSDCreation bool) (bool, error) { + mon cephv1.MonSpec, startWithAllNodes bool, rbdMirrorWorkers int, skipOSDCreation bool, rookVersion string) (bool, error) { var err error k8sversion := h.k8shelper.GetK8sServerVersion() @@ -410,7 +461,6 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st } } else { onamespace = SystemNamespace(namespace) - err := h.CreateCephOperator(onamespace) if err != nil { logger.Errorf("Rook Operator not installed ,error -> %v", err) @@ -449,6 +499,13 @@ func (h *CephInstaller) InstallRook(namespace, storeType string, usePVC bool, st return false, err } logger.Infof("installed rook operator and cluster : %s on k8s %s", namespace, h.k8sVersion) + + if rookVersion != Version1_2 { + if !h.k8shelper.IsPodInExpectedState("rook-ceph-admission-controller", onamespace, "Running") { + assert.Fail(h.T(), "admission controller is not running") + } + } + return true, nil } @@ -546,6 +603,7 @@ func (h *CephInstaller) UninstallRookFromMultipleNS(systemNamespace string, name } checkError(h.T(), err, "cannot uninstall rook-operator") + h.k8shelper.Clientset.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations().Delete("rook-ceph-webhook", nil) h.k8shelper.Clientset.RbacV1beta1().RoleBindings(systemNamespace).Delete("rook-ceph-system", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-global", nil) h.k8shelper.Clientset.RbacV1beta1().ClusterRoleBindings().Delete("rook-ceph-mgr-cluster", nil) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 95103aeaf..848859ae8 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -1586,6 +1586,35 @@ subjects: name: rook-csi-cephfs-plugin-sa namespace: ` + namespace + ` --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-admission-controller + namespace: ` + namespace + ` +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-role +rules: + - apiGroups: ["ceph.rook.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-admission-controller-rolebinding +subjects: + - kind: ServiceAccount + name: rook-ceph-admission-controller + apiGroup: "" + namespace: ` + namespace + ` +roleRef: + kind: ClusterRole + name: rook-ceph-admission-controller-role + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index c53a8ee9a..f55528713 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -1122,8 +1122,10 @@ func (k8sh *K8sHelper) IsPodInExpectedState(podNamePattern string, namespace str podList, err := k8sh.Clientset.CoreV1().Pods(namespace).List(listOpts) if err == nil { if len(podList.Items) >= 1 { - if podList.Items[0].Status.Phase == v1.PodPhase(state) { - return true + for _, pod := range podList.Items { + if pod.Status.Phase == v1.PodPhase(state) { + return true + } } } } diff --git a/tests/integration/ceph_base_deploy_test.go b/tests/integration/ceph_base_deploy_test.go index 41125dab2..194bca954 100644 --- a/tests/integration/ceph_base_deploy_test.go +++ b/tests/integration/ceph_base_deploy_test.go @@ -166,9 +166,8 @@ func StartTestCluster(t func() *testing.T, cluster *TestCluster) (*TestCluster, func (op *TestCluster) Setup() { // Turn on DEBUG logging capnslog.SetGlobalLogLevel(capnslog.DEBUG) - isRookInstalled, err := op.installer.InstallRook(op.namespace, op.storeType, op.usePVC, op.storageClassName, - cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation) + cephv1.MonSpec{Count: op.mons, AllowMultiplePerNode: true}, false /* startWithAllNodes */, op.rbdMirrorWorkers, op.skipOSDCreation, op.rookVersion) if !isRookInstalled || err != nil { logger.Errorf("Rook was not installed successfully: %v", err) diff --git a/tests/scripts/deploy_admission_controller.sh b/tests/scripts/deploy_admission_controller.sh new file mode 100755 index 000000000..027cabe7d --- /dev/null +++ b/tests/scripts/deploy_admission_controller.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# deploy_admission_controller.sh +# Sets up the environment for the admission controller webhook in the active cluster. +set -eo pipefail + +# Set our known directories and parameters. +BASE_DIR=$(cd "$(dirname "$0")"; pwd) + +# Set the variables for webhook. Any change here for namespace and service name should be done in pkg/operator/admission/init.go also +[ -z "${NAMESPACE}" ] && NAMESPACE="rook-ceph" +WEBHOOK_CONFIG_NAME="rook-ceph-webhook" +SERVICE_NAME="rook-ceph-admission-controller" + +INSTALL_SELF_SIGNED_CERT=true +echo "$BASE_DIR" + +if [ "${INSTALL_SELF_SIGNED_CERT}" == true ]; then + "${BASE_DIR}"/webhook-create-signed-cert.sh --namespace ${NAMESPACE} +fi +echo "Deploying webhook config" +export NAMESPACE +export WEBHOOK_CONFIG_NAME +export SERVICE_NAME +cat ${BASE_DIR}/webhook-config.yaml | \ + "${BASE_DIR}"/webhook-patch-ca-bundle.sh | \ + sed -e "s|\${NAMESPACE}|${NAMESPACE}|g" | \ + sed -e "s|\${WEBHOOK_CONFIG_NAME}|${WEBHOOK_CONFIG_NAME}|g" | \ + sed -e "s|\${SERVICE_NAME}|${SERVICE_NAME}|g" | \ + kubectl create -f - +echo "Webhook deployed! Please start the rook operator to create the service and admission controller pods" diff --git a/tests/scripts/deploy_admission_controller_test.sh b/tests/scripts/deploy_admission_controller_test.sh new file mode 100755 index 000000000..3ba1d0bb1 --- /dev/null +++ b/tests/scripts/deploy_admission_controller_test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +BASE_DIR=$(cd "$(dirname "$0")"; pwd) + +if [ -z "$KUBECONFIG" ];then + if [ -f "$HOME/.kube/config" ]; then + export KUBECONFIG="$HOME/.kube/config" + else + sudo cp /etc/kubernetes/admin.conf $HOME/ + sudo chown $(id -u):$(id -g) $HOME/admin.conf + export KUBECONFIG=$HOME/admin.conf + fi +fi + +if [ -f "${BASE_DIR}"/deploy_admission_controller.sh ];then + bash "${BASE_DIR}"/deploy_admission_controller.sh +else + echo ""${BASE_DIR}"/deploy_admission_controller.sh not found !" +fi \ No newline at end of file diff --git a/tests/scripts/webhook-config.yaml b/tests/scripts/webhook-config.yaml new file mode 100644 index 000000000..0101a14fd --- /dev/null +++ b/tests/scripts/webhook-config.yaml @@ -0,0 +1,22 @@ +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: ${WEBHOOK_CONFIG_NAME} + namespace: ${NAMESPACE} +webhooks: + - name: ${SERVICE_NAME}.${NAMESPACE}.svc + rules: + - apiGroups: ["ceph.rook.io"] + apiVersions: ["v1"] + operations: ["CREATE","UPDATE","DELETE"] + resources: ["*"] + clientConfig: + service: + name: ${SERVICE_NAME} + namespace: ${NAMESPACE} + path: /validate + caBundle: ${CA_BUNDLE} + admissionReviewVersions: ["v1beta1"] + sideEffects: None + timeoutSeconds: 5 + diff --git a/tests/scripts/webhook-create-signed-cert.sh b/tests/scripts/webhook-create-signed-cert.sh new file mode 100755 index 000000000..caca0f806 --- /dev/null +++ b/tests/scripts/webhook-create-signed-cert.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# Original script found at: https://github.com/morvencao/kube-mutating-webhook-tutorial/blob/master/deployment/webhook-create-signed-cert.sh + +set -e + +usage() { + cat <> ${tmpdir}/csr.conf +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = ${service} +DNS.2 = ${service}.${namespace} +DNS.3 = ${service}.${namespace}.svc +EOF + +openssl genrsa -out ${tmpdir}/server-key.pem 2048 +openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf + +cat ${tmpdir}/server.csr +# clean-up any previously created CSR for our service. Ignore errors if not present. +kubectl delete csr ${csrName} 2>/dev/null || true + +# create server cert/key CSR and send to k8s API +cat <&2 + exit 1 +fi +echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem + + +kubectl create secret generic ${secret} \ + --from-file=tls.key=${tmpdir}/server-key.pem \ + --from-file=tls.crt=${tmpdir}/server-cert.pem \ + --dry-run -o yaml | + kubectl -n ${namespace} apply -f - + diff --git a/tests/scripts/webhook-patch-ca-bundle.sh b/tests/scripts/webhook-patch-ca-bundle.sh new file mode 100755 index 000000000..a41388ec1 --- /dev/null +++ b/tests/scripts/webhook-patch-ca-bundle.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Original script found at: https://github.com/morvencao/kube-mutating-webhook-tutorial/blob/master/deployment/webhook-patch-ca-bundle.sh + +set -o errexit +set -o nounset +set -o pipefail + +export CA_BUNDLE=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n') + +if command -v envsubst >/dev/null 2>&1; then + envsubst +else + sed -e "s|\${CA_BUNDLE}|${CA_BUNDLE}|g" +fi