Merge pull request #9934 from subhamkrai/automate-adm
core: start admission controller without any script
This commit is contained in:
20 files changed
+984
-227
No files matched your search
@@ -1,32 +0,0 @@
|
||||
---
|
||||
title: Admission Controller
|
||||
weight: 2030
|
||||
indent: true
|
||||
---
|
||||
|
||||
# Admission Controller
|
||||
|
||||
An admission controller intercepts requests to the Kubernetes API server prior to persistence of the object, but after the request is authenticated and authorized.
|
||||
|
||||
Enabling the Rook admission controller is recommended to provide an additional level of validation that Rook is configured correctly with the custom resource (CR) settings.
|
||||
|
||||
## Quick Start
|
||||
|
||||
To deploy the Rook admission controllers we have a helper script that will automate the configuration.
|
||||
|
||||
This script will help us achieve the following tasks
|
||||
1. Creates certificate using cert-manager.
|
||||
2. Creates ValidatingWebhookConfig and fills the CA bundle with the appropriate value from the cluster.
|
||||
|
||||
Run the following commands:
|
||||
|
||||
```console
|
||||
kubectl create -f deploy/examples/crds.yaml -f deploy/examples/common.yaml
|
||||
tests/scripts/deploy_admission_controller.sh
|
||||
```
|
||||
Now that the Secrets have been deployed, we can deploy the operator:
|
||||
```console
|
||||
kubectl create -f deploy/examples/operator.yaml
|
||||
```
|
||||
|
||||
At this point the operator will start the admission controller Deployment automatically and the Webhook will start intercepting requests for Rook resources.
|
||||
@@ -36,6 +36,16 @@ lsblk -f
|
||||
|
||||
If the `FSTYPE` field is not empty, there is a filesystem on top of the corresponding device. In this example, you can use `vdb` for Ceph and can't use `vda` or its partitions.
|
||||
|
||||
## Admission Controller
|
||||
|
||||
Enabling the Rook admission controller is recommended to provide an additional level of validation that Rook is configured correctly with the custom resource (CR) settings. An admission controller intercepts requests to the Kubernetes API server prior to persistence of the object, but after the request is authenticated and authorized.
|
||||
|
||||
To deploy the Rook admission controllers, install the cert manager before Rook is installed:
|
||||
|
||||
```console
|
||||
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.7.1/cert-manager.yaml
|
||||
```.
|
||||
|
||||
## LVM package
|
||||
|
||||
Ceph OSDs have a dependency on LVM in the following scenarios:
|
||||
|
||||
@@ -481,6 +481,9 @@ rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["admissionregistration.k8s.io"]
|
||||
resources: ["validatingwebhookconfigurations"]
|
||||
verbs: ["create", "get"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
@@ -863,6 +866,14 @@ rules:
|
||||
- persistentvolumeclaims
|
||||
verbs:
|
||||
- delete
|
||||
- apiGroups:
|
||||
- cert-manager.io
|
||||
resources:
|
||||
- certificates
|
||||
- issuers
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
||||
@@ -18,6 +18,9 @@ rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["admissionregistration.k8s.io"]
|
||||
resources: ["validatingwebhookconfigurations"]
|
||||
verbs: ["create", "get", "delete"]
|
||||
---
|
||||
# The cluster role for managing all the cluster-specific resources in a namespace
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
||||
@@ -77,6 +77,8 @@ spec:
|
||||
value: "{{ .Values.disableDeviceHotplug }}"
|
||||
- name: ROOK_ENABLE_DISCOVERY_DAEMON
|
||||
value: "{{ .Values.enableDiscoveryDaemon }}"
|
||||
- name: ROOK_DISABLE_ADMISSION_CONTROLLER
|
||||
value: "{{ .Values.disableAdmissionController }}"
|
||||
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
|
||||
@@ -44,6 +44,15 @@ rules:
|
||||
- cronjobs
|
||||
verbs:
|
||||
- delete
|
||||
- apiGroups:
|
||||
- cert-manager.io
|
||||
resources:
|
||||
- certificates
|
||||
- issuers
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- delete
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
||||
@@ -28,7 +28,7 @@ nodeSelector: {}
|
||||
# For more info, see https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# disktype: ssd
|
||||
|
||||
# Tolerations for the rook-ceph-operator to allow it to run on nodes with particular taints
|
||||
# Tolerations for the rook-ceph-operator to allow it to run on nodes with particular taints
|
||||
tolerations: []
|
||||
|
||||
# Delay to use in node.kubernetes.io/unreachable toleration
|
||||
@@ -329,6 +329,8 @@ cephCommandsTimeoutSeconds: "15"
|
||||
# For more details see https://github.com/rook/rook/issues/2417
|
||||
enableSelinuxRelabeling: true
|
||||
|
||||
disableAdmissionController: false
|
||||
|
||||
# Writing to the hostPath is required for the Ceph mon and osd pods. Given the restricted permissions in OpenShift with SELinux,
|
||||
# the pod must be running privileged in order to write to the hostPath volume, this must be set to true then.
|
||||
hostpathRequiresPrivileged: false
|
||||
|
||||
@@ -568,6 +568,9 @@ rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["admissionregistration.k8s.io"]
|
||||
resources: ["validatingwebhookconfigurations"]
|
||||
verbs: ["create", "get", "delete"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
@@ -1047,6 +1050,15 @@ rules:
|
||||
- cronjobs
|
||||
verbs:
|
||||
- delete
|
||||
- apiGroups:
|
||||
- cert-manager.io
|
||||
resources:
|
||||
- certificates
|
||||
- issuers
|
||||
verbs:
|
||||
- get
|
||||
- create
|
||||
- delete
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
||||
@@ -461,6 +461,9 @@ spec:
|
||||
- name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS
|
||||
value: "5"
|
||||
|
||||
- name: ROOK_DISABLE_ADMISSION_CONTROLLER
|
||||
value: "false"
|
||||
|
||||
# The name of the node to pass with the downward API
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
|
||||
@@ -4,17 +4,18 @@ go 1.16
|
||||
|
||||
require (
|
||||
github.com/IBM/keyprotect-go-client v0.7.0
|
||||
github.com/aws/aws-sdk-go v1.37.19
|
||||
github.com/aws/aws-sdk-go v1.40.21
|
||||
github.com/banzaicloud/k8s-objectmatcher v1.1.0
|
||||
github.com/ceph/go-ceph v0.12.0
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
|
||||
github.com/go-ini/ini v1.51.1
|
||||
github.com/google/go-cmp v0.5.5
|
||||
github.com/google/uuid v1.1.2
|
||||
github.com/google/go-cmp v0.5.6
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/vault v1.8.5
|
||||
github.com/hashicorp/vault-plugin-secrets-kv v0.9.0
|
||||
github.com/hashicorp/vault/api v1.1.2-0.20210713235431-1fc8af4c041f
|
||||
github.com/hashicorp/vault/sdk v0.2.2-0.20211101151547-6654f4b913f9
|
||||
github.com/jetstack/cert-manager v1.6.2
|
||||
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.0
|
||||
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20220105185820-c1da9586e05b
|
||||
github.com/libopenstorage/secrets v0.0.0-20210709082113-dde442ea20ec
|
||||
@@ -24,12 +25,12 @@ require (
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.46.0
|
||||
github.com/prometheus-operator/prometheus-operator/pkg/client v0.46.0
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/spf13/cobra v1.2.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.171+incompatible // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
gopkg.in/ini.v1 v1.57.0
|
||||
gopkg.in/ini.v1 v1.62.0
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
k8s.io/api v0.22.2
|
||||
k8s.io/apimachinery v0.22.2
|
||||
|
||||
@@ -183,7 +183,7 @@ func (o *Operator) startCRDManager(context context.Context, mgrErrorCh chan erro
|
||||
}
|
||||
|
||||
// Add webhook if needed
|
||||
isPresent, err := isSecretPresent(context, o.context)
|
||||
isPresent, err := createWebhook(context, o.context)
|
||||
if err != nil {
|
||||
mgrErrorCh <- errors.Wrap(err, "failed to retrieve admission webhook secret")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
/*
|
||||
Copyright 2022 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package operator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
api "github.com/jetstack/cert-manager/pkg/apis/certmanager/v1"
|
||||
v1 "github.com/jetstack/cert-manager/pkg/apis/meta/v1"
|
||||
cs "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/typed/certmanager/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
admv1 "k8s.io/api/admissionregistration/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
issuerName = "selfsigned-issuer"
|
||||
certificateName = "rook-admission-controller-cert"
|
||||
webhookConfigName = "rook-ceph-webhook"
|
||||
)
|
||||
|
||||
func fetchorCreateIssuer(ctx context.Context, certMgrClient *cs.CertmanagerV1Client) (*api.Issuer, error) {
|
||||
logger.Infof("Fetching Issuer %s/%s.", namespace, issuerName)
|
||||
issuer, err := certMgrClient.Issuers(namespace).Get(ctx, issuerName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return createIssuers(ctx, certMgrClient)
|
||||
}
|
||||
|
||||
logger.Errorf("failed to get issuer %s. %v", issuerName, err)
|
||||
return issuer, err
|
||||
}
|
||||
|
||||
return issuer, nil
|
||||
}
|
||||
|
||||
func createIssuers(ctx context.Context, certMgrClient *cs.CertmanagerV1Client) (*api.Issuer, error) {
|
||||
logger.Infof("Creating Issuer %s/%s.", namespace, issuerName)
|
||||
issuer := &api.Issuer{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: issuerName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: api.IssuerSpec{
|
||||
IssuerConfig: api.IssuerConfig{
|
||||
SelfSigned: &api.SelfSignedIssuer{},
|
||||
},
|
||||
},
|
||||
}
|
||||
issuers, err := certMgrClient.Issuers(namespace).Create(ctx, issuer, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create issuer %s. %v", issuerName, err)
|
||||
}
|
||||
|
||||
sleepTime := 1
|
||||
attempts := 10
|
||||
|
||||
for i := 0; i < attempts; i++ {
|
||||
issuers, err = certMgrClient.Issuers(namespace).Get(ctx, issuerName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get issuer %s. %v", issuerName, err)
|
||||
return issuers, err
|
||||
}
|
||||
|
||||
if len(issuers.Status.Conditions) != 0 {
|
||||
if issuers.Status.Conditions[0].Reason == "IsReady" && issuers.Status.Conditions[0].Status == "True" {
|
||||
return issuers, nil
|
||||
}
|
||||
logger.Debugf("webhook config %q status=%+v", issuerName, issuers.Status.Conditions[0].Status)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
logger.Infof("waiting for webhook config %q to be in ready status", issuerName)
|
||||
}
|
||||
return issuers, nil
|
||||
}
|
||||
|
||||
func fetchorCreateCertificate(ctx context.Context, certMgrClient *cs.CertmanagerV1Client, issuer *api.Issuer) error {
|
||||
logger.Infof("Fetching certificate %s/%s.", namespace, certificateName)
|
||||
_, err := certMgrClient.Certificates(namespace).Get(ctx, certificateName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return createCertificate(ctx, certMgrClient, issuer)
|
||||
}
|
||||
logger.Errorf("failed to get certificate %s. %v", certificateName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createCertificate(ctx context.Context, certMgrClient *cs.CertmanagerV1Client, issuer *api.Issuer) error {
|
||||
logger.Infof("Creating certificate %s/%s.", namespace, certificateName)
|
||||
certificate := &api.Certificate{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: certificateName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: api.CertificateSpec{
|
||||
DNSNames: []string{
|
||||
admissionControllerAppName,
|
||||
fmt.Sprintf("%s.%s.svc", admissionControllerAppName, namespace),
|
||||
fmt.Sprintf("%s.%s.svc.cluster.local", admissionControllerAppName, namespace)},
|
||||
IssuerRef: v1.ObjectReference{
|
||||
Name: issuer.Name,
|
||||
Kind: "Issuer",
|
||||
},
|
||||
SecretName: admissionControllerAppName,
|
||||
},
|
||||
}
|
||||
|
||||
_, err := certMgrClient.Certificates(namespace).Create(ctx, certificate, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create certificate %s. %v", certificateName, err)
|
||||
}
|
||||
sleepTime := 1
|
||||
attempts := 10
|
||||
|
||||
for i := 0; i < attempts; i++ {
|
||||
cert, err := certMgrClient.Certificates(namespace).Get(ctx, certificateName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Errorf("failed to get certificate %s. %v", certificateName, err)
|
||||
}
|
||||
if len(cert.Status.Conditions) != 0 {
|
||||
if cert.Status.Conditions[0].Reason == "Ready" && cert.Status.Conditions[0].Status == "True" {
|
||||
return nil
|
||||
}
|
||||
logger.Debugf("webhook config %q status=%+v", certificateName, cert.Status.Conditions[0].Status)
|
||||
}
|
||||
|
||||
logger.Infof("waiting for webhook config %q to be in ready status", certificateName)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchValidatingWebhookConfig(ctx context.Context, clusterdContext *clusterd.Context) error {
|
||||
|
||||
logger.Infof("Fetching webhook %s/%s.", namespace, webhookConfigName)
|
||||
_, err := clusterdContext.Clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, webhookConfigName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return createValidatingWebhookConfig(ctx, clusterdContext)
|
||||
}
|
||||
logger.Errorf("failed to get webhook %s. %v", webhookConfigName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createValidatingWebhookConfig(ctx context.Context, clusterdContext *clusterd.Context) error {
|
||||
sideEffects := admv1.SideEffectClassNone
|
||||
var timeout int32 = 5
|
||||
serviceCephClusterPath := "/validate-ceph-rook-io-v1-cephcluster"
|
||||
serviceCephBlockPoolPath := "/validate-ceph-rook-io-v1-cephblockpool"
|
||||
serviceCephObjectStorePath := "/validate-ceph-rook-io-v1-cephobjectstore"
|
||||
|
||||
logger.Infof("Creating webhook %s/%s.", namespace, webhookConfigName)
|
||||
|
||||
validatingWebhook := &admv1.ValidatingWebhookConfiguration{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: webhookConfigName,
|
||||
Namespace: namespace,
|
||||
Annotations: map[string]string{
|
||||
"cert-manager.io/inject-ca-from": fmt.Sprintf("%s/rook-admission-controller-cert", namespace),
|
||||
},
|
||||
},
|
||||
Webhooks: []admv1.ValidatingWebhook{
|
||||
{
|
||||
Name: fmt.Sprintf("cephcluster-wh-%s-%s.rook.io", admissionControllerAppName, namespace),
|
||||
Rules: []admv1.RuleWithOperations{
|
||||
{
|
||||
Rule: admv1.Rule{
|
||||
APIGroups: []string{"ceph.rook.io"},
|
||||
APIVersions: []string{"v1"},
|
||||
Resources: []string{"cephclusters"},
|
||||
},
|
||||
Operations: []admv1.OperationType{
|
||||
admv1.Update,
|
||||
admv1.Create,
|
||||
admv1.Delete,
|
||||
},
|
||||
},
|
||||
},
|
||||
ClientConfig: admv1.WebhookClientConfig{
|
||||
Service: &admv1.ServiceReference{
|
||||
Name: admissionControllerAppName,
|
||||
Namespace: namespace,
|
||||
Path: &serviceCephClusterPath,
|
||||
},
|
||||
},
|
||||
AdmissionReviewVersions: []string{"v1", "v1beta1"},
|
||||
SideEffects: &sideEffects,
|
||||
TimeoutSeconds: &timeout,
|
||||
},
|
||||
{
|
||||
Name: fmt.Sprintf("cephblockpool-wh-%s-%s.rook.io", admissionControllerAppName, namespace),
|
||||
Rules: []admv1.RuleWithOperations{
|
||||
{
|
||||
Rule: admv1.Rule{
|
||||
APIGroups: []string{"ceph.rook.io"},
|
||||
APIVersions: []string{"v1"},
|
||||
Resources: []string{"cephblockpools"},
|
||||
},
|
||||
Operations: []admv1.OperationType{
|
||||
admv1.Update,
|
||||
admv1.Create,
|
||||
admv1.Delete,
|
||||
},
|
||||
},
|
||||
},
|
||||
ClientConfig: admv1.WebhookClientConfig{
|
||||
Service: &admv1.ServiceReference{
|
||||
Name: admissionControllerAppName,
|
||||
Namespace: namespace,
|
||||
Path: &serviceCephBlockPoolPath,
|
||||
},
|
||||
},
|
||||
AdmissionReviewVersions: []string{"v1", "v1beta1"},
|
||||
SideEffects: &sideEffects,
|
||||
TimeoutSeconds: &timeout,
|
||||
},
|
||||
{
|
||||
Name: fmt.Sprintf("cephobjectstore-wh-%s-%s.rook.io", admissionControllerAppName, namespace),
|
||||
Rules: []admv1.RuleWithOperations{
|
||||
{
|
||||
Rule: admv1.Rule{
|
||||
APIGroups: []string{"ceph.rook.io"},
|
||||
APIVersions: []string{"v1"},
|
||||
Resources: []string{"cephobjectstores"},
|
||||
},
|
||||
Operations: []admv1.OperationType{
|
||||
admv1.Update,
|
||||
admv1.Create,
|
||||
admv1.Delete,
|
||||
},
|
||||
},
|
||||
},
|
||||
ClientConfig: admv1.WebhookClientConfig{
|
||||
Service: &admv1.ServiceReference{
|
||||
Name: admissionControllerAppName,
|
||||
Namespace: namespace,
|
||||
Path: &serviceCephObjectStorePath,
|
||||
},
|
||||
},
|
||||
AdmissionReviewVersions: []string{"v1", "v1beta1"},
|
||||
SideEffects: &sideEffects,
|
||||
TimeoutSeconds: &timeout,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := clusterdContext.Clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, validatingWebhook, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
logger.Errorf("failed to create validating webhook %s. %v", webhookConfigName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteIssuerAndCetificate(ctx context.Context, certMgrClient *cs.CertmanagerV1Client, clusterdContext *clusterd.Context) error {
|
||||
|
||||
err := clusterdContext.Clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, webhookConfigName, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
logger.Errorf("failed to delete validating webhook %s. %v", webhookConfigName, err)
|
||||
return err
|
||||
}
|
||||
err = certMgrClient.Certificates(namespace).Delete(ctx, certificateName, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
logger.Errorf("failed to delete issuer %s. %v", issuerName, err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = certMgrClient.Issuers(namespace).Delete(ctx, issuerName, metav1.DeleteOptions{})
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
logger.Errorf("failed to delete certificate %s. %v", certificateName, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -18,11 +18,11 @@ package operator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
cs "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/typed/certmanager/v1"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
@@ -36,29 +36,71 @@ import (
|
||||
const (
|
||||
admissionControllerAppName = "rook-ceph-admission-controller"
|
||||
tlsPort int32 = 443
|
||||
webhookEnv = "ROOK_DISABLE_ADMISSION_CONTROLLER"
|
||||
)
|
||||
|
||||
var (
|
||||
namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
|
||||
certManagerWebhookName = "cert-manager-webhook"
|
||||
)
|
||||
|
||||
func isSecretPresent(ctx context.Context, context *clusterd.Context) (bool, error) {
|
||||
func createWebhook(ctx context.Context, context *clusterd.Context) (bool, error) {
|
||||
certMgrClient, err := cs.NewForConfig(context.KubeConfig)
|
||||
if err != nil {
|
||||
logger.Errorf("failed to set config for cert-manager. %v", err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if os.Getenv(webhookEnv) == "true" {
|
||||
logger.Info("delete Issuer and Certificate since secret is not found")
|
||||
if err = deleteIssuerAndCetificate(ctx, certMgrClient, context); err != nil {
|
||||
logger.Errorf("failed to delete issuer or certificate. %v", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
logger.Infof("Fetching webhook %s to see if cert-manager is installed.", certManagerWebhookName)
|
||||
_, err = context.Clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, certManagerWebhookName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
logger.Info("failed to get cert manager")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
issuer, err := fetchorCreateIssuer(ctx, certMgrClient)
|
||||
if err != nil {
|
||||
logger.Errorf("issuer creation failed %v", err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
err = fetchorCreateCertificate(ctx, certMgrClient, issuer)
|
||||
if err != nil {
|
||||
logger.Errorf("certificate creation failed %v", err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
logger.Infof("looking for admission webhook secret %q", admissionControllerAppName)
|
||||
s, err := context.Clientset.CoreV1().Secrets(namespace).Get(ctx, admissionControllerAppName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
// If secret is not found. All good ! Proceed with rook without admission controllers
|
||||
if apierrors.IsNotFound(err) {
|
||||
// If secret is not found. All good ! Proceed with rook without admission controllers
|
||||
logger.Info("delete Issuer and Certificate since secret is not found")
|
||||
if err = deleteIssuerAndCetificate(ctx, certMgrClient, context); err != nil {
|
||||
logger.Infof("could not delete issuer or certificate. %v", err)
|
||||
}
|
||||
logger.Infof("admission webhook secret %q not found. proceeding without the admission controller", admissionControllerAppName)
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Search for any previous admission controller deployment and if so removing it
|
||||
logger.Debug("searching for old admission controller deployment")
|
||||
removeOldAdmissionControllerDeployment(ctx, context)
|
||||
|
||||
logger.Infof("admission webhook secret %q found", admissionControllerAppName)
|
||||
|
||||
err = fetchValidatingWebhookConfig(ctx, context)
|
||||
if err != nil {
|
||||
logger.Errorf("webhook creation failed %v", err)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for k, data := range s.Data {
|
||||
filePath := path.Join(certDir, k)
|
||||
// We must use 0600 mode so that the files can be overridden each time the Secret is fetched
|
||||
@@ -100,25 +142,3 @@ func createWebhookService(ctx context.Context, context *clusterd.Context) error
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeOldAdmissionControllerDeployment(ctx context.Context, context *clusterd.Context) {
|
||||
opts := metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", admissionControllerAppName)}
|
||||
d, err := context.Clientset.AppsV1().Deployments(namespace).List(ctx, opts)
|
||||
if err != nil {
|
||||
logger.Warningf("failed to get old admission controller deployment. %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(d.Items) > 0 {
|
||||
for _, deploy := range d.Items {
|
||||
var gracePeriod int64
|
||||
propagation := metav1.DeletePropagationForeground
|
||||
deleteOpts := metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
|
||||
if err = context.Clientset.AppsV1().Deployments(namespace).Delete(ctx, deploy.Name, deleteOpts); err != nil {
|
||||
logger.Warningf("failed to delete admission controller deployment %q. %v", deploy.Name, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
logger.Info("successfully removed old admission controller deployment. please remove old service account, cluster role and bindings manually")
|
||||
}
|
||||
}
|
||||
@@ -190,7 +190,7 @@ func (h *CephInstaller) startAdmissionController() error {
|
||||
if err != nil {
|
||||
return errors.Errorf("failed to find user home directory. %v", err)
|
||||
}
|
||||
scriptPath := path.Join(rootPath, "tests/scripts/deploy_admission_controller_test.sh")
|
||||
scriptPath := path.Join(rootPath, "tests/scripts/deploy_cert_manager.sh")
|
||||
err = h.k8shelper.MakeContext().Executor.ExecuteCommandWithEnv([]string{fmt.Sprintf("NAMESPACE=%s", h.settings.OperatorNamespace), fmt.Sprintf("HOME=%s", userHome)}, "bash", scriptPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -494,7 +494,6 @@ func (h *CephInstaller) GetNodeHostnames() ([]string, error) {
|
||||
}
|
||||
|
||||
func (h *CephInstaller) installRookOperator() (bool, error) {
|
||||
ctx := context.TODO()
|
||||
var err error
|
||||
|
||||
h.k8shelper.CreateAnonSystemClusterBinding()
|
||||
@@ -520,13 +519,13 @@ func (h *CephInstaller) installRookOperator() (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
discovery, err := h.k8shelper.Clientset.AppsV1().DaemonSets(h.settings.OperatorNamespace).Get(ctx, "rook-discover", metav1.GetOptions{})
|
||||
if h.settings.EnableDiscovery {
|
||||
assert.NoError(h.T(), err)
|
||||
assert.NotNil(h.T(), discovery)
|
||||
} else {
|
||||
assert.Error(h.T(), err)
|
||||
assert.True(h.T(), kerrors.IsNotFound(err))
|
||||
if h.k8shelper.IsPodInExpectedState("rook-discover", h.settings.OperatorNamespace, "Running") {
|
||||
assert.NoError(h.T(), err)
|
||||
} else {
|
||||
assert.Error(h.T(), err)
|
||||
assert.True(h.T(), kerrors.IsNotFound(err))
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
|
||||
@@ -62,6 +62,10 @@ func (s *TestCephSettings) ApplyEnvVars() {
|
||||
if os.Getenv("SKIP_CLEANUP_POLICY") == "false" {
|
||||
s.SkipCleanupPolicy = false
|
||||
}
|
||||
err := os.Setenv("ROOK_DISABLE_ADMISSION_CONTROLLER", "false")
|
||||
if err != nil {
|
||||
logger.Errorf("failed to set ROOK_DISABLE_ADMISSION_CONTROLLER. %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestCephSettings) readManifest(filename string) string {
|
||||
|
||||
@@ -66,7 +66,7 @@ func (h *HelmSuite) SetupSuite() {
|
||||
UsePVC: false,
|
||||
Mons: 1,
|
||||
SkipOSDCreation: false,
|
||||
EnableAdmissionController: false,
|
||||
EnableAdmissionController: true,
|
||||
EnableDiscovery: true,
|
||||
ChangeHostName: true,
|
||||
ConnectionsEncrypted: true,
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
BASE_DIR=$(cd "$(dirname "$0")" && pwd)
|
||||
|
||||
if [ -z "$KUBECONFIG" ];then
|
||||
if [ -f "$HOME/.kube/config" ]; then
|
||||
export KUBECONFIG="$HOME/.kube/config"
|
||||
else
|
||||
sudo cp /etc/kubernetes/admin.conf "$HOME"/
|
||||
sudo chown "$(id -u)":"$(id -g)" "$HOME"/admin.conf
|
||||
export KUBECONFIG=$HOME/admin.conf
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -f "${BASE_DIR}"/deploy_admission_controller.sh ];then
|
||||
bash "${BASE_DIR}"/deploy_admission_controller.sh
|
||||
else
|
||||
echo "${BASE_DIR}/deploy_admission_controller.sh not found !"
|
||||
fi
|
||||
Executable → Regular
+1
-21
@@ -1,17 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy_admission_controller.sh
|
||||
# Sets up the environment for the admission controller webhook in the active cluster.
|
||||
# Sets up the cert-manager for CI
|
||||
set -eEo pipefail
|
||||
|
||||
function cleanup() {
|
||||
set +e
|
||||
kubectl -n rook-ceph delete validatingwebhookconfigurations "$WEBHOOK_CONFIG_NAME"
|
||||
kubectl -n rook-ceph delete certificate rook-admission-controller-cert
|
||||
kubectl -n rook-ceph delete issuers selfsigned-issuer
|
||||
kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/"$CERT_VERSION"/cert-manager.yaml
|
||||
set -e
|
||||
}
|
||||
|
||||
function error_log() {
|
||||
set +e -x
|
||||
kubectl -n rook-ceph get issuer
|
||||
@@ -23,10 +13,8 @@ function error_log() {
|
||||
kubectl -n cert-manager logs deploy/cert-manager-webhook --tail=10
|
||||
kubectl -n cert-manager logs deploy/cert-manager-cainjector --tail=10
|
||||
set -e +x
|
||||
cleanup
|
||||
}
|
||||
|
||||
trap cleanup SIGINT
|
||||
trap error_log ERR
|
||||
|
||||
# Minimum 1.16.0 kubernetes version is required to start the admission controller
|
||||
@@ -80,11 +68,3 @@ timeout 25 bash <<-'EOF'
|
||||
EOF
|
||||
|
||||
echo "Successfully deployed cert-manager"
|
||||
|
||||
echo "Creating Issuer, Certificate and Deploying webhook config"
|
||||
echo "Deploying webhook config"
|
||||
|
||||
envsubst < "${BASE_DIR}"/webhook-config.yaml |
|
||||
kubectl create -f -
|
||||
|
||||
echo "Webhook deployed! Please start the rook operator to create the service and admission controller pods"
|
||||
@@ -1,72 +0,0 @@
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: selfsigned-issuer
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
selfSigned: {}
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: rook-admission-controller-cert
|
||||
namespace: ${NAMESPACE}
|
||||
spec:
|
||||
dnsNames:
|
||||
- ${SERVICE_NAME}
|
||||
- ${SERVICE_NAME}.${NAMESPACE}.svc
|
||||
- ${SERVICE_NAME}.${NAMESPACE}.svc.cluster.local
|
||||
issuerRef:
|
||||
kind: Issuer
|
||||
name: selfsigned-issuer
|
||||
secretName: rook-ceph-admission-controller
|
||||
---
|
||||
apiVersion: admissionregistration.k8s.io/v1
|
||||
kind: ValidatingWebhookConfiguration
|
||||
metadata:
|
||||
name: ${WEBHOOK_CONFIG_NAME}
|
||||
annotations:
|
||||
cert-manager.io/inject-ca-from: ${NAMESPACE}/rook-admission-controller-cert
|
||||
webhooks:
|
||||
- name: cephcluster-wh-${SERVICE_NAME}-${NAMESPACE}.rook.io
|
||||
rules:
|
||||
- apiGroups: ["ceph.rook.io"]
|
||||
apiVersions: ["v1"]
|
||||
operations: ["CREATE", "UPDATE", "DELETE"]
|
||||
resources: ["cephclusters"]
|
||||
clientConfig:
|
||||
service:
|
||||
name: ${SERVICE_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
path: /validate-ceph-rook-io-v1-cephcluster
|
||||
admissionReviewVersions: ["v1", "v1beta1"] # API server will try to use first version in the list which it supports.
|
||||
sideEffects: None
|
||||
timeoutSeconds: 5
|
||||
- name: cephblockpool-wh-${SERVICE_NAME}-${NAMESPACE}.rook.io
|
||||
rules:
|
||||
- apiGroups: ["ceph.rook.io"]
|
||||
apiVersions: ["v1"]
|
||||
operations: ["CREATE", "UPDATE", "DELETE"]
|
||||
resources: ["cephblockpools"]
|
||||
clientConfig:
|
||||
service:
|
||||
name: ${SERVICE_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
path: /validate-ceph-rook-io-v1-cephblockpool
|
||||
admissionReviewVersions: ["v1", "v1beta1"] # API server will try to use first version in the list which it supports.
|
||||
sideEffects: None
|
||||
timeoutSeconds: 5
|
||||
- name: cephobjectstore-wh-${SERVICE_NAME}-${NAMESPACE}.rook.io
|
||||
rules:
|
||||
- apiGroups: ["ceph.rook.io"]
|
||||
apiVersions: ["v1"]
|
||||
operations: ["CREATE", "UPDATE", "DELETE"]
|
||||
resources: ["cephobjectstores"]
|
||||
clientConfig:
|
||||
service:
|
||||
name: ${SERVICE_NAME}
|
||||
namespace: ${NAMESPACE}
|
||||
path: /validate-ceph-rook-io-v1-cephobjectstore
|
||||
admissionReviewVersions: ["v1", "v1beta1"] # API server will try to use first version in the list which it supports.
|
||||
sideEffects: None
|
||||
timeoutSeconds: 5
|
||||
Reference in new issue
Block a user