From 2ef83c4027433236fa14c12e479ca8ae6b4ffe31 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Thu, 27 Jun 2019 13:58:04 -0600 Subject: [PATCH] ceph: use cmd-reporter for ceph version job Use cmd-reporter to detect Ceph version. The job to detect Ceph version needs to run in a security account context capable of getting/creating ConfigMaps, so a new security account, role, and role binding are created for the cmd-runner jobs. In the future, the Ceph mgr may create jobs in this service account for running `ceph-volume` on nodes, and this also keeps Rook operators controlling multiple Ceph clusters independent. Fixes #2686 - ceph version job not working correctly with graylog Signed-off-by: Blaine Gardner --- Documentation/ceph-psp.md | 16 +++- cluster/charts/rook-ceph/templates/role.yaml | 18 +++++ .../rook-ceph/templates/rolebinding.yaml | 14 ++++ .../rook-ceph/templates/serviceaccount.yaml | 9 +++ cluster/examples/kubernetes/ceph/common.yaml | 54 +++++++++++-- pkg/operator/ceph/cluster/cluster.go | 76 ++++++------------- pkg/operator/ceph/cluster/controller.go | 4 +- pkg/operator/ceph/cluster/migration_test.go | 2 +- pkg/operator/ceph/operator.go | 4 +- tests/framework/installer/ceph_manifests.go | 40 ++++++++++ tests/integration/upgrade_test.go | 46 ++++++++++- 11 files changed, 219 insertions(+), 64 deletions(-) diff --git a/Documentation/ceph-psp.md b/Documentation/ceph-psp.md index 949412e0f..d3226ceac 100644 --- a/Documentation/ceph-psp.md +++ b/Documentation/ceph-psp.md @@ -127,5 +127,19 @@ subjects: - kind: ServiceAccount name: rook-ceph-mgr namespace: rook-ceph - +--- +# Allow the rook-ceph-cmd-reporter serviceAccount to use the privileged PSP +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rook-ceph-cmd-reporter-psp + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:rook +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph ``` diff --git a/cluster/charts/rook-ceph/templates/role.yaml b/cluster/charts/rook-ceph/templates/role.yaml index 92cb1e76b..528b52a07 100644 --- a/cluster/charts/rook-ceph/templates/role.yaml +++ b/cluster/charts/rook-ceph/templates/role.yaml @@ -76,4 +76,22 @@ rules: - "*" verbs: - "*" +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete {{- end }} diff --git a/cluster/charts/rook-ceph/templates/rolebinding.yaml b/cluster/charts/rook-ceph/templates/rolebinding.yaml index bbaa6a084..c9b7bc2d3 100644 --- a/cluster/charts/rook-ceph/templates/rolebinding.yaml +++ b/cluster/charts/rook-ceph/templates/rolebinding.yaml @@ -76,4 +76,18 @@ subjects: - kind: ServiceAccount name: rook-ceph-mgr namespace: {{ .Release.Namespace }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: {{ .Release.Namespace }} {{- end }} diff --git a/cluster/charts/rook-ceph/templates/serviceaccount.yaml b/cluster/charts/rook-ceph/templates/serviceaccount.yaml index fbe3d3347..2f04334bd 100644 --- a/cluster/charts/rook-ceph/templates/serviceaccount.yaml +++ b/cluster/charts/rook-ceph/templates/serviceaccount.yaml @@ -27,3 +27,12 @@ metadata: operator: rook storage-backend: ceph chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + labels: + operator: rook + storage-backend: ceph + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" diff --git a/cluster/examples/kubernetes/ceph/common.yaml b/cluster/examples/kubernetes/ceph/common.yaml index c625d31e6..d04e41ba7 100644 --- a/cluster/examples/kubernetes/ceph/common.yaml +++ b/cluster/examples/kubernetes/ceph/common.yaml @@ -494,8 +494,16 @@ metadata: name: rook-ceph-mgr namespace: rook-ceph # OLM: END SERVICE ACCOUNT MGR -# OLM: BEGIN CLUSTER ROLE --- +# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph +# OLM: END CMD REPORTER SERVICE ACCOUNT +--- +# OLM: BEGIN CLUSTER ROLE kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -569,10 +577,30 @@ rules: verbs: - "*" # OLM: END CLUSTER ROLE -# -# OLM: BEGIN CLUSTER ROLEBINDING --- - # Allow the operator to create resources in this cluster's namespace +# OLM: BEGIN CMD REPORTER ROLE +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +# OLM: END CMD REPORTER ROLE +--- +# OLM: BEGIN CLUSTER ROLEBINDING +# Allow the operator to create resources in this cluster's namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -645,5 +673,21 @@ subjects: - kind: ServiceAccount name: rook-ceph-mgr namespace: rook-ceph +# OLM: END CLUSTER ROLEBINDING +--- +# OLM: BEGIN CMD REPORTER ROLEBINDING +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph +# OLM: END CMD REPORTER ROLEBINDING --- -# OLM: END CLUSTER ROLEBINDING \ No newline at end of file diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index c49ea5bc0..df9878655 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -24,6 +24,8 @@ import ( "sync" "time" + "github.com/rook/rook/pkg/operator/k8sutil/cmdreporter" + "github.com/google/go-cmp/cmp" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2" @@ -36,7 +38,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/cluster/rbd" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" - batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -83,64 +84,35 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster { } } -func (c *cluster) detectCephVersion(image string, timeout time.Duration) (*cephver.CephVersion, error) { - // get the major ceph version by running "ceph --version" in the ceph image - podSpec := v1.PodSpec{ - Containers: []v1.Container{ - { - Command: []string{"ceph"}, - Args: []string{"--version"}, - Name: "version", - Image: image, - }, - }, - RestartPolicy: v1.RestartPolicyOnFailure, - } - - // apply "mon" placement - cephv1.GetMonPlacement(c.Spec.Placement).ApplyToPodSpec(&podSpec) - - job := &batch.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: detectVersionName, - Namespace: c.Namespace, - }, - Spec: batch.JobSpec{ - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "job": detectVersionName, - }, - }, - Spec: podSpec, - }, - }, - } - k8sutil.AddRookVersionLabelToJob(job) - k8sutil.SetOwnerRef(&job.ObjectMeta, &c.ownerRef) - - // run the job to detect the version - if err := k8sutil.RunReplaceableJob(c.context.Clientset, job, true); err != nil { - return nil, fmt.Errorf("failed to start version job. %+v", err) - } - - if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil { - return nil, fmt.Errorf("failed to complete version job. %+v", err) - } - - log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName) +func (c *cluster) detectCephVersion(rookImage, cephImage string, timeout time.Duration) (*cephver.CephVersion, error) { + versionReporter, err := cmdreporter.New( + c.context.Clientset, &c.ownerRef, + detectVersionName, detectVersionName, c.Namespace, + []string{"ceph"}, []string{"--version"}, + rookImage, cephImage, + ) if err != nil { - return nil, fmt.Errorf("failed to get version job log to detect version. %+v", err) + return nil, fmt.Errorf("failed to set up ceph version job. %+v", err) } - version, err := cephver.ExtractCephVersion(log) + job := versionReporter.Job() + job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter" + + stdout, stderr, retcode, err := versionReporter.Run(timeout) + if err != nil { + return nil, fmt.Errorf("failed to complete ceph version job. %+v", err) + } + if retcode != 0 { + return nil, fmt.Errorf(`ceph version job returned failure with retcode %d. + stdout: %s + stderr: %s`, retcode, stdout, stderr) + } + + version, err := cephver.ExtractCephVersion(stdout) if err != nil { return nil, fmt.Errorf("failed to extract ceph version. %+v", err) } - // delete the job since we're done with it - k8sutil.DeleteBatchJob(c.context.Clientset, c.Namespace, job.Name, false) - logger.Infof("Detected ceph image version: %s", version) return version, nil } diff --git a/pkg/operator/ceph/cluster/controller.go b/pkg/operator/ceph/cluster/controller.go index 4da8ab078..637db4308 100644 --- a/pkg/operator/ceph/cluster/controller.go +++ b/pkg/operator/ceph/cluster/controller.go @@ -283,7 +283,7 @@ func (c *ClusterController) onAdd(obj interface{}) { // Start the Rook cluster components. Retry several times in case of failure. validOrchestration := true err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) { - cephVersion, err := cluster.detectCephVersion(cluster.Spec.CephVersion.Image, 15*time.Minute) + cephVersion, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, 15*time.Minute) if err != nil { logger.Errorf("unknown ceph major version. %+v", err) return false, nil @@ -496,7 +496,7 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) { // if the image changed, we need to detect the new image version if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image { logger.Infof("the ceph version changed. detecting the new image version...") - version, err := cluster.detectCephVersion(newClust.Spec.CephVersion.Image, 15*time.Minute) + version, err := cluster.detectCephVersion(c.rookImage, newClust.Spec.CephVersion.Image, 15*time.Minute) if err != nil { logger.Errorf("unknown ceph major version. %+v", err) return diff --git a/pkg/operator/ceph/cluster/migration_test.go b/pkg/operator/ceph/cluster/migration_test.go index 3d9194f59..e4f16df03 100644 --- a/pkg/operator/ceph/cluster/migration_test.go +++ b/pkg/operator/ceph/cluster/migration_test.go @@ -28,7 +28,7 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment" testop "github.com/rook/rook/pkg/operator/test" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/operator/ceph/operator.go b/pkg/operator/ceph/operator.go index a11c443c0..fd90589ff 100644 --- a/pkg/operator/ceph/operator.go +++ b/pkg/operator/ceph/operator.go @@ -33,12 +33,12 @@ import ( "github.com/rook/rook/pkg/operator/ceph/csi" "github.com/rook/rook/pkg/operator/ceph/file" "github.com/rook/rook/pkg/operator/ceph/object" - "github.com/rook/rook/pkg/operator/ceph/object/user" + objectuser "github.com/rook/rook/pkg/operator/ceph/object/user" "github.com/rook/rook/pkg/operator/ceph/pool" "github.com/rook/rook/pkg/operator/ceph/provisioner" "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/sig-storage-lib-external-provisioner/controller" ) diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 774520c41..bde0e1a38 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -964,6 +964,12 @@ metadata: name: rook-ceph-mgr namespace: ` + namespace + ` --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +--- kind: Role apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: @@ -1008,6 +1014,25 @@ rules: verbs: - "*" --- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- # Allow the operator to create resources in this cluster's namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -1067,6 +1092,21 @@ subjects: - kind: ServiceAccount name: rook-ceph-mgr namespace: ` + namespace + ` +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +--- ` } diff --git a/tests/integration/upgrade_test.go b/tests/integration/upgrade_test.go index e7381359d..541b59eb5 100644 --- a/tests/integration/upgrade_test.go +++ b/tests/integration/upgrade_test.go @@ -152,5 +152,49 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { // Update the clusterroles that have been modified in master from the previous release func (s *UpgradeSuite) updateClusterRoles() error { logger.Infof("Placeholder: create the new resources that have been added since 1.0") - return nil + namespace := s.namespace + newResources := ` +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: ` + namespace + ` +--- +` + logger.Infof("creating the new resources that have been added since 1.0") + return s.k8sh.ResourceOperation("create", newResources) }