ceph: use cmd-reporter for ceph version job

Use cmd-reporter to detect Ceph version.

The job to detect Ceph version needs to run in a security account context
capable of getting/creating ConfigMaps, so a new security account, role,
and role binding are created for the cmd-runner jobs. In the future, the
Ceph mgr may create jobs in this service account for running
`ceph-volume` on nodes, and this also keeps Rook operators controlling
multiple Ceph clusters independent.

Fixes #2686 - ceph version job not working correctly with graylog

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>
This commit is contained in:
Blaine Gardner committed 2019-07-05 15:37:14 -06:00
1 parent 0a58d3dfdd
commit 2ef83c4027
11 files changed
+219 -64

No files matched your search

+15 -1
View File
@@ -127,5 +127,19 @@ subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: rook-ceph
---
# Allow the rook-ceph-cmd-reporter serviceAccount to use the privileged PSP
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rook-ceph-cmd-reporter-psp
namespace: rook-ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: psp:rook
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: rook-ceph
```
@@ -76,4 +76,22 @@ rules:
- "*"
verbs:
- "*"
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
{{- end }}
@@ -76,4 +76,18 @@ subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: {{ .Release.Namespace }}
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: {{ .Release.Namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: {{ .Release.Namespace }}
{{- end }}
@@ -27,3 +27,12 @@ metadata:
operator: rook
storage-backend: ceph
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
labels:
operator: rook
storage-backend: ceph
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+49 -5
View File
@@ -494,8 +494,16 @@ metadata:
name: rook-ceph-mgr
namespace: rook-ceph
# OLM: END SERVICE ACCOUNT MGR
# OLM: BEGIN CLUSTER ROLE
---
# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
namespace: rook-ceph
# OLM: END CMD REPORTER SERVICE ACCOUNT
---
# OLM: BEGIN CLUSTER ROLE
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
@@ -569,10 +577,30 @@ rules:
verbs:
- "*"
# OLM: END CLUSTER ROLE
#
# OLM: BEGIN CLUSTER ROLEBINDING
---
# Allow the operator to create resources in this cluster's namespace
# OLM: BEGIN CMD REPORTER ROLE
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: rook-ceph
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
# OLM: END CMD REPORTER ROLE
---
# OLM: BEGIN CLUSTER ROLEBINDING
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
@@ -645,5 +673,21 @@ subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: rook-ceph
# OLM: END CLUSTER ROLEBINDING
---
# OLM: BEGIN CMD REPORTER ROLEBINDING
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: rook-ceph
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: rook-ceph
# OLM: END CMD REPORTER ROLEBINDING
---
# OLM: END CLUSTER ROLEBINDING
+24 -52
View File
@@ -24,6 +24,8 @@ import (
"sync"
"time"
"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
"github.com/google/go-cmp/cmp"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
@@ -36,7 +38,6 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"github.com/rook/rook/pkg/operator/k8sutil"
batch "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
@@ -83,64 +84,35 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
}
}
func (c *cluster) detectCephVersion(image string, timeout time.Duration) (*cephver.CephVersion, error) {
// get the major ceph version by running "ceph --version" in the ceph image
podSpec := v1.PodSpec{
Containers: []v1.Container{
{
Command: []string{"ceph"},
Args: []string{"--version"},
Name: "version",
Image: image,
},
},
RestartPolicy: v1.RestartPolicyOnFailure,
}
// apply "mon" placement
cephv1.GetMonPlacement(c.Spec.Placement).ApplyToPodSpec(&podSpec)
job := &batch.Job{
ObjectMeta: metav1.ObjectMeta{
Name: detectVersionName,
Namespace: c.Namespace,
},
Spec: batch.JobSpec{
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"job": detectVersionName,
},
},
Spec: podSpec,
},
},
}
k8sutil.AddRookVersionLabelToJob(job)
k8sutil.SetOwnerRef(&job.ObjectMeta, &c.ownerRef)
// run the job to detect the version
if err := k8sutil.RunReplaceableJob(c.context.Clientset, job, true); err != nil {
return nil, fmt.Errorf("failed to start version job. %+v", err)
}
if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil {
return nil, fmt.Errorf("failed to complete version job. %+v", err)
}
log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName)
func (c *cluster) detectCephVersion(rookImage, cephImage string, timeout time.Duration) (*cephver.CephVersion, error) {
versionReporter, err := cmdreporter.New(
c.context.Clientset, &c.ownerRef,
detectVersionName, detectVersionName, c.Namespace,
[]string{"ceph"}, []string{"--version"},
rookImage, cephImage,
)
if err != nil {
return nil, fmt.Errorf("failed to get version job log to detect version. %+v", err)
return nil, fmt.Errorf("failed to set up ceph version job. %+v", err)
}
version, err := cephver.ExtractCephVersion(log)
job := versionReporter.Job()
job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter"
stdout, stderr, retcode, err := versionReporter.Run(timeout)
if err != nil {
return nil, fmt.Errorf("failed to complete ceph version job. %+v", err)
}
if retcode != 0 {
return nil, fmt.Errorf(`ceph version job returned failure with retcode %d.
stdout: %s
stderr: %s`, retcode, stdout, stderr)
}
version, err := cephver.ExtractCephVersion(stdout)
if err != nil {
return nil, fmt.Errorf("failed to extract ceph version. %+v", err)
}
// delete the job since we're done with it
k8sutil.DeleteBatchJob(c.context.Clientset, c.Namespace, job.Name, false)
logger.Infof("Detected ceph image version: %s", version)
return version, nil
}
+2 -2
View File
@@ -283,7 +283,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
// Start the Rook cluster components. Retry several times in case of failure.
validOrchestration := true
err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) {
cephVersion, err := cluster.detectCephVersion(cluster.Spec.CephVersion.Image, 15*time.Minute)
cephVersion, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, 15*time.Minute)
if err != nil {
logger.Errorf("unknown ceph major version. %+v", err)
return false, nil
@@ -496,7 +496,7 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
// if the image changed, we need to detect the new image version
if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image {
logger.Infof("the ceph version changed. detecting the new image version...")
version, err := cluster.detectCephVersion(newClust.Spec.CephVersion.Image, 15*time.Minute)
version, err := cluster.detectCephVersion(c.rookImage, newClust.Spec.CephVersion.Image, 15*time.Minute)
if err != nil {
logger.Errorf("unknown ceph major version. %+v", err)
return
+1 -1
View File
@@ -28,7 +28,7 @@ import (
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
testop "github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+2 -2
View File
@@ -33,12 +33,12 @@ import (
"github.com/rook/rook/pkg/operator/ceph/csi"
"github.com/rook/rook/pkg/operator/ceph/file"
"github.com/rook/rook/pkg/operator/ceph/object"
"github.com/rook/rook/pkg/operator/ceph/object/user"
objectuser "github.com/rook/rook/pkg/operator/ceph/object/user"
"github.com/rook/rook/pkg/operator/ceph/pool"
"github.com/rook/rook/pkg/operator/ceph/provisioner"
"github.com/rook/rook/pkg/operator/discover"
"github.com/rook/rook/pkg/operator/k8sutil"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"sigs.k8s.io/sig-storage-lib-external-provisioner/controller"
)
@@ -964,6 +964,12 @@ metadata:
name: rook-ceph-mgr
namespace: ` + namespace + `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
@@ -1008,6 +1014,25 @@ rules:
verbs:
- "*"
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
---
# Allow the operator to create resources in this cluster's namespace
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
@@ -1067,6 +1092,21 @@ subjects:
- kind: ServiceAccount
name: rook-ceph-mgr
namespace: ` + namespace + `
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
`
}
+45 -1
View File
@@ -152,5 +152,49 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
// Update the clusterroles that have been modified in master from the previous release
func (s *UpgradeSuite) updateClusterRoles() error {
logger.Infof("Placeholder: create the new resources that have been added since 1.0")
return nil
namespace := s.namespace
newResources := `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
`
logger.Infof("creating the new resources that have been added since 1.0")
return s.k8sh.ResourceOperation("create", newResources)
}