ceph: use cmd-reporter for ceph version job
Use cmd-reporter to detect Ceph version. The job to detect Ceph version needs to run in a security account context capable of getting/creating ConfigMaps, so a new security account, role, and role binding are created for the cmd-runner jobs. In the future, the Ceph mgr may create jobs in this service account for running `ceph-volume` on nodes, and this also keeps Rook operators controlling multiple Ceph clusters independent. Fixes #2686 - ceph version job not working correctly with graylog Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>
This commit is contained in:
11 files changed
+219
-64
No files matched your search
@@ -127,5 +127,19 @@ subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-mgr
|
||||
namespace: rook-ceph
|
||||
|
||||
---
|
||||
# Allow the rook-ceph-cmd-reporter serviceAccount to use the privileged PSP
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter-psp
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: psp:rook
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: rook-ceph
|
||||
```
|
||||
@@ -76,4 +76,22 @@ rules:
|
||||
- "*"
|
||||
verbs:
|
||||
- "*"
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
{{- end }}
|
||||
@@ -76,4 +76,18 @@ subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-mgr
|
||||
namespace: {{ .Release.Namespace }}
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cmd-reporter
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
@@ -27,3 +27,12 @@ metadata:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
labels:
|
||||
operator: rook
|
||||
storage-backend: ceph
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
|
||||
@@ -494,8 +494,16 @@ metadata:
|
||||
name: rook-ceph-mgr
|
||||
namespace: rook-ceph
|
||||
# OLM: END SERVICE ACCOUNT MGR
|
||||
# OLM: BEGIN CLUSTER ROLE
|
||||
---
|
||||
# OLM: BEGIN CMD REPORTER SERVICE ACCOUNT
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: rook-ceph
|
||||
# OLM: END CMD REPORTER SERVICE ACCOUNT
|
||||
---
|
||||
# OLM: BEGIN CLUSTER ROLE
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
@@ -569,10 +577,30 @@ rules:
|
||||
verbs:
|
||||
- "*"
|
||||
# OLM: END CLUSTER ROLE
|
||||
#
|
||||
# OLM: BEGIN CLUSTER ROLEBINDING
|
||||
---
|
||||
# Allow the operator to create resources in this cluster's namespace
|
||||
# OLM: BEGIN CMD REPORTER ROLE
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: rook-ceph
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
# OLM: END CMD REPORTER ROLE
|
||||
---
|
||||
# OLM: BEGIN CLUSTER ROLEBINDING
|
||||
# Allow the operator to create resources in this cluster's namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
@@ -645,5 +673,21 @@ subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-mgr
|
||||
namespace: rook-ceph
|
||||
# OLM: END CLUSTER ROLEBINDING
|
||||
---
|
||||
# OLM: BEGIN CMD REPORTER ROLEBINDING
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: rook-ceph
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cmd-reporter
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: rook-ceph
|
||||
# OLM: END CMD REPORTER ROLEBINDING
|
||||
---
|
||||
# OLM: END CLUSTER ROLEBINDING
|
||||
@@ -24,6 +24,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/operator/k8sutil/cmdreporter"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
rookv1alpha2 "github.com/rook/rook/pkg/apis/rook.io/v1alpha2"
|
||||
@@ -36,7 +38,6 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
batch "k8s.io/api/batch/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
@@ -83,64 +84,35 @@ func newCluster(c *cephv1.CephCluster, context *clusterd.Context) *cluster {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cluster) detectCephVersion(image string, timeout time.Duration) (*cephver.CephVersion, error) {
|
||||
// get the major ceph version by running "ceph --version" in the ceph image
|
||||
podSpec := v1.PodSpec{
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Command: []string{"ceph"},
|
||||
Args: []string{"--version"},
|
||||
Name: "version",
|
||||
Image: image,
|
||||
},
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyOnFailure,
|
||||
}
|
||||
|
||||
// apply "mon" placement
|
||||
cephv1.GetMonPlacement(c.Spec.Placement).ApplyToPodSpec(&podSpec)
|
||||
|
||||
job := &batch.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: detectVersionName,
|
||||
Namespace: c.Namespace,
|
||||
},
|
||||
Spec: batch.JobSpec{
|
||||
Template: v1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"job": detectVersionName,
|
||||
},
|
||||
},
|
||||
Spec: podSpec,
|
||||
},
|
||||
},
|
||||
}
|
||||
k8sutil.AddRookVersionLabelToJob(job)
|
||||
k8sutil.SetOwnerRef(&job.ObjectMeta, &c.ownerRef)
|
||||
|
||||
// run the job to detect the version
|
||||
if err := k8sutil.RunReplaceableJob(c.context.Clientset, job, true); err != nil {
|
||||
return nil, fmt.Errorf("failed to start version job. %+v", err)
|
||||
}
|
||||
|
||||
if err := k8sutil.WaitForJobCompletion(c.context.Clientset, job, timeout); err != nil {
|
||||
return nil, fmt.Errorf("failed to complete version job. %+v", err)
|
||||
}
|
||||
|
||||
log, err := k8sutil.GetPodLog(c.context.Clientset, c.Namespace, "job="+detectVersionName)
|
||||
func (c *cluster) detectCephVersion(rookImage, cephImage string, timeout time.Duration) (*cephver.CephVersion, error) {
|
||||
versionReporter, err := cmdreporter.New(
|
||||
c.context.Clientset, &c.ownerRef,
|
||||
detectVersionName, detectVersionName, c.Namespace,
|
||||
[]string{"ceph"}, []string{"--version"},
|
||||
rookImage, cephImage,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get version job log to detect version. %+v", err)
|
||||
return nil, fmt.Errorf("failed to set up ceph version job. %+v", err)
|
||||
}
|
||||
|
||||
version, err := cephver.ExtractCephVersion(log)
|
||||
job := versionReporter.Job()
|
||||
job.Spec.Template.Spec.ServiceAccountName = "rook-ceph-cmd-reporter"
|
||||
|
||||
stdout, stderr, retcode, err := versionReporter.Run(timeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to complete ceph version job. %+v", err)
|
||||
}
|
||||
if retcode != 0 {
|
||||
return nil, fmt.Errorf(`ceph version job returned failure with retcode %d.
|
||||
stdout: %s
|
||||
stderr: %s`, retcode, stdout, stderr)
|
||||
}
|
||||
|
||||
version, err := cephver.ExtractCephVersion(stdout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to extract ceph version. %+v", err)
|
||||
}
|
||||
|
||||
// delete the job since we're done with it
|
||||
k8sutil.DeleteBatchJob(c.context.Clientset, c.Namespace, job.Name, false)
|
||||
|
||||
logger.Infof("Detected ceph image version: %s", version)
|
||||
return version, nil
|
||||
}
|
||||
|
||||
@@ -283,7 +283,7 @@ func (c *ClusterController) onAdd(obj interface{}) {
|
||||
// Start the Rook cluster components. Retry several times in case of failure.
|
||||
validOrchestration := true
|
||||
err = wait.Poll(clusterCreateInterval, clusterCreateTimeout, func() (bool, error) {
|
||||
cephVersion, err := cluster.detectCephVersion(cluster.Spec.CephVersion.Image, 15*time.Minute)
|
||||
cephVersion, err := cluster.detectCephVersion(c.rookImage, cluster.Spec.CephVersion.Image, 15*time.Minute)
|
||||
if err != nil {
|
||||
logger.Errorf("unknown ceph major version. %+v", err)
|
||||
return false, nil
|
||||
@@ -496,7 +496,7 @@ func (c *ClusterController) onUpdate(oldObj, newObj interface{}) {
|
||||
// if the image changed, we need to detect the new image version
|
||||
if oldClust.Spec.CephVersion.Image != newClust.Spec.CephVersion.Image {
|
||||
logger.Infof("the ceph version changed. detecting the new image version...")
|
||||
version, err := cluster.detectCephVersion(newClust.Spec.CephVersion.Image, 15*time.Minute)
|
||||
version, err := cluster.detectCephVersion(c.rookImage, newClust.Spec.CephVersion.Image, 15*time.Minute)
|
||||
if err != nil {
|
||||
logger.Errorf("unknown ceph major version. %+v", err)
|
||||
return
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
|
||||
testop "github.com/rook/rook/pkg/operator/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
@@ -33,12 +33,12 @@ import (
|
||||
"github.com/rook/rook/pkg/operator/ceph/csi"
|
||||
"github.com/rook/rook/pkg/operator/ceph/file"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object"
|
||||
"github.com/rook/rook/pkg/operator/ceph/object/user"
|
||||
objectuser "github.com/rook/rook/pkg/operator/ceph/object/user"
|
||||
"github.com/rook/rook/pkg/operator/ceph/pool"
|
||||
"github.com/rook/rook/pkg/operator/ceph/provisioner"
|
||||
"github.com/rook/rook/pkg/operator/discover"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"k8s.io/api/core/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"sigs.k8s.io/sig-storage-lib-external-provisioner/controller"
|
||||
)
|
||||
|
||||
|
||||
@@ -964,6 +964,12 @@ metadata:
|
||||
name: rook-ceph-mgr
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
@@ -1008,6 +1014,25 @@ rules:
|
||||
verbs:
|
||||
- "*"
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
# Allow the operator to create resources in this cluster's namespace
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
@@ -1067,6 +1092,21 @@ subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-mgr
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cmd-reporter
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
`
|
||||
}
|
||||
|
||||
|
||||
@@ -152,5 +152,49 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
// Update the clusterroles that have been modified in master from the previous release
|
||||
func (s *UpgradeSuite) updateClusterRoles() error {
|
||||
logger.Infof("Placeholder: create the new resources that have been added since 1.0")
|
||||
return nil
|
||||
namespace := s.namespace
|
||||
newResources := `
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- configmaps
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
- create
|
||||
- update
|
||||
- delete
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1beta1
|
||||
metadata:
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: rook-ceph-cmd-reporter
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rook-ceph-cmd-reporter
|
||||
namespace: ` + namespace + `
|
||||
---
|
||||
`
|
||||
logger.Infof("creating the new resources that have been added since 1.0")
|
||||
return s.k8sh.ResourceOperation("create", newResources)
|
||||
}
|
||||
Reference in new issue
Block a user