Merge pull request #9567 from Madhu-1/fix-9220

csi: add custom ceph.conf for csi pods
This commit is contained in:
Travis Nielsen authored and GitHub committed 2022-03-01 08:19:02 -07:00
commit 08aed0213c
14 files changed
+259 -4

No files matched your search

@@ -17,6 +17,7 @@ storage cluster.
* [Separate Storage Groups](#separate-storage-groups)
* [Configuring Pools](#configuring-pools)
* [Custom ceph.conf Settings](#custom-cephconf-settings)
* [Custom CSI ceph.conf Settings](#custom-csi-cephconf-settings)
* [OSD CRUSH Settings](#osd-crush-settings)
* [OSD Dedicated Network](#osd-dedicated-network)
* [Phantom OSD Removal](#phantom-osd-removal)
@@ -293,6 +294,36 @@ data:
osd pool default size = 2
```
## Custom CSI ceph.conf Settings
> **WARNING**: It is highly recommended to use the default setting that comes with
> CephCSI and this can only be used when absolutely necessary.
> The `ceph.conf` should be reset back to default values if/when the configurations are no
> longer necessary.
If the `csi-ceph-conf-override` ConfigMap is created before the cluster is
started, the CephCSI pods will automatically pick up the settings. If you
add the settings to the ConfigMap after the cluster has been initialized,
you can restart the Rook operator pod and wait for Rook to recreate CSI pods
to take immediate effect.
After the CSI pods are restarted, the new settings should be in effect.
### Example
In this [Example](https://github.com/rook/rook/tree/{{ branchName }}/deploy/csi-ceph-conf-override.yaml) we
will set the `rbd_validate_pool` to `false` to skip rbd pool validation.
> **WARNING**: Modify Ceph settings carefully to avoid modifying the default
> configuration.
> Changing the settings could result in unexpected results if used incorrectly.
```console
kubectl create -f csi-ceph-conf-override.yaml
```
Restart the Rook operator pod and wait for CSI pods to be recreated.
## OSD CRUSH Settings
A useful view of the [CRUSH Map](http://docs.ceph.com/docs/master/rados/operations/crush-map/)
+3
View File
@@ -6,3 +6,6 @@
Pr: https://github.com/rook/rook/pull/9550
## Features
* Add support for custom ceph.conf for csi pods.
Pr: https://github.com/rook/rook/pull/9567
@@ -0,0 +1,12 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: csi-ceph-conf-override
namespace: rook-ceph # namespace:operator
data:
ceph.conf: |
[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
rbd_validate_pool = false
+45
View File
@@ -0,0 +1,45 @@
/*
Copyright 2022 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"context"
"github.com/pkg/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
const (
configName = "csi-ceph-conf-override"
)
// checkCsiCephConfigMapExists checks if the csi-ceph-conf-override configmap
// exists in the given namespace.
func checkCsiCephConfigMapExists(ctx context.Context, clientset kubernetes.Interface, namespace string) (bool, error) {
_, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{})
if err != nil {
if !k8serrors.IsNotFound(err) {
return false, errors.Wrapf(err, "failed to get csi ceph.conf configmap %q (in %q)", configName, namespace)
}
return false, nil
}
logger.Infof("csi ceph.conf configmap %q exists", configName)
return true, nil
}
+56
View File
@@ -0,0 +1,56 @@
/*
Copyright 2022 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package csi
import (
"context"
"testing"
"github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/test"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func Test_checkCsiCephConfigMapExists(t *testing.T) {
namespace := "test-namespace"
ctx := context.TODO()
// create mocked client
clientset := test.New(t, 3)
ok, err := checkCsiCephConfigMapExists(ctx, clientset, namespace)
assert.NoError(t, err)
assert.False(t, ok)
configMap := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: configName,
Namespace: namespace,
},
}
configMap.Data = map[string]string{
client.DefaultConfigFile: "",
}
_, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{})
assert.NoError(t, err)
ok, err = checkCsiCephConfigMapExists(ctx, clientset, namespace)
assert.NoError(t, err)
assert.True(t, ok)
}
+6
View File
@@ -194,6 +194,12 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
}
exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace)
if err != nil {
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap")
}
CustomCSICephConfigExists = exists
err = r.validateAndConfigureDrivers(serverVersion, ownerInfo)
if err != nil {
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi")
+9 -4
View File
@@ -59,6 +59,7 @@ type Param struct {
EnableCephFSSnapshotter bool
EnableVolumeReplicationSideCar bool
EnableCSIAddonsSideCar bool
MountCustomCephConf bool
LogLevel uint8
CephFSGRPCMetricsPort uint16
CephFSLivenessMetricsPort uint16
@@ -79,10 +80,11 @@ type templateParam struct {
var (
CSIParam Param
EnableRBD = false
EnableCephFS = false
EnableCSIGRPCMetrics = false
AllowUnsupported = false
EnableRBD = false
EnableCephFS = false
EnableCSIGRPCMetrics = false
AllowUnsupported = false
CustomCSICephConfigExists = false
//driver names
CephFSDriverName string
@@ -347,6 +349,9 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
}
}
if CustomCSICephConfigExists {
tp.MountCustomCephConf = v.SupportsCustomCephConf()
}
tp.ProvisionerReplicas = defaultProvisionerReplicas
nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{})
if err == nil {
@@ -130,6 +130,11 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
{{ if .MountCustomCephConf }}
- name: ceph-config
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
{{ end }}
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
@@ -174,3 +179,11 @@ spec:
emptyDir: {
medium: "Memory"
}
{{ if .MountCustomCephConf }}
- name: ceph-config
configMap:
name: csi-ceph-conf-override
items:
- key: ceph.conf
path: ceph.conf
{{ end }}
@@ -110,6 +110,11 @@ spec:
mountPath: /etc/selinux
readOnly: true
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
{{ end }}
- name: liveness-prometheus
securityContext:
privileged: true
@@ -176,3 +181,11 @@ spec:
hostPath:
path: /etc/selinux
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
configMap:
name: csi-ceph-conf-override
items:
- key: ceph.conf
path: ceph.conf
{{ end }}
@@ -220,6 +220,11 @@ spec:
mountPath: /etc/ceph-csi-config/
- name: keys-tmp-dir
mountPath: /tmp/csi/keys
{{ if .MountCustomCephConf }}
- name: ceph-config
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
{{ end }}
- name: liveness-prometheus
image: {{ .CSIPluginImage }}
args:
@@ -269,6 +274,14 @@ spec:
items:
- key: csi-mapping-config-json
path: cluster-mapping.json
{{ if .MountCustomCephConf }}
- name: ceph-config
configMap:
name: csi-ceph-conf-override
items:
- key: ceph.conf
path: ceph.conf
{{ end }}
- name: keys-tmp-dir
emptyDir: {
medium: "Memory"
@@ -118,6 +118,11 @@ spec:
mountPath: /etc/selinux
readOnly: true
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
{{ end }}
{{ if .EnableCSIAddonsSideCar }}
- name: csi-addons
securityContext:
@@ -238,3 +243,11 @@ spec:
hostPath:
path: /etc/selinux
{{ end }}
{{ if .MountCustomCephConf }}
- name: ceph-config
configMap:
name: csi-ceph-conf-override
items:
- key: ceph.conf
path: ceph.conf
{{ end }}
+14
View File
@@ -35,6 +35,10 @@ var (
releasev340,
releasev350,
}
// custom ceph.conf is supported in v3.5.0+
cephConfSupportedVersion = CephCSIVersion{3, 5, 0}
// for parsing the output of `cephcsi`
versionCSIPattern = regexp.MustCompile(`v(\d+)\.(\d+)\.(\d+)`)
)
@@ -112,3 +116,13 @@ func extractCephCSIVersion(src string) (*CephCSIVersion, error) {
return &CephCSIVersion{major, minor, bugfix}, nil
}
// SupportsCustomCephConf checks if the detected version supports custom ceph.conf
func (v *CephCSIVersion) SupportsCustomCephConf() bool {
// if AllowUnsupported is set also a csi-image greater than the supported ones are allowed
if AllowUnsupported {
return true
}
return v.isAtLeast(&cephConfSupportedVersion)
}
+16
View File
@@ -89,3 +89,19 @@ func Test_extractCephCSIVersion(t *testing.T) {
assert.Nil(t, version)
assert.Contains(t, err.Error(), "failed to parse version from")
}
func TestSupportsCustomCephConf(t *testing.T) {
AllowUnsupported = true
ret := testMinVersion.SupportsCustomCephConf()
assert.True(t, ret)
AllowUnsupported = false
ret = testMinVersion.SupportsCustomCephConf()
assert.False(t, ret)
ret = testReleaseV340.SupportsCustomCephConf()
assert.False(t, ret)
ret = testReleaseV350.SupportsCustomCephConf()
assert.True(t, ret)
}
@@ -258,6 +258,21 @@ func (h *CephInstaller) CreateCephCluster() error {
return errors.Errorf("failed to create custom ceph.conf. %+v", err)
}
customCSISettings := map[string]string{
"ceph.conf": "[global]\nauth_client_required = cephx",
}
customCSICM := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "csi-ceph-conf-override",
Namespace: h.settings.Namespace,
},
Data: customCSISettings,
}
if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCSICM, metav1.CreateOptions{}); err != nil {
return errors.Errorf("failed to create custom csi ceph.conf. %+v", err)
}
logger.Info("Starting Rook Cluster")
rookCluster := h.Manifests.GetCephCluster()
logger.Info(rookCluster)