Merge pull request #9567 from Madhu-1/fix-9220
csi: add custom ceph.conf for csi pods
This commit is contained in:
14 files changed
+259
-4
No files matched your search
@@ -17,6 +17,7 @@ storage cluster.
|
||||
* [Separate Storage Groups](#separate-storage-groups)
|
||||
* [Configuring Pools](#configuring-pools)
|
||||
* [Custom ceph.conf Settings](#custom-cephconf-settings)
|
||||
* [Custom CSI ceph.conf Settings](#custom-csi-cephconf-settings)
|
||||
* [OSD CRUSH Settings](#osd-crush-settings)
|
||||
* [OSD Dedicated Network](#osd-dedicated-network)
|
||||
* [Phantom OSD Removal](#phantom-osd-removal)
|
||||
@@ -293,6 +294,36 @@ data:
|
||||
osd pool default size = 2
|
||||
```
|
||||
|
||||
## Custom CSI ceph.conf Settings
|
||||
|
||||
> **WARNING**: It is highly recommended to use the default setting that comes with
|
||||
> CephCSI and this can only be used when absolutely necessary.
|
||||
> The `ceph.conf` should be reset back to default values if/when the configurations are no
|
||||
> longer necessary.
|
||||
|
||||
If the `csi-ceph-conf-override` ConfigMap is created before the cluster is
|
||||
started, the CephCSI pods will automatically pick up the settings. If you
|
||||
add the settings to the ConfigMap after the cluster has been initialized,
|
||||
you can restart the Rook operator pod and wait for Rook to recreate CSI pods
|
||||
to take immediate effect.
|
||||
|
||||
After the CSI pods are restarted, the new settings should be in effect.
|
||||
|
||||
### Example
|
||||
|
||||
In this [Example](https://github.com/rook/rook/tree/{{ branchName }}/deploy/csi-ceph-conf-override.yaml) we
|
||||
will set the `rbd_validate_pool` to `false` to skip rbd pool validation.
|
||||
|
||||
> **WARNING**: Modify Ceph settings carefully to avoid modifying the default
|
||||
> configuration.
|
||||
> Changing the settings could result in unexpected results if used incorrectly.
|
||||
|
||||
```console
|
||||
kubectl create -f csi-ceph-conf-override.yaml
|
||||
```
|
||||
|
||||
Restart the Rook operator pod and wait for CSI pods to be recreated.
|
||||
|
||||
## OSD CRUSH Settings
|
||||
|
||||
A useful view of the [CRUSH Map](http://docs.ceph.com/docs/master/rados/operations/crush-map/)
|
||||
|
||||
@@ -6,3 +6,6 @@
|
||||
Pr: https://github.com/rook/rook/pull/9550
|
||||
|
||||
## Features
|
||||
|
||||
* Add support for custom ceph.conf for csi pods.
|
||||
Pr: https://github.com/rook/rook/pull/9567
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: csi-ceph-conf-override
|
||||
namespace: rook-ceph # namespace:operator
|
||||
data:
|
||||
ceph.conf: |
|
||||
[global]
|
||||
auth_cluster_required = cephx
|
||||
auth_service_required = cephx
|
||||
auth_client_required = cephx
|
||||
rbd_validate_pool = false
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
Copyright 2022 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
configName = "csi-ceph-conf-override"
|
||||
)
|
||||
|
||||
// checkCsiCephConfigMapExists checks if the csi-ceph-conf-override configmap
|
||||
// exists in the given namespace.
|
||||
func checkCsiCephConfigMapExists(ctx context.Context, clientset kubernetes.Interface, namespace string) (bool, error) {
|
||||
_, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if !k8serrors.IsNotFound(err) {
|
||||
return false, errors.Wrapf(err, "failed to get csi ceph.conf configmap %q (in %q)", configName, namespace)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
logger.Infof("csi ceph.conf configmap %q exists", configName)
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
Copyright 2022 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package csi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/test"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func Test_checkCsiCephConfigMapExists(t *testing.T) {
|
||||
namespace := "test-namespace"
|
||||
ctx := context.TODO()
|
||||
// create mocked client
|
||||
clientset := test.New(t, 3)
|
||||
|
||||
ok, err := checkCsiCephConfigMapExists(ctx, clientset, namespace)
|
||||
assert.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: configName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
configMap.Data = map[string]string{
|
||||
client.DefaultConfigFile: "",
|
||||
}
|
||||
_, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ok, err = checkCsiCephConfigMapExists(ctx, clientset, namespace)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
}
|
||||
@@ -194,6 +194,12 @@ func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, e
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
|
||||
}
|
||||
|
||||
exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace)
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap")
|
||||
}
|
||||
CustomCSICephConfigExists = exists
|
||||
|
||||
err = r.validateAndConfigureDrivers(serverVersion, ownerInfo)
|
||||
if err != nil {
|
||||
return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi")
|
||||
|
||||
@@ -59,6 +59,7 @@ type Param struct {
|
||||
EnableCephFSSnapshotter bool
|
||||
EnableVolumeReplicationSideCar bool
|
||||
EnableCSIAddonsSideCar bool
|
||||
MountCustomCephConf bool
|
||||
LogLevel uint8
|
||||
CephFSGRPCMetricsPort uint16
|
||||
CephFSLivenessMetricsPort uint16
|
||||
@@ -79,10 +80,11 @@ type templateParam struct {
|
||||
var (
|
||||
CSIParam Param
|
||||
|
||||
EnableRBD = false
|
||||
EnableCephFS = false
|
||||
EnableCSIGRPCMetrics = false
|
||||
AllowUnsupported = false
|
||||
EnableRBD = false
|
||||
EnableCephFS = false
|
||||
EnableCSIGRPCMetrics = false
|
||||
AllowUnsupported = false
|
||||
CustomCSICephConfigExists = false
|
||||
|
||||
//driver names
|
||||
CephFSDriverName string
|
||||
@@ -347,6 +349,9 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
|
||||
}
|
||||
}
|
||||
|
||||
if CustomCSICephConfigExists {
|
||||
tp.MountCustomCephConf = v.SupportsCustomCephConf()
|
||||
}
|
||||
tp.ProvisionerReplicas = defaultProvisionerReplicas
|
||||
nodes, err := r.context.Clientset.CoreV1().Nodes().List(r.opManagerContext, metav1.ListOptions{})
|
||||
if err == nil {
|
||||
|
||||
@@ -130,6 +130,11 @@ spec:
|
||||
mountPath: /etc/ceph-csi-config/
|
||||
- name: keys-tmp-dir
|
||||
mountPath: /tmp/csi/keys
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
{{ end }}
|
||||
- name: liveness-prometheus
|
||||
image: {{ .CSIPluginImage }}
|
||||
args:
|
||||
@@ -174,3 +179,11 @@ spec:
|
||||
emptyDir: {
|
||||
medium: "Memory"
|
||||
}
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
configMap:
|
||||
name: csi-ceph-conf-override
|
||||
items:
|
||||
- key: ceph.conf
|
||||
path: ceph.conf
|
||||
{{ end }}
|
||||
@@ -110,6 +110,11 @@ spec:
|
||||
mountPath: /etc/selinux
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
{{ end }}
|
||||
- name: liveness-prometheus
|
||||
securityContext:
|
||||
privileged: true
|
||||
@@ -176,3 +181,11 @@ spec:
|
||||
hostPath:
|
||||
path: /etc/selinux
|
||||
{{ end }}
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
configMap:
|
||||
name: csi-ceph-conf-override
|
||||
items:
|
||||
- key: ceph.conf
|
||||
path: ceph.conf
|
||||
{{ end }}
|
||||
@@ -220,6 +220,11 @@ spec:
|
||||
mountPath: /etc/ceph-csi-config/
|
||||
- name: keys-tmp-dir
|
||||
mountPath: /tmp/csi/keys
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
{{ end }}
|
||||
- name: liveness-prometheus
|
||||
image: {{ .CSIPluginImage }}
|
||||
args:
|
||||
@@ -269,6 +274,14 @@ spec:
|
||||
items:
|
||||
- key: csi-mapping-config-json
|
||||
path: cluster-mapping.json
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
configMap:
|
||||
name: csi-ceph-conf-override
|
||||
items:
|
||||
- key: ceph.conf
|
||||
path: ceph.conf
|
||||
{{ end }}
|
||||
- name: keys-tmp-dir
|
||||
emptyDir: {
|
||||
medium: "Memory"
|
||||
|
||||
@@ -118,6 +118,11 @@ spec:
|
||||
mountPath: /etc/selinux
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
{{ end }}
|
||||
{{ if .EnableCSIAddonsSideCar }}
|
||||
- name: csi-addons
|
||||
securityContext:
|
||||
@@ -238,3 +243,11 @@ spec:
|
||||
hostPath:
|
||||
path: /etc/selinux
|
||||
{{ end }}
|
||||
{{ if .MountCustomCephConf }}
|
||||
- name: ceph-config
|
||||
configMap:
|
||||
name: csi-ceph-conf-override
|
||||
items:
|
||||
- key: ceph.conf
|
||||
path: ceph.conf
|
||||
{{ end }}
|
||||
@@ -35,6 +35,10 @@ var (
|
||||
releasev340,
|
||||
releasev350,
|
||||
}
|
||||
|
||||
// custom ceph.conf is supported in v3.5.0+
|
||||
cephConfSupportedVersion = CephCSIVersion{3, 5, 0}
|
||||
|
||||
// for parsing the output of `cephcsi`
|
||||
versionCSIPattern = regexp.MustCompile(`v(\d+)\.(\d+)\.(\d+)`)
|
||||
)
|
||||
@@ -112,3 +116,13 @@ func extractCephCSIVersion(src string) (*CephCSIVersion, error) {
|
||||
|
||||
return &CephCSIVersion{major, minor, bugfix}, nil
|
||||
}
|
||||
|
||||
// SupportsCustomCephConf checks if the detected version supports custom ceph.conf
|
||||
func (v *CephCSIVersion) SupportsCustomCephConf() bool {
|
||||
// if AllowUnsupported is set also a csi-image greater than the supported ones are allowed
|
||||
if AllowUnsupported {
|
||||
return true
|
||||
}
|
||||
|
||||
return v.isAtLeast(&cephConfSupportedVersion)
|
||||
}
|
||||
@@ -89,3 +89,19 @@ func Test_extractCephCSIVersion(t *testing.T) {
|
||||
assert.Nil(t, version)
|
||||
assert.Contains(t, err.Error(), "failed to parse version from")
|
||||
}
|
||||
|
||||
func TestSupportsCustomCephConf(t *testing.T) {
|
||||
AllowUnsupported = true
|
||||
ret := testMinVersion.SupportsCustomCephConf()
|
||||
assert.True(t, ret)
|
||||
|
||||
AllowUnsupported = false
|
||||
ret = testMinVersion.SupportsCustomCephConf()
|
||||
assert.False(t, ret)
|
||||
|
||||
ret = testReleaseV340.SupportsCustomCephConf()
|
||||
assert.False(t, ret)
|
||||
|
||||
ret = testReleaseV350.SupportsCustomCephConf()
|
||||
assert.True(t, ret)
|
||||
}
|
||||
@@ -258,6 +258,21 @@ func (h *CephInstaller) CreateCephCluster() error {
|
||||
return errors.Errorf("failed to create custom ceph.conf. %+v", err)
|
||||
}
|
||||
|
||||
customCSISettings := map[string]string{
|
||||
"ceph.conf": "[global]\nauth_client_required = cephx",
|
||||
}
|
||||
customCSICM := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "csi-ceph-conf-override",
|
||||
Namespace: h.settings.Namespace,
|
||||
},
|
||||
Data: customCSISettings,
|
||||
}
|
||||
|
||||
if _, err := h.k8shelper.Clientset.CoreV1().ConfigMaps(h.settings.Namespace).Create(ctx, customCSICM, metav1.CreateOptions{}); err != nil {
|
||||
return errors.Errorf("failed to create custom csi ceph.conf. %+v", err)
|
||||
}
|
||||
|
||||
logger.Info("Starting Rook Cluster")
|
||||
rookCluster := h.Manifests.GetCephCluster()
|
||||
logger.Info(rookCluster)
|
||||
|
||||
Reference in new issue
Block a user