From 07ba73bb8f19e516cf3d0bbd9854b1352f101b1a Mon Sep 17 00:00:00 2001 From: Sergey Kaydalov Date: Tue, 24 Sep 2019 19:59:53 +0300 Subject: [PATCH] EdgeFS: support for rtkvs disk engine and Samsung KVSSD Signed-off-by: Sergey Kaydalov --- Documentation/edgefs-cluster-crd.md | 61 ++++-- .../kubernetes/edgefs/cluster_kvssd.yaml | 195 ++++++++++++++++++ pkg/apis/edgefs.rook.io/v1/cluster.go | 2 + pkg/apis/edgefs.rook.io/v1/devices.go | 25 ++- pkg/apis/edgefs.rook.io/v1/types.go | 3 + .../cluster/cluster_deployment_config.go | 84 +++++++- pkg/operator/edgefs/cluster/configmap.go | 25 ++- .../edgefs/cluster/target/config/config.go | 25 ++- pkg/operator/edgefs/cluster/target/layout.go | 67 +++++- pkg/operator/edgefs/cluster/target/pod.go | 21 +- pkg/operator/edgefs/cluster/utils.go | 16 +- 11 files changed, 487 insertions(+), 37 deletions(-) create mode 100644 cluster/examples/kubernetes/edgefs/cluster_kvssd.yaml diff --git a/Documentation/edgefs-cluster-crd.md b/Documentation/edgefs-cluster-crd.md index 46aad3d89..b206270af 100644 --- a/Documentation/edgefs-cluster-crd.md +++ b/Documentation/edgefs-cluster-crd.md @@ -71,6 +71,8 @@ Settings can be specified at the global level to apply to the cluster as a whole - **WARNING**: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by `dataDirHostPath` must be deleted. Otherwise, stale information and other config will remain from the previous cluster and the new target will fail to start. If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes [empty dir docs](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir). - `dataVolumeSize`: Alternative to `dataDirHostPath`. If defined then Cluster CRD operator will disregard `dataDirHostPath` setting and instead will automatically claim persistent volume. If `storage` settings not provided then provisioned volume will also be used as a storage device for Target pods (automatic provisioning via `rtlfs`). +- `sysRepCount`: overrides the default (3) system replication count value. Can be set to 1 or 2 for a cluster with limited number of failure domains. For example, a signle node setup with two disks can provide up to 2 replicas per chunk and requires `sysRepCount` to be set to 1 or 2. +- `failureDomain`: identifies the way chunk replicas are distributed accross cluster's disks. The `device` domain requires each replicas to resides on different disks, the `host` domains implies one replica per node and the `zone` domain is for one replica per zone. If `failureDomain` option isn't specified, then the failure domain is set to `host` or `zone` depending on nodes config (see below). The `failureDomain` allows to specify the failure domain explicitly. For example, a single-node cluster with multiple nodes requires the `failureDomain` set to `device` if the `sysRepCount` > 1. - `dashboard`: This specification may be used to override and enable additional [EdgeFS UI Dashboard](edgefs-ui.md) functionality. - `localAddr`: Specifies local IP address to be used as Kubernetes external IP. - `network`: [network configuration settings](#network-configuration-settings) @@ -110,7 +112,7 @@ If a node does not specify any configuration then it will inherit the cluster le ### Storage Selection Settings Below are the settings available, both at the cluster and individual node level, for selecting which storage resources will be included in the cluster. -- `useAllDevices`: `true` or `false`, indicating whether all devices found on nodes in the cluster should be automatically consumed by Targets. This is recommended for controlled environments where you will not risk formatting of devices with existing data. When `true`, all devices will be used except those with partitions created or a local filesystem. This can be overridden by `deviceFilter`. +- `useAllDevices`: `true` or `false`, indicating whether all devices found on nodes in the cluster should be automatically consumed by Targets. This is recommended for controlled environments where you will not risk formatting of devices with existing data. When `true`, all devices will be used except those with partitions created or a local filesystem. This can be overridden by `deviceFilter`. **warning** Don't set this option to `true` for RTKVS disk backend. - `deviceFilter`: A regular expression that allows selection of devices to be consumed by target. If individual devices have been specified for a node then this filter will be ignored. This field uses [golang regular expression syntax](https://golang.org/pkg/regexp/syntax/). For example: - `sdb`: Only selects the `sdb` device if found - `^sd.`: Selects all devices starting with `sd` @@ -121,32 +123,34 @@ Below are the settings available, both at the cluster and individual node level, - `name`: The name of the device (e.g., `sda`). - `fullpath`: The full path to the device (e.g., `/dev/disk/by-id/scsi-35000c5008335c83f`). If specified then `name` can be omitted. - `config`: Device-specific config settings. See the [config settings](#target-configuration-settings) below. -- `directories`: A list of directory paths on the nodes that will be included in the storage cluster. Note that using two directories on the same physical device can cause a negative performance impact. Mixing of `devices` and `directories` on the same node isn't supported. Since EdgeFS is leveraging StatefulSet, directories can only be defined at cluster level. +- `directories`: A list of directory paths on the nodes that will be included in the storage cluster. Note that using two directories on the same physical device can cause a negative performance impact. Since EdgeFS is leveraging StatefulSet, directories can only be defined at cluster level. Mixing of `devices` and `directories` on the same node isn't supported unless `rtkvs` disk engine is used. ** note **: For the `rtkvs` disk engine, at least one directory needs to be provided in order to store some small amount of metadata. For performance reasons, it would be better to have one directory per disk. - `path`: The path on disk of the directory (e.g., `/rook/storage-dir`). - `config`: Directory-specific config settings. See the [config settings](#target-configuration-settings) below. ### Storage Configuration Settings -The following storage selection settings are specific to EdgeFS and do not apply to other backends. All variables are key-value pairs represented as strings. While EdgeFS supports multiple backends, it is not recommended to mix them within same cluster. In case of `devices` (physical or emulated raw disks), EdgeFS will automatically use `rtrd` backend. In all other cases `rtlfs` (local file system) will be used. +The following storage selection settings are specific to EdgeFS and do not apply to other backends. All variables are key-value pairs represented as strings. While EdgeFS supports multiple backends, it is not recommended to mix them within same cluster. In case of `devices` (physical or emulated raw disks), EdgeFS will automatically use `rtrd` backend unless `useRtkvsBackend` is specified. In the latter case, the `rtkvs` engine will be chosen. In all other cases `rtlfs` (local file system) will be used. **IMPORTANT** Keys needs to be case-sensitive and values has to be provided as strings. - - `useMetadataOffload`: Dynamically detect appropriate SSD/NVMe device to use for the metadata on each node. Performance can be improved by using a low latency device as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Typical and recommended proportion is in range of 1:1 - 1:6. Default is false. Applicable only to rtrd. - - `useMetadataMask`: Defines what parts of metadata needs to be stored on offloaded devices. Default is 0xff, offload all metadata. To save SSD/NVMe capacity, set it to 0x7d to offload all except second level manifests. Applicable only to rtrd. - - `useBCache`: When `useMetadataOffload` is true, enable use of BCache. Default is false. Applicable only to rtrd and when host has "bcache" kernel module preloaded. - - `useBCacheWB`: When `useMetadataOffload` and `useBCache` is true, this option can enable use of BCache write-back cache. By default BCache only used as read cache in front of HDD. Applicable only to rtrd. - - `useAllSSD`: When set to true, only SSD/NVMe non rotational devices will be used. Default is false and if `useMetadataOffload` not defined then only rotational devices (HDDs) will be picked up during node provisioning phase. - - `rtPLevelOverride`: In case of large devices or directories, it will be automatically partitioned into smaller parts around 500GB each. In case of embedded use cases, lowering the value would allow to operate with smaller memory footprint devices at the cost of performance. This option allows partitioning number override. Default is automatic. Typical and recommended range is 1 - 32. - - `hddReadAhead`: For all HDD or hybrid (SSD/HDD) use cases, adjusting hddReadAhead may provide significant boost in performance. Set to a value higher then 0, in KBs. - - `mdReserved`: For hybrid (SSD/HDD) use case, adjusting mdReserved can be necessary when combined with BCache read/write caches. Allowed range 10-99% of automatically calcuated slice. + - `useRtkvsBackend`: forces the cluster use the `rtkvs` disk engine and setting's value selects a key-value backend to be used. At the moment there is only backend named the `kvssd` for Samsung's KV SSD. The usage of `rtkvs` engine implies definition of one or several `device.name` or `device.fullpath` settings which have to point to backend's disk entries (see cluster_kvsdd.yaml). + - `walMode`: allows to enable/disable the Write-Ahead log (WAL). For `rtlfs` and `rtkvs` there are two options: `on` to enable or `off` to disable. It's better to keep it `on` unless `useAllSSD` or `useRtkvsBackend` are used. For `rtkvs`, there is an extra option: `metadata` which implies usage of WAL for data types which aren't stored on `rtkvs` backend (a KVSSD). + - `useMetadataOffload`: Dynamically detect appropriate SSD/NVMe device to use for the metadata on each node. Performance can be improved by using a low latency device as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Typical and recommended proportion is in range of 1:1 - 1:6. Default is false. Applicable only to `rtrd`. + - `useMetadataMask`: Defines what parts of metadata needs to be stored on offloaded devices. Default is 0xff, offload all metadata. To save SSD/NVMe capacity, set it to 0x7d to offload all except second level manifests. Applicable only to `rtrd`. + - `useBCache`: When `useMetadataOffload` is true, enable use of BCache. Default is false. Applicable only to `rtrd` and when host has "bcache" kernel module preloaded. + - `useBCacheWB`: When `useMetadataOffload` and `useBCache` is true, this option can enable use of BCache write-back cache. By default BCache only used as read cache in front of HDD. Applicable only to `rtrd`. + - `useAllSSD`: When set to true, only SSD/NVMe non rotational devices will be used. Default is false and if `useMetadataOffload` not defined then only rotational devices (HDDs) will be picked up during node provisioning phase. Is not applicabel to `rtkvs`. + - `rtPLevelOverride`: In case of large devices or directories, it will be automatically partitioned into smaller parts around 500GB each. In case of embedded use cases, lowering the value would allow to operate with smaller memory footprint devices at the cost of performance. This option allows partitioning number override. Default is automatic. Typical and recommended range is 1 - 32. For `rtkvs` recomended plevel is 16. + - `hddReadAhead`: For all HDD or hybrid (SSD/HDD) use cases, adjusting hddReadAhead may provide significant boost in performance. Set to a value higher then 0, in KBs. Not applicable to `rtkvs` + - `mdReserved`: For hybrid (SSD/HDD) use case, adjusting mdReserved can be necessary when combined with BCache read/write caches. Allowed range 10-99% of automatically calcuated slice. Not applicable to `rtkvs` - `rtVerifyChid`: Verify transferred or read payload. Payload can be data or metadata chunk of flexible size between 4K and 8MB. EdgeFS uses SHA-3 variant to cryptographically sign each chunk and uses it for self validation, self healing and FlexHash addressing. In case of low CPU systems verification after networking transfer prior to write can be disabled by setting this parameter to 0. In case of high CPU systems, verification after read but before networking transfer can be enabled by setting this parameter to 2. Default is 1, i.e. verify after networking transfer only. Setting it to 0 may improve CPU utilization at the cost of reduced availability. However, for objects with 3 or more replicas, availability isn't going to be visibly affected. - - `lmdbPageSize`: Defines default LMDB page size in bytes. Default is 16384. For capacity (all HDD) or hybrid (HDD/SSD) systems consider to increase this value to 32768 to achieve higher throughput performance. For all SSD and small database workloads, consider to decrease this to 8192 to achieve lower latency and higher IOPS. Please be advised that smaller values MAY cause fragmentation. Acceptable values are 4096, 8192, 16384 and 32768. - - `lmdbMdPageSize`: Defines SSD metadata offload LMDB page size in bytes. Default is 8192. For large amount of small objects or files, consider to decrease this to 4096 to achieve better SSD capacity utilization. Acceptable values are 4096, 8192, 16384 and 32768. + - `lmdbPageSize`: Defines default LMDB page size in bytes. Default is 16384. For capacity (all HDD) or hybrid (HDD/SSD) systems consider to increase this value to 32768 to achieve higher throughput performance. For all SSD and small database workloads, consider to decrease this to 8192 to achieve lower latency and higher IOPS. Please be advised that smaller values MAY cause fragmentation. Acceptable values are 4096, 8192, 16384 and 32768. Not applicable to `rtkvs` + - `lmdbMdPageSize`: Defines SSD metadata offload LMDB page size in bytes. Default is 8192. For large amount of small objects or files, consider to decrease this to 4096 to achieve better SSD capacity utilization. Acceptable values are 4096, 8192, 16384 and 32768. Not applicable to `rtkvs` - `sync`: Defines default behavior of write operations at device or directory level. Acceptable values are 0, 1 (default), 2, 3. - `0`: No syncing will happen. Highest performance possible and good for HPC scratch types of deployments. This option will still sustain crash of pods or software bugs. It will not sustain server power loss an may cause node / device level inconsistency. - `1`: Default method. Will guarantee node / device consistency in case of power loss with reduced durability. - `2`: Provides better durability in case of power loss at the cost of extra metadata syncing. - `3`: Most durable and reliable option at the cost of significant performance impact. - - `maxSizeGB`: Defines maximum allowed size to use per directory in gigabytes. Applicable only to rtlfs. - - `zone`: Enables the node's failure domain number. Default value is 0 (no zoning). Zoning number is a logical failure domain tagging mechanism and if enabled then it has to be set for all the nodes in the cluster. + - `maxSizeGB`: For `rtlfs`, defines maximum allowed size to use per directory in gigabytes. For `rtkvs` this is the maximum space the disk's metadata table can occupy. + - `zone`: Enables the node's failure domain number. Default value is 0 (no zoning). Zoning number is a logical failure domain tagging mechanism and if enabled then it has to be set for all the nodes in the cluster. See also, the `failureDomain` ### Placement Configuration Settings Placement configuration for the cluster services. It includes the following keys: `mgr`, `target` and `all`. Each service will have its placement configuration generated by merging the generic configuration under `all` with the most specific one (which will override any attributes). @@ -248,6 +252,33 @@ spec: - name: "172.17.4.301" deviceFilter: "^sd." ``` +### Storage Configuration: Samsung's KV SSD +A single nodes configuration with 2 KV SSDs. The host's /media directory should have at least 64GB of free space for metadata. + +```yaml +spec: + edgefsImageName: edgefs/edgefs:1.2.0 + serviceAccount: rook-edgefs-cluster + dataDirHostPath: /var/lib/edgefs + sysRepCount: 2 + failureDomain: "device" + storage: + useAllNodes: false + directories: + - path: /media + useAllDevices: false + config: + useRtkvsBackend: kvssd + rtPLevelOverride: "16" + maxSizeGB: "32" + sync: "0" + walMode: "off" + nodes: + - name: "node1" + devices: + - fullpath: "/dev/disk/by-id/nvme-SAMSUNG_MZQLB3T8HALS-000AZ_S3VJNY0J600450" + - fullpath: "/dev/disk/by-id/nvme-SAMSUNG_MZQLB3T8HALS-000AZ_S3VJNY0K303383" +``` ### Node Affinity To control where various services will be scheduled by Kubernetes, use the placement configuration sections below. diff --git a/cluster/examples/kubernetes/edgefs/cluster_kvssd.yaml b/cluster/examples/kubernetes/edgefs/cluster_kvssd.yaml new file mode 100644 index 000000000..ef7417654 --- /dev/null +++ b/cluster/examples/kubernetes/edgefs/cluster_kvssd.yaml @@ -0,0 +1,195 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: rook-edgefs +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-edgefs-cluster + namespace: rook-edgefs +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-edgefs-cluster + namespace: rook-edgefs +rules: +- apiGroups: [""] + resources: ["configmaps", "endpoints"] + verbs: [ "get", "list", "watch", "create", "update", "delete" ] +- apiGroups: ["edgefs.rook.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: [""] + resources: ["pods"] + verbs: [ "get", "list" ] +- apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: [ "get", "update" ] +--- +# Allow the operator to create resources in this cluster's namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-edgefs-cluster-mgmt + namespace: rook-edgefs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-edgefs-cluster-mgmt +subjects: +- kind: ServiceAccount + name: rook-edgefs-system + namespace: rook-edgefs-system +--- +# Allow the pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-edgefs-cluster + namespace: rook-edgefs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-edgefs-cluster +subjects: +- kind: ServiceAccount + name: rook-edgefs-cluster + namespace: rook-edgefs +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged +spec: + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' + allowedCapabilities: + - '*' + hostPID: true + hostIPC: true + hostNetwork: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: privileged-psp-user +rules: +- apiGroups: + - apps + resources: + - podsecuritypolicies + resourceNames: + - privileged + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-edgefs-system-psp + namespace: rook-edgefs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: privileged-psp-user +subjects: +- kind: ServiceAccount + name: rook-edgefs-system + namespace: rook-edgefs-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rook-edgefs-cluster-psp + namespace: rook-edgefs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: privileged-psp-user +subjects: +- kind: ServiceAccount + name: rook-edgefs-cluster + namespace: rook-edgefs +--- +apiVersion: edgefs.rook.io/v1 +kind: Cluster +metadata: + name: rook-edgefs + namespace: rook-edgefs +spec: + edgefsImageName: edgefs/edgefs:1.2.64 # specify version here, i.e. edgefs/edgefs:1.1.0 etc + serviceAccount: rook-edgefs-cluster + dataDirHostPath: /var/lib/edgefs + sysRepCount: 2 # number of chunk replicas to keep. Overrides default one (3) + failureDomain: device # failure domain: "device", "host" or "zone". Overrides automatically calculated one + #devicesResurrectMode: "restoreZapWait" + #dashboard: + # localAddr: 10.3.30.75 + #network: # cluster level networking configuration + # provider: host + # selectors: + # server: "enp2s0f0" + # broker: "enp2s0f0" + #skipHostPrepare: true + #trlogProcessingInterval: 2 # set transaction log processing interval to 2s to speed up ISGW Link delivery + #trlogKeepDays: 2 # keep up to 2 days of transaction log interval batches to reduce local storage overhead + #useHostLocalTime: true + storage: # cluster level storage configuration and selection + useAllNodes: false + directories: + - path: /media # if the RTKVS is chosen, then the directory will be used as a journal to store metadata. Mandatory for RTKVS. + useAllDevices: false + config: + useRtkvsBackend: kvssd # Use the RTKVS store driver. Value is the backend ID. The "kvssd" is for Samsung's KV SSD + # rtVerifyChid: "0" # may improve CPU utilization + rtPLevelOverride: "16" # enable large device partitioning, only needed if automatic not working + maxSizeGB: "32" # RTKVS: max.space allocated for a journal + sync: "0" # highest performance, consistent on pod/software failures, not-consistent on power failures + walMode: "off" # Write-ahead-log mode. From a set: "on" - enabled, "off" - disabled, "metadata" - use the WAL for data type stored on the metadata SSD (RTKVS only) + # zone: "1" # defines failure domain's zone number for all edgefs nodes + nodes: + - name: "msl-colo-client20" + devices: + - name: "nvme0n1" # for the RTKVS store, the user needs to specify explicitly devices (KV SSD drives for in this example) to be used. + - name: "nvme1n1" + # config: # configuration can be specified at the node level which overrides the cluster level config + # rtPLevelOverride: 8 + # zone: "2" # defines failure domain's zone number for specific node(node3074ub16) + #resources: + # limits: + # cpu: "2" + # memory: "4096Mi" + # requests: + # cpu: "2" + # memory: "4096Mi" + # A key value list of annotations + #annotations: + # all: + # key: value + # mgr: + # prepare: + # target: + #placement: + # all: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: nodekey + # operator: In + # values: + # - edgefs-target + # tolerations: + # - key: taintKey + # operator: Exists diff --git a/pkg/apis/edgefs.rook.io/v1/cluster.go b/pkg/apis/edgefs.rook.io/v1/cluster.go index 1aa846323..307bdbd7e 100644 --- a/pkg/apis/edgefs.rook.io/v1/cluster.go +++ b/pkg/apis/edgefs.rook.io/v1/cluster.go @@ -23,6 +23,7 @@ const ( DeploymentRtlfs = "rtlfs" DeploymentRtrd = "rtrd" DeploymentAutoRtlfs = "autoRtlfs" + DeploymentRtkvs = "rtkvs" ) type ClusterDeploymentConfig struct { @@ -78,6 +79,7 @@ type DevicesConfig struct { Rtrd RTDevices RtrdSlaves []RTDevices Rtlfs RtlfsDevices + Rtkvs RtkvsDevices Zone int IsGatewayNode bool } diff --git a/pkg/apis/edgefs.rook.io/v1/devices.go b/pkg/apis/edgefs.rook.io/v1/devices.go index 4923f281c..e9dd9a8e3 100644 --- a/pkg/apis/edgefs.rook.io/v1/devices.go +++ b/pkg/apis/edgefs.rook.io/v1/devices.go @@ -33,6 +33,7 @@ type RTDevice struct { BcacheWritearound int `json:"bcache_writearound"` PlevelOverride int `json:"plevel_override,omitempty"` Sync int `json:"sync"` + WalDisabled int `json:"wal_disabled,omitempty"` } type RtlfsDevices struct { @@ -48,10 +49,31 @@ type RtlfsDevice struct { PlevelOverride int `json:"plevel_override,omitempty"` CheckMountpoint int `json:"check_mountpoint"` Sync int `json:"sync"` + WalDisabled int `json:"wal_disabled,omitempty"` +} + +type RtkvsDevices struct { + Backend string `json:"backend"` + Devices []RtkvsDevice `json:"devices"` +} + +type RtkvsDevice struct { + Name string `json:"name"` + Path string `json:"path"` + JornalPath string `json:"journal"` + JournalMaxsize uint64 `json:"jenvsize,omitempty"` + VerifyChid int `json:"verify_chid"` + PlevelOverride int `json:"plevel_override,omitempty"` + Sync int `json:"sync"` + WalDisabled int `json:"wal_disabled,omitempty"` } type CcowTenant struct { - FailureDomain int `json:"failure_domain"` + FailureDomain int `json:"failure_domain"` + ReplicationCount int `json:"replication_count,omitempty"` + SyncPut int `json:"sync_put,omitempty"` + SyncPutNamed int `json:"sync_put_named"` + CommitWait int `json:"sync_put_commit_wait"` } type CcowNetwork struct { @@ -104,5 +126,6 @@ type SetupNode struct { Rtrd RTDevices `json:"rtrd"` RtrdSlaves []RTDevices `json:"rtrdslaves"` Rtlfs RtlfsDevices `json:"rtlfs"` + Rtkvs RtkvsDevices `json:"rtkvs"` NodeType string `json:"nodeType"` } diff --git a/pkg/apis/edgefs.rook.io/v1/types.go b/pkg/apis/edgefs.rook.io/v1/types.go index e6d132fb0..d363353d0 100644 --- a/pkg/apis/edgefs.rook.io/v1/types.go +++ b/pkg/apis/edgefs.rook.io/v1/types.go @@ -73,6 +73,9 @@ type ClusterSpec struct { ChunkCacheSize resource.Quantity `json:"chunkCacheSize,omitempty"` TrlogProcessingInterval int `json:"trlogProcessingInterval,omitempty"` TrlogKeepDays int `json:"trlogKeepDays,omitempty"` + SystemReplicationCount int `json:"sysRepCount,omitempty"` + FailureDomain string `json:"failureDomain,omitempty"` + CommitNWait int `json:"commitNWait,omitempty"` MaxContainerCapacity resource.Quantity `json:"maxContainerCapacity,omitempty"` UseHostLocalTime bool `json:"useHostLocalTime,omitempty"` } diff --git a/pkg/operator/edgefs/cluster/cluster_deployment_config.go b/pkg/operator/edgefs/cluster/cluster_deployment_config.go index 0ea02bd02..cb1548ede 100644 --- a/pkg/operator/edgefs/cluster/cluster_deployment_config.go +++ b/pkg/operator/edgefs/cluster/cluster_deployment_config.go @@ -98,7 +98,11 @@ func (c *cluster) createClusterReconfigurationSpec(existingConfig edgefsv1.Clust func (c *cluster) getClusterDeploymentType() (string, error) { - if len(c.Spec.Storage.Directories) > 0 && (len(c.Spec.DataDirHostPath) > 0 || c.Spec.DataVolumeSize.Value() != 0) { + storeConfig := config.ToStoreConfig(c.Spec.Storage.Config) + if len(storeConfig.UseRtkvsBackend) > 0 { + logger.Warningf("Using rtkvs") + return edgefsv1.DeploymentRtkvs, nil + } else if len(c.Spec.Storage.Directories) > 0 && (len(c.Spec.DataDirHostPath) > 0 || c.Spec.DataVolumeSize.Value() != 0) { return edgefsv1.DeploymentRtlfs, nil } else if c.HasDevicesSpecification() && (len(c.Spec.DataDirHostPath) > 0 || c.Spec.DataVolumeSize.Value() != 0) { return edgefsv1.DeploymentRtrd, nil @@ -116,7 +120,7 @@ func (c *cluster) needPrivelege(deploymentType string) bool { needPriveleges = true } - if deploymentType == edgefsv1.DeploymentRtrd { + if deploymentType == edgefsv1.DeploymentRtrd || deploymentType == edgefsv1.DeploymentRtkvs { needPriveleges = true } return needPriveleges @@ -124,7 +128,9 @@ func (c *cluster) needPrivelege(deploymentType string) bool { func getClusterTransportKey(deploymentType string) string { transportKey := edgefsv1.DeploymentRtrd - if deploymentType == edgefsv1.DeploymentRtlfs || deploymentType == edgefsv1.DeploymentAutoRtlfs { + if deploymentType == edgefsv1.DeploymentRtkvs { + transportKey = edgefsv1.DeploymentRtkvs + } else if deploymentType == edgefsv1.DeploymentRtlfs || deploymentType == edgefsv1.DeploymentAutoRtlfs { transportKey = edgefsv1.DeploymentRtlfs } return transportKey @@ -136,6 +142,7 @@ func (c *cluster) createDevicesConfig(deploymentType string, node rookv1alpha2.N devicesConfig.Rtrd.Devices = make([]edgefsv1.RTDevice, 0) devicesConfig.RtrdSlaves = make([]edgefsv1.RTDevices, 0) devicesConfig.Rtlfs.Devices = make([]edgefsv1.RtlfsDevice, 0) + devicesConfig.Rtkvs.Devices = make([]edgefsv1.RtkvsDevice, 0) n := c.resolveNode(node.Name) if n == nil { @@ -165,7 +172,65 @@ func (c *cluster) createDevicesConfig(deploymentType string, node rookv1alpha2.N return devicesConfig, nil } - if deploymentType == edgefsv1.DeploymentRtrd { + if deploymentType == edgefsv1.DeploymentRtkvs { + rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar) + nodeDevices, _ := discover.ListDevices(c.context, rookSystemNS, n.Name) + logger.Infof("[%s] available devices: ", n.Name) + for _, dev := range nodeDevices[n.Name] { + logger.Infof("\tName: %s, Size: %s, Type: %s, Rotational: %t, Empty: %t", dev.Name, edgefsv1.ByteCountBinary(dev.Size), dev.Type, dev.Rotational, dev.Empty) + } + disks := []string{} + if storeConfig.UseRtkvsBackend == "kvssd" { + // For KVSSD backend make sure the user selected devices are attached + // TODO: make sure disks are truly KVSSD devices + availDevs, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, + n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices()) + + if deviceErr != nil { + // Devices were specified but we couldn't find any. + // User needs to fix CRD. + return devicesConfig, fmt.Errorf("failed to create DevicesConfig for node %s cluster %s: %v", n.Name, c.Namespace, deviceErr) + } + + if len(availDevs) == 0 { + // For the rtkvs backed user has to explicitly specify NVME devices + return devicesConfig, fmt.Errorf("failed to create DevicesConfig for node %s cluster %s: no NVME (KVSSD) devices in CRD", n.Name, c.Namespace) + } + // Make sure selected disks exist + for _, dev := range availDevs { + for _, disk := range nodeDevices[n.Name] { + if disk.Name == dev.Name { + fullPath := dev.FullPath + if len(fullPath) == 0 { + fullPath = "/dev/disk/by-id/" + target.GetIdDevLinkName(disk.DevLinks) + } + disks = append(disks, fullPath) + logger.Infof("\t%s: using %s (%s) as a KVSSD drive", n.Name, fullPath, dev.Name) + } + } + } + if len(disks) == 0 { + return devicesConfig, fmt.Errorf("failed to create DevicesConfig for node %s cluster %s: "+ + "none of specified KVSSD devices were detected", n.Name, c.Namespace) + } + } else { + return devicesConfig, fmt.Errorf("failed to create DevicesConfig for node %s cluster %s:"+ + " rtkvs backend %s isn't supported", n.Name, c.Namespace, storeConfig.UseRtkvsBackend) + } + + // storage.directories need to contain a mountpoint(s) for rtkvs journal/metadata + // The user can specify single or several mountpoints. In the former case all VDEVs will share the same mountpoint, + // in the latter each VDEV will get its own. + // NOTE: directories are scooped at a cluster level and cannot be overridden on a per-node basis. + + if len(c.Spec.Storage.Directories) == 0 { + return devicesConfig, fmt.Errorf("failed to create DevicesConfig for node %s cluster %s: "+ + "couldn't find KVSSD journals mountpoints in the CRD. Use storage.directories for this purpose", + n.Name, c.Namespace) + } + devicesConfig.Rtkvs = target.GetRtkvsDevices(disks, c.Spec.Storage.Directories, &storeConfig) + + } else if deploymentType == edgefsv1.DeploymentRtrd { rookSystemNS := os.Getenv(k8sutil.PodNamespaceEnvVar) nodeDevices, _ := discover.ListDevices(c.context, rookSystemNS, n.Name) @@ -247,7 +312,16 @@ func (c *cluster) validateDeploymentConfig(deploymentConfig edgefsv1.ClusterDepl } deploymentNodesCount := len(deploymentConfig.DevConfig) - if deploymentConfig.TransportKey == edgefsv1.DeploymentRtlfs { + if deploymentConfig.TransportKey == edgefsv1.DeploymentRtkvs { + if len(c.Spec.Storage.Directories) == 0 { + return fmt.Errorf("RTKVS configuration error: storage.directory has to " + + "contain at least one host's directory. It will be used to store certain metadata types.") + } + if c.Spec.Storage.UseAllNodes || (c.Spec.Storage.UseAllDevices != nil && *c.Spec.Storage.UseAllDevices) { + return fmt.Errorf("RTKVS configuration error: storage.useAllNodes and storage.useAllDevices options aren't allowed." + + " A per-node NVME KVSSD disks list is expected") + } + } else if deploymentConfig.TransportKey == edgefsv1.DeploymentRtlfs { // Check directories devices count on all nodes if len(c.Spec.Storage.Directories)*deploymentNodesCount < 3 { return fmt.Errorf("Rtlfs devices should be more then 3 on all nodes summary") diff --git a/pkg/operator/edgefs/cluster/configmap.go b/pkg/operator/edgefs/cluster/configmap.go index e92ae35b5..4ba06da06 100644 --- a/pkg/operator/edgefs/cluster/configmap.go +++ b/pkg/operator/edgefs/cluster/configmap.go @@ -125,7 +125,22 @@ func (c *cluster) createClusterConfigMap(deploymentConfig edgefsv1.ClusterDeploy if devConfig.Zone > 0 { failureDomain = 2 } - + if len(c.Spec.FailureDomain) > 0 { + switch c.Spec.FailureDomain { + case "device": + failureDomain = 0 + case "host": + failureDomain = 1 + case "zone": + failureDomain = 2 + default: + logger.Infof("Unknow failure domain %s, skipped", c.Spec.FailureDomain) + } + } + commitWait := 1 + if c.Spec.CommitNWait > 0 { + commitWait = 0 + } nodeConfig := edgefsv1.SetupNode{ Ccow: edgefsv1.CcowConf{ Trlog: edgefsv1.CcowTrlog{ @@ -133,6 +148,7 @@ func (c *cluster) createClusterConfigMap(deploymentConfig edgefsv1.ClusterDeploy }, Tenant: edgefsv1.CcowTenant{ FailureDomain: failureDomain, + CommitWait: commitWait, }, Network: edgefsv1.CcowNetwork{ BrokerInterfaces: brokerIfName, @@ -161,6 +177,7 @@ func (c *cluster) createClusterConfigMap(deploymentConfig edgefsv1.ClusterDeploy Rtlfs: edgefsv1.RtlfsDevices{ Devices: rtlfsDevices, }, + Rtkvs: devConfig.Rtkvs, Ipv4Autodetect: 1, RtlfsAutodetect: rtlfsAutoDetectPath, ClusterNodes: dnsRecords, @@ -176,6 +193,12 @@ func (c *cluster) createClusterConfigMap(deploymentConfig edgefsv1.ClusterDeploy nodeConfig.Ccowd.BgConfig.SpeculativeBackrefTimeout = c.Spec.TrlogKeepDays * 24 * 3600 * 1000 } + if c.Spec.SystemReplicationCount > 0 { + nodeConfig.Ccow.Tenant.ReplicationCount = c.Spec.SystemReplicationCount + nodeConfig.Ccow.Tenant.SyncPut = c.Spec.SystemReplicationCount + nodeConfig.Ccow.Tenant.SyncPutNamed = c.Spec.SystemReplicationCount + } + cm[nodeName] = nodeConfig logger.Debugf("Resolved Node %s = %+v", nodeName, cm[nodeName]) diff --git a/pkg/operator/edgefs/cluster/target/config/config.go b/pkg/operator/edgefs/cluster/target/config/config.go index 9dbf14dc4..63787b390 100644 --- a/pkg/operator/edgefs/cluster/target/config/config.go +++ b/pkg/operator/edgefs/cluster/target/config/config.go @@ -50,6 +50,8 @@ const ( RtPlevelOverrideKey = "rtPLevelOverride" SyncKey = "sync" ZoneKey = "zone" + UseRtkvsBackendKey = "useRtkvsBackend" + WalModeKey = "walMode" ) type StoreConfig struct { @@ -63,7 +65,7 @@ type StoreConfig struct { MDReserved int `json:"mdReserved,omitempty"` // applies to data chunks on HDD partitions, in KBs HDDReadAhead int `json:"hddReadAhead,omitempty"` - // rtlfs only, max size to use per directory, in bytes + // rtlfs/rtkvs only, max size to use per directory (journal in rtkvs), in bytes MaxSize uint64 `json:"maxsize,omitempty"` // enable use of bcache UseBCache bool `json:"useBCache,omitempty"` @@ -81,6 +83,10 @@ type StoreConfig struct { Sync int `json:"sync"` // apply edgefs cluster zones id to whole cluster or node if zone value > 0 Zone int `json:"zone,omitempty"` + // Use RTKVS IO engine with specified backend name + UseRtkvsBackend string `json:"useRtkvsBackend,omitempty"` + // Write-ahead-log mode + WalMode int `json:"walMode,omitempty"` } func DefaultStoreConfig() StoreConfig { @@ -98,6 +104,8 @@ func DefaultStoreConfig() StoreConfig { RtPLevelOverride: 0, Sync: 1, Zone: 0, + UseRtkvsBackend: "", + WalMode: 0, } } @@ -108,6 +116,12 @@ var validLmbdPageSize = map[int]bool{ 32768: true, } +var walModeMap = map[string]int{ + "on": 0, + "off": 2, + "metadata": 1, +} + func ToStoreConfig(config map[string]string) StoreConfig { storeConfig := DefaultStoreConfig() for k, v := range config { @@ -178,8 +192,15 @@ func ToStoreConfig(config map[string]string) StoreConfig { if value > 0 { storeConfig.Zone = value } + case UseRtkvsBackendKey: + storeConfig.UseRtkvsBackend = v + case WalModeKey: + if val, ok := walModeMap[v]; !ok { + logger.Warningf("Incorrect 'walMode' value %s ignored", v) + } else { + storeConfig.WalMode = val + } } - } return storeConfig diff --git a/pkg/operator/edgefs/cluster/target/layout.go b/pkg/operator/edgefs/cluster/target/layout.go index ad3d297ff..81999491a 100644 --- a/pkg/operator/edgefs/cluster/target/layout.go +++ b/pkg/operator/edgefs/cluster/target/layout.go @@ -41,7 +41,7 @@ func CreateQualifiedHeadlessServiceName(replicaNum int, namespace string) string } // EdgeFS RT-RD driver needs SCSI-3, ATA, NVMe by-id link -func getIdDevLinkName(dls string) (dl string) { +func GetIdDevLinkName(dls string) (dl string) { dlsArr := strings.Split(dls, " ") for i := range dlsArr { s := strings.Replace(dlsArr[i], "/dev/disk/by-id/", "", 1) @@ -182,6 +182,11 @@ func GetContainersRTDevices(nodeName string, maxContainerCapacity int64, nodeDis func getRTDevices(cntDevs ContainerDevices, storeConfig *config.StoreConfig) (rtDevices []edgefsv1.RTDevice, err error) { rtDevices = make([]edgefsv1.RTDevice, 0) + walDisabled := 0 + if storeConfig.WalMode > 0 { + walDisabled = 1 + } + if storeConfig.UseAllSSD { // // All flush media case (High Performance) @@ -192,14 +197,14 @@ func getRTDevices(cntDevs ContainerDevices, storeConfig *config.StoreConfig) (rt if storeConfig.UseMetadataOffload { fmt.Println("Warning: useMetadataOffload parameter is ignored due to use useAllSSD=true") } - for i := range cntDevs.Ssds { rtdev := edgefsv1.RTDevice{ - Name: getIdDevLinkName(cntDevs.Ssds[i].DevLinks), - Device: "/dev/" + cntDevs.Ssds[i].Name, - Psize: storeConfig.LmdbPageSize, - VerifyChid: storeConfig.RtVerifyChid, - Sync: storeConfig.Sync, + Name: GetIdDevLinkName(cntDevs.Ssds[i].DevLinks), + Device: "/dev/" + cntDevs.Ssds[i].Name, + Psize: storeConfig.LmdbPageSize, + VerifyChid: storeConfig.RtVerifyChid, + Sync: storeConfig.Sync, + WalDisabled: walDisabled, } if storeConfig.RtPLevelOverride != 0 { rtdev.PlevelOverride = storeConfig.RtPLevelOverride @@ -219,12 +224,13 @@ func getRTDevices(cntDevs ContainerDevices, storeConfig *config.StoreConfig) (rt // for i := range cntDevs.Hdds { rtdev := edgefsv1.RTDevice{ - Name: getIdDevLinkName(cntDevs.Hdds[i].DevLinks), + Name: GetIdDevLinkName(cntDevs.Hdds[i].DevLinks), Device: "/dev/" + cntDevs.Hdds[i].Name, Psize: storeConfig.LmdbPageSize, VerifyChid: storeConfig.RtVerifyChid, HDDReadAhead: storeConfig.HDDReadAhead, Sync: storeConfig.Sync, + WalDisabled: walDisabled, } if storeConfig.RtPLevelOverride != 0 { rtdev.PlevelOverride = storeConfig.RtPLevelOverride @@ -259,17 +265,18 @@ func getRTDevices(cntDevs ContainerDevices, storeConfig *config.StoreConfig) (rt for i := range hdds_divided { for j := range hdds_divided[i] { rtdev := edgefsv1.RTDevice{ - Name: getIdDevLinkName(hdds_divided[i][j].DevLinks), + Name: GetIdDevLinkName(hdds_divided[i][j].DevLinks), Device: "/dev/" + hdds_divided[i][j].Name, Psize: storeConfig.LmdbPageSize, MdPsize: storeConfig.LmdbMdPageSize, VerifyChid: storeConfig.RtVerifyChid, HDDReadAhead: storeConfig.HDDReadAhead, BcacheWritearound: (map[bool]int{true: 0, false: 1})[storeConfig.UseBCacheWB], - Journal: getIdDevLinkName(cntDevs.Ssds[i].DevLinks), - Metadata: getIdDevLinkName(cntDevs.Ssds[i].DevLinks) + "," + storeConfig.UseMetadataMask, + Journal: GetIdDevLinkName(cntDevs.Ssds[i].DevLinks), + Metadata: GetIdDevLinkName(cntDevs.Ssds[i].DevLinks) + "," + storeConfig.UseMetadataMask, Bcache: 0, Sync: storeConfig.Sync, + WalDisabled: walDisabled, } if storeConfig.UseBCache { @@ -290,6 +297,10 @@ func getRTDevices(cntDevs ContainerDevices, storeConfig *config.StoreConfig) (rt func GetRtlfsDevices(directories []rookalpha.Directory, storeConfig *config.StoreConfig) []edgefsv1.RtlfsDevice { rtlfsDevices := make([]edgefsv1.RtlfsDevice, 0) + walDisabled := 0 + if storeConfig.WalMode > 0 { + walDisabled = 1 + } for _, dir := range directories { rtlfsDevice := edgefsv1.RtlfsDevice{ Name: filepath.Base(dir.Path), @@ -298,6 +309,7 @@ func GetRtlfsDevices(directories []rookalpha.Directory, storeConfig *config.Stor Psize: storeConfig.LmdbPageSize, VerifyChid: storeConfig.RtVerifyChid, Sync: storeConfig.Sync, + WalDisabled: walDisabled, } if storeConfig.MaxSize != 0 { rtlfsDevice.Maxsize = storeConfig.MaxSize @@ -309,3 +321,36 @@ func GetRtlfsDevices(directories []rookalpha.Directory, storeConfig *config.Stor } return rtlfsDevices } + +func GetRtkvsDevices(disks []string, directories []rookalpha.Directory, storeConfig *config.StoreConfig) edgefsv1.RtkvsDevices { + rc := edgefsv1.RtkvsDevices{} + rc.Devices = make([]edgefsv1.RtkvsDevice, 0) + rc.Backend = storeConfig.UseRtkvsBackend + sharedDir := len(disks) > len(directories) + index := 0 + for _, disk := range disks { + journalPath := directories[0].Path + if !sharedDir { + journalPath = directories[index].Path + } + devIDs := strings.Split(disk, "/") + devID := devIDs[len(devIDs)-1] + rtkvsDevice := edgefsv1.RtkvsDevice{ + Name: devID, + Path: disk, + JornalPath: journalPath, + VerifyChid: storeConfig.RtVerifyChid, + Sync: storeConfig.Sync, + WalDisabled: storeConfig.WalMode, + } + if storeConfig.MaxSize != 0 { + rtkvsDevice.JournalMaxsize = storeConfig.MaxSize + } + if storeConfig.RtPLevelOverride != 0 { + rtkvsDevice.PlevelOverride = storeConfig.RtPLevelOverride + } + rc.Devices = append(rc.Devices, rtkvsDevice) + index++ + } + return rc +} diff --git a/pkg/operator/edgefs/cluster/target/pod.go b/pkg/operator/edgefs/cluster/target/pod.go index 3087cfe51..a681ac333 100644 --- a/pkg/operator/edgefs/cluster/target/pod.go +++ b/pkg/operator/edgefs/cluster/target/pod.go @@ -38,6 +38,7 @@ const ( dataVolumeName = "edgefs-datadir" stateVolumeFolder = ".state" etcVolumeFolder = ".etc" + kvsJournalFolder = "kvsjournaldir" ) func (c *Cluster) createAppLabels() map[string]string { @@ -250,6 +251,13 @@ func (c *Cluster) makeDaemonContainer(containerImage string, dro edgefsv1.Device for _, device := range rtlfsDevices { volumeMounts = append(volumeMounts, v1.VolumeMount{Name: device.Name, MountPath: device.Path}) } + } else if c.deploymentConfig.DeploymentType == edgefsv1.DeploymentRtkvs { + fmt.Printf("\tExporting data dirs:\n") + for i := 0; i < len(c.Storage.Directories); i++ { + fmt.Printf("\t\t%v\n", c.Storage.Directories[i].Path) + name := kvsJournalFolder + strconv.Itoa(i) + volumeMounts = append(volumeMounts, v1.VolumeMount{Name: name, MountPath: c.Storage.Directories[i].Path}) + } } name := "daemon" @@ -433,7 +441,18 @@ func (c *Cluster) createPodSpec(rookImage string, dro edgefsv1.DevicesResurrectO }) } - + } else if c.deploymentConfig.DeploymentType == edgefsv1.DeploymentRtkvs { + for i := 0; i < len(c.Storage.Directories); i++ { + volumes = append(volumes, v1.Volume{ + Name: kvsJournalFolder + strconv.Itoa(i), + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: c.Storage.Directories[i].Path, + Type: &hostPathDirectoryOrCreate, + }, + }, + }) + } } var containers []v1.Container diff --git a/pkg/operator/edgefs/cluster/utils.go b/pkg/operator/edgefs/cluster/utils.go index 00e77c1cb..8ecc4bd05 100644 --- a/pkg/operator/edgefs/cluster/utils.go +++ b/pkg/operator/edgefs/cluster/utils.go @@ -154,7 +154,10 @@ func (c *cluster) retrieveDeploymentConfig() (edgefsv1.ClusterDeploymentConfig, // we can't detect deployment type on gw node, move to next one if !devicesConfig.IsGatewayNode && !deploymentTypeAchived { - if len(nodeConfig.Rtrd.Devices) > 0 { + if len(nodeConfig.Rtkvs.Devices) > 0 { + deploymentConfig.DeploymentType = edgefsv1.DeploymentRtkvs + deploymentConfig.TransportKey = edgefsv1.DeploymentRtkvs + } else if len(nodeConfig.Rtrd.Devices) > 0 { deploymentConfig.DeploymentType = edgefsv1.DeploymentRtrd deploymentConfig.TransportKey = edgefsv1.DeploymentRtrd deploymentConfig.NeedPrivileges = true @@ -206,6 +209,15 @@ func (c *cluster) PrintRtlfsDevices(containerIndex int, rtlfsDevices edgefsv1.Rt } } +func (c *cluster) PrintRtkvsDevices(containerIndex int, rtkvsDevices edgefsv1.RtkvsDevices) { + for _, device := range rtkvsDevices.Devices { + logger.Infof("\t\tContainer[%d] Path: %s, Name: %s, Backend: %s, JournalPath: %s, JournalMaxSize: %s,"+ + " plevelOverride: %d, sync: %d, wal_disabled: %d", + containerIndex, device.Path, device.Name, rtkvsDevices.Backend, device.JornalPath, + edgefsv1.ByteCountBinary(device.JournalMaxsize), device.PlevelOverride, device.Sync, device.WalDisabled) + } +} + func (c *cluster) PrintDeploymentConfig(deploymentConfig *edgefsv1.ClusterDeploymentConfig) { logger.Infof("[%s] DeploymentConfig: ", c.Namespace) logger.Infof("DeploymentType: %s", deploymentConfig.DeploymentType) @@ -228,6 +240,8 @@ func (c *cluster) PrintDeploymentConfig(deploymentConfig *edgefsv1.ClusterDeploy } case edgefsv1.DeploymentRtlfs: c.PrintRtlfsDevices(0, nodeDevConfig.Rtlfs) + case edgefsv1.DeploymentRtkvs: + c.PrintRtkvsDevices(0, nodeDevConfig.Rtkvs) case edgefsv1.DeploymentAutoRtlfs: logger.Infof("\t\tContainer[0] Path: /mnt/disks/disk0") logger.Infof("\t\tContainer[0] Path: /mnt/disks/disk1")