Merge pull request #10950 from BlaineEXE/nfs-kerberos-implementation
nfs: add kerberos client security support
This commit is contained in:
25 files changed
+5206
-258
No files matched your search
@@ -53,6 +53,20 @@ spec:
|
||||
logLevel: NIV_INFO
|
||||
|
||||
security:
|
||||
kerberos:
|
||||
principalName: "nfs"
|
||||
|
||||
configFiles:
|
||||
volumeSource:
|
||||
configMap:
|
||||
name: my-krb5-config-files
|
||||
|
||||
keytabFile:
|
||||
volumeSource:
|
||||
secret:
|
||||
secretName: my-nfs-keytab
|
||||
defaultMode: 0600 # mode must be 0600
|
||||
|
||||
sssd:
|
||||
sidecar:
|
||||
image: registry.access.redhat.com/rhel7/sssd:latest
|
||||
@@ -64,6 +78,8 @@ spec:
|
||||
defaultMode: 0600 # mode must be 0600
|
||||
|
||||
debugLevel: 0
|
||||
|
||||
resources: {}
|
||||
```
|
||||
|
||||
## NFS Settings
|
||||
@@ -90,15 +106,37 @@ The `server` spec sets configuration for Rook-created NFS-Ganesha server pods.
|
||||
|
||||
The `security` spec sets security configuration for the NFS cluster.
|
||||
|
||||
* `kerberos`: Kerberos configures NFS-Ganesha to secure NFS client connections with Kerberos.
|
||||
* `principalName`: this value is combined with (a) the namespace and name of the CephNFS (with a
|
||||
hyphen between) and (b) the Realm configured in the user-provided kerberos config file(s) to
|
||||
determine the full service principal name: `<principalName>/<namespace>-<name>@<realm>`.
|
||||
e.g., nfs/rook-ceph-my-nfs@example.net. For full details, see the
|
||||
[NFS security doc](../Storage-Configuration/NFS/nfs-security.md#nfs-service-principals).
|
||||
* `configFiles`: defines where the Kerberos configuration should be sourced from. Config
|
||||
files will be placed into the `/etc/krb5.conf.rook/` directory. For advanced usage, see the
|
||||
[NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-configuration).
|
||||
* `volumeSource`: this is a standard Kubernetes VolumeSource for Kerberos configuration files
|
||||
like what is normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret,
|
||||
or HostPath. The volume may contain multiple files, all of which will be loaded.
|
||||
* `keytabFile`: defines where the Kerberos keytab should be sourced from. The keytab file will be
|
||||
placed into `/etc/krb5.keytab`. For advanced usage, see the
|
||||
[NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-configuration).
|
||||
* `volumeSource`: this is a standard Kubernetes VolumeSource for the Kerberos keytab file like
|
||||
what is normally used to configure Volumes for a Pod. For example, a Secret or HostPath.
|
||||
There are two requirements for the source's content:
|
||||
1. The config file must be mountable via `subPath: krb5.keytab`. For example, in a Secret, the
|
||||
data item must be named `krb5.keytab`, or `items` must be defined to select the key and
|
||||
give it path `krb5.keytab`. A HostPath directory must have the `krb5.keytab` file.
|
||||
2. The volume or config file must have mode 0600.
|
||||
|
||||
* `sssd`: SSSD enables integration with System Security Services Daemon (SSSD). See also:
|
||||
[ID mapping via SSSD](../Storage-Configuration/NFS/nfs-security.md#id-mapping-via-sssd).
|
||||
* `sidecar`: Specifying this configuration tells Rook to run SSSD in a sidecar alongside the NFS
|
||||
server in each NFS pod.
|
||||
* `image`: defines the container image that should be used for the SSSD sidecar.
|
||||
* `sssdConfigFile`: defines where the SSSD configuration should be sourced from. The
|
||||
config file will be placed into `/etc/sssd/sssd.conf`. If this is left empty, Rook will not
|
||||
add the file. This allows you to inject your own, for example by using
|
||||
[Vault's agent injector](https://www.vaultproject.io/docs/platform/k8s/injector).
|
||||
config file will be placed into `/etc/sssd/sssd.conf`. For advanced usage, see the
|
||||
[NFS security doc](../Storage-Configuration/NFS/nfs-security.md#sssd-configuration).
|
||||
* `volumeSource`: this is a standard Kubernetes
|
||||
[VolumeSource](https://pkg.go.dev/k8s.io/api/core/v1#VolumeSource) like what is normally
|
||||
used to configure Volumes for a Pod. For example, a ConfigMap, Secret, or HostPath.
|
||||
@@ -110,6 +148,7 @@ The `security` spec sets security configuration for the NFS cluster.
|
||||
* `debugLevel`: sets the debug level for SSSD. If unset or `0`, Rook does nothing. Otherwise,
|
||||
this may be a value between 1 and 10. See the
|
||||
[SSSD docs](https://sssd.io/troubleshooting/basics.html#sssd-debug-logs) for more info.
|
||||
* `resources`: Kubernetes resource requests and limits to set on NFS server containers
|
||||
|
||||
|
||||
## Scaling the active server count
|
||||
|
||||
@@ -2,19 +2,24 @@
|
||||
title: Security
|
||||
---
|
||||
|
||||
Rook provides security for CephNFS server clusters through two high-level features:
|
||||
[user ID mapping](#user-id-mapping) and [user authentication](#user-authentication).
|
||||
|
||||
!!! attention
|
||||
All features in this document are experimental and may not support upgrades to future versions.
|
||||
|
||||
!!! attention
|
||||
Some configurations of these features may break the ability to
|
||||
[mount NFS storage to pods via PVCs](./nfs-csi-driver.md#attaching-an-export-to-a-pod).
|
||||
The NFS CSI driver may not be able to mount exports for pods when ID mapping is configured.
|
||||
|
||||
|
||||
## User ID mapping
|
||||
|
||||
User ID mapping allows the NFS server to map connected NFS client IDs to a different user domain,
|
||||
allowing NFS clients to be associated with a particular user in your organization. For example,
|
||||
users stored in LDAP can be associated with NFS users and vice versa.
|
||||
|
||||
!!! attention
|
||||
This feature is experimental and may not support upgrades to future versions.
|
||||
|
||||
!!! attention
|
||||
Some configurations of this feature may break the ability to
|
||||
[mount NFS storage to pods via PVCs](./nfs-csi-driver.md#attaching-an-export-to-a-pod).
|
||||
The NFS CSI driver may not be able to mount exports for pods when ID mapping is configured.
|
||||
|
||||
### ID mapping via SSSD
|
||||
|
||||
[SSSD](https://sssd.io) is the System Security Services Daemon. It can be used to provide user ID
|
||||
@@ -26,7 +31,7 @@ LDAP has been tested.
|
||||
package is included in `quay.io/ceph/ceph` in v17.2.4 and newer. For older Ceph versions you may
|
||||
build your own Ceph image which adds `RUN yum install sssd-client && yum clean all`.
|
||||
|
||||
#### **SSSD configuration**
|
||||
#### SSSD configuration
|
||||
|
||||
SSSD requires a configuration file in order to configure its connection to the user ID mapping
|
||||
system (e.g., LDAP). The file follows the `sssd.conf` format documented in its
|
||||
@@ -35,11 +40,14 @@ system (e.g., LDAP). The file follows the `sssd.conf` format documented in its
|
||||
Methods of providing the configuration file are documented in the
|
||||
[NFS CRD security section](../../CRDs/ceph-nfs-crd.md#security).
|
||||
|
||||
The SSSD configuration file may be omitted from the CephNFS spec if desired. In this case, Rook will
|
||||
not set the `/etc/sssd/sssd.conf` in any way. This allows you to manage the `sssd.conf` file
|
||||
yourself however you wish. For example, you may build it into your custom Ceph container image, or
|
||||
use the [Vault agent injector](https://www.vaultproject.io/docs/platform/k8s/injector) to securely
|
||||
add the file via annotations on the CephNFS spec (passed to the NFS server pods).
|
||||
Recommendations:
|
||||
- The SSSD sidecar only requires the namespace switch (a.k.a. "nsswitch" or "nss"). We recommend
|
||||
enabling only the `nss` service to lower CPU usage.
|
||||
- NFS-Ganesha does not require user enumeration. We recommend leaving this option unset or setting
|
||||
`enumerate = false` to speed up lookups and reduce RAM usage.
|
||||
- NFS exports created via documented methods do not require listing all members of groups. We
|
||||
recommend setting `ignore_group_members = true` to speed up LDAP lookups. Only customized exports
|
||||
that set `manage_gids` need to consider this option.
|
||||
|
||||
A sample `sssd.conf` file is shown below.
|
||||
```ini
|
||||
@@ -54,24 +62,104 @@ filter_users = root
|
||||
|
||||
[domain/default]
|
||||
id_provider = ldap
|
||||
ldap_uri = ldap://server.address
|
||||
ldap_search_base = dc=company,dc=net
|
||||
ldap_default_bind_dn = cn=admin,dc=company,dc=net
|
||||
ldap_uri = ldap://server-address.example.net
|
||||
ldap_search_base = dc=example,dc=net
|
||||
ldap_default_bind_dn = cn=admin,dc=example,dc=net
|
||||
ldap_default_authtok_type = password
|
||||
ldap_default_authtok = my-password
|
||||
ldap_user_search_base = ou=users,dc=company,dc=net
|
||||
ldap_group_search_base = ou=groups,dc=company,dc=net
|
||||
ldap_access_filter = memberOf=cn=rook,ou=groups,dc=company,dc=net
|
||||
ldap_user_search_base = ou=users,dc=example,dc=net
|
||||
ldap_group_search_base = ou=groups,dc=example,dc=net
|
||||
ldap_access_filter = memberOf=cn=rook,ou=groups,dc=example,dc=net
|
||||
# recommended options for speeding up LDAP lookups:
|
||||
enumerate = false
|
||||
ignore_group_members = true
|
||||
```
|
||||
|
||||
Notes:
|
||||
- The SSSD sidecar only requires the namespace switch (a.k.a. "nsswitch" or "nss"). We recommend
|
||||
enabling only the `nss` service to lower CPU usage.
|
||||
- NFS-Ganesha does not require user enumeration. We recommend leaving this option unset or setting
|
||||
`enumerate = false` to speed up lookups and reduce RAM usage.
|
||||
- NFS exports created via documented methods do not require listing all members of groups. We
|
||||
recommend setting `ignore_group_members = true` to speed up LDAP lookups. Only customized exports
|
||||
that set `manage_gids` need to consider this option.
|
||||
The SSSD configuration file may be omitted from the CephNFS spec if desired. In this case, Rook will
|
||||
not set `/etc/sssd/sssd.conf` in any way. This allows you to manage the `sssd.conf` file yourself
|
||||
however you wish. For example, you may build it into your custom Ceph container image, or use the
|
||||
[Vault agent injector](https://www.vaultproject.io/docs/platform/k8s/injector) to securely add the
|
||||
file via annotations on the CephNFS spec (passed to the NFS server pods).
|
||||
|
||||
|
||||
## User authentication
|
||||
|
||||
User authentication allows NFS clients and the Rook CephNFS servers to authenticate with each other
|
||||
to ensure security.
|
||||
|
||||
### Authentication through Kerberos
|
||||
|
||||
Kerberos is the authentication mechanism natively supported by NFS-Ganesha. With NFSv4, individual
|
||||
users are authenticated and not merely client machines.
|
||||
|
||||
#### Kerberos configuration
|
||||
|
||||
Kerberos authentication requires configuration files in order for the NFS-Ganesha server to
|
||||
authenticate to the Kerberos server (KDC). The requirements are two-parted:
|
||||
1. one or more kerberos configuration files that configures the connection to the Kerberos server.
|
||||
This file follows the `krb5.conf` format documented in its
|
||||
[man pages](https://linux.die.net/man/5/krb5.conf).
|
||||
2. a keytab file that provides credentials for the
|
||||
[service principal](#nfs-service-principals) that NFS-Ganesha will use to authenticate with the
|
||||
Kerberos server.
|
||||
|
||||
Methods of providing the configuration files are documented in the
|
||||
[NFS CRD security section](../../CRDs/ceph-nfs-crd.md#security).
|
||||
|
||||
Recommendations:
|
||||
- Rook configures Kerberos to log to stderr. We suggest removing logging sections from config files
|
||||
to avoid consuming unnecessary disk space from logging to files.
|
||||
|
||||
A sample Kerberos config file is shown below.
|
||||
```ini
|
||||
[libdefaults]
|
||||
default_realm = EXAMPLE.NET
|
||||
|
||||
[realms]
|
||||
EXAMPLE.NET = {
|
||||
kdc = kdc.example.net:88
|
||||
admin_server = kdc.example.net:749
|
||||
}
|
||||
|
||||
[domain_realm]
|
||||
.example.net = EXAMPLE.NET
|
||||
example.net = EXAMPLE.NET
|
||||
```
|
||||
|
||||
The Kerberos config files (`configFiles`) may be omitted from the Ceph NFS spec if desired. In this
|
||||
case, Rook will not add any config files to `/etc/krb5.conf.rook/`, but it will still configure
|
||||
Kerberos to load any config files it finds there. This allows you to manage these files yourself
|
||||
however you wish.
|
||||
|
||||
Similarly, the keytab file (`keytabFile`) may be omitted from the CephNFS spec if desired. In this
|
||||
case, Rook will not set `/etc/krb5.keytab` in any way. This allows you to manage the `krb5.keytab`
|
||||
file yourself however you wish.
|
||||
|
||||
As an example for either of the above cases, you may build files into your custom Ceph container
|
||||
image or use the [Vault agent injector](https://www.vaultproject.io/docs/platform/k8s/injector) to
|
||||
securely add files via annotations on the CephNFS spec (passed to the NFS server pods).
|
||||
|
||||
#### NFS service principals
|
||||
|
||||
The Kerberos service principal used by Rook's CephNFS servers to authenticate with the Kerberos
|
||||
server is built up from 3 components:
|
||||
1. the configured from `spec.security.kerberos.principalName` that acts as the service name
|
||||
2. the hostname of the server on which NFS-Ganesha is running which is in turn built up from the
|
||||
namespace and name of the CephNFS resource, joined by a hyphen. e.g., `rooknamespace-nfsname`
|
||||
3. the realm as configured by the kerberos config file(s) from `spec.security.kerberos.configFiles`
|
||||
|
||||
The full service principal name is constructed as `<principalName>/<namespace>-<name>@<realm>`. For
|
||||
ease of scaling up or down CephNFS clusters, this principal is used for all servers in the CephNFS
|
||||
cluster.
|
||||
|
||||
Users must add this service principal to their Kerberos server configuration.
|
||||
|
||||
!!! example
|
||||
For a CephNFS named "fileshare" in the "business-unit" Kubernetes namespace that has a
|
||||
`principalName` of "sales-apac" and where the Kerberos realm is "EXAMPLE.NET", the full
|
||||
principal name will be `sales-apac/business-unit-fileshare@EXAMPLE.NET`.
|
||||
|
||||
!!! advanced
|
||||
`spec.security.kerberos.principalName` corresponds directly to NFS-Ganesha's
|
||||
NFS_KRB5:PrincipalName config. See the
|
||||
[NFS-Ganesha wiki](https://github.com/nfs-ganesha/nfs-ganesha/wiki/RPCSEC_GSS) for more details.
|
||||
@@ -64,7 +64,7 @@ The Ceph CLI can be used from the Rook toolbox pod to create and manage NFS expo
|
||||
ensure the necessary Ceph mgr modules are enabled, if necessary, and that the Ceph orchestrator
|
||||
backend is set to Rook.
|
||||
|
||||
#### **Enable the Ceph orchestrator if necessary**
|
||||
#### Enable the Ceph orchestrator if necessary
|
||||
|
||||
* Required for Ceph v16.2.7 and below
|
||||
* Optional for Ceph v16.2.8 and above
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+1868
-1
File diff suppressed because it is too large.
Load diff
+69
-22
@@ -62,23 +62,35 @@ spec:
|
||||
# Configure security options for the NFS cluster. See docs for more information:
|
||||
# https://rook.github.io/docs/rook/latest/Storage-Configuration/NFS/nfs-security/
|
||||
security:
|
||||
# sssd:
|
||||
# sidecar:
|
||||
# image: registry.access.redhat.com/rhel7/sssd:latest
|
||||
# sssdConfigFile:
|
||||
# volumeSource:
|
||||
# configMap:
|
||||
# name: my-nfs-sssd-config
|
||||
# defaultMode: 0600 # mode must be 0600
|
||||
# # debugLevel: 6
|
||||
#
|
||||
# resources:
|
||||
# limits:
|
||||
# cpu: "2"
|
||||
# memory: "1Gi"
|
||||
# requests:
|
||||
# cpu: "2"
|
||||
# memory: "1Gi"
|
||||
# kerberos:
|
||||
# principalName: "nfs"
|
||||
# configFiles:
|
||||
# volumeSource:
|
||||
# configMap:
|
||||
# name: krb5-conf
|
||||
# defaultMode: 0644 # required?
|
||||
# keytabFile:
|
||||
# volumeSource:
|
||||
# secret:
|
||||
# secretName: keytab
|
||||
# defaultMode: 0600 # required
|
||||
#
|
||||
# sssd:
|
||||
# sidecar:
|
||||
# image: registry.access.redhat.com/rhel7/sssd:latest
|
||||
# sssdConfigFile:
|
||||
# volumeSource:
|
||||
# configMap:
|
||||
# name: my-nfs-sssd-config
|
||||
# defaultMode: 0600 # mode must be 0600
|
||||
# # debugLevel: 6
|
||||
# resources:
|
||||
# limits:
|
||||
# cpu: "2"
|
||||
# memory: "1Gi"
|
||||
# requests:
|
||||
# cpu: "2"
|
||||
# memory: "1Gi"
|
||||
# ---
|
||||
# # The built-in Ceph pool ".nfs" is used for storing configuration for all CephNFS clusters. If this
|
||||
# # shared pool needs to be configured with alternate settings, create this pool (once) with any of
|
||||
@@ -97,6 +109,7 @@ spec:
|
||||
# replicated:
|
||||
# size: 3
|
||||
# requireSafeReplicaSize: true
|
||||
|
||||
# ---
|
||||
# # Example configmap for providing sssd.conf file to the SSSD sidecar
|
||||
# # Note that this example uses an obfuscated password that may still not be as secure as desired
|
||||
@@ -119,13 +132,47 @@ spec:
|
||||
# [domain/default]
|
||||
# id_provider = ldap
|
||||
# ldap_uri = ldap://server.address
|
||||
# ldap_search_base = dc=company,dc=net
|
||||
# ldap_default_bind_dn = cn=admin,dc=company,dc=net
|
||||
# ldap_search_base = dc=example,dc=net
|
||||
# ldap_default_bind_dn = cn=admin,dc=example,dc=net
|
||||
# ldap_default_authtok_type = obfuscated_password
|
||||
# ldap_default_authtok = some-obfuscated-password
|
||||
# ldap_user_search_base = ou=users,dc=company,dc=net
|
||||
# ldap_group_search_base = ou=groups,dc=company,dc=net
|
||||
# ldap_access_filter = memberOf=cn=rook,ou=groups,dc=company,dc=net
|
||||
# ldap_user_search_base = ou=users,dc=example,dc=net
|
||||
# ldap_group_search_base = ou=groups,dc=example,dc=net
|
||||
# ldap_access_filter = memberOf=cn=rook,ou=groups,dc=example,dc=net
|
||||
# # recommended options for speeding up LDAP lookups:
|
||||
# enumerate = false
|
||||
# ignore_group_members = true
|
||||
|
||||
# # Example secret and configmap providing krb5.keytab and krb5 config files
|
||||
# ---
|
||||
# apiVersion: v1
|
||||
# kind: Secret
|
||||
# metadata:
|
||||
# name: keytab
|
||||
# namespace: rook-ceph
|
||||
# data:
|
||||
# # e.g., Keytab containing principal nfs/rook-ceph-my-nfs@EXAMPLE.NET
|
||||
# krb5.keytab: # your keytab here
|
||||
# ---
|
||||
# # suggest not putting [logging] section in here
|
||||
# apiVersion: v1
|
||||
# kind: ConfigMap
|
||||
# metadata:
|
||||
# name: krb5-conf
|
||||
# namespace: rook-ceph
|
||||
# data:
|
||||
# example-net.conf: |
|
||||
# [libdefaults]
|
||||
# default_realm = EXAMPLE.NET
|
||||
#
|
||||
# [realms]
|
||||
# EXAMPLE.NET = {
|
||||
# kdc = kerberos-server.default.svc:88
|
||||
# admin_server = kerberos-server.default.svc:749
|
||||
# }
|
||||
#
|
||||
# [domain_realm]
|
||||
# .example.net = EXAMPLE.NET
|
||||
# example.net = EXAMPLE.NET
|
||||
# kerberos-server.default.svc = EXAMPLE.NET # e.g., kerberos server with a k8s service endpoint
|
||||
# kerberos-server = EXAMPLE.NET
|
||||
@@ -146,7 +146,7 @@ spec:
|
||||
keytabFile:
|
||||
volumeSource:
|
||||
secret:
|
||||
name: rook-ceph-nfs-organization-keytab
|
||||
secretName: rook-ceph-nfs-organization-keytab
|
||||
defaultMode: 0600 # required
|
||||
```
|
||||
|
||||
@@ -259,7 +259,7 @@ dnsConfig:
|
||||
#### Volumes
|
||||
Volumes that should be mounted into nfs-ganesha container to support Kerberos:
|
||||
1. `keytabFile` volume: use `subPath` on the mount to add the `krb5.keytab` file to `/etc/krb5.keytab`
|
||||
2. `configFiles` volume: mount (without `subPath`) to `/etc/krb5.conf.d/` to allow all files to
|
||||
2. `configFiles` volume: mount (without `subPath`) to `/etc/krb5.conf.rook/` to allow all files to
|
||||
be mounted (e.g., if a ConfigMap has multiple data items or hostPath has multiple conf.d files)
|
||||
|
||||
#### Ganesha config
|
||||
@@ -299,7 +299,7 @@ the configuration they require.
|
||||
|
||||
The minimal `/etc/krb5.conf` file Rook will create is as so:
|
||||
```ini
|
||||
includedir /etc/krb5.conf.d/ # include all user-defined config files
|
||||
includedir /etc/krb5.conf.rook/ # include all user-defined config files
|
||||
|
||||
[logging]
|
||||
default = STDERR # only log to stderr by default
|
||||
|
||||
@@ -24,6 +24,25 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
)
|
||||
|
||||
// KerberosEnabled returns true if Kerberos is enabled from the spec.
|
||||
func (n *NFSSecuritySpec) KerberosEnabled() bool {
|
||||
if n == nil {
|
||||
return false
|
||||
}
|
||||
if n.Kerberos != nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetPrincipalName gets the principal name for the Kerberos spec or the default value if it is unset.
|
||||
func (k *KerberosSpec) GetPrincipalName() string {
|
||||
if k.PrincipalName == "" {
|
||||
return "nfs"
|
||||
}
|
||||
return k.PrincipalName
|
||||
}
|
||||
|
||||
func (n *CephNFS) ValidateCreate() error {
|
||||
return n.Spec.Security.Validate()
|
||||
}
|
||||
@@ -51,11 +70,25 @@ func (sec *NFSSecuritySpec) Validate() error {
|
||||
return errors.New("System Security Services Daemon (SSSD) sidecar is enabled, but no image is specified")
|
||||
}
|
||||
|
||||
volSource := sidecar.SSSDConfigFile.VolumeSource
|
||||
if volSource != nil && reflect.DeepEqual(*volSource, v1.VolumeSource{}) {
|
||||
if volSourceExistsAndIsEmpty(sidecar.SSSDConfigFile.VolumeSource) {
|
||||
return errors.New("System Security Services Daemon (SSSD) sidecar is enabled with config from a VolumeSource, but no source is specified")
|
||||
}
|
||||
}
|
||||
|
||||
krb := sec.Kerberos
|
||||
if krb != nil {
|
||||
if volSourceExistsAndIsEmpty(krb.ConfigFiles.VolumeSource) {
|
||||
return errors.New("Kerberos is enabled with config from a VolumeSource, but no source is specified")
|
||||
}
|
||||
|
||||
if volSourceExistsAndIsEmpty(krb.KeytabFile.VolumeSource) {
|
||||
return errors.New("Kerberos is enabled with keytab from a VolumeSource, but no source is specified")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func volSourceExistsAndIsEmpty(v *v1.VolumeSource) bool {
|
||||
return v != nil && reflect.DeepEqual(*v, v1.VolumeSource{})
|
||||
}
|
||||
@@ -19,6 +19,7 @@ package v1
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
@@ -96,3 +97,52 @@ func TestNFSSecuritySpec_Validate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNFSSecuritySpec_KerberosEnabled(t *testing.T) {
|
||||
t.Run("nil security spec", func(t *testing.T) {
|
||||
var sec *NFSSecuritySpec
|
||||
assert.False(t, sec.KerberosEnabled())
|
||||
})
|
||||
|
||||
t.Run("empty security spec", func(t *testing.T) {
|
||||
sec := &NFSSecuritySpec{}
|
||||
assert.False(t, sec.KerberosEnabled())
|
||||
})
|
||||
|
||||
t.Run("empty kerberos spec", func(t *testing.T) {
|
||||
sec := &NFSSecuritySpec{
|
||||
Kerberos: &KerberosSpec{},
|
||||
}
|
||||
assert.True(t, sec.KerberosEnabled())
|
||||
})
|
||||
|
||||
t.Run("filled in kerberos spec", func(t *testing.T) {
|
||||
sec := &NFSSecuritySpec{
|
||||
Kerberos: &KerberosSpec{
|
||||
PrincipalName: "mom",
|
||||
},
|
||||
}
|
||||
assert.True(t, sec.KerberosEnabled())
|
||||
})
|
||||
}
|
||||
|
||||
func TestKerberosSpec_GetPrincipalName(t *testing.T) {
|
||||
t.Run("empty kerberos spec", func(t *testing.T) {
|
||||
k := &KerberosSpec{}
|
||||
assert.Equal(t, "nfs", k.GetPrincipalName())
|
||||
})
|
||||
|
||||
t.Run("principal name nfs", func(t *testing.T) {
|
||||
k := &KerberosSpec{
|
||||
PrincipalName: "nfs",
|
||||
}
|
||||
assert.Equal(t, "nfs", k.GetPrincipalName())
|
||||
})
|
||||
|
||||
t.Run("principal name set", func(t *testing.T) {
|
||||
k := &KerberosSpec{
|
||||
PrincipalName: "set",
|
||||
}
|
||||
assert.Equal(t, "set", k.GetPrincipalName())
|
||||
})
|
||||
}
|
||||
@@ -1976,6 +1976,66 @@ type NFSSecuritySpec struct {
|
||||
// +optional
|
||||
// +nullable
|
||||
SSSD *SSSDSpec `json:"sssd,omitempty"`
|
||||
|
||||
// Kerberos configures NFS-Ganesha to secure NFS client connections with Kerberos.
|
||||
// +optional
|
||||
// +nullable
|
||||
Kerberos *KerberosSpec `json:"kerberos,omitempty"`
|
||||
}
|
||||
|
||||
// KerberosSpec represents configuration for Kerberos.
|
||||
type KerberosSpec struct {
|
||||
// PrincipalName corresponds directly to NFS-Ganesha's NFS_KRB5:PrincipalName config. In
|
||||
// practice, this is the service prefix of the principal name. The default is "nfs".
|
||||
// This value is combined with (a) the namespace and name of the CephNFS (with a hyphen between)
|
||||
// and (b) the Realm configured in the user-provided krb5.conf to determine the full principal
|
||||
// name: <principalName>/<namespace>-<name>@<realm>. e.g., nfs/rook-ceph-my-nfs@example.net.
|
||||
// See https://github.com/nfs-ganesha/nfs-ganesha/wiki/RPCSEC_GSS for more detail.
|
||||
// +optional
|
||||
// +kubebuilder:default="nfs"
|
||||
PrincipalName string `json:"principalName"`
|
||||
|
||||
// ConfigFiles defines where the Kerberos configuration should be sourced from. Config files
|
||||
// will be placed into the `/etc/krb5.conf.rook/` directory.
|
||||
//
|
||||
// If this is left empty, Rook will not add any files. This allows you to manage the files
|
||||
// yourself however you wish. For example, you may build them into your custom Ceph container
|
||||
// image or use the Vault agent injector to securely add the files via annotations on the
|
||||
// CephNFS spec (passed to the NFS server pods).
|
||||
//
|
||||
// Rook configures Kerberos to log to stderr. We suggest removing logging sections from config
|
||||
// files to avoid consuming unnecessary disk space from logging to files.
|
||||
// +optional
|
||||
ConfigFiles KerberosConfigFiles `json:"configFiles"`
|
||||
|
||||
// KeytabFile defines where the Kerberos keytab should be sourced from. The keytab file will be
|
||||
// placed into `/etc/krb5.keytab`. If this is left empty, Rook will not add the file.
|
||||
// This allows you to manage the `krb5.keytab` file yourself however you wish. For example, you
|
||||
// may build it into your custom Ceph container image or use the Vault agent injector to
|
||||
// securely add the file via annotations on the CephNFS spec (passed to the NFS server pods).
|
||||
// +optional
|
||||
KeytabFile KerberosKeytabFile `json:"keytabFile"`
|
||||
}
|
||||
|
||||
// KerberosConfigFiles represents the source(s) from which Kerberos configuration should come.
|
||||
type KerberosConfigFiles struct {
|
||||
// VolumeSource accepts a standard Kubernetes VolumeSource for Kerberos configuration files like
|
||||
// what is normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret, or
|
||||
// HostPath. The volume may contain multiple files, all of which will be loaded.
|
||||
VolumeSource *v1.VolumeSource `json:"volumeSource,omitempty"`
|
||||
}
|
||||
|
||||
// KerberosKeytabFile represents the source(s) from which the Kerberos keytab file should come.
|
||||
type KerberosKeytabFile struct {
|
||||
// VolumeSource accepts a standard Kubernetes VolumeSource for the Kerberos keytab file like
|
||||
// what is normally used to configure Volumes for a Pod. For example, a Secret or HostPath.
|
||||
// There are two requirements for the source's content:
|
||||
// 1. The config file must be mountable via `subPath: krb5.keytab`. For example, in a
|
||||
// Secret, the data item must be named `krb5.keytab`, or `items` must be defined to
|
||||
// select the key and give it path `krb5.keytab`. A HostPath directory must have the
|
||||
// `krb5.keytab` file.
|
||||
// 2. The volume or config file must have mode 0600.
|
||||
VolumeSource *v1.VolumeSource `json:"volumeSource,omitempty"`
|
||||
}
|
||||
|
||||
// SSSDSpec represents configuration for System Security Services Daemon (SSSD).
|
||||
@@ -1992,11 +2052,11 @@ type SSSDSidecar struct {
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
Image string `json:"image"`
|
||||
|
||||
// Config defines where the SSSD configuration should be sourced from. The config file will be
|
||||
// placed into `/etc/sssd/sssd.conf`. If this is left empty, Rook will not add the file. This
|
||||
// allows you to manage the `sssd.conf` file yourself however you wish. For example, you may
|
||||
// build it into your custom Ceph container image or use the Vault agent injector to securely
|
||||
// add the file via annotations on the CephNFS spec (passed to the NFS server pods).
|
||||
// SSSDConfigFile defines where the SSSD configuration should be sourced from. The config file
|
||||
// will be placed into `/etc/sssd/sssd.conf`. If this is left empty, Rook will not add the file.
|
||||
// This allows you to manage the `sssd.conf` file yourself however you wish. For example, you
|
||||
// may build it into your custom Ceph container image or use the Vault agent injector to
|
||||
// securely add the file via annotations on the CephNFS spec (passed to the NFS server pods).
|
||||
// +optional
|
||||
SSSDConfigFile SSSDSidecarConfigFile `json:"sssdConfigFile"`
|
||||
|
||||
@@ -2025,18 +2085,6 @@ type SSSDSidecarConfigFile struct {
|
||||
VolumeSource *v1.VolumeSource `json:"volumeSource,omitempty"`
|
||||
}
|
||||
|
||||
// SSSDSidecarConfigFileConfigMap represents the option to use a ConfigMap as the source for SSSD configuration.
|
||||
type SSSDSidecarConfigFileConfigMap struct {
|
||||
// Name accepts the name of a ConfigMap which should contain the SSSD configuration. The
|
||||
// ConfigMap must contain the SSSD configuration in a data entry named 'sssd.conf'. The
|
||||
// ConfigMap must be created in the same namespace as the CephNFS (and CephCluster).
|
||||
// The user-supplied configuration follows the sssd.conf format documented here:
|
||||
// https://linux.die.net/man/5/sssd.conf
|
||||
// +kubebuilder:validation:Required
|
||||
// +kubebuilder:validation:MinLength=1
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// NetworkSpec for Ceph includes backward compatibility code
|
||||
type NetworkSpec struct {
|
||||
// Provider is what provides network connectivity to the cluster e.g. "host" or "multus"
|
||||
|
||||
@@ -2450,6 +2450,66 @@ func (in *KafkaEndpointSpec) DeepCopy() *KafkaEndpointSpec {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KerberosConfigFiles) DeepCopyInto(out *KerberosConfigFiles) {
|
||||
*out = *in
|
||||
if in.VolumeSource != nil {
|
||||
in, out := &in.VolumeSource, &out.VolumeSource
|
||||
*out = new(corev1.VolumeSource)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KerberosConfigFiles.
|
||||
func (in *KerberosConfigFiles) DeepCopy() *KerberosConfigFiles {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(KerberosConfigFiles)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KerberosKeytabFile) DeepCopyInto(out *KerberosKeytabFile) {
|
||||
*out = *in
|
||||
if in.VolumeSource != nil {
|
||||
in, out := &in.VolumeSource, &out.VolumeSource
|
||||
*out = new(corev1.VolumeSource)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KerberosKeytabFile.
|
||||
func (in *KerberosKeytabFile) DeepCopy() *KerberosKeytabFile {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(KerberosKeytabFile)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KerberosSpec) DeepCopyInto(out *KerberosSpec) {
|
||||
*out = *in
|
||||
in.ConfigFiles.DeepCopyInto(&out.ConfigFiles)
|
||||
in.KeytabFile.DeepCopyInto(&out.KeytabFile)
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KerberosSpec.
|
||||
func (in *KerberosSpec) DeepCopy() *KerberosSpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(KerberosSpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *KeyManagementServiceSpec) DeepCopyInto(out *KeyManagementServiceSpec) {
|
||||
*out = *in
|
||||
@@ -2809,6 +2869,11 @@ func (in *NFSSecuritySpec) DeepCopyInto(out *NFSSecuritySpec) {
|
||||
*out = new(SSSDSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
if in.Kerberos != nil {
|
||||
in, out := &in.Kerberos, &out.Kerberos
|
||||
*out = new(KerberosSpec)
|
||||
(*in).DeepCopyInto(*out)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -3655,22 +3720,6 @@ func (in *SSSDSidecarConfigFile) DeepCopy() *SSSDSidecarConfigFile {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *SSSDSidecarConfigFileConfigMap) DeepCopyInto(out *SSSDSidecarConfigFileConfigMap) {
|
||||
*out = *in
|
||||
return
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SSSDSidecarConfigFileConfigMap.
|
||||
func (in *SSSDSidecarConfigFileConfigMap) DeepCopy() *SSSDSidecarConfigFileConfigMap {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(SSSDSidecarConfigFileConfigMap)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *SSSDSpec) DeepCopyInto(out *SSSDSpec) {
|
||||
*out = *in
|
||||
|
||||
@@ -39,6 +39,8 @@ const (
|
||||
CephTool = "ceph"
|
||||
// RBDTool is the name of the CLI tool for 'rbd'
|
||||
RBDTool = "rbd"
|
||||
// RadosTool is the name of the CLI tool for 'rados'
|
||||
RadosTool = "rados"
|
||||
// Kubectl is the name of the CLI tool for 'kubectl'
|
||||
Kubectl = "kubectl"
|
||||
// CrushTool is the name of the CLI tool for 'crushtool'
|
||||
@@ -59,15 +61,14 @@ func CephConfFilePath(configDir, clusterName string) string {
|
||||
|
||||
// FinalizeCephCommandArgs builds the command line to be called
|
||||
func FinalizeCephCommandArgs(command string, clusterInfo *ClusterInfo, args []string, configDir string) (string, []string) {
|
||||
// the rbd client tool does not support the '--connect-timeout' option
|
||||
// so we only use it for the 'ceph' command
|
||||
// Also, there is no point of adding that option to 'crushtool' since that CLI does not connect to anything
|
||||
// 'crushtool' is a utility that lets you create, compile, decompile and test CRUSH map files.
|
||||
|
||||
// we could use a slice and iterate over it but since we have only 3 elements
|
||||
// I don't think this is worth a loop
|
||||
timeout := strconv.Itoa(int(exec.CephCommandsTimeout.Seconds()))
|
||||
if command != "rbd" && command != "crushtool" && command != "radosgw-admin" {
|
||||
|
||||
// some tools not support the '--connect-timeout' option
|
||||
// so we only use it for the 'ceph' command
|
||||
switch command {
|
||||
case RBDTool, CrushTool, RadosTool, "radosgw-admin":
|
||||
// do not add timeout flag
|
||||
default:
|
||||
args = append(args, "--connect-timeout="+timeout)
|
||||
}
|
||||
|
||||
@@ -127,6 +128,18 @@ func NewRBDCommand(context *clusterd.Context, clusterInfo *ClusterInfo, args []s
|
||||
return cmd
|
||||
}
|
||||
|
||||
func NewRadosCommand(context *clusterd.Context, clusterInfo *ClusterInfo, args []string) *CephToolCommand {
|
||||
cmd := newCephToolCommand(RadosTool, context, clusterInfo, args)
|
||||
cmd.JsonOutput = false
|
||||
|
||||
// When Multus is enabled, the rados tool should run inside the proxy container
|
||||
if clusterInfo.NetworkSpec.IsMultus() {
|
||||
cmd.RemoteExecution = true
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func (c *CephToolCommand) run() ([]byte, error) {
|
||||
// Return if the context has been canceled
|
||||
if c.clusterInfo.Context.Err() != nil {
|
||||
@@ -153,7 +166,10 @@ func (c *CephToolCommand) run() ([]byte, error) {
|
||||
args = append(args, "--format", "json")
|
||||
} else {
|
||||
// the `rbd` tool doesn't use special flag for plain format
|
||||
if c.tool != RBDTool {
|
||||
switch c.tool {
|
||||
case RBDTool, RadosTool:
|
||||
// do not add format option
|
||||
default:
|
||||
args = append(args, "--format", "plain")
|
||||
}
|
||||
}
|
||||
@@ -163,7 +179,7 @@ func (c *CephToolCommand) run() ([]byte, error) {
|
||||
|
||||
// NewRBDCommand does not use the --out-file option so we only check for remote execution here
|
||||
// Still forcing the check for the command if the behavior changes in the future
|
||||
if command == RBDTool {
|
||||
if command == RBDTool || command == RadosTool {
|
||||
if c.RemoteExecution {
|
||||
output, stderr, err = c.context.RemoteExecutor.ExecCommandInContainerWithFullOutputWithTimeout(c.clusterInfo.Context, ProxyAppLabel, CommandProxyInitContainerName, c.clusterInfo.Namespace, append([]string{command}, args...)...)
|
||||
if stderr != "" || err != nil {
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
Copyright 2022 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package client
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
)
|
||||
|
||||
/*
|
||||
Example Lock info:
|
||||
|
||||
{
|
||||
"name": "rook-ceph-nfs",
|
||||
"type": "exclusive",
|
||||
"tag": "rook-ceph-nfs",
|
||||
"lockers": [
|
||||
{
|
||||
"name": "client.28945",
|
||||
"cookie": "test-cookie",
|
||||
"description": "",
|
||||
"expiration": "2022-09-08T23:56:57.924802+0000",
|
||||
"addr": "10.244.0.44:0/960227889"
|
||||
}
|
||||
]
|
||||
}
|
||||
*/
|
||||
type radosLockInfo struct {
|
||||
Name string `json:"name"`
|
||||
Tag string `json:"tag"`
|
||||
Lockers []radosLockerInfo `json:"lockers"`
|
||||
}
|
||||
|
||||
type radosLockerInfo struct {
|
||||
Name string `json:"name"`
|
||||
Cookie string `json:"cookie"`
|
||||
}
|
||||
|
||||
// RadosLockObject locks a rados object in a given pool and namespace and returns a lock "cookie"
|
||||
// that can be used to identify this unique lock.
|
||||
func RadosLockObject(
|
||||
context *clusterd.Context, clusterInfo *ClusterInfo,
|
||||
pool, namespace, objectName, lockName string,
|
||||
lockTimeout time.Duration,
|
||||
) (string, error) {
|
||||
// generate a random "cookie" that identifies this lock "session"
|
||||
cookieLen := 12
|
||||
b := make([]byte, cookieLen)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", errors.Wrapf(err, "failed to generate cookie for lock %q on rados object rados://%s/%s/%s",
|
||||
lockName, pool, namespace, objectName)
|
||||
}
|
||||
cookie := fmt.Sprintf("%x", b)[:cookieLen]
|
||||
|
||||
cmd := NewRadosCommand(context, clusterInfo, []string{
|
||||
"--pool", pool,
|
||||
"--namespace", namespace,
|
||||
"lock", "get", objectName, lockName,
|
||||
"--lock-tag", lockName, // assume we aren't making many locks; use lock name as the tag
|
||||
"--lock-cookie", cookie,
|
||||
"--lock-duration", fmt.Sprintf("%d", int(math.Ceil(lockTimeout.Seconds()))),
|
||||
})
|
||||
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
||||
return "", errors.Wrapf(err, "failed to acquire lock %q on rados object rados://%s/%s/%s",
|
||||
lockName, pool, namespace, objectName)
|
||||
}
|
||||
|
||||
return cookie, nil
|
||||
}
|
||||
|
||||
// RadosUnlockObject unlocks a rados object in a given pool and namespace by searching for locks on
|
||||
// the object that match the "cookie" obtained from a RadosLockObject() call.
|
||||
func RadosUnlockObject(
|
||||
context *clusterd.Context, clusterInfo *ClusterInfo,
|
||||
pool, namespace, objectName, lockName string,
|
||||
lockCookie string,
|
||||
) error {
|
||||
lockInfo, err := radosObjectLockInfo(context, clusterInfo, pool, namespace, objectName, lockName, lockCookie)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to unlock object")
|
||||
}
|
||||
|
||||
if lockInfo.Name != lockName || lockInfo.Tag != lockName {
|
||||
// some other lock has locked the object, but that means the lock with the given cookie no
|
||||
// longer has it locked; treat this as success
|
||||
logger.Infof("rados object rados://%s/%s/%s is not locked by lock %q but is locked by %q",
|
||||
pool, namespace, objectName, lockName, lockInfo.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(lockInfo.Lockers) == 0 {
|
||||
logger.Infof("rados object rados://%s/%s/%s is already fully unlocked (lock: %q, cookie: %q)",
|
||||
pool, namespace, objectName, lockName, lockCookie)
|
||||
return nil
|
||||
}
|
||||
|
||||
locker := findLockerWithCookie(lockInfo.Lockers, lockCookie)
|
||||
if locker == nil {
|
||||
// this cookie is not locking the object but another is; still treat this as success but log it
|
||||
logger.Infof("rados object rados://%s/%s/%s is not locked by lock %q with cookie %q but is locked: %+v",
|
||||
pool, namespace, objectName, lockName, lockCookie, lockInfo.Lockers)
|
||||
return nil
|
||||
}
|
||||
|
||||
cmd := NewRadosCommand(context, clusterInfo, []string{
|
||||
"--pool", pool,
|
||||
"--namespace", namespace,
|
||||
"lock", "break", objectName, lockName, locker.Name, // breaking the lock also requires locker name
|
||||
"--lock-tag", lockName, // assume we aren't making many locks; use lock name as the tag
|
||||
"--lock-cookie", lockCookie,
|
||||
})
|
||||
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
||||
return errors.Wrapf(err, "failed to unlock lock %q on rados object rados://%s/%s/%s",
|
||||
lockName, pool, namespace, objectName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func radosObjectLockInfo(
|
||||
context *clusterd.Context, clusterInfo *ClusterInfo,
|
||||
pool, namespace, objectName, lockName string,
|
||||
lockCookie string,
|
||||
) (radosLockInfo, error) {
|
||||
cmd := NewRadosCommand(context, clusterInfo, []string{
|
||||
"--pool", pool,
|
||||
"--namespace", namespace,
|
||||
"lock", "info", objectName, lockName,
|
||||
"--format", "json",
|
||||
})
|
||||
cmd.JsonOutput = true
|
||||
rawInfo, err := cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err != nil {
|
||||
return radosLockInfo{}, errors.Wrapf(err, "failed to get lock %q info for rados object rados://%s/%s/%s",
|
||||
lockName, pool, namespace, objectName)
|
||||
}
|
||||
|
||||
lockInfo := radosLockInfo{}
|
||||
if err := json.Unmarshal(rawInfo, &lockInfo); err != nil {
|
||||
return radosLockInfo{}, errors.Wrapf(err, "failed to parse lock %q info for rados object rados://%s/%s/%s",
|
||||
lockName, pool, namespace, objectName)
|
||||
}
|
||||
|
||||
return lockInfo, nil
|
||||
}
|
||||
|
||||
func findLockerWithCookie(lockers []radosLockerInfo, cookie string) *radosLockerInfo {
|
||||
for _, locker := range lockers {
|
||||
if locker.Cookie == cookie {
|
||||
return &locker
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RadosRemoveObject idempotently removes a rados object from the given pool and namespace.
|
||||
func RadosRemoveObject(
|
||||
context *clusterd.Context, clusterInfo *ClusterInfo,
|
||||
pool, namespace, objectName string,
|
||||
) error {
|
||||
cmd := NewRadosCommand(context, clusterInfo, []string{
|
||||
"--pool", pool,
|
||||
"--namespace", namespace,
|
||||
"stat", objectName,
|
||||
})
|
||||
_, err := cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err != nil {
|
||||
if exec.IsTimeout(err) {
|
||||
return errors.Wrapf(err, "failed to determine if rados object rados://%s/%s/%s exists before removing it",
|
||||
pool, namespace, objectName)
|
||||
}
|
||||
// assume any other error means the object already doesn't exist
|
||||
logger.Debugf("rados object rados:/%s/%s/%s being removed is assumed to not exist after stat-ing: %v",
|
||||
pool, namespace, objectName, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
cmd = NewRadosCommand(context, clusterInfo, []string{
|
||||
"--pool", pool,
|
||||
"--namespace", namespace,
|
||||
"rm", objectName,
|
||||
})
|
||||
_, err = cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to remove rados object rados://%s/%s/%s", pool, namespace, objectName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
Copyright 2022 The Rook Authors. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
"github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRadosRemoveObject(t *testing.T) {
|
||||
newTest := func(mockExec *test.MockExecutor) (*clusterd.Context, *ClusterInfo) {
|
||||
ctx := &clusterd.Context{
|
||||
Executor: mockExec,
|
||||
}
|
||||
info := &ClusterInfo{
|
||||
Context: context.Background(),
|
||||
}
|
||||
return ctx, info
|
||||
}
|
||||
|
||||
t.Run("timeout stat-ing", func(t *testing.T) {
|
||||
me := &test.MockExecutor{
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
return "", test.FakeTimeoutError("")
|
||||
},
|
||||
}
|
||||
c, i := newTest(me)
|
||||
err := RadosRemoveObject(c, i, "mypool", "myns", "someobj")
|
||||
assert.Error(t, err)
|
||||
assert.True(t, exec.IsTimeout(err))
|
||||
})
|
||||
|
||||
t.Run("stat err means the object is already gone", func(t *testing.T) {
|
||||
me := &test.MockExecutor{
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
subCommand := arg[4]
|
||||
assert.Equal(t, subCommand, "stat")
|
||||
return "", errors.New("induced stat error")
|
||||
},
|
||||
}
|
||||
c, i := newTest(me)
|
||||
err := RadosRemoveObject(c, i, "mypool", "myns", "someobj")
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("rm err is an error", func(t *testing.T) {
|
||||
me := &test.MockExecutor{
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
assert.Equal(t, arg[0], "--pool")
|
||||
assert.Equal(t, arg[1], "mypool")
|
||||
assert.Equal(t, arg[2], "--namespace")
|
||||
assert.Equal(t, arg[3], "myns")
|
||||
subCommand := arg[4]
|
||||
assert.Equal(t, arg[5], "someobj")
|
||||
|
||||
if subCommand == "stat" {
|
||||
return "", nil // successful stat means obj exists
|
||||
} else if subCommand == "rm" {
|
||||
return "", errors.New("induced rm error")
|
||||
} else {
|
||||
panic(fmt.Sprintf("unhandled subcommand %q", subCommand))
|
||||
}
|
||||
},
|
||||
}
|
||||
c, i := newTest(me)
|
||||
err := RadosRemoveObject(c, i, "mypool", "myns", "someobj")
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("rm successful", func(t *testing.T) {
|
||||
me := &test.MockExecutor{
|
||||
MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
assert.Equal(t, arg[0], "--pool")
|
||||
assert.Equal(t, arg[1], "mypool")
|
||||
assert.Equal(t, arg[2], "--namespace")
|
||||
assert.Equal(t, arg[3], "myns")
|
||||
subCommand := arg[4]
|
||||
assert.Equal(t, arg[5], "someobj")
|
||||
|
||||
if subCommand == "stat" || subCommand == "rm" {
|
||||
return "", nil
|
||||
} else {
|
||||
panic(fmt.Sprintf("unhandled subcommand %q", subCommand))
|
||||
}
|
||||
},
|
||||
}
|
||||
c, i := newTest(me)
|
||||
err := RadosRemoveObject(c, i, "mypool", "myns", "someobj")
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
@@ -18,15 +18,22 @@ package nfs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
"github.com/rook/rook/pkg/clusterd"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config/keyring"
|
||||
cephver "github.com/rook/rook/pkg/operator/ceph/version"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
)
|
||||
|
||||
const kerberosRadosObjectName = "kerberos"
|
||||
|
||||
const (
|
||||
keyringTemplate = `
|
||||
[%s]
|
||||
@@ -48,18 +55,18 @@ func getNFSNodeID(n *cephv1.CephNFS, name string) string {
|
||||
return fmt.Sprintf("%s.%s", n.Name, name)
|
||||
}
|
||||
|
||||
func getGaneshaConfigObject(n *cephv1.CephNFS, version cephver.CephVersion, name string) string {
|
||||
func getGaneshaConfigObject(n *cephv1.CephNFS) string {
|
||||
return fmt.Sprintf("conf-nfs.%s", n.Name)
|
||||
}
|
||||
|
||||
func getRadosURL(n *cephv1.CephNFS, version cephver.CephVersion, name string) string {
|
||||
func getRadosURL(n *cephv1.CephNFS) string {
|
||||
url := fmt.Sprintf("rados://%s/", n.Spec.RADOS.Pool)
|
||||
|
||||
if n.Spec.RADOS.Namespace != "" {
|
||||
url += n.Spec.RADOS.Namespace + "/"
|
||||
}
|
||||
|
||||
url += getGaneshaConfigObject(n, version, name)
|
||||
url += getGaneshaConfigObject(n)
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -87,7 +94,7 @@ func (r *ReconcileCephNFS) generateKeyring(n *cephv1.CephNFS, name string) error
|
||||
func getGaneshaConfig(n *cephv1.CephNFS, version cephver.CephVersion, name string) string {
|
||||
nodeID := getNFSNodeID(n, name)
|
||||
userID := getNFSUserID(nodeID)
|
||||
url := getRadosURL(n, version, name)
|
||||
url := getRadosURL(n)
|
||||
return `
|
||||
NFS_CORE_PARAM {
|
||||
Enable_NLM = false;
|
||||
@@ -126,3 +133,242 @@ RADOS_URLS {
|
||||
%url ` + url + `
|
||||
`
|
||||
}
|
||||
|
||||
func ganeshaKrbConfigBlock(kerberosSpec *cephv1.KerberosSpec) string {
|
||||
return fmt.Sprintf(`NFS_KRB5 {
|
||||
PrincipalName = "%s" ;
|
||||
KeytabPath = /etc/krb5.keytab ;
|
||||
Active_krb5 = YES ;
|
||||
}
|
||||
`, kerberosSpec.GetPrincipalName())
|
||||
}
|
||||
|
||||
func ganeshaConfigIncludeKrbBlock(nfs *cephv1.CephNFS, radosObjectName string) string {
|
||||
// don't use sprintf b/c %u on front makes compiler confused
|
||||
return `%url "rados://` + nfs.Spec.RADOS.Pool + `/` + nfs.Spec.RADOS.Namespace + `/` + radosObjectName + `"` + "\n\n"
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNFS) setRadosConfig(nfs *cephv1.CephNFS) error {
|
||||
if nfs.Spec.Security.KerberosEnabled() {
|
||||
return setKerberosRadosConfig(r.context, r.clusterInfo, nfs)
|
||||
}
|
||||
return removeKerberosRadosConfig(r.context, r.clusterInfo, nfs)
|
||||
}
|
||||
|
||||
func setKerberosRadosConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, nfs *cephv1.CephNFS) error {
|
||||
radosPool := nfs.Spec.RADOS.Pool
|
||||
radosNs := nfs.Spec.RADOS.Namespace
|
||||
radosInfoStr := fmt.Sprintf("rados://%s/%s/", radosPool, radosNs)
|
||||
|
||||
logger.Infof("ensuring kerberos configuration exists in rados namespace %s", radosInfoStr)
|
||||
|
||||
// write ganesha kerberos configuration block into a temp file
|
||||
krbBlockFile, err := ioutil.TempFile("", "krb-block-file")
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create temp file for ganesha kerberos configuration block for %s", radosInfoStr)
|
||||
}
|
||||
defer krbBlockFile.Close()
|
||||
_, err = krbBlockFile.WriteString(ganeshaKrbConfigBlock(nfs.Spec.Security.Kerberos))
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed write ganesha kerberos configuration block temp file for %s", radosInfoStr)
|
||||
}
|
||||
|
||||
radosFlags := []string{
|
||||
"--pool", radosPool,
|
||||
"--namespace", radosNs,
|
||||
}
|
||||
|
||||
// write ganesha kerberos configuration block to rados object from temp file
|
||||
cmd := cephclient.NewRadosCommand(context, clusterInfo,
|
||||
append(radosFlags, "put", kerberosRadosObjectName, krbBlockFile.Name()))
|
||||
_, err = cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create or update the ganesha kerberos config object %s/%s",
|
||||
radosInfoStr, kerberosRadosObjectName)
|
||||
}
|
||||
|
||||
// prepend the config block that includes the kerberos config object to the ganesha config object
|
||||
ganeshaConfigObjName := getGaneshaConfigObject(nfs)
|
||||
krbIncludeBlock := ganeshaConfigIncludeKrbBlock(nfs, kerberosRadosObjectName)
|
||||
err = atomicPrependToConfigObject(context, clusterInfo, radosPool, radosNs, ganeshaConfigObjName, krbIncludeBlock)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to update the ganesha config object to include the kerberos object config")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeKerberosRadosConfig(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, nfs *cephv1.CephNFS) error {
|
||||
radosPool := nfs.Spec.RADOS.Pool
|
||||
radosNs := nfs.Spec.RADOS.Namespace
|
||||
radosInfoStr := fmt.Sprintf("rados://%s/%s/", radosPool, radosNs)
|
||||
|
||||
logger.Infof("ensuring kerberos configuration is removed from rados namespace %s", radosInfoStr)
|
||||
|
||||
// remove config block that includes the kerberos config object from the ganesha config object
|
||||
ganeshaConfigObjName := getGaneshaConfigObject(nfs)
|
||||
krbIncludeBlock := ganeshaConfigIncludeKrbBlock(nfs, kerberosRadosObjectName)
|
||||
err := atomicRemoveFromConfigObject(context, clusterInfo, radosPool, radosNs, ganeshaConfigObjName, krbIncludeBlock)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to update the ganesha config object to remove the kerberos object config")
|
||||
}
|
||||
|
||||
// remove the kerberos config rados object
|
||||
err = cephclient.RadosRemoveObject(context, clusterInfo, radosPool, radosNs, kerberosRadosObjectName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to remove the ganesha kerberos config object")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepend a config block to the rados object in an atomic function
|
||||
// uses rados locks to ensure the changes we make here aren't going to race with changes that might
|
||||
// be made by ceph mgr, ceph itself, or a user manually
|
||||
func atomicPrependToConfigObject(
|
||||
context *clusterd.Context, clusterInfo *cephclient.ClusterInfo,
|
||||
radosPool, radosNamespace, objectName, configBlock string,
|
||||
) error {
|
||||
tmpFilePattern := fmt.Sprintf("%s_%s_%s_prepend", radosPool, radosNamespace, objectName)
|
||||
objInfoString := fmt.Sprintf("rados://%s/%s/%s", radosPool, radosNamespace, objectName)
|
||||
|
||||
// read object into temp file
|
||||
tempFile, err := ioutil.TempFile("", tmpFilePattern)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create temp file for %s", objInfoString)
|
||||
}
|
||||
defer tempFile.Close()
|
||||
|
||||
radosFlags := []string{
|
||||
"--pool", radosPool,
|
||||
"--namespace", radosNamespace,
|
||||
}
|
||||
|
||||
// acquire lock to ensure no other processes (user, ceph, or rook) are racing each other
|
||||
// most common contender will be CSI when creating/removing NFS exports
|
||||
lockName := AppName
|
||||
|
||||
// planning to perform 2 commands after the lock: get object, and optionally write object, so
|
||||
// use lock timeout of 2x the normal ceph command timeout
|
||||
logger.Infof("locking rados object %q", objInfoString)
|
||||
cookie, err := cephclient.RadosLockObject(context, clusterInfo,
|
||||
radosPool, radosNamespace, objectName, lockName, exec.CephCommandsTimeout*2)
|
||||
if err != nil {
|
||||
return err // already a good err message
|
||||
}
|
||||
logger.Infof("successfully locked rados object %q", objInfoString)
|
||||
defer func() {
|
||||
logger.Infof("unlocking rados object %q", objInfoString)
|
||||
err := cephclient.RadosUnlockObject(context, clusterInfo,
|
||||
radosPool, radosNamespace, objectName, lockName, cookie)
|
||||
if err != nil {
|
||||
logger.Infof("failed to unlock rados object %q, but since the lock has a timeout, we will continue. %v", objInfoString, err)
|
||||
}
|
||||
logger.Info("successfully unlocked rados object %q", objInfoString)
|
||||
}()
|
||||
|
||||
cmd := cephclient.NewRadosCommand(context, clusterInfo,
|
||||
append(radosFlags, "get", objectName, tempFile.Name()))
|
||||
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
||||
return errors.Wrapf(err, "failed to get object %s", objInfoString)
|
||||
}
|
||||
|
||||
rawObj, err := ioutil.ReadAll(tempFile)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to read object %s from temp file", objInfoString)
|
||||
}
|
||||
|
||||
if strings.Contains(string(rawObj), configBlock) {
|
||||
logger.Debugf("rados object %s already has config block: %s", objInfoString, configBlock)
|
||||
return nil
|
||||
}
|
||||
logger.Debugf("rados object %s will have config block prepended: %s", objInfoString, configBlock)
|
||||
|
||||
newConfig := fmt.Sprintf("%s%s", configBlock, string(rawObj))
|
||||
if err := ioutil.WriteFile(tempFile.Name(), []byte(newConfig), fs.FileMode(0644)); err != nil {
|
||||
return errors.Wrapf(err, "failed to write new config content for object %s to temp file", objInfoString)
|
||||
}
|
||||
|
||||
cmd = cephclient.NewRadosCommand(context, clusterInfo,
|
||||
append(radosFlags, "put", objectName, tempFile.Name()))
|
||||
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
||||
return errors.Wrapf(err, "failed to set new config content on object %s", objInfoString)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// remove a config block from the rados object in an atomic function
|
||||
// uses rados locks to ensure the changes we make here aren't going to race with changes that might
|
||||
// be made by ceph mgr, ceph itself, or a user manually
|
||||
func atomicRemoveFromConfigObject(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo,
|
||||
radosPool, radosNamespace, objectName, configBlock string,
|
||||
) error {
|
||||
tmpFilePattern := fmt.Sprintf("%s_%s_%s_remove", radosPool, radosNamespace, objectName)
|
||||
objInfoString := fmt.Sprintf("rados://%s/%s/%s", radosPool, radosNamespace, objectName)
|
||||
|
||||
// read object into temp file
|
||||
tempFile, err := ioutil.TempFile("", tmpFilePattern)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create temp file for %s", objInfoString)
|
||||
}
|
||||
defer tempFile.Close()
|
||||
|
||||
radosFlags := []string{
|
||||
"--pool", radosPool,
|
||||
"--namespace", radosNamespace,
|
||||
}
|
||||
|
||||
// acquire lock to ensure no other processes (user, ceph, or rook) are racing each other
|
||||
// most common contender will be CSI when creating/removing NFS exports
|
||||
lockName := AppName
|
||||
|
||||
// planning to perform 2 commands after the lock: get object, and optionally write object, so
|
||||
// use lock timeout of 2x the normal ceph command timeout
|
||||
logger.Infof("locking rados object %q", objInfoString)
|
||||
cookie, err := cephclient.RadosLockObject(context, clusterInfo,
|
||||
radosPool, radosNamespace, objectName, lockName, exec.CephCommandsTimeout*2)
|
||||
if err != nil {
|
||||
return err // already a good err message
|
||||
}
|
||||
logger.Infof("successfully locked rados object %q", objInfoString)
|
||||
defer func() {
|
||||
logger.Infof("unlocking rados object %q", objInfoString)
|
||||
err := cephclient.RadosUnlockObject(context, clusterInfo,
|
||||
radosPool, radosNamespace, objectName, lockName, cookie)
|
||||
if err != nil {
|
||||
logger.Infof("failed to unlock rados object %q, but since the lock has a timeout, we will continue. %v", objInfoString, err)
|
||||
}
|
||||
logger.Infof("successfully unlocked rados object %q", objInfoString)
|
||||
}()
|
||||
|
||||
cmd := cephclient.NewRadosCommand(context, clusterInfo,
|
||||
append(radosFlags, "get", objectName, tempFile.Name()))
|
||||
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
||||
return errors.Wrapf(err, "failed to get object %s", objInfoString)
|
||||
}
|
||||
|
||||
rawObj, err := ioutil.ReadAll(tempFile)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to read object %s from temp file", objInfoString)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(rawObj), configBlock) {
|
||||
logger.Debugf("rados object %q already does not have config block: %s", objInfoString, configBlock)
|
||||
return nil
|
||||
}
|
||||
logger.Debugf("rados object %s will have config block removed: %s", objInfoString, configBlock)
|
||||
|
||||
newConfig := strings.ReplaceAll(string(rawObj), configBlock, "")
|
||||
if err := ioutil.WriteFile(tempFile.Name(), []byte(newConfig), fs.FileMode(0644)); err != nil {
|
||||
return errors.Wrapf(err, "failed to write new config content for object %s to temp file", objInfoString)
|
||||
}
|
||||
|
||||
cmd = cephclient.NewRadosCommand(context, clusterInfo,
|
||||
append(radosFlags, "put", objectName, tempFile.Name()))
|
||||
if _, err := cmd.RunWithTimeout(exec.CephCommandsTimeout); err != nil {
|
||||
return errors.Wrapf(err, "failed to set new config content on object %s", objInfoString)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/pkg/capnslog"
|
||||
"github.com/pkg/errors"
|
||||
@@ -170,6 +171,12 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul
|
||||
updateStatus(k8sutil.ObservedGenerationNotAvailable, r.client, request.NamespacedName, k8sutil.EmptyStatus)
|
||||
}
|
||||
|
||||
if err := cephNFS.Spec.Security.Validate(); err != nil {
|
||||
return reconcile.Result{Requeue: true, RequeueAfter: 15 * time.Second}, *cephNFS,
|
||||
errors.Wrapf(err, "failed to validate security spec for CephNFS %q",
|
||||
types.NamespacedName{Namespace: cephNFS.Namespace, Name: cephNFS.Name})
|
||||
}
|
||||
|
||||
// Make sure a CephCluster is present otherwise do nothing
|
||||
cephCluster, isReadyToReconcile, cephClusterExists, reconcileResponse := opcontroller.IsReadyToReconcile(r.opManagerContext, r.client, request.NamespacedName, controllerName)
|
||||
if !isReadyToReconcile {
|
||||
|
||||
@@ -238,6 +238,29 @@ func TestCephNFSController(t *testing.T) {
|
||||
assert.True(t, res.Requeue)
|
||||
})
|
||||
|
||||
t.Run("error - security spec invalid", func(t *testing.T) {
|
||||
t.Run("security.sssd empty should error", func(t *testing.T) {
|
||||
cCtx := newContext(baseExecutor())
|
||||
nfs := baseCephNFS()
|
||||
nfs.Spec.Security = &cephv1.NFSSecuritySpec{
|
||||
SSSD: &cephv1.SSSDSpec{},
|
||||
}
|
||||
cl := newControllerClient(nfs, cephClusterNotReady())
|
||||
r := newReconcile(cCtx, cl)
|
||||
fakeRecorder := record.NewFakeRecorder(5)
|
||||
r.recorder = fakeRecorder
|
||||
|
||||
res, err := r.Reconcile(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, res.Requeue)
|
||||
|
||||
assert.Len(t, fakeRecorder.Events, 1)
|
||||
event := <-fakeRecorder.Events
|
||||
assert.Contains(t, event, // verify the security spec calls the Validate() method
|
||||
"System Security Services Daemon (SSSD) is enabled, but no runtime option is specified")
|
||||
})
|
||||
})
|
||||
|
||||
assertCephNFSReady := func(t *testing.T, r *ReconcileCephNFS, names ...string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -478,10 +501,9 @@ func TestGetGaneshaConfigObject(t *testing.T) {
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
nodeid := "a"
|
||||
expectedName := "conf-nfs.my-nfs"
|
||||
|
||||
res := getGaneshaConfigObject(cephNFS, version.CephVersion{Major: 16}, nodeid)
|
||||
res := getGaneshaConfigObject(cephNFS)
|
||||
logger.Infof("Config Object for Pacific is %s", res)
|
||||
assert.Equal(t, expectedName, res)
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
opmon "github.com/rook/rook/pkg/operator/ceph/cluster/mon"
|
||||
"github.com/rook/rook/pkg/operator/ceph/config"
|
||||
"github.com/rook/rook/pkg/operator/k8sutil"
|
||||
"github.com/rook/rook/pkg/util/exec"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -64,11 +65,15 @@ func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error {
|
||||
return errors.Wrap(err, "failed to create config")
|
||||
}
|
||||
|
||||
err = r.addRADOSConfigFile(n, id)
|
||||
err = r.addRADOSConfigFile(n)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create RADOS config object")
|
||||
}
|
||||
|
||||
if err := r.setRadosConfig(n); err != nil {
|
||||
return errors.Wrap(err, "failed to set RADOS config options")
|
||||
}
|
||||
|
||||
cfg := daemonConfig{
|
||||
ID: id,
|
||||
ConfigConfigMap: configName,
|
||||
@@ -133,22 +138,28 @@ func (r *ReconcileCephNFS) upCephNFS(n *cephv1.CephNFS) error {
|
||||
}
|
||||
|
||||
// Create empty config file for new ganesha server
|
||||
func (r *ReconcileCephNFS) addRADOSConfigFile(n *cephv1.CephNFS, name string) error {
|
||||
config := getGaneshaConfigObject(n, r.clusterInfo.CephVersion, name)
|
||||
cmd := "rados"
|
||||
args := []string{
|
||||
func (r *ReconcileCephNFS) addRADOSConfigFile(n *cephv1.CephNFS) error {
|
||||
config := getGaneshaConfigObject(n)
|
||||
|
||||
flags := []string{
|
||||
"--pool", n.Spec.RADOS.Pool,
|
||||
"--namespace", n.Spec.RADOS.Namespace,
|
||||
"--conf", cephclient.CephConfFilePath(r.context.ConfigDir, n.Namespace),
|
||||
}
|
||||
err := r.context.Executor.ExecuteCommand(cmd, append(args, "stat", config)...)
|
||||
|
||||
cmd := cephclient.NewRadosCommand(r.context, r.clusterInfo, append(flags, "stat", config))
|
||||
_, err := cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err == nil {
|
||||
// If stat works then we assume it's present already
|
||||
return nil
|
||||
}
|
||||
|
||||
// try to create it
|
||||
return r.context.Executor.ExecuteCommand(cmd, append(args, "create", config)...)
|
||||
cmd = cephclient.NewRadosCommand(r.context, r.clusterInfo, append(flags, "create", config))
|
||||
_, err = cmd.RunWithTimeout(exec.CephCommandsTimeout)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create initial rados config object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ReconcileCephNFS) addServerToDatabase(nfs *cephv1.CephNFS, name string) error {
|
||||
|
||||
@@ -19,7 +19,6 @@ package nfs
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -33,15 +32,16 @@ func (r *ReconcileCephNFS) addSecurityConfigsToPod(nfs *cephv1.CephNFS, pod *v1.
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := sec.Validate(); err != nil {
|
||||
return errors.Wrapf(err, "failed to set up security for CephNFS %q", nsName)
|
||||
}
|
||||
|
||||
if sec.SSSD != nil {
|
||||
logger.Debugf("configuring system security services daemon (SSSD) for CephNFS %q", nsName)
|
||||
addSSSDConfigsToPod(r, nfs, pod)
|
||||
}
|
||||
|
||||
if sec.Kerberos != nil {
|
||||
logger.Debugf("configuring Kerberos for CephNFS %q", nsName)
|
||||
addKerberosConfigsToPod(r, nfs, pod)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -69,6 +69,31 @@ func addSSSDConfigsToPod(r *ReconcileCephNFS, nfs *cephv1.CephNFS, pod *v1.PodSp
|
||||
}
|
||||
}
|
||||
|
||||
func addKerberosConfigsToPod(r *ReconcileCephNFS, nfs *cephv1.CephNFS, pod *v1.PodSpec) {
|
||||
init, volume, ganeshaMount := generateKrbConfResources(r, nfs)
|
||||
|
||||
pod.InitContainers = append(pod.InitContainers, *init)
|
||||
pod.Volumes = append(pod.Volumes, *volume)
|
||||
// assume the first container is the NFS-Ganesha container
|
||||
pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, *ganeshaMount)
|
||||
|
||||
configVolSrc := nfs.Spec.Security.Kerberos.ConfigFiles.VolumeSource
|
||||
if configVolSrc != nil {
|
||||
vol, mnt := kerberosConfigFilesVolAndMount(*configVolSrc)
|
||||
|
||||
pod.Volumes = append(pod.Volumes, vol)
|
||||
pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, mnt)
|
||||
}
|
||||
|
||||
keytabVolSrc := nfs.Spec.Security.Kerberos.KeytabFile.VolumeSource
|
||||
if keytabVolSrc != nil {
|
||||
vol, mnt := keytabVolAndMount(*keytabVolSrc)
|
||||
|
||||
pod.Volumes = append(pod.Volumes, vol)
|
||||
pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, mnt)
|
||||
}
|
||||
}
|
||||
|
||||
func generateSssdSidecarResources(sidecarCfg *cephv1.SSSDSidecar) (
|
||||
init *v1.Container,
|
||||
sidecar *v1.Container,
|
||||
@@ -109,7 +134,7 @@ func generateSssdSidecarResources(sidecarCfg *cephv1.SSSDSidecar) (
|
||||
|
||||
volSource := sidecarCfg.SSSDConfigFile.VolumeSource
|
||||
if volSource != nil {
|
||||
vol, mount := configVolAndMount(*volSource)
|
||||
vol, mount := sssdConfigVolAndMount(*volSource)
|
||||
|
||||
volumes = append(volumes, vol)
|
||||
sssdMounts = append(sssdMounts, mount)
|
||||
@@ -153,7 +178,51 @@ ls --all --recursive /tmp/var/lib/sss/pipes`,
|
||||
return init, sidecar, volumes, ganeshaMounts
|
||||
}
|
||||
|
||||
func configVolAndMount(volSource v1.VolumeSource) (v1.Volume, v1.VolumeMount) {
|
||||
func generateKrbConfResources(r *ReconcileCephNFS, nfs *cephv1.CephNFS) (
|
||||
init *v1.Container,
|
||||
volume *v1.Volume, // add these volumes to the pod
|
||||
ganeshaMount *v1.VolumeMount, // add these volume mounts to the nfs-ganesha container
|
||||
) {
|
||||
generatedKrbConfVolName := "generated-krb5-conf"
|
||||
|
||||
volume = &v1.Volume{
|
||||
Name: generatedKrbConfVolName,
|
||||
VolumeSource: v1.VolumeSource{
|
||||
EmptyDir: &v1.EmptyDirVolumeSource{},
|
||||
},
|
||||
}
|
||||
ganeshaMount = &v1.VolumeMount{
|
||||
Name: generatedKrbConfVolName,
|
||||
MountPath: "/etc/krb5.conf",
|
||||
SubPath: "krb5.conf",
|
||||
}
|
||||
|
||||
// the init container is needed to copy the starting content from the /var/lib/sss/pipes
|
||||
// directory into the shared sockets dir so that SSSD has the content it needs to start up
|
||||
init = &v1.Container{
|
||||
Name: "generate-krb5-conf",
|
||||
Command: []string{
|
||||
"bash", "-c",
|
||||
`set -ex
|
||||
cat << EOF > /tmp/etc/krb5.conf
|
||||
[logging]
|
||||
default = STDERR
|
||||
|
||||
includedir /etc/krb5.conf.rook/
|
||||
EOF
|
||||
cat /tmp/etc/krb5.conf`,
|
||||
},
|
||||
VolumeMounts: []v1.VolumeMount{
|
||||
{Name: generatedKrbConfVolName, MountPath: "/tmp/etc"},
|
||||
},
|
||||
Image: r.cephClusterSpec.CephVersion.Image,
|
||||
Resources: nfs.Spec.Server.Resources,
|
||||
}
|
||||
|
||||
return init, volume, ganeshaMount
|
||||
}
|
||||
|
||||
func sssdConfigVolAndMount(volSource v1.VolumeSource) (v1.Volume, v1.VolumeMount) {
|
||||
volName := "sssd-conf"
|
||||
vol := v1.Volume{
|
||||
Name: volName,
|
||||
@@ -212,3 +281,32 @@ cat /tmp/etc/nsswitch.conf`,
|
||||
|
||||
return init, podVol, nfsGaneshaContainerMount
|
||||
}
|
||||
|
||||
func kerberosConfigFilesVolAndMount(volSource v1.VolumeSource) (v1.Volume, v1.VolumeMount) {
|
||||
volName := "krb5-conf-d"
|
||||
vol := v1.Volume{
|
||||
Name: volName,
|
||||
VolumeSource: volSource,
|
||||
}
|
||||
mount := v1.VolumeMount{
|
||||
Name: volName,
|
||||
MountPath: "/etc/krb5.conf.rook/",
|
||||
}
|
||||
|
||||
return vol, mount
|
||||
}
|
||||
|
||||
func keytabVolAndMount(volSource v1.VolumeSource) (v1.Volume, v1.VolumeMount) {
|
||||
volName := "krb5-keytab"
|
||||
vol := v1.Volume{
|
||||
Name: volName,
|
||||
VolumeSource: volSource,
|
||||
}
|
||||
mount := v1.VolumeMount{
|
||||
Name: volName,
|
||||
MountPath: "/etc/krb5.keytab",
|
||||
SubPath: "krb5.keytab",
|
||||
}
|
||||
|
||||
return vol, mount
|
||||
}
|
||||
@@ -127,22 +127,6 @@ func TestReconcileCephNFS_addSecurityConfigsToPod(t *testing.T) {
|
||||
assert.Empty(t, pod)
|
||||
})
|
||||
|
||||
t.Run("security.sssd empty should error", func(t *testing.T) {
|
||||
nfs := baseCephNFS(name, namespace)
|
||||
nfs.Spec.Security = &cephv1.NFSSecuritySpec{
|
||||
SSSD: &cephv1.SSSDSpec{},
|
||||
}
|
||||
|
||||
pod := &v1.PodSpec{}
|
||||
err := mockReconcile().addSecurityConfigsToPod(nfs, pod)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to set up security for CephNFS")
|
||||
assert.Contains(t, err.Error(), // verify the security spec calls the Validate() method
|
||||
"System Security Services Daemon (SSSD) is enabled, but no runtime option is specified")
|
||||
assert.Empty(t, pod)
|
||||
})
|
||||
|
||||
t.Run("security.sssd.sidecar with configMap", func(t *testing.T) {
|
||||
// this test is long but essentially makes sure that the sssd sidecar is added, including
|
||||
// any additional init containers needed, verifies that the right container images are used
|
||||
|
||||
@@ -17,6 +17,8 @@ limitations under the License.
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
||||
cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
|
||||
@@ -128,6 +130,15 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
|
||||
},
|
||||
HostNetwork: r.cephClusterSpec.Network.IsHost(),
|
||||
PriorityClassName: nfs.Spec.Server.PriorityClassName,
|
||||
// for kerberos, nfs-ganesha uses the hostname via getaddrinfo() and uses that when
|
||||
// connecting to the krb server. give all ganesha servers the same hostname so they can all
|
||||
// use the same krb credentials to auth
|
||||
Hostname: fmt.Sprintf("%s-%s", nfs.Namespace, nfs.Name),
|
||||
DNSConfig: &v1.PodDNSConfig{
|
||||
// for getaddrinfo() to get the hostname defined above, need to add localhost to
|
||||
// searches in resolv.conf
|
||||
Searches: []string{"localhost"},
|
||||
},
|
||||
}
|
||||
// Replace default unreachable node toleration
|
||||
k8sutil.AddUnreachableNodeToleration(&podSpec)
|
||||
|
||||
+233
-112
@@ -36,6 +36,65 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
func newDeploymentSpecTest(t *testing.T) (*ReconcileCephNFS, daemonConfig) {
|
||||
clientset := optest.New(t, 1)
|
||||
c := &clusterd.Context{
|
||||
Executor: &exectest.MockExecutor{},
|
||||
RookClientset: rookclient.NewSimpleClientset(),
|
||||
Clientset: clientset,
|
||||
}
|
||||
|
||||
s := scheme.Scheme
|
||||
object := []runtime.Object{&cephv1.CephNFS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
RADOS: cephv1.GaneshaRADOSSpec{
|
||||
Pool: "foo",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
|
||||
r := &ReconcileCephNFS{
|
||||
client: cl,
|
||||
scheme: scheme.Scheme,
|
||||
context: c,
|
||||
clusterInfo: &cephclient.ClusterInfo{
|
||||
FSID: "myfsid",
|
||||
CephVersion: cephver.Quincy,
|
||||
},
|
||||
cephClusterSpec: &cephv1.ClusterSpec{
|
||||
CephVersion: cephv1.CephVersionSpec{
|
||||
Image: "quay.io/ceph/ceph:v17",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
id := "i"
|
||||
configName := "rook-ceph-nfs-my-nfs-i"
|
||||
cfg := daemonConfig{
|
||||
ID: id,
|
||||
ConfigConfigMap: configName,
|
||||
ConfigConfigMapHash: "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101",
|
||||
DataPathMap: &config.DataPathMap{
|
||||
HostDataDir: "", // nfs daemon does not store data on host, ...
|
||||
ContainerDataDir: cephclient.DefaultConfigDir, // does share data in containers using emptyDir, ...
|
||||
HostLogAndCrashDir: "", // and does not log to /var/log/ceph dir nor creates crash dumps
|
||||
},
|
||||
}
|
||||
|
||||
return r, cfg
|
||||
}
|
||||
|
||||
func TestDeploymentSpec(t *testing.T) {
|
||||
t.Run("basic", func(t *testing.T) {
|
||||
nfs := &cephv1.CephNFS{
|
||||
@@ -65,60 +124,7 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
clientset := optest.New(t, 1)
|
||||
c := &clusterd.Context{
|
||||
Executor: &exectest.MockExecutor{},
|
||||
RookClientset: rookclient.NewSimpleClientset(),
|
||||
Clientset: clientset,
|
||||
}
|
||||
|
||||
s := scheme.Scheme
|
||||
object := []runtime.Object{&cephv1.CephNFS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
RADOS: cephv1.GaneshaRADOSSpec{
|
||||
Pool: "foo",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
|
||||
r := &ReconcileCephNFS{
|
||||
client: cl,
|
||||
scheme: scheme.Scheme,
|
||||
context: c,
|
||||
clusterInfo: &cephclient.ClusterInfo{
|
||||
FSID: "myfsid",
|
||||
CephVersion: cephver.Quincy,
|
||||
},
|
||||
cephClusterSpec: &cephv1.ClusterSpec{
|
||||
CephVersion: cephv1.CephVersionSpec{
|
||||
Image: "quay.io/ceph/ceph:v15",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
id := "i"
|
||||
configName := "rook-ceph-nfs-my-nfs-i"
|
||||
cfg := daemonConfig{
|
||||
ID: id,
|
||||
ConfigConfigMap: configName,
|
||||
ConfigConfigMapHash: "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101",
|
||||
DataPathMap: &config.DataPathMap{
|
||||
HostDataDir: "", // nfs daemon does not store data on host, ...
|
||||
ContainerDataDir: cephclient.DefaultConfigDir, // does share data in containers using emptyDir, ...
|
||||
HostLogAndCrashDir: "", // and does not log to /var/log/ceph dir nor creates crash dumps
|
||||
},
|
||||
}
|
||||
r, cfg := newDeploymentSpecTest(t)
|
||||
|
||||
d, err := r.makeDeployment(nfs, cfg)
|
||||
assert.NoError(t, err)
|
||||
@@ -148,10 +154,6 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
RADOS: cephv1.GaneshaRADOSSpec{
|
||||
Pool: "myfs-data0",
|
||||
Namespace: "nfs-test-ns",
|
||||
},
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 3,
|
||||
Resources: v1.ResourceRequirements{
|
||||
@@ -191,60 +193,7 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
clientset := optest.New(t, 1)
|
||||
c := &clusterd.Context{
|
||||
Executor: &exectest.MockExecutor{},
|
||||
RookClientset: rookclient.NewSimpleClientset(),
|
||||
Clientset: clientset,
|
||||
}
|
||||
|
||||
s := scheme.Scheme
|
||||
object := []runtime.Object{&cephv1.CephNFS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
},
|
||||
TypeMeta: controllerTypeMeta,
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
RADOS: cephv1.GaneshaRADOSSpec{
|
||||
Pool: "foo",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
|
||||
|
||||
r := &ReconcileCephNFS{
|
||||
client: cl,
|
||||
scheme: scheme.Scheme,
|
||||
context: c,
|
||||
clusterInfo: &cephclient.ClusterInfo{
|
||||
FSID: "myfsid",
|
||||
CephVersion: cephver.Quincy,
|
||||
},
|
||||
cephClusterSpec: &cephv1.ClusterSpec{
|
||||
CephVersion: cephv1.CephVersionSpec{
|
||||
Image: "quay.io/ceph/ceph:v15",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
id := "i"
|
||||
configName := "rook-ceph-nfs-my-nfs-i"
|
||||
cfg := daemonConfig{
|
||||
ID: id,
|
||||
ConfigConfigMap: configName,
|
||||
ConfigConfigMapHash: "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101",
|
||||
DataPathMap: &config.DataPathMap{
|
||||
HostDataDir: "", // nfs daemon does not store data on host, ...
|
||||
ContainerDataDir: cephclient.DefaultConfigDir, // does share data in containers using emptyDir, ...
|
||||
HostLogAndCrashDir: "", // and does not log to /var/log/ceph dir nor creates crash dumps
|
||||
},
|
||||
}
|
||||
r, cfg := newDeploymentSpecTest(t)
|
||||
|
||||
d, err := r.makeDeployment(nfs, cfg)
|
||||
assert.NoError(t, err)
|
||||
@@ -281,4 +230,176 @@ func TestDeploymentSpec(t *testing.T) {
|
||||
assert.Contains(t, contNames, "sssd")
|
||||
})
|
||||
|
||||
t.Run("with kerberos", func(t *testing.T) {
|
||||
nfs := &cephv1.CephNFS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nfs",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 3,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
},
|
||||
Security: &cephv1.NFSSecuritySpec{
|
||||
Kerberos: &cephv1.KerberosSpec{
|
||||
ConfigFiles: cephv1.KerberosConfigFiles{
|
||||
VolumeSource: &v1.VolumeSource{
|
||||
ConfigMap: &v1.ConfigMapVolumeSource{},
|
||||
},
|
||||
},
|
||||
KeytabFile: cephv1.KerberosKeytabFile{
|
||||
VolumeSource: &v1.VolumeSource{
|
||||
Secret: &v1.SecretVolumeSource{},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
r, cfg := newDeploymentSpecTest(t)
|
||||
|
||||
d, err := r.makeDeployment(nfs, cfg)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, d.Spec.Template.Annotations)
|
||||
assert.Equal(t, "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101", d.Spec.Template.Annotations["config-hash"])
|
||||
|
||||
// Deployment should have Ceph labels
|
||||
optest.AssertLabelsContainRookRequirements(t, d.ObjectMeta.Labels, AppName)
|
||||
|
||||
// this also ensures that the things added for the Kerberos don't include duplicate
|
||||
// volumes/mounts and that all volumes/mounts have a corresponding mount/volume
|
||||
podTemplate := optest.NewPodTemplateSpecTester(t, &d.Spec.Template)
|
||||
podTemplate.RunFullSuite(
|
||||
AppName,
|
||||
optest.ResourceLimitExpectations{
|
||||
CPUResourceLimit: "500",
|
||||
MemoryResourceLimit: "1Ki",
|
||||
CPUResourceRequest: "200",
|
||||
MemoryResourceRequest: "512",
|
||||
},
|
||||
)
|
||||
|
||||
initNames := []string{}
|
||||
for _, init := range d.Spec.Template.Spec.InitContainers {
|
||||
initNames = append(initNames, init.Name)
|
||||
}
|
||||
assert.Contains(t, initNames, "generate-krb5-conf")
|
||||
|
||||
contNames := []string{}
|
||||
for _, cont := range d.Spec.Template.Spec.Containers {
|
||||
contNames = append(contNames, cont.Name)
|
||||
}
|
||||
assert.NotContains(t, contNames, "sssd")
|
||||
})
|
||||
|
||||
t.Run("with sssd sidecar and kerberos", func(t *testing.T) {
|
||||
nfs := &cephv1.CephNFS{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-nfs",
|
||||
Namespace: "rook-ceph-test-ns",
|
||||
},
|
||||
Spec: cephv1.NFSGaneshaSpec{
|
||||
RADOS: cephv1.GaneshaRADOSSpec{
|
||||
Pool: "myfs-data0",
|
||||
Namespace: "nfs-test-ns",
|
||||
},
|
||||
Server: cephv1.GaneshaServerSpec{
|
||||
Active: 3,
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
},
|
||||
Security: &cephv1.NFSSecuritySpec{
|
||||
Kerberos: &cephv1.KerberosSpec{
|
||||
ConfigFiles: cephv1.KerberosConfigFiles{
|
||||
VolumeSource: &v1.VolumeSource{
|
||||
ConfigMap: &v1.ConfigMapVolumeSource{},
|
||||
},
|
||||
},
|
||||
KeytabFile: cephv1.KerberosKeytabFile{
|
||||
VolumeSource: &v1.VolumeSource{
|
||||
Secret: &v1.SecretVolumeSource{},
|
||||
},
|
||||
},
|
||||
},
|
||||
SSSD: &cephv1.SSSDSpec{
|
||||
Sidecar: &cephv1.SSSDSidecar{
|
||||
Image: "quay.io/sssd/sssd:latest",
|
||||
DebugLevel: 6,
|
||||
SSSDConfigFile: cephv1.SSSDSidecarConfigFile{
|
||||
VolumeSource: &v1.VolumeSource{
|
||||
ConfigMap: &v1.ConfigMapVolumeSource{},
|
||||
},
|
||||
},
|
||||
Resources: v1.ResourceRequirements{
|
||||
Limits: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI),
|
||||
},
|
||||
Requests: v1.ResourceList{
|
||||
v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI),
|
||||
v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
r, cfg := newDeploymentSpecTest(t)
|
||||
|
||||
d, err := r.makeDeployment(nfs, cfg)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, d.Spec.Template.Annotations)
|
||||
assert.Equal(t, "dcb0d2f5f5e86ec4929d8243cd640b8154165f8ff9b89809964fc7993e9b0101", d.Spec.Template.Annotations["config-hash"])
|
||||
|
||||
// Deployment should have Ceph labels
|
||||
optest.AssertLabelsContainRookRequirements(t, d.ObjectMeta.Labels, AppName)
|
||||
|
||||
// this also ensures that the things added for the SSSD sidecar don't include duplicate
|
||||
// volumes/mounts and that all volumes/mounts have a corresponding mount/volume
|
||||
podTemplate := optest.NewPodTemplateSpecTester(t, &d.Spec.Template)
|
||||
podTemplate.RunFullSuite(
|
||||
AppName,
|
||||
optest.ResourceLimitExpectations{
|
||||
CPUResourceLimit: "500",
|
||||
MemoryResourceLimit: "1Ki",
|
||||
CPUResourceRequest: "200",
|
||||
MemoryResourceRequest: "512",
|
||||
},
|
||||
)
|
||||
|
||||
initNames := []string{}
|
||||
for _, init := range d.Spec.Template.Spec.InitContainers {
|
||||
initNames = append(initNames, init.Name)
|
||||
}
|
||||
assert.Contains(t, initNames, "generate-nsswitch-conf")
|
||||
assert.Contains(t, initNames, "copy-sssd-sockets")
|
||||
assert.Contains(t, initNames, "generate-krb5-conf")
|
||||
|
||||
contNames := []string{}
|
||||
for _, cont := range d.Spec.Template.Spec.Containers {
|
||||
contNames = append(contNames, cont.Name)
|
||||
}
|
||||
assert.Contains(t, contNames, "sssd")
|
||||
})
|
||||
}
|
||||
+15
-5
@@ -34,6 +34,9 @@ import (
|
||||
kexec "k8s.io/utils/exec"
|
||||
)
|
||||
|
||||
// TimeoutWaitingForMessage can be used to identify if an error is due to a timeout.
|
||||
const TimeoutWaitingForMessage = "exec timeout waiting for"
|
||||
|
||||
var (
|
||||
CephCommandsTimeout = 15 * time.Second
|
||||
)
|
||||
@@ -71,6 +74,13 @@ func (*CommandExecutor) ExecuteCommandWithEnv(env []string, command string, arg
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsTimeout returns true if the error is due to a timeout in the exec function. Note that it cannot
|
||||
// determine if a command timed out due to behavior of the command itself; for example if a
|
||||
// '--timeout' flag was passed to the command.
|
||||
func IsTimeout(err error) bool {
|
||||
return strings.Contains(err.Error(), TimeoutWaitingForMessage)
|
||||
}
|
||||
|
||||
// ExecuteCommandWithTimeout starts a process and wait for its completion with timeout.
|
||||
func (*CommandExecutor) ExecuteCommandWithTimeout(timeout time.Duration, command string, arg ...string) (string, error) {
|
||||
logCommand(command, arg...)
|
||||
@@ -95,18 +105,18 @@ func (*CommandExecutor) ExecuteCommandWithTimeout(timeout time.Duration, command
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
if interruptSent {
|
||||
logger.Infof("timeout waiting for process %s to return after interrupt signal was sent. Sending kill signal to the process", command)
|
||||
logger.Infof("%s process %s to return after interrupt signal was sent. Sending kill signal to the process", TimeoutWaitingForMessage, command)
|
||||
var e error
|
||||
if err := cmd.Process.Kill(); err != nil {
|
||||
logger.Errorf("Failed to kill process %s: %v", command, err)
|
||||
e = fmt.Errorf("timeout waiting for the command %s to return after interrupt signal was sent. Tried to kill the process but that failed: %v", command, err)
|
||||
e = fmt.Errorf("%s the command %s to return after interrupt signal was sent. Tried to kill the process but that failed: %v", TimeoutWaitingForMessage, command, err)
|
||||
} else {
|
||||
e = fmt.Errorf("timeout waiting for the command %s to return", command)
|
||||
e = fmt.Errorf("%s the command %s to return", TimeoutWaitingForMessage, command)
|
||||
}
|
||||
return strings.TrimSpace(b.String()), e
|
||||
}
|
||||
|
||||
logger.Infof("timeout waiting for process %s to return. Sending interrupt signal to the process", command)
|
||||
logger.Infof("%s process %s to return. Sending interrupt signal to the process", TimeoutWaitingForMessage, command)
|
||||
if err := cmd.Process.Signal(os.Interrupt); err != nil {
|
||||
logger.Errorf("Failed to send interrupt signal to process %s: %v", command, err)
|
||||
// kill signal will be sent next loop
|
||||
@@ -117,7 +127,7 @@ func (*CommandExecutor) ExecuteCommandWithTimeout(timeout time.Duration, command
|
||||
return strings.TrimSpace(b.String()), err
|
||||
}
|
||||
if interruptSent {
|
||||
return strings.TrimSpace(b.String()), fmt.Errorf("timeout waiting for the command %s to return", command)
|
||||
return strings.TrimSpace(b.String()), fmt.Errorf("%s the command %s to return", TimeoutWaitingForMessage, command)
|
||||
}
|
||||
return strings.TrimSpace(b.String()), nil
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
exectest "github.com/rook/rook/pkg/util/exec/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
kexec "k8s.io/utils/exec"
|
||||
@@ -110,3 +111,8 @@ func TestExtractExitCode(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeTimeoutError(t *testing.T) {
|
||||
assert.True(t, IsTimeout(exectest.FakeTimeoutError("blah")))
|
||||
assert.True(t, IsTimeout(exectest.FakeTimeoutError("")))
|
||||
}
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -126,3 +127,7 @@ func TestMockExecHelperProcess(t *testing.T) {
|
||||
}
|
||||
os.Exit(rc)
|
||||
}
|
||||
|
||||
func FakeTimeoutError(text string) error {
|
||||
return errors.New(fmt.Sprintf("exec timeout waiting for %s", text))
|
||||
}
|
||||
Reference in new issue
Block a user