forked from rook/rook
Santosh
c6836c474a
Support using SSE-S3 encryption with RGW using vault Agent auth. RGW sends requests to the agent instead of directly to Vault, and the agent transparently injects the authentication token. This eliminates using and managing a static token Signed-off-by: Santosh <sapillai@redhat.com>
1289 lines
47 KiB
Go
1289 lines
47 KiB
Go
/*
|
|
Copyright 2018 The Rook Authors. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package object
|
|
|
|
import (
|
|
"bytes"
|
|
_ "embed"
|
|
"fmt"
|
|
"net/url"
|
|
"path"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"text/template"
|
|
|
|
"github.com/hashicorp/vault/api"
|
|
"github.com/libopenstorage/secrets/vault"
|
|
"github.com/pkg/errors"
|
|
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
|
|
"github.com/rook/rook/pkg/daemon/ceph/osd/kms"
|
|
cephconfig "github.com/rook/rook/pkg/operator/ceph/config"
|
|
"github.com/rook/rook/pkg/operator/ceph/controller"
|
|
"github.com/rook/rook/pkg/operator/k8sutil"
|
|
"github.com/rook/rook/pkg/util/log"
|
|
apps "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
discoveryv1 "k8s.io/api/discovery/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/apimachinery/pkg/util/validation"
|
|
"k8s.io/utils/ptr"
|
|
)
|
|
|
|
const (
|
|
serviceAccountName = "rook-ceph-rgw"
|
|
sseKMS = "ssekms"
|
|
sseS3 = "sses3"
|
|
vaultPrefix = "/v1/"
|
|
|
|
// Read Affinity settings for RGW clients to reduce cross-zone traffic
|
|
radosReadReplicaPolicy = "rados_replica_read_policy"
|
|
// read from a random OSD from the PG's active set
|
|
balanceReadReplicaPolicy = "balance"
|
|
// read from the primary OSD
|
|
defaultReadReplicaPolicy = "default"
|
|
|
|
//nolint:gosec // since this is not leaking any hardcoded details
|
|
setupVaultTokenFile = `
|
|
set -e
|
|
|
|
VAULT_TOKEN_OLD_PATH=%s
|
|
VAULT_TOKEN_NEW_PATH=%s
|
|
if [ -d $VAULT_TOKEN_OLD_PATH/ssekms ]
|
|
then
|
|
cp --recursive --verbose $VAULT_TOKEN_OLD_PATH/ssekms/..data/. $VAULT_TOKEN_NEW_PATH/ssekms
|
|
chmod --recursive --verbose 400 $VAULT_TOKEN_NEW_PATH/ssekms/*
|
|
chmod --verbose 700 $VAULT_TOKEN_NEW_PATH/ssekms
|
|
fi
|
|
if [ -d $VAULT_TOKEN_OLD_PATH/sses3 ]
|
|
then
|
|
cp --recursive --verbose $VAULT_TOKEN_OLD_PATH/sses3/..data/. $VAULT_TOKEN_NEW_PATH/sses3
|
|
chmod --recursive --verbose 400 $VAULT_TOKEN_NEW_PATH/sses3/*
|
|
chmod --verbose 700 $VAULT_TOKEN_NEW_PATH/sses3
|
|
fi
|
|
chmod --verbose 700 $VAULT_TOKEN_NEW_PATH
|
|
chown --recursive --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
|
|
`
|
|
)
|
|
|
|
var (
|
|
//go:embed rgw-probe.sh
|
|
rgwProbeScriptTemplate string
|
|
|
|
rgwAPIwithoutS3 = []string{"s3website", "swift", "swift_auth", "admin", "sts", "iam", "notifications"}
|
|
|
|
// used to pass Pod name and namespaces to Pod container arguments.
|
|
podNameEnvVars = []v1.EnvVar{
|
|
{
|
|
Name: "POD_NS",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "metadata.namespace",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "POD_NAME",
|
|
ValueFrom: &v1.EnvVarSource{
|
|
FieldRef: &v1.ObjectFieldSelector{
|
|
FieldPath: "metadata.name",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
// Filename to store RGW ops log per pod.
|
|
// Relies on the on environment variables to get the pod name and namespace.
|
|
// podNameEnvVars have to be passed to to log collector container.
|
|
opsLogFilename = "ops-log.$(POD_NS).$(POD_NAME).log"
|
|
opsLogAbsFilename = path.Join(cephconfig.VarLogCephDir, opsLogFilename)
|
|
)
|
|
|
|
type (
|
|
ProbeType string
|
|
ProtocolType string
|
|
)
|
|
|
|
const (
|
|
StartupProbeType ProbeType = "startup"
|
|
ReadinessProbeType ProbeType = "readiness"
|
|
|
|
HTTPProtocol ProtocolType = "HTTP"
|
|
HTTPSProtocol ProtocolType = "HTTPS"
|
|
)
|
|
|
|
type rgwProbeConfig struct {
|
|
ProbeType ProbeType
|
|
|
|
Protocol ProtocolType
|
|
Port string
|
|
Path string
|
|
}
|
|
|
|
// supportsCurlCaBundle returns true if the Ceph version supports CURL_CA_BUNDLE
|
|
// environment variable for CA bundle handling (Tentacle v20.0.0+).
|
|
// TODO: Remove this helper after Ceph Squid is no longer supported
|
|
func supportsCurlCaBundle(c *clusterConfig) bool {
|
|
return c.clusterInfo.CephVersion.IsAtLeastTentacle()
|
|
}
|
|
|
|
func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment, error) {
|
|
pod, err := c.makeRGWPodSpec(rgwConfig)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
strategy := apps.DeploymentStrategy{
|
|
Type: apps.RecreateDeploymentStrategyType,
|
|
}
|
|
// Use the same keyring and have dedicated rgw instances reflected in the service map
|
|
replicas := c.store.Spec.Gateway.Instances
|
|
|
|
strategy.Type = apps.RollingUpdateDeploymentStrategyType
|
|
strategy.RollingUpdate = &apps.RollingUpdateDeployment{
|
|
MaxUnavailable: &intstr.IntOrString{IntVal: int32(1)},
|
|
MaxSurge: &intstr.IntOrString{IntVal: int32(0)},
|
|
}
|
|
d := &apps.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: rgwConfig.ResourceName,
|
|
Namespace: c.store.Namespace,
|
|
Labels: getLabels(c.store.Name, c.store.Namespace, true),
|
|
},
|
|
Spec: apps.DeploymentSpec{
|
|
RevisionHistoryLimit: controller.RevisionHistoryLimit(),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: getLabels(c.store.Name, c.store.Namespace, false),
|
|
},
|
|
Template: pod,
|
|
Replicas: &replicas,
|
|
Strategy: strategy,
|
|
},
|
|
}
|
|
k8sutil.AddRookVersionLabelToDeployment(d)
|
|
c.store.Spec.Gateway.Annotations.ApplyToObjectMeta(&d.ObjectMeta)
|
|
c.store.Spec.Gateway.Labels.ApplyToObjectMeta(&d.ObjectMeta)
|
|
controller.AddCephVersionLabelToDeployment(c.clusterInfo.CephVersion, d)
|
|
|
|
return d, nil
|
|
}
|
|
|
|
func (c *clusterConfig) makeRGWPodSpec(rgwConfig *rgwConfig) (v1.PodTemplateSpec, error) {
|
|
rgwDaemonContainer, err := c.makeDaemonContainer(rgwConfig)
|
|
if err != nil {
|
|
return v1.PodTemplateSpec{}, err
|
|
}
|
|
|
|
hostNetwork := c.store.Spec.IsHostNetwork(c.clusterSpec)
|
|
podSpec := v1.PodSpec{
|
|
InitContainers: []v1.Container{
|
|
c.makeChownInitContainer(rgwConfig),
|
|
},
|
|
Containers: []v1.Container{rgwDaemonContainer},
|
|
RestartPolicy: v1.RestartPolicyAlways,
|
|
Volumes: append(
|
|
controller.DaemonVolumes(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
|
|
c.mimeTypesVolume(),
|
|
),
|
|
HostNetwork: hostNetwork,
|
|
PriorityClassName: c.store.Spec.Gateway.PriorityClassName,
|
|
SecurityContext: &v1.PodSecurityContext{},
|
|
ServiceAccountName: serviceAccountName,
|
|
}
|
|
|
|
if opsLogSidecar := c.store.Spec.Gateway.OpsLogSidecar; opsLogSidecar != nil {
|
|
// Add the side-car container named ops-log
|
|
podSpec.Containers = append(podSpec.Containers,
|
|
*controller.RgwOpsLogSidecarContainer(opsLogFilename,
|
|
c.clusterInfo.Namespace, *c.clusterSpec, podNameEnvVars,
|
|
opsLogSidecar.Resources))
|
|
}
|
|
|
|
// If the log collector is enabled we add the side-car container
|
|
if c.clusterSpec.LogCollector.Enabled {
|
|
shareProcessNamespace := true
|
|
podSpec.ShareProcessNamespace = &shareProcessNamespace
|
|
podSpec.Containers = append(podSpec.Containers,
|
|
*controller.LogCollectorContainer(getDaemonName(rgwConfig),
|
|
c.clusterInfo.Namespace, *c.clusterSpec, podNameEnvVars, opsLogAbsFilename))
|
|
}
|
|
|
|
// Replace default unreachable node toleration
|
|
k8sutil.AddUnreachableNodeToleration(&podSpec)
|
|
|
|
// Set the ssl cert if specified
|
|
if c.store.Spec.Gateway.SecurePort != 0 {
|
|
secretVolSrc, err := c.generateVolumeSourceWithTLSSecret()
|
|
if err != nil {
|
|
return v1.PodTemplateSpec{}, err
|
|
}
|
|
certVol := v1.Volume{
|
|
Name: certVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
Secret: secretVolSrc,
|
|
},
|
|
}
|
|
podSpec.Volumes = append(podSpec.Volumes, certVol)
|
|
}
|
|
// Check custom caBundle provided
|
|
if c.store.Spec.Gateway.CaBundleRef != "" {
|
|
customCaBundleVolSrc, err := c.generateVolumeSourceWithCaBundleSecret()
|
|
if err != nil {
|
|
return v1.PodTemplateSpec{}, err
|
|
}
|
|
customCaBundleVol := v1.Volume{
|
|
Name: caBundleVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
Secret: customCaBundleVolSrc,
|
|
},
|
|
}
|
|
podSpec.Volumes = append(podSpec.Volumes, customCaBundleVol)
|
|
// Determine CA bundle handling method based on Ceph version
|
|
// Newer Ceph versions support CURL_CA_BUNDLE env var (see PR
|
|
// https://github.com/ceph/ceph/pull/44283)
|
|
// Older versions require system CA trust update via init container
|
|
// TODO: Remove this logic after Ceph Squid is no longer supported
|
|
if !supportsCurlCaBundle(c) {
|
|
updatedCaBundleVol := v1.Volume{
|
|
Name: caBundleUpdatedVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
}
|
|
podSpec.Volumes = append(podSpec.Volumes, updatedCaBundleVol)
|
|
podSpec.InitContainers = append(podSpec.InitContainers,
|
|
c.createCaBundleUpdateInitContainer(rgwConfig))
|
|
}
|
|
}
|
|
kmsEnabled, err := c.CheckRGWKMS()
|
|
if err != nil {
|
|
return v1.PodTemplateSpec{}, err
|
|
}
|
|
s3Enabled, err := c.CheckRGWSSES3Enabled()
|
|
if err != nil {
|
|
return v1.PodTemplateSpec{}, err
|
|
}
|
|
if kmsEnabled || s3Enabled {
|
|
v := v1.Volume{
|
|
Name: rgwVaultVolumeName,
|
|
VolumeSource: v1.VolumeSource{
|
|
EmptyDir: &v1.EmptyDirVolumeSource{},
|
|
},
|
|
}
|
|
podSpec.Volumes = append(podSpec.Volumes, v)
|
|
|
|
kmsNeedsToken := kmsEnabled && c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled()
|
|
s3NeedsToken := s3Enabled && c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled()
|
|
if kmsNeedsToken {
|
|
vaultFileVol, _ := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.KeyManagementService.ConnectionDetails,
|
|
c.store.Spec.Security.KeyManagementService.TokenSecretName, sseKMS)
|
|
podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
|
|
}
|
|
if s3NeedsToken {
|
|
vaultFileVol, _ := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails,
|
|
c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName, sseS3)
|
|
podSpec.Volumes = append(podSpec.Volumes, vaultFileVol)
|
|
}
|
|
|
|
// Only add the vault token init container if at least one of KMS or SSE-S3 needs token-based auth.
|
|
// When SSE-S3 uses agent auth, no token init container is needed for that part.
|
|
if kmsNeedsToken || s3NeedsToken {
|
|
podSpec.InitContainers = append(podSpec.InitContainers,
|
|
c.vaultTokenInitContainer(rgwConfig, kmsNeedsToken, s3NeedsToken))
|
|
}
|
|
}
|
|
c.store.Spec.Gateway.Placement.ApplyToPodSpec(&podSpec)
|
|
|
|
// If host networking is not enabled, preferred pod anti-affinity is added to the rgw daemons
|
|
labels := getLabels(c.store.Name, c.store.Namespace, false)
|
|
k8sutil.SetNodeAntiAffinityForPod(&podSpec, c.store.Spec.IsHostNetwork(c.clusterSpec), k8sutil.LabelHostname(), labels, nil)
|
|
|
|
podTemplateSpec := v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: rgwConfig.ResourceName,
|
|
Labels: getLabels(c.store.Name, c.store.Namespace, true),
|
|
},
|
|
Spec: podSpec,
|
|
}
|
|
c.store.Spec.Gateway.Annotations.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
|
c.store.Spec.Gateway.Labels.ApplyToObjectMeta(&podTemplateSpec.ObjectMeta)
|
|
|
|
if hostNetwork {
|
|
podTemplateSpec.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
|
|
} else if c.clusterSpec.Network.IsMultus() {
|
|
if err := k8sutil.ApplyMultus(c.clusterInfo.Namespace, &c.clusterSpec.Network, &podTemplateSpec.ObjectMeta); err != nil {
|
|
return podTemplateSpec, err
|
|
}
|
|
}
|
|
|
|
addVols, addMounts := c.store.Spec.Gateway.AdditionalVolumeMounts.GenerateVolumesAndMounts("/var/rgw/")
|
|
podTemplateSpec.Spec.Volumes = append(podTemplateSpec.Spec.Volumes, addVols...)
|
|
podTemplateSpec.Spec.Containers[0].VolumeMounts = append(podTemplateSpec.Spec.Containers[0].VolumeMounts, addMounts...)
|
|
|
|
return podTemplateSpec, nil
|
|
}
|
|
|
|
func (c *clusterConfig) createCaBundleUpdateInitContainer(rgwConfig *rgwConfig) v1.Container {
|
|
caBundleMount := v1.VolumeMount{Name: caBundleVolumeName, MountPath: caBundleSourceCustomDir, ReadOnly: true}
|
|
volumeMounts := append(controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), caBundleMount)
|
|
updatedCaBundleDir := "/tmp/new-ca-bundle/"
|
|
updatedBundleMount := v1.VolumeMount{Name: caBundleUpdatedVolumeName, MountPath: updatedCaBundleDir, ReadOnly: false}
|
|
volumeMounts = append(volumeMounts, updatedBundleMount)
|
|
return v1.Container{
|
|
Name: "update-ca-bundle-initcontainer",
|
|
Command: []string{"/bin/bash", "-c"},
|
|
// copy all content of caBundleExtractedDir to avoid directory mount itself
|
|
Args: []string{
|
|
fmt.Sprintf("/usr/bin/update-ca-trust extract; cp -rf %s/* %s", caBundleExtractedDir, updatedCaBundleDir),
|
|
},
|
|
Image: c.clusterSpec.CephVersion.Image,
|
|
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
|
VolumeMounts: volumeMounts,
|
|
Resources: c.store.Spec.Gateway.Resources,
|
|
SecurityContext: controller.DefaultContainerSecurityContext(),
|
|
}
|
|
}
|
|
|
|
// The vault token is passed as Secret for rgw container. So it is mounted as read only.
|
|
// RGW has restrictions over vault token file, it should owned by same user (ceph) which
|
|
// rgw daemon runs and all other permission should be nil or zero. Here ownership can be
|
|
// changed with help of FSGroup but in openshift environments for security reasons it has
|
|
// predefined value, so it won't work there. Hence the token file and certs (if present)
|
|
// are copied to other volume from mounted secrets then ownership/permissions are changed
|
|
// accordingly with help of an init container.
|
|
func (c *clusterConfig) vaultTokenInitContainer(rgwConfig *rgwConfig, kmsEnabled, s3Enabled bool) v1.Container {
|
|
var vaultVolMounts []v1.VolumeMount
|
|
|
|
tmpVaultMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
|
|
vaultVolMounts = append(vaultVolMounts, tmpVaultMount)
|
|
if kmsEnabled {
|
|
_, ssekmsVaultVolMount := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.KeyManagementService.ConnectionDetails, "", sseKMS)
|
|
vaultVolMounts = append(vaultVolMounts, ssekmsVaultVolMount)
|
|
}
|
|
if s3Enabled {
|
|
_, sses3VaultVolMount := kms.VaultVolumeAndMountWithCustomName(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, "", sseS3)
|
|
vaultVolMounts = append(vaultVolMounts, sses3VaultVolMount)
|
|
}
|
|
return v1.Container{
|
|
Name: "vault-initcontainer-token-file-setup",
|
|
Command: []string{
|
|
"/bin/bash",
|
|
"-c",
|
|
fmt.Sprintf(setupVaultTokenFile,
|
|
kms.EtcVaultDir, rgwVaultDirName),
|
|
},
|
|
Image: c.clusterSpec.CephVersion.Image,
|
|
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
|
VolumeMounts: append(
|
|
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath), vaultVolMounts...),
|
|
Resources: c.store.Spec.Gateway.Resources,
|
|
SecurityContext: controller.DefaultContainerSecurityContext(),
|
|
}
|
|
}
|
|
|
|
func (c *clusterConfig) makeChownInitContainer(rgwConfig *rgwConfig) v1.Container {
|
|
return controller.ChownCephDataDirsInitContainer(
|
|
*c.DataPathMap,
|
|
c.clusterSpec.CephVersion.Image,
|
|
controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
|
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
|
|
c.store.Spec.Gateway.Resources,
|
|
controller.DefaultContainerSecurityContext(),
|
|
"",
|
|
)
|
|
}
|
|
|
|
func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container, error) {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
|
|
// start the rgw daemon in the foreground
|
|
startupProbe, err := c.defaultStartupProbe()
|
|
if err != nil {
|
|
return v1.Container{}, errors.Wrap(err, "failed to generate default startup probe")
|
|
}
|
|
readinessProbe, err := c.defaultReadinessProbe()
|
|
if err != nil {
|
|
return v1.Container{}, errors.Wrap(err, "failed to generate default readiness probe")
|
|
}
|
|
|
|
container := v1.Container{
|
|
Name: "rgw",
|
|
Image: c.clusterSpec.CephVersion.Image,
|
|
ImagePullPolicy: controller.GetContainerImagePullPolicy(c.clusterSpec.CephVersion.ImagePullPolicy),
|
|
Command: []string{
|
|
"radosgw",
|
|
},
|
|
Args: append(
|
|
controller.DaemonFlags(c.clusterInfo, c.clusterSpec,
|
|
strings.TrimPrefix(generateCephXUser(rgwConfig.ResourceName), "client.")),
|
|
"--foreground",
|
|
cephconfig.NewFlag("rgw frontends", fmt.Sprintf("%s %s", rgwFrontendName, c.portString())),
|
|
cephconfig.NewFlag("rgw-mime-types-file", mimeTypesMountPath()),
|
|
cephconfig.NewFlag("rgw realm", rgwConfig.Realm),
|
|
cephconfig.NewFlag("rgw zonegroup", rgwConfig.ZoneGroup),
|
|
cephconfig.NewFlag("rgw zone", rgwConfig.Zone),
|
|
),
|
|
VolumeMounts: append(
|
|
controller.DaemonVolumeMounts(c.DataPathMap, rgwConfig.ResourceName, c.clusterSpec.DataDirHostPath),
|
|
c.mimeTypesVolumeMount(),
|
|
),
|
|
Env: controller.DaemonEnvVars(c.clusterSpec),
|
|
Resources: c.store.Spec.Gateway.Resources,
|
|
StartupProbe: startupProbe,
|
|
LivenessProbe: noLivenessProbe(),
|
|
ReadinessProbe: readinessProbe,
|
|
SecurityContext: controller.DefaultContainerSecurityContext(),
|
|
WorkingDir: cephconfig.VarLogCephDir,
|
|
}
|
|
|
|
// If the startup probe is enabled
|
|
container = cephconfig.ConfigureStartupProbe(container, c.store.Spec.HealthCheck.StartupProbe)
|
|
// If the readiness probe is enabled
|
|
configureReadinessProbe(&container, c.store.Spec.HealthCheck)
|
|
if c.store.Spec.IsTLSEnabled() {
|
|
// Add a volume mount for the ssl certificate
|
|
mount := v1.VolumeMount{Name: certVolumeName, MountPath: certDir, ReadOnly: true}
|
|
container.VolumeMounts = append(container.VolumeMounts, mount)
|
|
}
|
|
if c.store.Spec.Gateway.CaBundleRef != "" {
|
|
// Use same CA bundle handling method as determined in makeRGWPodSpec
|
|
if supportsCurlCaBundle(c) {
|
|
customCaBundleMount := v1.VolumeMount{Name: caBundleVolumeName, MountPath: caBundleMountPath, ReadOnly: true}
|
|
container.VolumeMounts = append(container.VolumeMounts, customCaBundleMount)
|
|
container.Env = append(container.Env,
|
|
v1.EnvVar{
|
|
// Following PR introduced to specify ssl certificate for RGW
|
|
// admin operations with this CURL_CA_BUNDLE env variable.
|
|
// https://github.com/ceph/ceph/pull/44283
|
|
Name: "CURL_CA_BUNDLE",
|
|
Value: path.Join(caBundleMountPath, caBundleFileName),
|
|
},
|
|
)
|
|
} else {
|
|
// TODO: Remove this logic after Ceph Squid is no longer supported
|
|
updatedBundleMount := v1.VolumeMount{Name: caBundleUpdatedVolumeName, MountPath: caBundleExtractedDir, ReadOnly: true}
|
|
container.VolumeMounts = append(container.VolumeMounts, updatedBundleMount)
|
|
}
|
|
}
|
|
kmsEnabled, err := c.CheckRGWKMS()
|
|
if err != nil {
|
|
log.NamedError(nsName, logger, "failed to enable SSE-KMS. %v", err)
|
|
return v1.Container{}, err
|
|
}
|
|
if kmsEnabled {
|
|
log.NamedDebug(nsName, logger, "enabliing SSE-KMS. %v", c.store.Spec.Security.KeyManagementService)
|
|
container.Args = append(container.Args, c.sseKMSDefaultOptions(kmsEnabled)...)
|
|
if c.store.Spec.Security.KeyManagementService.IsTokenAuthEnabled() {
|
|
container.Args = append(container.Args, c.sseKMSVaultTokenOptions(kmsEnabled)...)
|
|
}
|
|
if c.store.Spec.Security.KeyManagementService.IsTLSEnabled() {
|
|
container.Args = append(container.Args, c.sseKMSVaultTLSOptions(kmsEnabled)...)
|
|
}
|
|
}
|
|
if c.store.Spec.Gateway.OpsLogSidecar != nil {
|
|
container.Env = append(container.Env, podNameEnvVars...)
|
|
container.Args = append(container.Args,
|
|
cephconfig.NewFlag("rgw_enable_ops_log", "true"),
|
|
cephconfig.NewFlag("rgw_ops_log_file_path", opsLogAbsFilename),
|
|
)
|
|
} else {
|
|
// older version of Rook was using mon db to store rgw ops log config params.
|
|
// cleanup previous value in case if ops log was enabled in older version:
|
|
// TODO: only needed in Rook v1.17. remove after
|
|
monStore := cephconfig.GetMonStore(c.context, c.clusterInfo)
|
|
who := generateCephXUser(rgwConfig.ResourceName)
|
|
prevVal, _ := monStore.Get(who, "rgw_enable_ops_log")
|
|
if prevOpsEnabled, _ := strconv.ParseBool(prevVal); prevOpsEnabled {
|
|
err := monStore.Delete(who, "rgw_enable_ops_log")
|
|
if err != nil {
|
|
return v1.Container{}, errors.Wrapf(err, "failed to delete rgw_enable_ops_log from mon db")
|
|
}
|
|
err = monStore.Delete(who, "rgw_ops_log_file_path")
|
|
if err != nil {
|
|
return v1.Container{}, errors.Wrapf(err, "failed to delete rgw_ops_log_file_path from mon db")
|
|
}
|
|
}
|
|
}
|
|
|
|
if flags := buildRGWConfigFlags(c.store); len(flags) != 0 {
|
|
container.Args = append(container.Args, flags...)
|
|
}
|
|
|
|
s3EncryptionEnabled, err := c.CheckRGWSSES3Enabled()
|
|
if err != nil {
|
|
return v1.Container{}, err
|
|
}
|
|
if s3EncryptionEnabled {
|
|
log.NamedDebug(nsName, logger, "enabliing SSE-S3. %v", c.store.Spec.Security.ServerSideEncryptionS3)
|
|
|
|
container.Args = append(container.Args, c.sseS3DefaultOptions(s3EncryptionEnabled)...)
|
|
if c.store.Spec.Security.ServerSideEncryptionS3.IsAgentAuthEnabled() {
|
|
container.Args = append(container.Args, c.sseS3VaultAgentOptions(s3EncryptionEnabled)...)
|
|
} else if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
|
|
container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3EncryptionEnabled)...)
|
|
}
|
|
if c.store.Spec.Security.ServerSideEncryptionS3.IsTLSEnabled() {
|
|
container.Args = append(container.Args, c.sseS3VaultTLSOptions(s3EncryptionEnabled)...)
|
|
}
|
|
}
|
|
|
|
if s3EncryptionEnabled || kmsEnabled {
|
|
vaultVolMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
|
|
container.VolumeMounts = append(container.VolumeMounts, vaultVolMount)
|
|
}
|
|
|
|
hostingOptions, err := c.addDNSNamesToRGWServer()
|
|
if err != nil {
|
|
return v1.Container{}, err
|
|
}
|
|
if hostingOptions != "" {
|
|
container.Args = append(container.Args, hostingOptions)
|
|
}
|
|
|
|
// set RGW read Affinity
|
|
isCrushLocationSet := false
|
|
if c.store.Spec.Gateway.ReadAffinity != nil {
|
|
if c.clusterInfo.CephVersion.IsAtLeastTentacle() {
|
|
container.Args = append(container.Args, cephconfig.NewFlag(radosReadReplicaPolicy, c.store.Spec.Gateway.ReadAffinity.Type))
|
|
// set crush location if `balance` and `default` mode are not set.
|
|
if c.store.Spec.Gateway.ReadAffinity.Type != balanceReadReplicaPolicy && c.store.Spec.Gateway.ReadAffinity.Type != defaultReadReplicaPolicy {
|
|
container.Command = []string{"bash", "-x", "-c", `exec radosgw --crush-location="host=${NODE_NAME//./-}" "$@"`}
|
|
isCrushLocationSet = true
|
|
}
|
|
} else {
|
|
log.NamedWarning(nsName, logger, "can't set RGW read affinity for ceph version below v20 (tentacle)")
|
|
}
|
|
}
|
|
|
|
if !isCrushLocationSet {
|
|
container.Args = append(container.Args, cephconfig.NewFlag("host", controller.ContainerEnvVarReference(k8sutil.PodNameEnvVar)))
|
|
}
|
|
|
|
// user configs are very last arguments so that they override what Rook might be setting before
|
|
for flag, val := range c.store.Spec.Gateway.RgwCommandFlags {
|
|
container.Args = append(container.Args, cephconfig.NewFlag(flag, val))
|
|
}
|
|
|
|
return container, nil
|
|
}
|
|
|
|
// configureReadinessProbe returns the desired readiness probe for a given daemon
|
|
func configureReadinessProbe(container *v1.Container, healthCheck cephv1.ObjectHealthCheckSpec) {
|
|
if ok := healthCheck.ReadinessProbe; ok != nil {
|
|
if !healthCheck.ReadinessProbe.Disabled {
|
|
probe := healthCheck.ReadinessProbe.Probe
|
|
// If the spec value is empty, let's use a default
|
|
if probe != nil {
|
|
// Set the readiness probe on the container to overwrite the default probe created by Rook
|
|
container.ReadinessProbe = cephconfig.GetProbeWithDefaults(probe, container.ReadinessProbe)
|
|
}
|
|
} else {
|
|
container.ReadinessProbe = nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// RGW has internal mechanisms for restarting its processing if it gets stuck. any failures detected
|
|
// in a liveness probe are likely to either (1) be resolved by the RGW internally or (2) be a result
|
|
// of connection issues to the Ceph cluster. in the first case, restarting is unnecessary. in the
|
|
// second case, restarting will only cause more load to the Ceph cluster by causing RGWs to attempt
|
|
// to re-connect, potentially causing more issues with the Ceph cluster. forcing a restart of the
|
|
// RGW is more likely to cause issues than solve them, so do not implement this probe.
|
|
func noLivenessProbe() *v1.Probe {
|
|
return nil
|
|
}
|
|
|
|
func (c *clusterConfig) defaultReadinessProbe() (*v1.Probe, error) {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
probePath, disableProbe := getRGWProbePath(c.store.Spec.Protocols)
|
|
if disableProbe {
|
|
log.NamedInfo(nsName, logger, "disabling startup probe for %q store", c.store.Name)
|
|
return nil, nil
|
|
}
|
|
proto, port := c.endpointInfo()
|
|
cfg := rgwProbeConfig{
|
|
ProbeType: ReadinessProbeType,
|
|
Protocol: proto,
|
|
Port: port.String(),
|
|
Path: probePath,
|
|
}
|
|
script, err := renderProbe(cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
probe := &v1.Probe{
|
|
ProbeHandler: v1.ProbeHandler{
|
|
Exec: &v1.ExecAction{
|
|
Command: []string{
|
|
"bash", "-c", script,
|
|
},
|
|
},
|
|
},
|
|
TimeoutSeconds: 5,
|
|
InitialDelaySeconds: 10,
|
|
// if RGWs aren't responding reliably, remove them from service routing until they are stable
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 3,
|
|
SuccessThreshold: 3, // don't re-add too soon to "flappy" RGWs from being rout-able
|
|
}
|
|
|
|
return probe, nil
|
|
}
|
|
|
|
// getRGWProbePath - returns custom path for RGW probe and returns true if probe should be disabled.
|
|
func getRGWProbePath(protocolSpec cephv1.ProtocolSpec) (path string, disable bool) {
|
|
enabledAPIs := buildRGWEnableAPIsConfigVal(protocolSpec)
|
|
if len(enabledAPIs) == 0 {
|
|
// all apis including s3 are enabled
|
|
// using default s3 Probe
|
|
return "", false
|
|
}
|
|
if slices.Contains(enabledAPIs, "s3") {
|
|
// using default s3 Probe
|
|
return "", false
|
|
}
|
|
if slices.Contains(enabledAPIs, "swift") {
|
|
// using swift api for probe
|
|
// calculate path for swift probe
|
|
prefix := "/swift/"
|
|
if protocolSpec.Swift != nil && protocolSpec.Swift.UrlPrefix != nil && *protocolSpec.Swift.UrlPrefix != "" {
|
|
prefix = *protocolSpec.Swift.UrlPrefix
|
|
if !strings.HasPrefix(prefix, "/") {
|
|
prefix = "/" + prefix
|
|
}
|
|
if !strings.HasSuffix(prefix, "/") {
|
|
prefix += "/"
|
|
}
|
|
}
|
|
prefix += "info"
|
|
return prefix, false
|
|
}
|
|
// both swift and s3 are disabled - disable probe.
|
|
return "", true
|
|
}
|
|
|
|
func (c *clusterConfig) defaultStartupProbe() (*v1.Probe, error) {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
|
|
probePath, disableProbe := getRGWProbePath(c.store.Spec.Protocols)
|
|
if disableProbe {
|
|
log.NamedInfo(nsName, logger, "disabling startup probe for %q store", c.store.Name)
|
|
return nil, nil
|
|
}
|
|
proto, port := c.endpointInfo()
|
|
cfg := rgwProbeConfig{
|
|
ProbeType: StartupProbeType,
|
|
Protocol: proto,
|
|
Port: port.String(),
|
|
Path: probePath,
|
|
}
|
|
|
|
script, err := renderProbe(cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
probe := &v1.Probe{
|
|
ProbeHandler: v1.ProbeHandler{
|
|
Exec: &v1.ExecAction{
|
|
Command: []string{
|
|
"bash", "-c", script,
|
|
},
|
|
},
|
|
},
|
|
TimeoutSeconds: 5,
|
|
InitialDelaySeconds: 10,
|
|
// RGW's default init timeout is 300 seconds; give an extra margin before the pod should be
|
|
// restarted by kubernetes
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 33,
|
|
}
|
|
|
|
return probe, nil
|
|
}
|
|
|
|
func (c *clusterConfig) endpointInfo() (ProtocolType, *intstr.IntOrString) {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
|
|
// The port the liveness probe needs to probe
|
|
// Assume we run on SDN by default
|
|
proto := HTTPProtocol
|
|
port := intstr.FromInt(int(rgwPortInternalPort))
|
|
|
|
// If Host Networking is enabled, the port from the spec must be reflected
|
|
if c.store.Spec.IsHostNetwork(c.clusterSpec) {
|
|
proto = HTTPProtocol
|
|
port = intstr.FromInt(int(c.store.Spec.Gateway.Port))
|
|
}
|
|
|
|
if c.store.Spec.Gateway.Port == 0 && c.store.Spec.IsTLSEnabled() {
|
|
proto = HTTPSProtocol
|
|
port = intstr.FromInt(int(c.store.Spec.Gateway.SecurePort))
|
|
}
|
|
|
|
log.NamedDebug(nsName, logger, "rgw %q probe port is %v", c.store.Namespace+"/"+c.store.Name, port)
|
|
return proto, &port
|
|
}
|
|
|
|
func (c *clusterConfig) generateService(cephObjectStore *cephv1.CephObjectStore) *v1.Service {
|
|
svc := &v1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: instanceName(cephObjectStore.Name),
|
|
Namespace: cephObjectStore.Namespace,
|
|
Labels: getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true),
|
|
},
|
|
}
|
|
|
|
if c.store.Spec.Gateway.Service != nil {
|
|
c.store.Spec.Gateway.Service.Annotations.ApplyToObjectMeta(&svc.ObjectMeta)
|
|
c.store.Spec.Gateway.Service.Labels.ApplyToObjectMeta(&svc.ObjectMeta)
|
|
}
|
|
if c.store.Spec.IsHostNetwork(c.clusterSpec) {
|
|
svc.Spec.ClusterIP = v1.ClusterIPNone
|
|
}
|
|
|
|
_, destPort := c.endpointInfo()
|
|
|
|
// When the cluster is external we must use the same one as the gateways are listening on
|
|
if cephObjectStore.Spec.IsExternal() {
|
|
destPort.IntVal = cephObjectStore.Spec.Gateway.Port
|
|
} else {
|
|
// If the cluster is not external we add the Selector
|
|
svc.Spec = v1.ServiceSpec{
|
|
Selector: getLabels(cephObjectStore.Name, cephObjectStore.Namespace, false),
|
|
}
|
|
}
|
|
|
|
addPort(svc, "http", cephObjectStore.Spec.Gateway.Port, destPort.IntVal)
|
|
addPort(svc, "https", cephObjectStore.Spec.Gateway.SecurePort, cephObjectStore.Spec.Gateway.SecurePort)
|
|
|
|
return svc
|
|
}
|
|
|
|
func (c *clusterConfig) generateEndpoint(cephObjectStore *cephv1.CephObjectStore) (*discoveryv1.EndpointSlice, error) {
|
|
labels := getLabels(cephObjectStore.Name, cephObjectStore.Namespace, true)
|
|
|
|
// Convert to string addresses
|
|
k8sEndpointAddrs := []string{}
|
|
for _, rookEndpoint := range cephObjectStore.Spec.Gateway.ExternalRgwEndpoints {
|
|
k8sEndpointAddrs = append(k8sEndpointAddrs, rookEndpoint.IP)
|
|
}
|
|
|
|
addressType, err := k8sutil.GetIpAddressType(k8sEndpointAddrs)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to get the addressType")
|
|
}
|
|
|
|
endpoints := &discoveryv1.EndpointSlice{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: instanceName(cephObjectStore.Name),
|
|
Namespace: cephObjectStore.Namespace,
|
|
Labels: labels,
|
|
},
|
|
AddressType: addressType,
|
|
Endpoints: []discoveryv1.Endpoint{
|
|
{
|
|
Addresses: k8sEndpointAddrs,
|
|
Conditions: discoveryv1.EndpointConditions{
|
|
Ready: ptr.To(true),
|
|
},
|
|
},
|
|
},
|
|
Ports: []discoveryv1.EndpointPort{},
|
|
}
|
|
|
|
addPortToEndpoint(endpoints, "http", cephObjectStore.Spec.Gateway.Port)
|
|
addPortToEndpoint(endpoints, "https", cephObjectStore.Spec.Gateway.SecurePort)
|
|
|
|
return endpoints, nil
|
|
}
|
|
|
|
func (c *clusterConfig) reconcileExternalEndpoint(cephObjectStore *cephv1.CephObjectStore) error {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
log.NamedInfo(nsName, logger, "reconciling external object store service")
|
|
|
|
endpoint, err := c.generateEndpoint(cephObjectStore)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to generate the endpoint %q", endpoint.Name)
|
|
}
|
|
// Set owner ref to the parent object
|
|
err = c.ownerInfo.SetControllerReference(endpoint)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to set owner reference to ceph object store endpoint %q", endpoint.Name)
|
|
}
|
|
|
|
_, err = k8sutil.CreateOrUpdateEndpointSlice(c.clusterInfo.Context, c.context.Clientset, cephObjectStore.Namespace, endpoint)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create or update object store %q endpoint", cephObjectStore.Name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *clusterConfig) reconcileService(store *cephv1.CephObjectStore) error {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
service := c.generateService(store)
|
|
// Set owner ref to the parent object
|
|
err := c.ownerInfo.SetControllerReference(service)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to set owner reference to ceph object store service %q", service.Name)
|
|
}
|
|
|
|
svc, err := k8sutil.CreateOrUpdateService(c.clusterInfo.Context, c.context.Clientset, store.Namespace, service)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "failed to create or update object store %q service", store.Name)
|
|
}
|
|
|
|
log.NamedInfo(nsName, logger, "ceph object store gateway service running at %s", svc.Spec.ClusterIP)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *clusterConfig) vaultPrefixRGW() string {
|
|
secretEngine := c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]
|
|
var vaultPrefixPath string
|
|
switch secretEngine {
|
|
case kms.VaultKVSecretEngineKey:
|
|
vaultPrefixPath = path.Join(vaultPrefix,
|
|
c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendPathKey], "/data")
|
|
case kms.VaultTransitSecretEngineKey:
|
|
vaultPrefixPath = path.Join(vaultPrefix, secretEngine)
|
|
}
|
|
|
|
return vaultPrefixPath
|
|
}
|
|
|
|
func (c *clusterConfig) CheckRGWKMS() (bool, error) {
|
|
nsName := controller.NsName(c.store.Namespace, c.store.Name)
|
|
|
|
if c.store.Spec.Security != nil && c.store.Spec.Security.KeyManagementService.IsEnabled() {
|
|
err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, &c.store.Spec.Security.KeyManagementService, c.store.Namespace)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
secretEngine := c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]
|
|
|
|
// currently RGW supports kv(version 2) and transit secret engines in vault for sse:kms
|
|
switch secretEngine {
|
|
case kms.VaultKVSecretEngineKey:
|
|
kvVers := c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey]
|
|
if kvVers != "" {
|
|
if kvVers != "v2" {
|
|
return false, errors.New("failed to validate vault kv version, only v2 is supported")
|
|
}
|
|
} else {
|
|
// If VAUL_BACKEND is not specified let's assume it's v2
|
|
log.NamedWarning(nsName, logger, "%s is not set, assuming the only supported version 2", vault.VaultBackendKey)
|
|
c.store.Spec.Security.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = "v2"
|
|
}
|
|
return true, nil
|
|
case kms.VaultTransitSecretEngineKey:
|
|
return true, nil
|
|
default:
|
|
return false, errors.New("failed to validate vault secret engine")
|
|
|
|
}
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
func (c *clusterConfig) CheckRGWSSES3Enabled() (bool, error) {
|
|
if c.store.Spec.Security != nil && c.store.Spec.Security.ServerSideEncryptionS3.IsEnabled() {
|
|
// When agent auth is used, tokenSecretName must not be set
|
|
if kms.GetParam(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, vault.AuthMethod) == "agent" &&
|
|
c.store.Spec.Security.ServerSideEncryptionS3.TokenSecretName != "" {
|
|
return false, errors.New("tokenSecretName must not be set when VAULT_AUTH_METHOD is agent for SSE-S3")
|
|
}
|
|
|
|
err := kms.ValidateConnectionDetails(c.clusterInfo.Context, c.context, &c.store.Spec.Security.ServerSideEncryptionS3, c.store.Namespace)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
// currently RGW supports only transit secret engines in vault for sse:s3
|
|
if c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey] != kms.VaultTransitSecretEngineKey {
|
|
return false, errors.New("vault secret engine is not transit")
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
func addPort(service *v1.Service, name string, port, destPort int32) {
|
|
if port == 0 || destPort == 0 {
|
|
return
|
|
}
|
|
service.Spec.Ports = append(service.Spec.Ports, v1.ServicePort{
|
|
Name: name,
|
|
Port: port,
|
|
TargetPort: intstr.FromInt(int(destPort)),
|
|
Protocol: v1.ProtocolTCP,
|
|
})
|
|
}
|
|
|
|
func addPortToEndpoint(endpoints *discoveryv1.EndpointSlice, name string, port int32) {
|
|
if port == 0 {
|
|
return
|
|
}
|
|
endpoints.Ports = append(endpoints.Ports, discoveryv1.EndpointPort{
|
|
Name: ptr.To(name),
|
|
Port: ptr.To(port),
|
|
Protocol: ptr.To(v1.ProtocolTCP),
|
|
})
|
|
}
|
|
|
|
func getLabels(name, namespace string, includeNewLabels bool) map[string]string {
|
|
labels := controller.CephDaemonAppLabels(AppName, namespace, cephconfig.RgwType, name, name, "cephobjectstores.ceph.rook.io", includeNewLabels)
|
|
labels["rook_object_store"] = name
|
|
return labels
|
|
}
|
|
|
|
func (c *clusterConfig) generateVolumeSourceWithTLSSecret() (*v1.SecretVolumeSource, error) {
|
|
// Keep the TLS secret as secure as possible in the container. Give only user read perms.
|
|
// Because the Secret mount is owned by "root" and fsGroup breaks on OCP since we cannot predict it
|
|
// Also, we don't want to change the SCC for fsGroup to RunAsAny since it has a major broader impact
|
|
// Let's open the permissions a bit more so that everyone can read the cert.
|
|
userReadOnly := int32(0o444)
|
|
var secretVolSrc *v1.SecretVolumeSource
|
|
if c.store.Spec.Gateway.SSLCertificateRef != "" {
|
|
secretVolSrc = &v1.SecretVolumeSource{
|
|
SecretName: c.store.Spec.Gateway.SSLCertificateRef,
|
|
}
|
|
secretType, err := c.rgwTLSSecretType(c.store.Spec.Gateway.SSLCertificateRef)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
switch secretType {
|
|
case v1.SecretTypeOpaque:
|
|
secretVolSrc.Items = []v1.KeyToPath{
|
|
{Key: certKeyName, Path: certFilename, Mode: &userReadOnly},
|
|
}
|
|
case v1.SecretTypeTLS:
|
|
secretVolSrc.Items = []v1.KeyToPath{
|
|
{Key: v1.TLSCertKey, Path: certFilename, Mode: &userReadOnly},
|
|
{Key: v1.TLSPrivateKeyKey, Path: certKeyFileName, Mode: &userReadOnly},
|
|
}
|
|
}
|
|
} else if c.store.Spec.GetServiceServingCert() != "" {
|
|
secretVolSrc = &v1.SecretVolumeSource{
|
|
SecretName: c.store.Spec.GetServiceServingCert(),
|
|
Items: []v1.KeyToPath{
|
|
{Key: v1.TLSCertKey, Path: certFilename, Mode: &userReadOnly},
|
|
{Key: v1.TLSPrivateKeyKey, Path: certKeyFileName, Mode: &userReadOnly},
|
|
},
|
|
}
|
|
} else {
|
|
return nil, errors.New("no TLS certificates found")
|
|
}
|
|
|
|
return secretVolSrc, nil
|
|
}
|
|
|
|
func (c *clusterConfig) generateVolumeSourceWithCaBundleSecret() (*v1.SecretVolumeSource, error) {
|
|
// RGW runs as 'ceph' user and needs access to the CA bundle.
|
|
readOnly := int32(0o444)
|
|
caBundleVolSrc := &v1.SecretVolumeSource{
|
|
SecretName: c.store.Spec.Gateway.CaBundleRef,
|
|
}
|
|
secretType, err := c.rgwTLSSecretType(c.store.Spec.Gateway.CaBundleRef)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if secretType != v1.SecretTypeOpaque {
|
|
return nil, errors.New("CaBundle secret should be 'Opaque' type")
|
|
}
|
|
caBundleVolSrc.Items = []v1.KeyToPath{
|
|
{Key: caBundleKeyName, Path: caBundleFileName, Mode: &readOnly},
|
|
}
|
|
return caBundleVolSrc, nil
|
|
}
|
|
|
|
func (c *clusterConfig) rgwTLSSecretType(secretName string) (v1.SecretType, error) {
|
|
rgwTlsSecret, err := c.context.Clientset.CoreV1().Secrets(c.clusterInfo.Namespace).Get(c.clusterInfo.Context, secretName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", errors.Wrapf(err, "failed to get Kubernetes secrets referring the TLS certificates")
|
|
}
|
|
return rgwTlsSecret.Type, nil
|
|
}
|
|
|
|
func getDaemonName(rgwConfig *rgwConfig) string {
|
|
return fmt.Sprintf("ceph-%s", generateCephXUser(rgwConfig.ResourceName))
|
|
}
|
|
|
|
// Following apis set the RGW options if requested, since they are used in unit tests for validating different scenarios
|
|
func (c *clusterConfig) sseKMSDefaultOptions(setOptions bool) []string {
|
|
if setOptions {
|
|
return []string{
|
|
cephconfig.NewFlag("rgw crypt s3 kms backend",
|
|
c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.Provider]),
|
|
cephconfig.NewFlag("rgw crypt vault addr",
|
|
c.store.Spec.Security.KeyManagementService.ConnectionDetails[api.EnvVaultAddress]),
|
|
}
|
|
}
|
|
return []string{}
|
|
}
|
|
|
|
func (c *clusterConfig) sseS3DefaultOptions(setOptions bool) []string {
|
|
if setOptions {
|
|
return []string{
|
|
cephconfig.NewFlag("rgw crypt sse s3 backend",
|
|
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.Provider]),
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault addr",
|
|
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[api.EnvVaultAddress]),
|
|
}
|
|
}
|
|
return []string{}
|
|
}
|
|
|
|
func (c *clusterConfig) sseKMSVaultTokenOptions(setOptions bool) []string {
|
|
if setOptions {
|
|
return []string{
|
|
cephconfig.NewFlag("rgw crypt vault auth", kms.KMSTokenSecretNameKey),
|
|
cephconfig.NewFlag("rgw crypt vault token file",
|
|
path.Join(rgwVaultDirName, sseKMS, kms.VaultFileName)),
|
|
cephconfig.NewFlag("rgw crypt vault prefix", c.vaultPrefixRGW()),
|
|
cephconfig.NewFlag("rgw crypt vault secret engine",
|
|
c.store.Spec.Security.KeyManagementService.ConnectionDetails[kms.VaultSecretEngineKey]),
|
|
}
|
|
}
|
|
return []string{}
|
|
}
|
|
|
|
func (c *clusterConfig) sseS3VaultTokenOptions(setOptions bool) []string {
|
|
if setOptions {
|
|
return []string{
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault auth", kms.KMSTokenSecretNameKey),
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault token file",
|
|
path.Join(rgwVaultDirName, sseS3, kms.VaultFileName)),
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault prefix",
|
|
path.Join(vaultPrefix, c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey])),
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault secret engine",
|
|
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey]),
|
|
}
|
|
}
|
|
return []string{}
|
|
}
|
|
|
|
func (c *clusterConfig) sseKMSVaultTLSOptions(setOptions bool) []string {
|
|
var rgwOptions []string
|
|
if setOptions {
|
|
rgwOptions = append(rgwOptions, cephconfig.NewFlag("rgw crypt vault verify ssl", "true"))
|
|
|
|
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientCert) != "" {
|
|
rgwOptions = append(rgwOptions,
|
|
cephconfig.NewFlag("rgw crypt vault ssl clientcert", path.Join(rgwVaultDirName, sseKMS, kms.VaultCertFileName)))
|
|
}
|
|
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultClientKey) != "" {
|
|
rgwOptions = append(rgwOptions,
|
|
cephconfig.NewFlag("rgw crypt vault ssl clientkey", path.Join(rgwVaultDirName, sseKMS, kms.VaultKeyFileName)))
|
|
}
|
|
if kms.GetParam(c.store.Spec.Security.KeyManagementService.ConnectionDetails, api.EnvVaultCACert) != "" {
|
|
rgwOptions = append(rgwOptions,
|
|
cephconfig.NewFlag("rgw crypt vault ssl cacert", path.Join(rgwVaultDirName, sseKMS, kms.VaultCAFileName)))
|
|
}
|
|
}
|
|
return rgwOptions
|
|
}
|
|
|
|
func (c *clusterConfig) sseS3VaultAgentOptions(setOptions bool) []string {
|
|
if setOptions {
|
|
return []string{
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault auth", "agent"),
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault prefix",
|
|
path.Join(vaultPrefix, c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey])),
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault secret engine",
|
|
c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails[kms.VaultSecretEngineKey]),
|
|
}
|
|
}
|
|
return []string{}
|
|
}
|
|
|
|
func (c *clusterConfig) sseS3VaultTLSOptions(setOptions bool) []string {
|
|
var rgwOptions []string
|
|
if setOptions {
|
|
rgwOptions = append(rgwOptions, cephconfig.NewFlag("rgw crypt sse s3 vault verify ssl", "true"))
|
|
|
|
if kms.GetParam(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, api.EnvVaultClientCert) != "" {
|
|
rgwOptions = append(rgwOptions,
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault ssl clientcert", path.Join(rgwVaultDirName, sseS3, kms.VaultCertFileName)))
|
|
}
|
|
if kms.GetParam(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, api.EnvVaultClientKey) != "" {
|
|
rgwOptions = append(rgwOptions,
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault ssl clientkey", path.Join(rgwVaultDirName, sseS3, kms.VaultKeyFileName)))
|
|
}
|
|
if kms.GetParam(c.store.Spec.Security.ServerSideEncryptionS3.ConnectionDetails, api.EnvVaultCACert) != "" {
|
|
rgwOptions = append(rgwOptions,
|
|
cephconfig.NewFlag("rgw crypt sse s3 vault ssl cacert", path.Join(rgwVaultDirName, sseS3, kms.VaultCAFileName)))
|
|
}
|
|
}
|
|
return rgwOptions
|
|
}
|
|
|
|
// Builds list of rgw config parameters which should be passed as CLI flags.
|
|
// Consider set config option as flag if BOTH criteria fulfilled:
|
|
// 1. config value is not secret
|
|
// 2. config change requires RGW daemon restart
|
|
//
|
|
// Otherwise set rgw config parameter to mon database in ./config.go -> setFlagsMonConfigStore()
|
|
// CLI flags override values from mon db: see ceph config docs: https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/#config-sources
|
|
func buildRGWConfigFlags(objectStore *cephv1.CephObjectStore) []string {
|
|
nsName := controller.NsName(objectStore.Namespace, objectStore.Name)
|
|
|
|
var res []string
|
|
// todo: move all flags here
|
|
if enableAPIs := buildRGWEnableAPIsConfigVal(objectStore.Spec.Protocols); len(enableAPIs) != 0 {
|
|
res = append(res, cephconfig.NewFlag("rgw_enable_apis", strings.Join(enableAPIs, ",")))
|
|
log.NamedDebug(nsName, logger, "Enabling APIs for RGW instance %q: %s", objectStore.Name, enableAPIs)
|
|
}
|
|
return res
|
|
}
|
|
|
|
func buildRGWEnableAPIsConfigVal(protocolSpec cephv1.ProtocolSpec) []string {
|
|
if len(protocolSpec.EnableAPIs) != 0 {
|
|
// handle explicit enabledAPIS spec
|
|
enabledAPIs := make([]string, len(protocolSpec.EnableAPIs))
|
|
for i, v := range protocolSpec.EnableAPIs {
|
|
enabledAPIs[i] = strings.TrimSpace(string(v))
|
|
}
|
|
return enabledAPIs
|
|
}
|
|
|
|
// if enabledAPIs not set, check if S3 should be disabled
|
|
if protocolSpec.S3 != nil && protocolSpec.S3.Enabled != nil && !*protocolSpec.S3.Enabled { //nolint // disable deprecation check
|
|
return rgwAPIwithoutS3
|
|
}
|
|
// see also https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings on disabling s3
|
|
// when using '/' as prefix
|
|
if protocolSpec.Swift != nil && protocolSpec.Swift.UrlPrefix != nil && *protocolSpec.Swift.UrlPrefix == "/" {
|
|
logger.Warning("Forcefully disabled S3 as the swift prefix is given as a slash /. Ignoring any S3 options (including Enabled=true)!")
|
|
return rgwAPIwithoutS3
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func renderProbe(cfg rgwProbeConfig) (string, error) {
|
|
var writer bytes.Buffer
|
|
name := string(cfg.ProbeType) + "-probe"
|
|
|
|
t := template.New(name)
|
|
t, err := t.Parse(rgwProbeScriptTemplate)
|
|
if err != nil {
|
|
return "", errors.Wrapf(err, "failed to parse template %q", name)
|
|
}
|
|
|
|
if err := t.Execute(&writer, cfg); err != nil {
|
|
return "", errors.Wrapf(err, "failed to render template %q", name)
|
|
}
|
|
|
|
return writer.String(), nil
|
|
}
|
|
|
|
func (c *clusterConfig) addDNSNamesToRGWServer() (string, error) {
|
|
if c.store.Spec.Hosting == nil {
|
|
return "", nil
|
|
}
|
|
if !c.store.AdvertiseEndpointIsSet() && len(c.store.Spec.Hosting.DNSNames) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
dnsNames := []string{}
|
|
|
|
if c.store.AdvertiseEndpointIsSet() {
|
|
dnsNames = append(dnsNames, c.store.Spec.Hosting.AdvertiseEndpoint.DnsName)
|
|
}
|
|
|
|
dnsNames = append(dnsNames, c.store.Spec.Hosting.DNSNames...)
|
|
|
|
// add default RGW service domain name to ensure RGW doesn't reject it
|
|
dnsNames = append(dnsNames, c.store.GetServiceDomainName())
|
|
|
|
// add custom endpoints from zone spec if exists
|
|
if c.store.Spec.Zone.Name != "" {
|
|
zone, err := c.context.RookClientset.CephV1().CephObjectZones(c.store.Namespace).Get(c.clusterInfo.Context, c.store.Spec.Zone.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
dnsNames = append(dnsNames, zone.Spec.CustomEndpoints...)
|
|
}
|
|
|
|
// validate dns names
|
|
var hostNames []string
|
|
for _, dnsName := range dnsNames {
|
|
hostName, err := GetHostnameFromEndpoint(dnsName)
|
|
if err != nil {
|
|
return "", errors.Wrap(err,
|
|
"failed to interpret endpoint from one of the following sources: CephObjectStore.spec.hosting.dnsNames, CephObjectZone.spec.customEndpoints")
|
|
}
|
|
hostNames = append(hostNames, hostName)
|
|
}
|
|
|
|
// remove duplicate host names
|
|
checkDuplicate := make(map[string]bool)
|
|
removeDuplicateHostNames := []string{}
|
|
for _, hostName := range hostNames {
|
|
if _, ok := checkDuplicate[hostName]; !ok {
|
|
checkDuplicate[hostName] = true
|
|
removeDuplicateHostNames = append(removeDuplicateHostNames, hostName)
|
|
}
|
|
}
|
|
|
|
return cephconfig.NewFlag("rgw dns name", strings.Join(removeDuplicateHostNames, ",")), nil
|
|
}
|
|
|
|
func GetHostnameFromEndpoint(endpoint string) (string, error) {
|
|
if len(endpoint) == 0 {
|
|
return "", fmt.Errorf("endpoint cannot be empty string")
|
|
}
|
|
|
|
// if endpoint doesn't end in '/', Ceph adds it
|
|
// Rook can do this also to get more accurate error results from this function
|
|
if !strings.HasSuffix(endpoint, "/") {
|
|
endpoint = endpoint + "/"
|
|
}
|
|
|
|
// url.Parse() requires a protocol to parse the host name correctly
|
|
if !strings.Contains(endpoint, "://") {
|
|
endpoint = "http://" + endpoint
|
|
}
|
|
|
|
// "net/url".Parse() assumes that a URL is a "path" with optional things surrounding it.
|
|
// For Ceph RGWs, we assume an endpoint is a "hostname" with optional things surrounding it.
|
|
// Because of this difference in fundamental assumption, Rook needs to adjust some endpoints
|
|
// used as input to url.Parse() to allow the function to extract a hostname reliably. Also,
|
|
// Rook needs to look at several parts of the url.Parse() output to identify more failure scenarios
|
|
parsedURL, err := url.Parse(endpoint)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// error in this case: url.Parse("https://http://hostname") parses without error with `Host = "http:"`
|
|
// also catches issue where user adds colon but no port number after
|
|
if strings.HasSuffix(parsedURL.Host, ":") {
|
|
return "", fmt.Errorf("host %q parsed from endpoint %q has invalid colon (:) placement", parsedURL.Host, endpoint)
|
|
}
|
|
|
|
hostname := parsedURL.Hostname()
|
|
dnsErrs := validation.IsDNS1123Subdomain(parsedURL.Hostname())
|
|
if len(dnsErrs) > 0 {
|
|
return "", fmt.Errorf("hostname %q parsed from endpoint %q is not a valid DNS-1123 subdomain: %v", hostname, endpoint, strings.Join(dnsErrs, ", "))
|
|
}
|
|
|
|
return hostname, nil
|
|
}
|